chore: Bump delta-producer-publication-graph-maintainer to 1.0.9-rc2 + enable other producers

This commit is contained in:
Laurent Deketelaere
2024-04-25 15:56:56 +02:00
parent 723527fb62
commit 94baeb539f
5 changed files with 372 additions and 271 deletions

View File

@@ -354,7 +354,7 @@ services:
restart: always
logging: *default-logging
delta-producer-publication-graph-maintainer:
image: lblod/delta-producer-publication-graph-maintainer:1.0.8
image: lblod/delta-producer-publication-graph-maintainer:1.0.9-rc2
environment:
MAX_BODY_SIZE: "50mb"
PRETTY_PRINT_DIFF_JSON: "true"
@@ -362,8 +362,6 @@ services:
PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql"
SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503"
CONFIG_SERVICES_JSON_PATH: '/config/publication-graph-maintainer/config.json'
LOG_DELTA_REWRITE: "true"
LOG_INCOMING_DELTA: "true"
volumes:
- ./config/delta-producer:/config
- ./data/files/:/share
@@ -375,106 +373,55 @@ services:
restart: always
logging: *default-logging
################################################################################
# DELTA ORGANIZATIONS: START
################################################################################
# delta-producer-bg-jobs-initiator-organizations:
# image: lblod/delta-producer-background-jobs-initiator:0.3.1
# environment:
# JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
# DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations"
# INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations"
# HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations"
# CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour
# CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM
# START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts.
# labels:
# - "logging=true"
# links:
# - db:database
# restart: always
# logging: *default-logging
# delta-producer-pub-graph-maintainer-organizations:
# image: lblod/delta-producer-publication-graph-maintainer:0.15.4
# environment:
# MAX_BODY_SIZE: "50mb"
# JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
# RELATIVE_FILE_PATH: "deltas/organizations"
# PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations"
# HEALING_JOB_OPERATION: "http://http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizationsredpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations"
# INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: ""
# QUEUE_POLL_INTERVAL: 3000
# USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
# SKIP_MU_AUTH_INITIAL_SYNC: "true"
# HEALING_PATCH_GRAPH_BATCH_SIZE: 500
# SERVE_DELTA_FILES: "true"
# PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations"
# DELTA_INTERVAL_MS: 10000
# PRETTY_PRINT_DIFF_JSON: "true"
# ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
# volumes:
# - ./config/delta-producer/organizations:/config
# - ./data/files/:/share
# links:
# - db:database
# - triplestore:virtuoso
# restart: always
# labels:
# - "logging=true"
# logging: *default-logging
################################################################################
# DELTA ORGANIZATIONS: END
################################################################################
################################################################################
# DELTA ORGANIZATIONS PUBLIC INFO: START
################################################################################
# delta-producer-bg-jobs-initiator-organizations-public-info:
# image: lblod/delta-producer-background-jobs-initiator:0.3.1
# environment:
# JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
# DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info"
# INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info"
# HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info"
# CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours
# CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM
# START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync
# labels:
# - "logging=true"
# links:
# - db:database
# restart: always
# logging: *default-logging
# delta-producer-pub-graph-maintainer-organizations-public-info:
# image: lblod/delta-producer-publication-graph-maintainer:0.15.4
# environment:
# MAX_BODY_SIZE: "50mb"
# JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
# RELATIVE_FILE_PATH: "deltas/organizations-public-info"
# PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info"
# HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info"
# INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info"
# QUEUE_POLL_INTERVAL: 3000
# USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
# SKIP_MU_AUTH_INITIAL_SYNC: "true"
# HEALING_PATCH_GRAPH_BATCH_SIZE: 500
# SERVE_DELTA_FILES: "true"
# PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info"
# DELTA_INTERVAL_MS: 10000
# PRETTY_PRINT_DIFF_JSON: "true"
# PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql"
# PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql"
# ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
# volumes:
# - ./config/delta-producer/organizations-public-info:/config
# - ./data/files/:/share
# links:
# - db:database
# - triplestore:virtuoso
# restart: always
# labels:
# - "logging=true"
# logging: *default-logging
delta-producer-bg-jobs-initiator-organizations-public-info:
image: lblod/delta-producer-background-jobs-initiator:0.3.1
environment:
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info"
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info"
HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info"
CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours
CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM
START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync
labels:
- "logging=true"
links:
- db:database
restart: always
logging: *default-logging
delta-producer-pub-graph-maintainer-organizations-public-info:
image: lblod/delta-producer-publication-graph-maintainer:0.15.4
environment:
MAX_BODY_SIZE: "50mb"
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
RELATIVE_FILE_PATH: "deltas/organizations-public-info"
PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info"
HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info"
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info"
QUEUE_POLL_INTERVAL: 3000
USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
SKIP_MU_AUTH_INITIAL_SYNC: "true"
HEALING_PATCH_GRAPH_BATCH_SIZE: 500
SERVE_DELTA_FILES: "true"
PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info"
DELTA_INTERVAL_MS: 10000
PRETTY_PRINT_DIFF_JSON: "true"
PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql"
PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql"
ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
volumes:
- ./config/delta-producer/organizations-public-info:/config
- ./data/files/:/share
links:
- db:database
- triplestore:virtuoso
restart: always
labels:
- "logging=true"
logging: *default-logging
################################################################################
# DELTA ORGANIZATIONS PUBLIC INFO: END
################################################################################
@@ -482,50 +429,50 @@ services:
################################################################################
# DELTA PUBLIC INFORMATION: START
################################################################################
# delta-producer-bg-jobs-initiator-public:
# image: lblod/delta-producer-background-jobs-initiator:0.3.1
# environment:
# JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
# DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public"
# INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public"
# HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public"
# CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours
# CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM
# START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts.
# labels:
# - "logging=true"
# links:
# - db:database
# restart: always
# logging: *default-logging
# delta-producer-pub-graph-maintainer-public:
# image: lblod/delta-producer-publication-graph-maintainer:0.15.4
# environment:
# MAX_BODY_SIZE: "50mb"
# JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
# RELATIVE_FILE_PATH: "deltas/public"
# PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public"
# HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public"
# INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public"
# QUEUE_POLL_INTERVAL: 3000
# USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
# SKIP_MU_AUTH_INITIAL_SYNC: "true"
# HEALING_PATCH_GRAPH_BATCH_SIZE: 500
# SERVE_DELTA_FILES: "true"
# PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public"
# DELTA_INTERVAL_MS: 10000
# PRETTY_PRINT_DIFF_JSON: "true"
# ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
# volumes:
# - ./config/delta-producer/public:/config
# - ./data/files/:/share
# links:
# - db:database
# - triplestore:virtuoso
# restart: always
# labels:
# - "logging=true"
# logging: *default-logging
delta-producer-bg-jobs-initiator-public:
image: lblod/delta-producer-background-jobs-initiator:0.3.1
environment:
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public"
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public"
HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public"
CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours
CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM
START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts.
labels:
- "logging=true"
links:
- db:database
restart: always
logging: *default-logging
delta-producer-pub-graph-maintainer-public:
image: lblod/delta-producer-publication-graph-maintainer:0.15.4
environment:
MAX_BODY_SIZE: "50mb"
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
RELATIVE_FILE_PATH: "deltas/public"
PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public"
HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public"
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public"
QUEUE_POLL_INTERVAL: 3000
USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
SKIP_MU_AUTH_INITIAL_SYNC: "true"
HEALING_PATCH_GRAPH_BATCH_SIZE: 500
SERVE_DELTA_FILES: "true"
PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public"
DELTA_INTERVAL_MS: 10000
PRETTY_PRINT_DIFF_JSON: "true"
ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
volumes:
- ./config/delta-producer/public:/config
- ./data/files/:/share
links:
- db:database
- triplestore:virtuoso
restart: always
labels:
- "logging=true"
logging: *default-logging
################################################################################
# DELTA PUBLIC INFORMATION: END
################################################################################
@@ -533,50 +480,50 @@ services:
################################################################################
# DELTA WORSHIP POSTS: START
################################################################################
# delta-producer-bg-jobs-initiator-worship-posts:
# image: lblod/delta-producer-background-jobs-initiator:0.3.1
# environment:
# JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
# DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts"
# INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts"
# HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts"
# CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours
# CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM
# START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts.
# labels:
# - "logging=true"
# links:
# - db:database
# restart: always
# logging: *default-logging
# delta-producer-pub-graph-maintainer-worship-posts:
# image: lblod/delta-producer-publication-graph-maintainer:0.15.4
# environment:
# MAX_BODY_SIZE: "50mb"
# JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
# RELATIVE_FILE_PATH: "deltas/worship-posts"
# PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts"
# HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts"
# INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts"
# QUEUE_POLL_INTERVAL: 3000
# USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
# SKIP_MU_AUTH_INITIAL_SYNC: "true"
# HEALING_PATCH_GRAPH_BATCH_SIZE: 500
# SERVE_DELTA_FILES: "true"
# PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts"
# DELTA_INTERVAL_MS: 10000
# PRETTY_PRINT_DIFF_JSON: "true"
# ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
# volumes:
# - ./config/delta-producer/worship-posts:/config
# - ./data/files/:/share
# links:
# - db:database
# - triplestore:virtuoso
# restart: always
# labels:
# - "logging=true"
# logging: *default-logging
delta-producer-bg-jobs-initiator-worship-posts:
image: lblod/delta-producer-background-jobs-initiator:0.3.1
environment:
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts"
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts"
HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts"
CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours
CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM
START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts.
labels:
- "logging=true"
links:
- db:database
restart: always
logging: *default-logging
delta-producer-pub-graph-maintainer-worship-posts:
image: lblod/delta-producer-publication-graph-maintainer:0.15.4
environment:
MAX_BODY_SIZE: "50mb"
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
RELATIVE_FILE_PATH: "deltas/worship-posts"
PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts"
HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts"
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts"
QUEUE_POLL_INTERVAL: 3000
USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
SKIP_MU_AUTH_INITIAL_SYNC: "true"
HEALING_PATCH_GRAPH_BATCH_SIZE: 500
SERVE_DELTA_FILES: "true"
PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts"
DELTA_INTERVAL_MS: 10000
PRETTY_PRINT_DIFF_JSON: "true"
ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
volumes:
- ./config/delta-producer/worship-posts:/config
- ./data/files/:/share
links:
- db:database
- triplestore:virtuoso
restart: always
labels:
- "logging=true"
logging: *default-logging
################################################################################
# DELTA WORSHIP POSTS: END
################################################################################