diff --git a/config/delta-producer/background-job-initiator/config.json b/config/delta-producer/background-job-initiator/config.json new file mode 100644 index 0000000..379c1c2 --- /dev/null +++ b/config/delta-producer/background-job-initiator/config.json @@ -0,0 +1,12 @@ +[ + { + "name": "organizations", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations", + "cronPatternDumpJob": "42 * * * *", + "cronPatternHealingJob": "0 0 4 * * *", + "startInitialSync": true + } +] \ No newline at end of file diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index bc7981c..8babcc4 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -3,7 +3,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", - "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true @@ -12,7 +12,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", - "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true diff --git a/config/delta-producer/publication-graph-maintainer/config.json b/config/delta-producer/publication-graph-maintainer/config.json new file mode 100644 index 0000000..a7d0a14 --- /dev/null +++ b/config/delta-producer/publication-graph-maintainer/config.json @@ -0,0 +1,25 @@ +{ + "organizations": { + "deltaInterval": 10000, + "deltaPath": "/organizations/delta", + "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations", + "exportConfigPath": "/config/organizations/export.json", + "filesPath": "/organizations/files", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", + "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations", + "queuePollInterval": 3000, + "relativeFilePath": "/deltas/organizations", + "serveDeltaFiles": true, + "skipMuAuthInitialSync": true, + "skipMuAuthHealing": true, + "useFileDiff": true, + "healingInitialBatchSizeInsert": 1000, + "useVirtuosoForExpensiveSelects": true, + "errorGraph": "http://mu.semte.ch/graphs/jobs", + "loginPath": "/organizations/login", + "logOutgoingDelta": true + } +} \ No newline at end of file diff --git a/config/delta/rules.js b/config/delta/rules.js index 3a24e37..376ce20 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -92,7 +92,7 @@ export default [ match: { }, callback: { - url: 'http://delta-producer-pub-graph-maintainer-organizations/delta', + url: 'http://delta-producer-publication-graph-maintainer/organizations/delta', method: 'POST' }, options: { diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 798d293..60d7ec2 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -361,7 +361,7 @@ defmodule Dispatcher do ################################################################# get "/sync/organizations/files/*path" do - Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/" + Proxy.forward conn, path, "http://delta-producer-publication-graph-maintainer/files/" end ################################################################# diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3384ef5..30580ce 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -28,6 +28,12 @@ services: SPRING_SECURITY_CONFIG: "/config/security-dev.yml" restart: "no" deltanotifier: + environment: + DEBUG_DELTA_SEND: "true" + DEBUG_DELTA_NOT_SENDING_EMPTY: "true" + DEBUG_DELTA_MATCH: "true" + DEBUG_TRIPLE_MATCHES_SPEC: "true" + DEBUG_DELTA_FOLD: "true" restart: "no" mocklogin: image: lblod/mock-login-service:latest @@ -51,9 +57,17 @@ services: restart: "no" uri-info: restart: "no" - delta-producer-bg-jobs-initiator-organizations: + delta-producer-background-jobs-initiator: + # delta-producer-bg-jobs-initiator-organizations: + ports: + - 9999:80 restart: "no" - delta-producer-pub-graph-maintainer-organizations: + delta-producer-publication-graph-maintainer: + build: ../delta-producer-publication-graph-maintainer + environment: + NODE_ENV: "development" + ports: + - 9229:9229 restart: "no" search: restart: "no" @@ -70,26 +84,30 @@ services: delta-producer-dump-file-publisher: restart: "no" organizations-sync-with-sharepoint-jobs-initiator: + # entrypoint: ["echo", "service disabled"] restart: "no" organizations-sync-with-sharepoint: + # entrypoint: ["echo", "service disabled"] restart: "no" construct-administrative-unit-relationships: restart: "no" - delta-producer-bg-jobs-initiator-worship-posts: - restart: "no" - delta-producer-pub-graph-maintainer-worship-posts: - restart: "no" - delta-producer-bg-jobs-initiator-public: - restart: "no" - delta-producer-pub-graph-maintainer-public: - restart: "no" + # delta-producer-pub-graph-maintainer-worship-posts: + # entrypoint: ["echo", "service disabled"] + # restart: "no" + # delta-producer-pub-graph-maintainer-public: + # entrypoint: ["echo", "service disabled"] + # restart: "no" mandatarissen-consumer: + entrypoint: ["echo", "service disabled"] restart: "no" leidinggevenden-consumer: + entrypoint: ["echo", "service disabled"] restart: "no" worship-services-sensitive-consumer: + entrypoint: ["echo", "service disabled"] restart: "no" ldes-backend: + entrypoint: ["echo", "service disabled"] environment: BASE_URL: "http://localhost/ldes/" kbo-data-sync: diff --git a/docker-compose.yml b/docker-compose.yml index f8c219e..4cbb7e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -206,10 +206,10 @@ services: image: semtech/mu-file-service:3.2.0 volumes: - ./data/files:/share - labels: - - "logging=true" links: - db:database + labels: + - "logging=true" restart: always logging: *default-logging jobs-controller: @@ -264,6 +264,8 @@ services: END_DATE_WORSHIP_GOVERNING_BODY: "2026-03-31T00:00:00" links: - db:database + labels: + - "logging=true" restart: always logging: *default-logging reasoner: @@ -326,6 +328,8 @@ services: volumes: - ./data/publication-triplestore:/data - ./config/publication-triplestore/virtuoso.ini:/data/virtuoso.ini + labels: + - "logging=true" restart: always logging: *default-logging delta-producer-dump-file-publisher: @@ -335,54 +339,89 @@ services: - ./data/files:/share links: - db:database + labels: + - "logging=true" + restart: always + logging: *default-logging + delta-producer-background-jobs-initiator: + image: lblod/delta-producer-background-jobs-initiator:1.0.1 + volumes: + - ./config/delta-producer/background-job-initiator:/config + links: + - db:database + labels: + - "logging=true" + restart: always + logging: *default-logging + delta-producer-publication-graph-maintainer: + image: lblod/delta-producer-publication-graph-maintainer:1.0.8 + environment: + MAX_BODY_SIZE: "50mb" + PRETTY_PRINT_DIFF_JSON: "true" + PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" + PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" + SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" + CONFIG_SERVICES_JSON_PATH: '/config/publication-graph-maintainer/config.json' + LOG_DELTA_REWRITE: "true" + LOG_INCOMING_DELTA: "true" + volumes: + - ./config/delta-producer:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + labels: + - "logging=true" restart: always logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS: START ################################################################################ - delta-producer-bg-jobs-initiator-organizations: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-organizations: - image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/organizations" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/organizations:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging + # delta-producer-bg-jobs-initiator-organizations: + # image: lblod/delta-producer-background-jobs-initiator:0.3.1 + # environment: + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" + # CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour + # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + # labels: + # - "logging=true" + # links: + # - db:database + # restart: always + # logging: *default-logging + # delta-producer-pub-graph-maintainer-organizations: + # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + # environment: + # MAX_BODY_SIZE: "50mb" + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # RELATIVE_FILE_PATH: "deltas/organizations" + # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" + # HEALING_JOB_OPERATION: "http://http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizationsredpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "" + # QUEUE_POLL_INTERVAL: 3000 + # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + # SKIP_MU_AUTH_INITIAL_SYNC: "true" + # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + # SERVE_DELTA_FILES: "true" + # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" + # DELTA_INTERVAL_MS: 10000 + # PRETTY_PRINT_DIFF_JSON: "true" + # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + # volumes: + # - ./config/delta-producer/organizations:/config + # - ./data/files/:/share + # links: + # - db:database + # - triplestore:virtuoso + # restart: always + # labels: + # - "logging=true" + # logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS: END ################################################################################ @@ -390,50 +429,52 @@ services: ################################################################################ # DELTA ORGANIZATIONS PUBLIC INFO: START ################################################################################ - delta-producer-bg-jobs-initiator-organizations-public-info: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-organizations-public-info: - image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/organizations-public-info" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" - PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/organizations-public-info:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging + # delta-producer-bg-jobs-initiator-organizations-public-info: + # image: lblod/delta-producer-background-jobs-initiator:0.3.1 + # environment: + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + # CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours + # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + # START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync + # labels: + # - "logging=true" + # links: + # - db:database + # restart: always + # logging: *default-logging + # delta-producer-pub-graph-maintainer-organizations-public-info: + # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + # environment: + # MAX_BODY_SIZE: "50mb" + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # RELATIVE_FILE_PATH: "deltas/organizations-public-info" + # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + # QUEUE_POLL_INTERVAL: 3000 + # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + # SKIP_MU_AUTH_INITIAL_SYNC: "true" + # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + # SERVE_DELTA_FILES: "true" + # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" + # DELTA_INTERVAL_MS: 10000 + # PRETTY_PRINT_DIFF_JSON: "true" + # PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" + # PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" + # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + # volumes: + # - ./config/delta-producer/organizations-public-info:/config + # - ./data/files/:/share + # links: + # - db:database + # - triplestore:virtuoso + # restart: always + # labels: + # - "logging=true" + # logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS PUBLIC INFO: END ################################################################################ @@ -441,48 +482,50 @@ services: ################################################################################ # DELTA PUBLIC INFORMATION: START ################################################################################ - delta-producer-bg-jobs-initiator-public: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-public: - image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/public" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/public:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging + # delta-producer-bg-jobs-initiator-public: + # image: lblod/delta-producer-background-jobs-initiator:0.3.1 + # environment: + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + # CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours + # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + # labels: + # - "logging=true" + # links: + # - db:database + # restart: always + # logging: *default-logging + # delta-producer-pub-graph-maintainer-public: + # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + # environment: + # MAX_BODY_SIZE: "50mb" + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # RELATIVE_FILE_PATH: "deltas/public" + # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + # QUEUE_POLL_INTERVAL: 3000 + # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + # SKIP_MU_AUTH_INITIAL_SYNC: "true" + # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + # SERVE_DELTA_FILES: "true" + # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" + # DELTA_INTERVAL_MS: 10000 + # PRETTY_PRINT_DIFF_JSON: "true" + # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + # volumes: + # - ./config/delta-producer/public:/config + # - ./data/files/:/share + # links: + # - db:database + # - triplestore:virtuoso + # restart: always + # labels: + # - "logging=true" + # logging: *default-logging ################################################################################ # DELTA PUBLIC INFORMATION: END ################################################################################ @@ -490,48 +533,50 @@ services: ################################################################################ # DELTA WORSHIP POSTS: START ################################################################################ - delta-producer-bg-jobs-initiator-worship-posts: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-worship-posts: - image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/worship-posts" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/worship-posts:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging + # delta-producer-bg-jobs-initiator-worship-posts: + # image: lblod/delta-producer-background-jobs-initiator:0.3.1 + # environment: + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + # CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours + # CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM + # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + # labels: + # - "logging=true" + # links: + # - db:database + # restart: always + # logging: *default-logging + # delta-producer-pub-graph-maintainer-worship-posts: + # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + # environment: + # MAX_BODY_SIZE: "50mb" + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # RELATIVE_FILE_PATH: "deltas/worship-posts" + # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + # QUEUE_POLL_INTERVAL: 3000 + # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + # SKIP_MU_AUTH_INITIAL_SYNC: "true" + # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + # SERVE_DELTA_FILES: "true" + # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" + # DELTA_INTERVAL_MS: 10000 + # PRETTY_PRINT_DIFF_JSON: "true" + # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + # volumes: + # - ./config/delta-producer/worship-posts:/config + # - ./data/files/:/share + # links: + # - db:database + # - triplestore:virtuoso + # restart: always + # labels: + # - "logging=true" + # logging: *default-logging ################################################################################ # DELTA WORSHIP POSTS: END ################################################################################