From fc4fcfddf89231df82f17e809d5315c15cb428e0 Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Tue, 23 Apr 2024 13:07:08 +0200 Subject: [PATCH 1/5] chore: Bum delta-producer organizations to v1.0.8 + disabled not need for testing purpose --- .../background-job-initiator/config.json | 12 + .../dump-file-publisher/config.json | 4 +- .../publication-graph-maintainer/config.json | 25 ++ config/delta/rules.js | 2 +- config/dispatcher/dispatcher.ex | 2 +- docker-compose.dev.yml | 38 +- docker-compose.yml | 389 ++++++++++-------- 7 files changed, 286 insertions(+), 186 deletions(-) create mode 100644 config/delta-producer/background-job-initiator/config.json create mode 100644 config/delta-producer/publication-graph-maintainer/config.json diff --git a/config/delta-producer/background-job-initiator/config.json b/config/delta-producer/background-job-initiator/config.json new file mode 100644 index 0000000..379c1c2 --- /dev/null +++ b/config/delta-producer/background-job-initiator/config.json @@ -0,0 +1,12 @@ +[ + { + "name": "organizations", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations", + "cronPatternDumpJob": "42 * * * *", + "cronPatternHealingJob": "0 0 4 * * *", + "startInitialSync": true + } +] \ No newline at end of file diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index bc7981c..8babcc4 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -3,7 +3,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", - "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true @@ -12,7 +12,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", - "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true diff --git a/config/delta-producer/publication-graph-maintainer/config.json b/config/delta-producer/publication-graph-maintainer/config.json new file mode 100644 index 0000000..a7d0a14 --- /dev/null +++ b/config/delta-producer/publication-graph-maintainer/config.json @@ -0,0 +1,25 @@ +{ + "organizations": { + "deltaInterval": 10000, + "deltaPath": "/organizations/delta", + "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations", + "exportConfigPath": "/config/organizations/export.json", + "filesPath": "/organizations/files", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", + "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations", + "queuePollInterval": 3000, + "relativeFilePath": "/deltas/organizations", + "serveDeltaFiles": true, + "skipMuAuthInitialSync": true, + "skipMuAuthHealing": true, + "useFileDiff": true, + "healingInitialBatchSizeInsert": 1000, + "useVirtuosoForExpensiveSelects": true, + "errorGraph": "http://mu.semte.ch/graphs/jobs", + "loginPath": "/organizations/login", + "logOutgoingDelta": true + } +} \ No newline at end of file diff --git a/config/delta/rules.js b/config/delta/rules.js index 3a24e37..376ce20 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -92,7 +92,7 @@ export default [ match: { }, callback: { - url: 'http://delta-producer-pub-graph-maintainer-organizations/delta', + url: 'http://delta-producer-publication-graph-maintainer/organizations/delta', method: 'POST' }, options: { diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 798d293..60d7ec2 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -361,7 +361,7 @@ defmodule Dispatcher do ################################################################# get "/sync/organizations/files/*path" do - Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/" + Proxy.forward conn, path, "http://delta-producer-publication-graph-maintainer/files/" end ################################################################# diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3384ef5..30580ce 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -28,6 +28,12 @@ services: SPRING_SECURITY_CONFIG: "/config/security-dev.yml" restart: "no" deltanotifier: + environment: + DEBUG_DELTA_SEND: "true" + DEBUG_DELTA_NOT_SENDING_EMPTY: "true" + DEBUG_DELTA_MATCH: "true" + DEBUG_TRIPLE_MATCHES_SPEC: "true" + DEBUG_DELTA_FOLD: "true" restart: "no" mocklogin: image: lblod/mock-login-service:latest @@ -51,9 +57,17 @@ services: restart: "no" uri-info: restart: "no" - delta-producer-bg-jobs-initiator-organizations: + delta-producer-background-jobs-initiator: + # delta-producer-bg-jobs-initiator-organizations: + ports: + - 9999:80 restart: "no" - delta-producer-pub-graph-maintainer-organizations: + delta-producer-publication-graph-maintainer: + build: ../delta-producer-publication-graph-maintainer + environment: + NODE_ENV: "development" + ports: + - 9229:9229 restart: "no" search: restart: "no" @@ -70,26 +84,30 @@ services: delta-producer-dump-file-publisher: restart: "no" organizations-sync-with-sharepoint-jobs-initiator: + # entrypoint: ["echo", "service disabled"] restart: "no" organizations-sync-with-sharepoint: + # entrypoint: ["echo", "service disabled"] restart: "no" construct-administrative-unit-relationships: restart: "no" - delta-producer-bg-jobs-initiator-worship-posts: - restart: "no" - delta-producer-pub-graph-maintainer-worship-posts: - restart: "no" - delta-producer-bg-jobs-initiator-public: - restart: "no" - delta-producer-pub-graph-maintainer-public: - restart: "no" + # delta-producer-pub-graph-maintainer-worship-posts: + # entrypoint: ["echo", "service disabled"] + # restart: "no" + # delta-producer-pub-graph-maintainer-public: + # entrypoint: ["echo", "service disabled"] + # restart: "no" mandatarissen-consumer: + entrypoint: ["echo", "service disabled"] restart: "no" leidinggevenden-consumer: + entrypoint: ["echo", "service disabled"] restart: "no" worship-services-sensitive-consumer: + entrypoint: ["echo", "service disabled"] restart: "no" ldes-backend: + entrypoint: ["echo", "service disabled"] environment: BASE_URL: "http://localhost/ldes/" kbo-data-sync: diff --git a/docker-compose.yml b/docker-compose.yml index f8c219e..4cbb7e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -206,10 +206,10 @@ services: image: semtech/mu-file-service:3.2.0 volumes: - ./data/files:/share - labels: - - "logging=true" links: - db:database + labels: + - "logging=true" restart: always logging: *default-logging jobs-controller: @@ -264,6 +264,8 @@ services: END_DATE_WORSHIP_GOVERNING_BODY: "2026-03-31T00:00:00" links: - db:database + labels: + - "logging=true" restart: always logging: *default-logging reasoner: @@ -326,6 +328,8 @@ services: volumes: - ./data/publication-triplestore:/data - ./config/publication-triplestore/virtuoso.ini:/data/virtuoso.ini + labels: + - "logging=true" restart: always logging: *default-logging delta-producer-dump-file-publisher: @@ -335,54 +339,89 @@ services: - ./data/files:/share links: - db:database + labels: + - "logging=true" + restart: always + logging: *default-logging + delta-producer-background-jobs-initiator: + image: lblod/delta-producer-background-jobs-initiator:1.0.1 + volumes: + - ./config/delta-producer/background-job-initiator:/config + links: + - db:database + labels: + - "logging=true" + restart: always + logging: *default-logging + delta-producer-publication-graph-maintainer: + image: lblod/delta-producer-publication-graph-maintainer:1.0.8 + environment: + MAX_BODY_SIZE: "50mb" + PRETTY_PRINT_DIFF_JSON: "true" + PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" + PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" + SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" + CONFIG_SERVICES_JSON_PATH: '/config/publication-graph-maintainer/config.json' + LOG_DELTA_REWRITE: "true" + LOG_INCOMING_DELTA: "true" + volumes: + - ./config/delta-producer:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + labels: + - "logging=true" restart: always logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS: START ################################################################################ - delta-producer-bg-jobs-initiator-organizations: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-organizations: - image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/organizations" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/organizations:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging + # delta-producer-bg-jobs-initiator-organizations: + # image: lblod/delta-producer-background-jobs-initiator:0.3.1 + # environment: + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" + # CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour + # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + # labels: + # - "logging=true" + # links: + # - db:database + # restart: always + # logging: *default-logging + # delta-producer-pub-graph-maintainer-organizations: + # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + # environment: + # MAX_BODY_SIZE: "50mb" + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # RELATIVE_FILE_PATH: "deltas/organizations" + # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" + # HEALING_JOB_OPERATION: "http://http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizationsredpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "" + # QUEUE_POLL_INTERVAL: 3000 + # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + # SKIP_MU_AUTH_INITIAL_SYNC: "true" + # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + # SERVE_DELTA_FILES: "true" + # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" + # DELTA_INTERVAL_MS: 10000 + # PRETTY_PRINT_DIFF_JSON: "true" + # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + # volumes: + # - ./config/delta-producer/organizations:/config + # - ./data/files/:/share + # links: + # - db:database + # - triplestore:virtuoso + # restart: always + # labels: + # - "logging=true" + # logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS: END ################################################################################ @@ -390,50 +429,52 @@ services: ################################################################################ # DELTA ORGANIZATIONS PUBLIC INFO: START ################################################################################ - delta-producer-bg-jobs-initiator-organizations-public-info: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-organizations-public-info: - image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/organizations-public-info" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" - PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/organizations-public-info:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging + # delta-producer-bg-jobs-initiator-organizations-public-info: + # image: lblod/delta-producer-background-jobs-initiator:0.3.1 + # environment: + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + # CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours + # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + # START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync + # labels: + # - "logging=true" + # links: + # - db:database + # restart: always + # logging: *default-logging + # delta-producer-pub-graph-maintainer-organizations-public-info: + # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + # environment: + # MAX_BODY_SIZE: "50mb" + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # RELATIVE_FILE_PATH: "deltas/organizations-public-info" + # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + # QUEUE_POLL_INTERVAL: 3000 + # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + # SKIP_MU_AUTH_INITIAL_SYNC: "true" + # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + # SERVE_DELTA_FILES: "true" + # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" + # DELTA_INTERVAL_MS: 10000 + # PRETTY_PRINT_DIFF_JSON: "true" + # PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" + # PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" + # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + # volumes: + # - ./config/delta-producer/organizations-public-info:/config + # - ./data/files/:/share + # links: + # - db:database + # - triplestore:virtuoso + # restart: always + # labels: + # - "logging=true" + # logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS PUBLIC INFO: END ################################################################################ @@ -441,48 +482,50 @@ services: ################################################################################ # DELTA PUBLIC INFORMATION: START ################################################################################ - delta-producer-bg-jobs-initiator-public: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-public: - image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/public" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/public:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging + # delta-producer-bg-jobs-initiator-public: + # image: lblod/delta-producer-background-jobs-initiator:0.3.1 + # environment: + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + # CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours + # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + # labels: + # - "logging=true" + # links: + # - db:database + # restart: always + # logging: *default-logging + # delta-producer-pub-graph-maintainer-public: + # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + # environment: + # MAX_BODY_SIZE: "50mb" + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # RELATIVE_FILE_PATH: "deltas/public" + # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + # QUEUE_POLL_INTERVAL: 3000 + # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + # SKIP_MU_AUTH_INITIAL_SYNC: "true" + # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + # SERVE_DELTA_FILES: "true" + # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" + # DELTA_INTERVAL_MS: 10000 + # PRETTY_PRINT_DIFF_JSON: "true" + # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + # volumes: + # - ./config/delta-producer/public:/config + # - ./data/files/:/share + # links: + # - db:database + # - triplestore:virtuoso + # restart: always + # labels: + # - "logging=true" + # logging: *default-logging ################################################################################ # DELTA PUBLIC INFORMATION: END ################################################################################ @@ -490,48 +533,50 @@ services: ################################################################################ # DELTA WORSHIP POSTS: START ################################################################################ - delta-producer-bg-jobs-initiator-worship-posts: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-worship-posts: - image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/worship-posts" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/worship-posts:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging + # delta-producer-bg-jobs-initiator-worship-posts: + # image: lblod/delta-producer-background-jobs-initiator:0.3.1 + # environment: + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + # CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours + # CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM + # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + # labels: + # - "logging=true" + # links: + # - db:database + # restart: always + # logging: *default-logging + # delta-producer-pub-graph-maintainer-worship-posts: + # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + # environment: + # MAX_BODY_SIZE: "50mb" + # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + # RELATIVE_FILE_PATH: "deltas/worship-posts" + # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" + # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + # QUEUE_POLL_INTERVAL: 3000 + # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + # SKIP_MU_AUTH_INITIAL_SYNC: "true" + # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + # SERVE_DELTA_FILES: "true" + # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" + # DELTA_INTERVAL_MS: 10000 + # PRETTY_PRINT_DIFF_JSON: "true" + # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + # volumes: + # - ./config/delta-producer/worship-posts:/config + # - ./data/files/:/share + # links: + # - db:database + # - triplestore:virtuoso + # restart: always + # labels: + # - "logging=true" + # logging: *default-logging ################################################################################ # DELTA WORSHIP POSTS: END ################################################################################ From 723527fb62a0d5077216ac2f044745a2f9cc3b69 Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Tue, 23 Apr 2024 13:08:58 +0200 Subject: [PATCH 2/5] chore: Disable more --- .../dump-file-publisher/config.json | 54 ----- config/delta/rules.js | 200 +++++------------- config/jobs-controller/config.json | 32 --- 3 files changed, 57 insertions(+), 229 deletions(-) diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 8babcc4..1e80d21 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -16,59 +16,5 @@ "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", - "fileBaseName": "dump-organizations-public-info", - "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", - "fileBaseName": "dump-organizations-public-info", - "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", - "fileBaseName": "dump-public", - "publicationGraphEndpoint": "http://triplestore:8890/sparql", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", - "fileBaseName": "dump-public", - "publicationGraphEndpoint": "http://triplestore:8890/sparql", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", - "fileBaseName": "dump-worship-posts", - "publicationGraphEndpoint": "http://triplestore:8890/sparql", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", - "fileBaseName": "dump-worship-posts", - "publicationGraphEndpoint": "http://triplestore:8890/sparql", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true } } diff --git a/config/delta/rules.js b/config/delta/rules.js index 376ce20..27414ad 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -1,40 +1,5 @@ export default [ - { - match: { - graph: { - type: "uri", - value: "http://mu.semte.ch/graphs/administrative-unit" - } - }, - callback: { - url: 'http://search/update', - method: 'POST' - }, - options: { - resourceFormat: "v0.0.1", - gracePeriod: 10000, - ignoreFromSelf: true - } - }, - { - match: { - graph: { - type: "uri", - value: "http://mu.semte.ch/graphs/worship-service" - } - }, - callback: { - url: 'http://search/update', - method: 'POST' - }, - options: { - resourceFormat: "v0.0.1", - gracePeriod: 10000, - ignoreFromSelf: true - } - }, - - { + /*{ match: { graph: { type: "uri", @@ -69,7 +34,7 @@ export default [ ignoreFromSelf: true } }, - +*/ { match: { predicate: { @@ -105,58 +70,7 @@ export default [ ] } }, - { - match: { - }, - callback: { - url: 'http://delta-producer-pub-graph-maintainer-organizations-public-info/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] - } - }, - { - match: { - }, - callback: { - url: 'http://delta-producer-pub-graph-maintainer-public/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] - } - }, - { - match: { - // anything - }, - callback: { - url: 'http://delta-producer-pub-graph-maintainer-worship-posts/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] - } - }, + { match: { predicate: { @@ -180,60 +94,60 @@ export default [ } }, // organizations-sync-with-sharepoint/delta : jobs come from the jobs graph, data from the two main graphs - { - match: { - graph: { - type: 'uri', - value: 'http://mu.semte.ch/graphs/jobs' - } - }, - callback: { - url: 'http://organizations-sync-with-sharepoint/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - } - }, - { - match: { - graph: { - type: 'uri', - value: 'http://mu.semte.ch/graphs/shared' - } - }, - callback: { - url: 'http://organizations-sync-with-sharepoint/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - } - }, - { - match: { - graph: { - type: 'uri', - value: 'http://mu.semte.ch/graphs/administrative-unit' - } - }, - callback: { - url: 'http://organizations-sync-with-sharepoint/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - } - } + // { + // match: { + // graph: { + // type: 'uri', + // value: 'http://mu.semte.ch/graphs/jobs' + // } + // }, + // callback: { + // url: 'http://organizations-sync-with-sharepoint/delta', + // method: 'POST' + // }, + // options: { + // resourceFormat: 'v0.0.1', + // gracePeriod: 1000, + // ignoreFromSelf: true, + // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + // } + // }, + // { + // match: { + // graph: { + // type: 'uri', + // value: 'http://mu.semte.ch/graphs/shared' + // } + // }, + // callback: { + // url: 'http://organizations-sync-with-sharepoint/delta', + // method: 'POST' + // }, + // options: { + // resourceFormat: 'v0.0.1', + // gracePeriod: 1000, + // ignoreFromSelf: true, + // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + // } + // }, + // { + // match: { + // graph: { + // type: 'uri', + // value: 'http://mu.semte.ch/graphs/administrative-unit' + // } + // }, + // callback: { + // url: 'http://organizations-sync-with-sharepoint/delta', + // method: 'POST' + // }, + // options: { + // resourceFormat: 'v0.0.1', + // gracePeriod: 1000, + // ignoreFromSelf: true, + // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + // } + // } /* Commenting ldes rules until ready to go to qa/prod. In the meantime, we override the configuration on dev. { diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 4115501..4863700 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -1,36 +1,4 @@ { - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units": { - "tasksConfiguration": [ - { - "currentOperation": null, - "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", - "nextIndex": "0" - }, - { - "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", - "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", - "nextIndex": "1" - } - ] - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units": { - "tasksConfiguration": [ - { - "currentOperation": null, - "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", - "nextIndex": "0" - } - ] - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units": { - "tasksConfiguration": [ - { - "currentOperation": null, - "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", - "nextIndex": "0" - } - ] - }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations": { "tasksConfiguration": [ { From 94baeb539f3d58f42e28d9968192d17fb1aecb2a Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Thu, 25 Apr 2024 15:56:56 +0200 Subject: [PATCH 3/5] chore: Bump delta-producer-publication-graph-maintainer to 1.0.9-rc2 + enable other producers --- .../dump-file-publisher/config.json | 54 +++ config/delta/rules.js | 200 +++++++---- config/jobs-controller/config.json | 32 ++ docker-compose.dev.yml | 34 +- docker-compose.yml | 323 ++++++++---------- 5 files changed, 372 insertions(+), 271 deletions(-) diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 1e80d21..8babcc4 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -16,5 +16,59 @@ "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "fileBaseName": "dump-organizations-public-info", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "fileBaseName": "dump-organizations-public-info", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "fileBaseName": "dump-public", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "fileBaseName": "dump-public", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", + "fileBaseName": "dump-worship-posts", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", + "fileBaseName": "dump-worship-posts", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true } } diff --git a/config/delta/rules.js b/config/delta/rules.js index 27414ad..376ce20 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -1,5 +1,40 @@ export default [ - /*{ + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/administrative-unit" + } + }, + callback: { + url: 'http://search/update', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 10000, + ignoreFromSelf: true + } + }, + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/worship-service" + } + }, + callback: { + url: 'http://search/update', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 10000, + ignoreFromSelf: true + } + }, + + { match: { graph: { type: "uri", @@ -34,7 +69,7 @@ export default [ ignoreFromSelf: true } }, -*/ + { match: { predicate: { @@ -70,7 +105,58 @@ export default [ ] } }, - + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-organizations-public-info/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-public/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, + { + match: { + // anything + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-worship-posts/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, { match: { predicate: { @@ -94,60 +180,60 @@ export default [ } }, // organizations-sync-with-sharepoint/delta : jobs come from the jobs graph, data from the two main graphs - // { - // match: { - // graph: { - // type: 'uri', - // value: 'http://mu.semte.ch/graphs/jobs' - // } - // }, - // callback: { - // url: 'http://organizations-sync-with-sharepoint/delta', - // method: 'POST' - // }, - // options: { - // resourceFormat: 'v0.0.1', - // gracePeriod: 1000, - // ignoreFromSelf: true, - // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - // } - // }, - // { - // match: { - // graph: { - // type: 'uri', - // value: 'http://mu.semte.ch/graphs/shared' - // } - // }, - // callback: { - // url: 'http://organizations-sync-with-sharepoint/delta', - // method: 'POST' - // }, - // options: { - // resourceFormat: 'v0.0.1', - // gracePeriod: 1000, - // ignoreFromSelf: true, - // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - // } - // }, - // { - // match: { - // graph: { - // type: 'uri', - // value: 'http://mu.semte.ch/graphs/administrative-unit' - // } - // }, - // callback: { - // url: 'http://organizations-sync-with-sharepoint/delta', - // method: 'POST' - // }, - // options: { - // resourceFormat: 'v0.0.1', - // gracePeriod: 1000, - // ignoreFromSelf: true, - // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - // } - // } + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/jobs' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/shared' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/administrative-unit' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + } /* Commenting ldes rules until ready to go to qa/prod. In the meantime, we override the configuration on dev. { diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 4863700..4115501 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -1,4 +1,36 @@ { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextIndex": "0" + }, + { + "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "1" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "0" + } + ] + }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations": { "tasksConfiguration": [ { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 30580ce..4eca35d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -28,12 +28,6 @@ services: SPRING_SECURITY_CONFIG: "/config/security-dev.yml" restart: "no" deltanotifier: - environment: - DEBUG_DELTA_SEND: "true" - DEBUG_DELTA_NOT_SENDING_EMPTY: "true" - DEBUG_DELTA_MATCH: "true" - DEBUG_TRIPLE_MATCHES_SPEC: "true" - DEBUG_DELTA_FOLD: "true" restart: "no" mocklogin: image: lblod/mock-login-service:latest @@ -58,16 +52,8 @@ services: uri-info: restart: "no" delta-producer-background-jobs-initiator: - # delta-producer-bg-jobs-initiator-organizations: - ports: - - 9999:80 restart: "no" delta-producer-publication-graph-maintainer: - build: ../delta-producer-publication-graph-maintainer - environment: - NODE_ENV: "development" - ports: - - 9229:9229 restart: "no" search: restart: "no" @@ -84,30 +70,26 @@ services: delta-producer-dump-file-publisher: restart: "no" organizations-sync-with-sharepoint-jobs-initiator: - # entrypoint: ["echo", "service disabled"] restart: "no" organizations-sync-with-sharepoint: - # entrypoint: ["echo", "service disabled"] restart: "no" construct-administrative-unit-relationships: restart: "no" - # delta-producer-pub-graph-maintainer-worship-posts: - # entrypoint: ["echo", "service disabled"] - # restart: "no" - # delta-producer-pub-graph-maintainer-public: - # entrypoint: ["echo", "service disabled"] - # restart: "no" + delta-producer-bg-jobs-initiator-worship-posts: + restart: "no" + delta-producer-pub-graph-maintainer-worship-posts: + restart: "no" + delta-producer-bg-jobs-initiator-public: + restart: "no" + delta-producer-pub-graph-maintainer-public: + restart: "no" mandatarissen-consumer: - entrypoint: ["echo", "service disabled"] restart: "no" leidinggevenden-consumer: - entrypoint: ["echo", "service disabled"] restart: "no" worship-services-sensitive-consumer: - entrypoint: ["echo", "service disabled"] restart: "no" ldes-backend: - entrypoint: ["echo", "service disabled"] environment: BASE_URL: "http://localhost/ldes/" kbo-data-sync: diff --git a/docker-compose.yml b/docker-compose.yml index 4cbb7e2..445be73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -354,7 +354,7 @@ services: restart: always logging: *default-logging delta-producer-publication-graph-maintainer: - image: lblod/delta-producer-publication-graph-maintainer:1.0.8 + image: lblod/delta-producer-publication-graph-maintainer:1.0.9-rc2 environment: MAX_BODY_SIZE: "50mb" PRETTY_PRINT_DIFF_JSON: "true" @@ -362,8 +362,6 @@ services: PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" CONFIG_SERVICES_JSON_PATH: '/config/publication-graph-maintainer/config.json' - LOG_DELTA_REWRITE: "true" - LOG_INCOMING_DELTA: "true" volumes: - ./config/delta-producer:/config - ./data/files/:/share @@ -375,106 +373,55 @@ services: restart: always logging: *default-logging - ################################################################################ - # DELTA ORGANIZATIONS: START - ################################################################################ - # delta-producer-bg-jobs-initiator-organizations: - # image: lblod/delta-producer-background-jobs-initiator:0.3.1 - # environment: - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - # CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour - # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - # labels: - # - "logging=true" - # links: - # - db:database - # restart: always - # logging: *default-logging - # delta-producer-pub-graph-maintainer-organizations: - # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - # environment: - # MAX_BODY_SIZE: "50mb" - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # RELATIVE_FILE_PATH: "deltas/organizations" - # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" - # HEALING_JOB_OPERATION: "http://http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizationsredpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "" - # QUEUE_POLL_INTERVAL: 3000 - # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - # SKIP_MU_AUTH_INITIAL_SYNC: "true" - # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - # SERVE_DELTA_FILES: "true" - # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" - # DELTA_INTERVAL_MS: 10000 - # PRETTY_PRINT_DIFF_JSON: "true" - # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - # volumes: - # - ./config/delta-producer/organizations:/config - # - ./data/files/:/share - # links: - # - db:database - # - triplestore:virtuoso - # restart: always - # labels: - # - "logging=true" - # logging: *default-logging - ################################################################################ - # DELTA ORGANIZATIONS: END - ################################################################################ - ################################################################################ # DELTA ORGANIZATIONS PUBLIC INFO: START ################################################################################ - # delta-producer-bg-jobs-initiator-organizations-public-info: - # image: lblod/delta-producer-background-jobs-initiator:0.3.1 - # environment: - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - # CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours - # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - # START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync - # labels: - # - "logging=true" - # links: - # - db:database - # restart: always - # logging: *default-logging - # delta-producer-pub-graph-maintainer-organizations-public-info: - # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - # environment: - # MAX_BODY_SIZE: "50mb" - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # RELATIVE_FILE_PATH: "deltas/organizations-public-info" - # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - # QUEUE_POLL_INTERVAL: 3000 - # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - # SKIP_MU_AUTH_INITIAL_SYNC: "true" - # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - # SERVE_DELTA_FILES: "true" - # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" - # DELTA_INTERVAL_MS: 10000 - # PRETTY_PRINT_DIFF_JSON: "true" - # PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" - # PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" - # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - # volumes: - # - ./config/delta-producer/organizations-public-info:/config - # - ./data/files/:/share - # links: - # - db:database - # - triplestore:virtuoso - # restart: always - # labels: - # - "logging=true" - # logging: *default-logging + delta-producer-bg-jobs-initiator-organizations-public-info: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-organizations-public-info: + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + RELATIVE_FILE_PATH: "deltas/organizations-public-info" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" + PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + volumes: + - ./config/delta-producer/organizations-public-info:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + labels: + - "logging=true" + logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS PUBLIC INFO: END ################################################################################ @@ -482,50 +429,50 @@ services: ################################################################################ # DELTA PUBLIC INFORMATION: START ################################################################################ - # delta-producer-bg-jobs-initiator-public: - # image: lblod/delta-producer-background-jobs-initiator:0.3.1 - # environment: - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - # CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours - # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - # labels: - # - "logging=true" - # links: - # - db:database - # restart: always - # logging: *default-logging - # delta-producer-pub-graph-maintainer-public: - # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - # environment: - # MAX_BODY_SIZE: "50mb" - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # RELATIVE_FILE_PATH: "deltas/public" - # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - # QUEUE_POLL_INTERVAL: 3000 - # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - # SKIP_MU_AUTH_INITIAL_SYNC: "true" - # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - # SERVE_DELTA_FILES: "true" - # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" - # DELTA_INTERVAL_MS: 10000 - # PRETTY_PRINT_DIFF_JSON: "true" - # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - # volumes: - # - ./config/delta-producer/public:/config - # - ./data/files/:/share - # links: - # - db:database - # - triplestore:virtuoso - # restart: always - # labels: - # - "logging=true" - # logging: *default-logging + delta-producer-bg-jobs-initiator-public: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-public: + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + RELATIVE_FILE_PATH: "deltas/public" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + volumes: + - ./config/delta-producer/public:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + labels: + - "logging=true" + logging: *default-logging ################################################################################ # DELTA PUBLIC INFORMATION: END ################################################################################ @@ -533,50 +480,50 @@ services: ################################################################################ # DELTA WORSHIP POSTS: START ################################################################################ - # delta-producer-bg-jobs-initiator-worship-posts: - # image: lblod/delta-producer-background-jobs-initiator:0.3.1 - # environment: - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - # CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours - # CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM - # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - # labels: - # - "logging=true" - # links: - # - db:database - # restart: always - # logging: *default-logging - # delta-producer-pub-graph-maintainer-worship-posts: - # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - # environment: - # MAX_BODY_SIZE: "50mb" - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # RELATIVE_FILE_PATH: "deltas/worship-posts" - # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - # QUEUE_POLL_INTERVAL: 3000 - # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - # SKIP_MU_AUTH_INITIAL_SYNC: "true" - # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - # SERVE_DELTA_FILES: "true" - # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" - # DELTA_INTERVAL_MS: 10000 - # PRETTY_PRINT_DIFF_JSON: "true" - # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - # volumes: - # - ./config/delta-producer/worship-posts:/config - # - ./data/files/:/share - # links: - # - db:database - # - triplestore:virtuoso - # restart: always - # labels: - # - "logging=true" - # logging: *default-logging + delta-producer-bg-jobs-initiator-worship-posts: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-worship-posts: + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + RELATIVE_FILE_PATH: "deltas/worship-posts" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + volumes: + - ./config/delta-producer/worship-posts:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + labels: + - "logging=true" + logging: *default-logging ################################################################################ # DELTA WORSHIP POSTS: END ################################################################################ From b5b8df6610d3a2e5b669ae30c52a4ede39375ab1 Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Fri, 26 Apr 2024 13:22:15 +0200 Subject: [PATCH 4/5] chore: Bump delta-producer-publication-graph-maintainer to v1.1.1-rc.1 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 445be73..06c5617 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -354,7 +354,7 @@ services: restart: always logging: *default-logging delta-producer-publication-graph-maintainer: - image: lblod/delta-producer-publication-graph-maintainer:1.0.9-rc2 + image: lblod/delta-producer-publication-graph-maintainer:1.1.1-rc.1 environment: MAX_BODY_SIZE: "50mb" PRETTY_PRINT_DIFF_JSON: "true" From e7129dc41df1b2a89f85ebc93fa2a0860eaf19df Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Mon, 29 Apr 2024 11:08:13 +0200 Subject: [PATCH 5/5] chore: Clear producer organizations graph from main triplestore --- .../20240429105000-clear-producer-organizations-graph.sparql | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 config/migrations/2024/20240429105000-clear-producer-organizations-graph.sparql diff --git a/config/migrations/2024/20240429105000-clear-producer-organizations-graph.sparql b/config/migrations/2024/20240429105000-clear-producer-organizations-graph.sparql new file mode 100644 index 0000000..5c90494 --- /dev/null +++ b/config/migrations/2024/20240429105000-clear-producer-organizations-graph.sparql @@ -0,0 +1,3 @@ +# The producer organizations graph is now in the publication-triplestore +# Lets remove it from the main triplestore +CLEAR GRAPH \ No newline at end of file