diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 1e80d21..8babcc4 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -16,5 +16,59 @@ "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "fileBaseName": "dump-organizations-public-info", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "fileBaseName": "dump-organizations-public-info", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "fileBaseName": "dump-public", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "fileBaseName": "dump-public", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", + "fileBaseName": "dump-worship-posts", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", + "fileBaseName": "dump-worship-posts", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true } } diff --git a/config/delta/rules.js b/config/delta/rules.js index 27414ad..376ce20 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -1,5 +1,40 @@ export default [ - /*{ + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/administrative-unit" + } + }, + callback: { + url: 'http://search/update', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 10000, + ignoreFromSelf: true + } + }, + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/worship-service" + } + }, + callback: { + url: 'http://search/update', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 10000, + ignoreFromSelf: true + } + }, + + { match: { graph: { type: "uri", @@ -34,7 +69,7 @@ export default [ ignoreFromSelf: true } }, -*/ + { match: { predicate: { @@ -70,7 +105,58 @@ export default [ ] } }, - + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-organizations-public-info/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-public/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, + { + match: { + // anything + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-worship-posts/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, { match: { predicate: { @@ -94,60 +180,60 @@ export default [ } }, // organizations-sync-with-sharepoint/delta : jobs come from the jobs graph, data from the two main graphs - // { - // match: { - // graph: { - // type: 'uri', - // value: 'http://mu.semte.ch/graphs/jobs' - // } - // }, - // callback: { - // url: 'http://organizations-sync-with-sharepoint/delta', - // method: 'POST' - // }, - // options: { - // resourceFormat: 'v0.0.1', - // gracePeriod: 1000, - // ignoreFromSelf: true, - // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - // } - // }, - // { - // match: { - // graph: { - // type: 'uri', - // value: 'http://mu.semte.ch/graphs/shared' - // } - // }, - // callback: { - // url: 'http://organizations-sync-with-sharepoint/delta', - // method: 'POST' - // }, - // options: { - // resourceFormat: 'v0.0.1', - // gracePeriod: 1000, - // ignoreFromSelf: true, - // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - // } - // }, - // { - // match: { - // graph: { - // type: 'uri', - // value: 'http://mu.semte.ch/graphs/administrative-unit' - // } - // }, - // callback: { - // url: 'http://organizations-sync-with-sharepoint/delta', - // method: 'POST' - // }, - // options: { - // resourceFormat: 'v0.0.1', - // gracePeriod: 1000, - // ignoreFromSelf: true, - // optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] - // } - // } + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/jobs' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/shared' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/administrative-unit' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + } /* Commenting ldes rules until ready to go to qa/prod. In the meantime, we override the configuration on dev. { diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 4863700..4115501 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -1,4 +1,36 @@ { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextIndex": "0" + }, + { + "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "1" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "0" + } + ] + }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations": { "tasksConfiguration": [ { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 30580ce..4eca35d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -28,12 +28,6 @@ services: SPRING_SECURITY_CONFIG: "/config/security-dev.yml" restart: "no" deltanotifier: - environment: - DEBUG_DELTA_SEND: "true" - DEBUG_DELTA_NOT_SENDING_EMPTY: "true" - DEBUG_DELTA_MATCH: "true" - DEBUG_TRIPLE_MATCHES_SPEC: "true" - DEBUG_DELTA_FOLD: "true" restart: "no" mocklogin: image: lblod/mock-login-service:latest @@ -58,16 +52,8 @@ services: uri-info: restart: "no" delta-producer-background-jobs-initiator: - # delta-producer-bg-jobs-initiator-organizations: - ports: - - 9999:80 restart: "no" delta-producer-publication-graph-maintainer: - build: ../delta-producer-publication-graph-maintainer - environment: - NODE_ENV: "development" - ports: - - 9229:9229 restart: "no" search: restart: "no" @@ -84,30 +70,26 @@ services: delta-producer-dump-file-publisher: restart: "no" organizations-sync-with-sharepoint-jobs-initiator: - # entrypoint: ["echo", "service disabled"] restart: "no" organizations-sync-with-sharepoint: - # entrypoint: ["echo", "service disabled"] restart: "no" construct-administrative-unit-relationships: restart: "no" - # delta-producer-pub-graph-maintainer-worship-posts: - # entrypoint: ["echo", "service disabled"] - # restart: "no" - # delta-producer-pub-graph-maintainer-public: - # entrypoint: ["echo", "service disabled"] - # restart: "no" + delta-producer-bg-jobs-initiator-worship-posts: + restart: "no" + delta-producer-pub-graph-maintainer-worship-posts: + restart: "no" + delta-producer-bg-jobs-initiator-public: + restart: "no" + delta-producer-pub-graph-maintainer-public: + restart: "no" mandatarissen-consumer: - entrypoint: ["echo", "service disabled"] restart: "no" leidinggevenden-consumer: - entrypoint: ["echo", "service disabled"] restart: "no" worship-services-sensitive-consumer: - entrypoint: ["echo", "service disabled"] restart: "no" ldes-backend: - entrypoint: ["echo", "service disabled"] environment: BASE_URL: "http://localhost/ldes/" kbo-data-sync: diff --git a/docker-compose.yml b/docker-compose.yml index 4cbb7e2..445be73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -354,7 +354,7 @@ services: restart: always logging: *default-logging delta-producer-publication-graph-maintainer: - image: lblod/delta-producer-publication-graph-maintainer:1.0.8 + image: lblod/delta-producer-publication-graph-maintainer:1.0.9-rc2 environment: MAX_BODY_SIZE: "50mb" PRETTY_PRINT_DIFF_JSON: "true" @@ -362,8 +362,6 @@ services: PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" CONFIG_SERVICES_JSON_PATH: '/config/publication-graph-maintainer/config.json' - LOG_DELTA_REWRITE: "true" - LOG_INCOMING_DELTA: "true" volumes: - ./config/delta-producer:/config - ./data/files/:/share @@ -375,106 +373,55 @@ services: restart: always logging: *default-logging - ################################################################################ - # DELTA ORGANIZATIONS: START - ################################################################################ - # delta-producer-bg-jobs-initiator-organizations: - # image: lblod/delta-producer-background-jobs-initiator:0.3.1 - # environment: - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - # CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour - # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - # labels: - # - "logging=true" - # links: - # - db:database - # restart: always - # logging: *default-logging - # delta-producer-pub-graph-maintainer-organizations: - # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - # environment: - # MAX_BODY_SIZE: "50mb" - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # RELATIVE_FILE_PATH: "deltas/organizations" - # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" - # HEALING_JOB_OPERATION: "http://http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizationsredpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "" - # QUEUE_POLL_INTERVAL: 3000 - # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - # SKIP_MU_AUTH_INITIAL_SYNC: "true" - # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - # SERVE_DELTA_FILES: "true" - # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" - # DELTA_INTERVAL_MS: 10000 - # PRETTY_PRINT_DIFF_JSON: "true" - # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - # volumes: - # - ./config/delta-producer/organizations:/config - # - ./data/files/:/share - # links: - # - db:database - # - triplestore:virtuoso - # restart: always - # labels: - # - "logging=true" - # logging: *default-logging - ################################################################################ - # DELTA ORGANIZATIONS: END - ################################################################################ - ################################################################################ # DELTA ORGANIZATIONS PUBLIC INFO: START ################################################################################ - # delta-producer-bg-jobs-initiator-organizations-public-info: - # image: lblod/delta-producer-background-jobs-initiator:0.3.1 - # environment: - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - # CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours - # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - # START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync - # labels: - # - "logging=true" - # links: - # - db:database - # restart: always - # logging: *default-logging - # delta-producer-pub-graph-maintainer-organizations-public-info: - # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - # environment: - # MAX_BODY_SIZE: "50mb" - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # RELATIVE_FILE_PATH: "deltas/organizations-public-info" - # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - # QUEUE_POLL_INTERVAL: 3000 - # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - # SKIP_MU_AUTH_INITIAL_SYNC: "true" - # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - # SERVE_DELTA_FILES: "true" - # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" - # DELTA_INTERVAL_MS: 10000 - # PRETTY_PRINT_DIFF_JSON: "true" - # PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" - # PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" - # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - # volumes: - # - ./config/delta-producer/organizations-public-info:/config - # - ./data/files/:/share - # links: - # - db:database - # - triplestore:virtuoso - # restart: always - # labels: - # - "logging=true" - # logging: *default-logging + delta-producer-bg-jobs-initiator-organizations-public-info: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-organizations-public-info: + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + RELATIVE_FILE_PATH: "deltas/organizations-public-info" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" + PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + volumes: + - ./config/delta-producer/organizations-public-info:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + labels: + - "logging=true" + logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS PUBLIC INFO: END ################################################################################ @@ -482,50 +429,50 @@ services: ################################################################################ # DELTA PUBLIC INFORMATION: START ################################################################################ - # delta-producer-bg-jobs-initiator-public: - # image: lblod/delta-producer-background-jobs-initiator:0.3.1 - # environment: - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - # CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours - # CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - # labels: - # - "logging=true" - # links: - # - db:database - # restart: always - # logging: *default-logging - # delta-producer-pub-graph-maintainer-public: - # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - # environment: - # MAX_BODY_SIZE: "50mb" - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # RELATIVE_FILE_PATH: "deltas/public" - # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - # QUEUE_POLL_INTERVAL: 3000 - # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - # SKIP_MU_AUTH_INITIAL_SYNC: "true" - # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - # SERVE_DELTA_FILES: "true" - # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" - # DELTA_INTERVAL_MS: 10000 - # PRETTY_PRINT_DIFF_JSON: "true" - # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - # volumes: - # - ./config/delta-producer/public:/config - # - ./data/files/:/share - # links: - # - db:database - # - triplestore:virtuoso - # restart: always - # labels: - # - "logging=true" - # logging: *default-logging + delta-producer-bg-jobs-initiator-public: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-public: + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + RELATIVE_FILE_PATH: "deltas/public" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + volumes: + - ./config/delta-producer/public:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + labels: + - "logging=true" + logging: *default-logging ################################################################################ # DELTA PUBLIC INFORMATION: END ################################################################################ @@ -533,50 +480,50 @@ services: ################################################################################ # DELTA WORSHIP POSTS: START ################################################################################ - # delta-producer-bg-jobs-initiator-worship-posts: - # image: lblod/delta-producer-background-jobs-initiator:0.3.1 - # environment: - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - # CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours - # CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM - # START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - # labels: - # - "logging=true" - # links: - # - db:database - # restart: always - # logging: *default-logging - # delta-producer-pub-graph-maintainer-worship-posts: - # image: lblod/delta-producer-publication-graph-maintainer:0.15.4 - # environment: - # MAX_BODY_SIZE: "50mb" - # JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - # RELATIVE_FILE_PATH: "deltas/worship-posts" - # PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" - # HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - # INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - # QUEUE_POLL_INTERVAL: 3000 - # USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - # SKIP_MU_AUTH_INITIAL_SYNC: "true" - # HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - # SERVE_DELTA_FILES: "true" - # PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" - # DELTA_INTERVAL_MS: 10000 - # PRETTY_PRINT_DIFF_JSON: "true" - # ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - # volumes: - # - ./config/delta-producer/worship-posts:/config - # - ./data/files/:/share - # links: - # - db:database - # - triplestore:virtuoso - # restart: always - # labels: - # - "logging=true" - # logging: *default-logging + delta-producer-bg-jobs-initiator-worship-posts: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-worship-posts: + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + RELATIVE_FILE_PATH: "deltas/worship-posts" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + volumes: + - ./config/delta-producer/worship-posts:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + labels: + - "logging=true" + logging: *default-logging ################################################################################ # DELTA WORSHIP POSTS: END ################################################################################