From e61f606def7e6c5ed5d90d95d665d5447c730293 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 6 Jan 2023 11:32:27 +0100 Subject: [PATCH] Add delta producer for posts --- .../dump-file-publisher/config.json | 12 ++++ config/delta-producer/posts/export.json | 49 +++++++++++++++ config/delta/rules.js | 35 +++++++++++ docker-compose.dev.yml | 4 ++ docker-compose.yml | 62 +++++++++++++++++++ 5 files changed, 162 insertions(+) create mode 100644 config/delta-producer/posts/export.json diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 5f0fd27..7456a19 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -34,5 +34,17 @@ "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", "compression": false + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/posts" : { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PostsCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/posts", + "fileBaseName": "dump-posts", + "compression": false + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/posts" : { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PostsCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/posts", + "fileBaseName": "dump-posts", + "compression": false } } diff --git a/config/delta-producer/posts/export.json b/config/delta-producer/posts/export.json new file mode 100644 index 0000000..df6a82a --- /dev/null +++ b/config/delta-producer/posts/export.json @@ -0,0 +1,49 @@ +{ + "export": [ + { + "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role", + "http://www.w3.org/ns/org#postIn", + "http://www.w3.org/ns/org#role" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://www.w3.org/ns/org#hasPost" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role", + "http://www.w3.org/ns/org#postIn", + "http://data.lblod.info/vocabularies/erediensten/functie", + "http://data.lblod.info/vocabularies/erediensten/behoort" + ] + } + ] +} diff --git a/config/delta/rules.js b/config/delta/rules.js index 15a2309..5fad7c0 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -156,6 +156,41 @@ export default [ optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] } }, + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-posts/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://redpencil.data.gift/id/deltas/producer/posts' + } + }, + callback: { + url: 'http://delta-producer-json-diff-publisher-posts/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, { match: { predicate: { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 33d98bd..8066f8a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -71,3 +71,7 @@ services: restart: "no" construct-administrative-unit-relationships: restart: "no" + delta-producer-bg-jobs-initiator-posts: + restart: "no" + delta-producer-pub-graph-maintainer-posts: + restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index 62232c0..c2b4c5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -404,6 +404,68 @@ services: # DELTA PUBLIC INFORMATION: END ################################################################################ + ################################################################################ + # DELTA POSTS: START + ################################################################################ + delta-producer-bg-jobs-initiator-posts: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/posts" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/posts" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/posts" + CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-posts: + image: lblod/delta-producer-publication-graph-maintainer:0.15.1 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + RELATIVE_FILE_PATH: "deltas/public" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/posts" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/posts" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/posts" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + volumes: + - ./config/delta-producer/posts:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + logging: *default-logging + delta-producer-json-diff-publisher-posts: + image: lblod/delta-producer-json-diff-file-publisher:0.2.3 + environment: + MAX_BODY_SIZE: "50mb" + ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + RELATIVE_FILE_PATH: "deltas/posts" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/posts" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/posts" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + volumes: + - ./data/files:/share + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + ################################################################################ + # DELTA POSTS: END + ################################################################################ + ################################################################################ # DELTA CONSUMERS ################################################################################