From 0e64b2589c4778eacec0bc88c756f0828ea344ac Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Fri, 6 Jan 2023 16:24:06 +0000 Subject: [PATCH] organizations publisher config update --- .gitignore | 1 + .../dump-file-publisher/config.json | 36 ++++++++++++------- ...20230106143000-delete-old-dump-info.sparql | 34 ++++++++++++++++++ ...43000-organization-producer-account.sparql | 16 +++++++++ docker-compose.yml | 9 +++-- 5 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql create mode 100644 config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql diff --git a/.gitignore b/.gitignore index cc42a73..d6b6a17 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ !/data/db/toLoad !/data/db/toLoad/* /data/files/* +/data/temp/* docker-compose.override.yml .idea *.iml diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 5f0fd27..a57e503 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -1,38 +1,50 @@ { - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", "fileBaseName": "dump-administrative-units", - "compression": true + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", "fileBaseName": "dump-administrative-units", - "compression": true + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", - "compression": true + "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", + "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizationsdeltas", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", - "compression": true + "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", + "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizationsdeltas", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", - "compression": false + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", - "compression": false + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true } } diff --git a/config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql b/config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql new file mode 100644 index 0000000..5be441c --- /dev/null +++ b/config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql @@ -0,0 +1,34 @@ +PREFIX nie: +PREFIX dct: +PREFIX dcat: +PREFIX prov: + + +DELETE { + GRAPH { + ?sub + dct:subject ; + dcat:distribution ?distribution. + ?distribution + dct:subject ?file ; + dct:created ?created . + ?phyiscalFile + nie:dataSource ?file + } +} +WHERE { + GRAPH { + ?sub + dct:subject ; + dcat:distribution ?distribution. + FILTER NOT EXISTS { + ?revision + prov:wasRevisionOf ?sub + } + ?distribution + dct:subject ?file ; + dct:created ?created . + ?phyiscalFile + nie:dataSource ?file + } +} \ No newline at end of file diff --git a/config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql b/config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql new file mode 100644 index 0000000..6e74965 --- /dev/null +++ b/config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql @@ -0,0 +1,16 @@ +PREFIX foaf: +INSERT DATA { + GRAPH { + + a foaf:OnlineAccount. + + + a foaf:Agent; + foaf:account . + + + a foaf:Group; + foaf:member ; + foaf:name "organizations". + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 62232c0..d43b246 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -91,7 +91,7 @@ services: LOGGING_LEVEL: "INFO" SPARQL_ENDPOINT: "http://db:8890/sparql" LIMIT_SIZE: "10000" - MU_AUTH_SUDO: "false" + MU_AUTH_SUDO: "true" labels: - "logging=true" restart: always @@ -208,13 +208,10 @@ services: # DELTA GENERAL ################################################################################ delta-producer-dump-file-publisher: - image: bdevloed/delta-producer-dump-file-publisher:0.8.0-beta + image: lblod/delta-producer-dump-file-publisher:0.9.0 volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share - environment: - FILES_GRAPH: "http://mu.semte.ch/graphs/public" # Note: this will be extracted in a config per job one day - DCAT_DATASET_GRAPH: "http://mu.semte.ch/graphs/public" links: - db:database restart: always @@ -312,6 +309,8 @@ services: USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + FILES_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-deltas" + ACCOUNT: "http://data.lblod.info/foaf/account/id/8df6f1e4-869f-4223-9834-961b01a7bd48" volumes: - ./config/delta-producer/organizations:/config - ./data/files/:/share