Merge pull request #210 from lblod/fix/publisher-config

organizations publisher config update
2023-01-09 16:41:09 +01:00
parent 71e024c007 fef430002b
commit 8e5d33f8fb
5 changed files with 85 additions and 19 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 !/data/db/toLoad
 !/data/db/toLoad/*
 /data/files/*
+/data/temp/*
 docker-compose.override.yml
 .idea
 *.iml
--- a/config/delta-producer/dump-file-publisher/config.json
+++ b/config/delta-producer/dump-file-publisher/config.json
@@ -1,50 +1,66 @@
 {
-  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" : {
+  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units": {
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units",
    "fileBaseName": "dump-administrative-units",
-    "compression": true
+    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
+    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
+    "cleanupOldDumps": true
  },
-  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" : {
+  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units": {
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units",
    "fileBaseName": "dump-administrative-units",
-    "compression": true
+    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
+    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
+    "cleanupOldDumps": true
  },
-  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" : {
+  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations": {
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
    "fileBaseName": "dump-organizations",
-    "compression": true
+    "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
+    "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizationsdeltas",
+    "cleanupOldDumps": true
  },
-  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" : {
+  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations": {
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
    "fileBaseName": "dump-organizations",
-    "compression": true
+    "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
+    "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizationsdeltas",
+    "cleanupOldDumps": true
  },
-  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" : {
+  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": {
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public",
    "fileBaseName": "dump-public",
-    "compression": false
+    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
+    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
+    "cleanupOldDumps": true
  },
-  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" : {
+  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": {
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public",
    "fileBaseName": "dump-public",
-    "compression": false
+    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
+    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
+    "cleanupOldDumps": true
  },
  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" : {
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts",
    "fileBaseName": "dump-worship-posts",
-    "compression": false
+    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
+    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
+    "cleanupOldDumps": true
  },
  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" : {
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts",
    "fileBaseName": "dump-worship-posts",
-    "compression": false
+    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
+    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
+    "cleanupOldDumps": true
  }
 }
--- a/config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql
+++ b/config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql
@@ -0,0 +1,34 @@
+PREFIX nie:  <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
+PREFIX dct:  <http://purl.org/dc/terms/>
+PREFIX dcat: <http://www.w3.org/ns/dcat#>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/public> {
+    ?sub
+      dct:subject <http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump> ;
+      dcat:distribution ?distribution.
+    ?distribution
+      dct:subject ?file ;
+      dct:created ?created .
+    ?phyiscalFile
+      nie:dataSource ?file
+  }
+}
+WHERE {
+  GRAPH <http://mu.semte.ch/graphs/public> {
+    ?sub
+      dct:subject <http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump> ;
+      dcat:distribution ?distribution.
+    FILTER NOT EXISTS {
+      ?revision
+        prov:wasRevisionOf ?sub
+    }
+    ?distribution
+      dct:subject ?file ;
+      dct:created ?created .
+    ?phyiscalFile
+      nie:dataSource ?file
+  }
+}
--- a/config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql
+++ b/config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql
@@ -0,0 +1,16 @@
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+INSERT DATA {
+  GRAPH <http://mu.semte.ch/graphs/accounts/deltas> {
+    <http://data.lblod.info/foaf/account/id/8df6f1e4-869f-4223-9834-961b01a7bd48>
+      a foaf:OnlineAccount.
+
+    <http://data.lblod.info/foaf/agent/id/3d204ad9-3f20-407a-ab40-81a3268d5594>
+      a foaf:Agent;
+      foaf:account <http://data.lblod.info/foaf/account/id/8df6f1e4-869f-4223-9834-961b01a7bd48>.
+
+    <http://data.lblod.info/foaf/group/id/44a6640d-b24d-43e2-a23d-7462f77e0adc>
+      a foaf:Group;
+      foaf:member <http://data.lblod.info/foaf/agent/id/3d204ad9-3f20-407a-ab40-81a3268d5594>;
+      foaf:name "organizations".
+  }
+}
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -91,7 +91,7 @@ services:
      LOGGING_LEVEL: "INFO"
      SPARQL_ENDPOINT: "http://db:8890/sparql"
      LIMIT_SIZE: "10000"
-      MU_AUTH_SUDO: "false"
+      MU_AUTH_SUDO: "true"
    labels:
      - "logging=true"
    restart: always
@@ -208,13 +208,10 @@ services:
  # DELTA GENERAL
  ################################################################################
  delta-producer-dump-file-publisher:
-    image: bdevloed/delta-producer-dump-file-publisher:0.8.0-beta
+    image: lblod/delta-producer-dump-file-publisher:0.9.0
    volumes:
      - ./config/delta-producer/dump-file-publisher:/config
      - ./data/files:/share
-    environment:
-      FILES_GRAPH: "http://mu.semte.ch/graphs/public" # Note: this will be extracted in a config per job one day
-      DCAT_DATASET_GRAPH: "http://mu.semte.ch/graphs/public"
    links:
      - db:database
    restart: always
@@ -312,6 +309,8 @@ services:
      USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
      SKIP_MU_AUTH_INITIAL_SYNC: "true"
      HEALING_PATCH_GRAPH_BATCH_SIZE: 500
+      FILES_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-deltas"
+      ACCOUNT: "http://data.lblod.info/foaf/account/id/8df6f1e4-869f-4223-9834-961b01a7bd48"
    volumes:
      - ./config/delta-producer/organizations:/config
      - ./data/files/:/share