Merge pull request #515 from lblod/hotfix/flush-eredienst-mandatarissen

Hotfix/flush eredienst mandatarissen
2025-02-14 16:45:59 +01:00
parent 9ea9d90751 f689f90e37
commit b273a77923
12 changed files with 344 additions and 8 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,64 @@
 # Changelog
+## 1.29.1 (2025-02-14)
+### Backend
+ - re-init `worship-services-sensitive-consumer` [OP-3483]
+### Deploy notes
+:warning: Ensure first loket did a full re-sync again of this delta-stream.
+```
+drc stop worship-services-main-info-consumer worship-services-private-info-consumer
+drc retart migrations
+```
+Update `docker-compose.override.yml` to remove the config of the consumers and replace it by:
+```
+  worship-services-main-info-consumer:
+    environment:
+      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
+      DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
+      DCR_SECRET_KEY: "key-of-the-producer"
+      DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
+      DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
+      DCR_DISABLE_INITIAL_SYNC: "false"
+      DCR_DISABLE_DELTA_INGEST: "false"
+  worship-services-private-info-consumer:
+    environment:
+      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
+      DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
+      DCR_SECRET_KEY: "key-of-the-producer"
+      DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
+      DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
+      DCR_DISABLE_INITIAL_SYNC: "false"
+      DCR_DISABLE_DELTA_INGEST: "false"
+```
+Then
+```
+drc up -d db mandatarissen-consumer leidinggevenden-consumer worship-services-main-info-consumer worship-services-private-info-consumer
+# Wait until success of the previous step
+```
+Then, update `docker-compose.override.yml` to:
+```
+  worship-services-main-info-consumer:
+    environment:
+      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
+      DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
+      DCR_SECRET_KEY: "key-of-the-producer"
+      DCR_LANDING_ZONE_DATABASE: "db"
+      DCR_REMAPPING_DATABASE: "db"
+      DCR_DISABLE_DELTA_INGEST: "false"
+      DCR_DISABLE_INITIAL_SYNC: "false"
+  worship-services-private-info-consumer:
+    environment:
+      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
+      DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
+      DCR_SECRET_KEY: "key-of-the-producer"
+      DCR_LANDING_ZONE_DATABASE: "database"
+      DCR_REMAPPING_DATABASE: "database"
+      DCR_DISABLE_DELTA_INGEST: "false"
+      DCR_DISABLE_INITIAL_SYNC: "false"
+```
+```
+drc up -d
+sh scripts/reset-elastic.sh
+```

 ## 1.29.0 (2025-02-07)
 ### Backend
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162924-flush-eredienst-mandatarissen-consumer-containers.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162924-flush-eredienst-mandatarissen-consumer-containers.sparql
@@ -0,0 +1,36 @@
+PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX oslc: <http://open-services.net/ns/core#>
+PREFIX cogs: <http://vocab.deri.ie/cogs#>
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
+PREFIX dbpedia: <http://dbpedia.org/resource/>
+
+
+DELETE {
+  GRAPH ?g {
+    ?container ?containerP ?containerO.
+  }
+} WHERE {
+  # Please note, if you ever copy/paste this migration the next time, there should be 4
+  # operation URI's in total. Since two consumers
+  VALUES ?operation {
+    <http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive>
+    <http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing>
+  }
+
+  GRAPH ?g {
+    ?job a <http://vocab.deri.ie/cogs#Job>;
+      task:operation ?operation.
+      
+    ?task dct:isPartOf ?job;
+      task:resultsContainer ?container.
+
+    ?container a nfo:DataContainer;
+      ?containerP ?containerO.
+  }
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162925-flush-eredienst-mandatarissen-consumer-tasks.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162925-flush-eredienst-mandatarissen-consumer-tasks.sparql
@@ -0,0 +1,33 @@
+PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX oslc: <http://open-services.net/ns/core#>
+PREFIX cogs: <http://vocab.deri.ie/cogs#>
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
+PREFIX dbpedia: <http://dbpedia.org/resource/>
+
+
+DELETE {
+  GRAPH ?g {
+    ?task ?taskP ?taskO.
+ }
+} WHERE {
+  # Please note, if you ever copy/paste this migration the next time, there should be 4
+  # operation URI's in total. Since two consumers
+  VALUES ?operation {
+    <http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive>
+    <http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing>
+  }
+
+  GRAPH ?g {
+    ?job a <http://vocab.deri.ie/cogs#Job>;
+      task:operation ?operation.
+
+    ?task dct:isPartOf ?job;
+      ?taskP ?taskO.
+  }
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162926-flush-eredienst-mandatarissen-consumer-jobs.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162926-flush-eredienst-mandatarissen-consumer-jobs.sparql
@@ -0,0 +1,31 @@
+  PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX oslc: <http://open-services.net/ns/core#>
+PREFIX cogs: <http://vocab.deri.ie/cogs#>
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
+PREFIX dbpedia: <http://dbpedia.org/resource/>
+
+
+DELETE {
+  GRAPH ?g {
+    ?job ?jobP ?jobO.
+ }
+} WHERE {
+  # Please note, if you ever copy/paste this migration the next time, there should be 4
+  # operation URI's in total. Since two consumers
+  VALUES ?operation {
+    <http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive>
+    <http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing>
+  }
+
+  GRAPH ?g {
+    ?job a <http://vocab.deri.ie/cogs#Job>;
+      task:operation ?operation;
+      ?jobP ?jobO.
+  }
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162927-remove-no-types.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162927-remove-no-types.sparql
@@ -0,0 +1,17 @@
+DELETE {
+  GRAPH ?g {
+    ?s ?p ?o
+  }
+}
+WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/worship-privacy-centric-graph>
+    <http://mu.semte.ch/graphs/worship-service>
+  }
+  GRAPH ?g {
+    ?s ?p ?o.
+    FILTER NOT EXISTS {
+      ?s a ?type.
+    }
+  }
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162928-remove-loket-persons-birthdates.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162928-remove-loket-persons-birthdates.sparql
@@ -0,0 +1,33 @@
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
+
+# Delete birthdates
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
+    ?birthDate ?p ?o .
+  }
+} WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/shared>
+    <http://mu.semte.ch/graphs/worship-service>
+  }
+
+  GRAPH ?g {
+    ?person a person:Person .
+  }
+
+  FILTER EXISTS {
+    GRAPH <http://mu.semte.ch/graphs/worship-service> {
+      ?agent a ch:AgentInPositie ;
+        mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
+    }
+  }
+
+  GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
+    ?person persoon:heeftGeboorte ?birthDate .
+    ?birthDate ?p ?o .
+  }
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162929-remove-loket-persons-identifiers.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162929-remove-loket-persons-identifiers.sparql
@@ -0,0 +1,54 @@
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
+
+# Delete identifiers
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
+    ?identifier ?pidentifier ?oidentifier .
+    ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
+  }
+} WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/shared>
+    <http://mu.semte.ch/graphs/worship-service>
+  }
+
+  GRAPH ?g {
+    ?person a person:Person .
+  }
+
+  FILTER EXISTS {
+    GRAPH <http://mu.semte.ch/graphs/worship-service> {
+      ?agent a ch:AgentInPositie ;
+        mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
+    }
+  }
+
+  GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
+    ?person persoon:registratie ?identifier .
+
+    ?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier ;
+      ?pidentifier ?oidentifier .
+
+    ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
+  }
+}
+
+;
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
+    ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
+  }
+} WHERE {
+
+  GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
+    ?structuredIdentifier a <https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator>;
+      ?pstructuredIdentifier ?ostructuredIdentifier.
+  }
+
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162930-remove-loket-persons.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162930-remove-loket-persons.sparql
@@ -0,0 +1,38 @@
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX person: <http://www.w3.org/ns/person#>
+
+# Delete persons
+DELETE {
+  GRAPH ?g {
+    ?person ?p ?o .
+  }
+
+  GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
+    ?person ?pprivate ?oprivate .
+  }
+} WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/shared>
+    <http://mu.semte.ch/graphs/worship-service>
+  }
+
+  GRAPH ?g {
+    ?person a person:Person ;
+      ?p ?o .
+  }
+
+  FILTER EXISTS {
+    GRAPH <http://mu.semte.ch/graphs/worship-service> {
+      ?agent a ch:AgentInPositie ;
+        mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
+    }
+  }
+
+  OPTIONAL {
+    GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
+      ?person ?pprivate ?oprivate .
+    }
+  }
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162931-contact-points.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162931-contact-points.sparql
@@ -0,0 +1,22 @@
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    ?contactPoint ?pcontactPoint ?ocontactPoint .
+    ?address ?paddress ?oaddress .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> { 
+    ?agent a <http://data.lblod.info/vocabularies/contacthub/AgentInPositie> .
+
+    OPTIONAL {
+      ?agent <http://schema.org/contactPoint> ?contactPoint .
+
+      ?contactPoint ?pcontactPoint ?ocontactPoint .
+
+      OPTIONAL {
+        ?contactPoint <http://www.w3.org/ns/locn#address> ?address .
+
+        ?address ?paddress ?oaddress .
+      }
+    }
+  }
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162932-remove-agents-in-position.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162932-remove-agents-in-position.sparql
@@ -0,0 +1,10 @@
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    ?agent ?pagent ?oagent .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> { 
+    ?agent a <http://data.lblod.info/vocabularies/contacthub/AgentInPositie> ;
+      ?pagent ?oagent .
+  }
+}
--- a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162933-flush-landing-zone.sparql
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162933-flush-landing-zone.sparql
@@ -0,0 +1,2 @@
+CLEAR GRAPH <http://mu.semte.ch/graphs/landing-zone/worship-services-private-info>;
+CLEAR GRAPH <http://mu.semte.ch/graphs/landing-zone/worship-services-main-info>;
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -370,7 +370,7 @@ services:
  # LOKET MANDATARISSEN CONSUMERS
  ################################################################################
  mandatarissen-consumer:
-    image: lblod/delta-consumer:0.1.0
+    image: lblod/delta-consumer:0.1.1
    environment:
      DCR_SERVICE_NAME: "mandatarissen-consumer"
      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -403,7 +403,7 @@ services:
  # LOKET LEIDINGGEVENDEN CONSUMERS
  ################################################################################
  leidinggevenden-consumer:
-    image: lblod/delta-consumer:0.1.0
+    image: lblod/delta-consumer:0.1.1
    environment:
      DCR_SERVICE_NAME: "leidinggevenden-consumer"
      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -436,13 +436,13 @@ services:
  # LOKET WORSHIP SENSITIVE CONSUMERS
  ################################################################################
  worship-services-main-info-consumer:
-    image: lblod/delta-consumer:0.1.0
+    image: lblod/delta-consumer:0.1.1
    environment:
      DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
      DCR_SERVICE_NAME: "worship-services-main-info-consumer"
      DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
-      DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
-      DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
+      DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive/main"
+      DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing/main"
      DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-main-info-consumer"
      DCR_DISABLE_INITIAL_SYNC: "true"
      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
@@ -469,13 +469,13 @@ services:
    logging: *default-logging

  worship-services-private-info-consumer:
-    image: lblod/delta-consumer:0.1.0
+    image: lblod/delta-consumer:0.1.1
    environment:
      DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
      DCR_SERVICE_NAME: "worship-services-private-info-consumer"
      DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
-      DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
-      DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
+      DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive/private"
+      DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing/private"
      DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-private-info-consumer"
      DCR_DISABLE_INITIAL_SYNC: "true"
      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"