diff --git a/CHANGELOG.md b/CHANGELOG.md
index eb68860..c811e39 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,64 @@
# Changelog
+## 1.29.1 (2025-02-14)
+### Backend
+ - re-init `worship-services-sensitive-consumer` [OP-3483]
+### Deploy notes
+:warning: Ensure first loket did a full re-sync again of this delta-stream.
+```
+drc stop worship-services-main-info-consumer worship-services-private-info-consumer
+drc retart migrations
+```
+Update `docker-compose.override.yml` to remove the config of the consumers and replace it by:
+```
+ worship-services-main-info-consumer:
+ environment:
+ DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
+ DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
+ DCR_SECRET_KEY: "key-of-the-producer"
+ DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
+ DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
+ DCR_DISABLE_INITIAL_SYNC: "false"
+ DCR_DISABLE_DELTA_INGEST: "false"
+ worship-services-private-info-consumer:
+ environment:
+ DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
+ DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
+ DCR_SECRET_KEY: "key-of-the-producer"
+ DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
+ DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
+ DCR_DISABLE_INITIAL_SYNC: "false"
+ DCR_DISABLE_DELTA_INGEST: "false"
+```
+Then
+```
+drc up -d db mandatarissen-consumer leidinggevenden-consumer worship-services-main-info-consumer worship-services-private-info-consumer
+# Wait until success of the previous step
+```
+Then, update `docker-compose.override.yml` to:
+```
+ worship-services-main-info-consumer:
+ environment:
+ DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
+ DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
+ DCR_SECRET_KEY: "key-of-the-producer"
+ DCR_LANDING_ZONE_DATABASE: "db"
+ DCR_REMAPPING_DATABASE: "db"
+ DCR_DISABLE_DELTA_INGEST: "false"
+ DCR_DISABLE_INITIAL_SYNC: "false"
+ worship-services-private-info-consumer:
+ environment:
+ DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
+ DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
+ DCR_SECRET_KEY: "key-of-the-producer"
+ DCR_LANDING_ZONE_DATABASE: "database"
+ DCR_REMAPPING_DATABASE: "database"
+ DCR_DISABLE_DELTA_INGEST: "false"
+ DCR_DISABLE_INITIAL_SYNC: "false"
+```
+```
+drc up -d
+sh scripts/reset-elastic.sh
+```
## 1.29.0 (2025-02-07)
### Backend
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162924-flush-eredienst-mandatarissen-consumer-containers.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162924-flush-eredienst-mandatarissen-consumer-containers.sparql
new file mode 100644
index 0000000..7ad2301
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162924-flush-eredienst-mandatarissen-consumer-containers.sparql
@@ -0,0 +1,36 @@
+PREFIX mu:
+PREFIX task:
+PREFIX dct:
+PREFIX prov:
+PREFIX nie:
+PREFIX ext:
+PREFIX oslc:
+PREFIX cogs:
+PREFIX adms:
+PREFIX nfo:
+PREFIX dbpedia:
+
+
+DELETE {
+ GRAPH ?g {
+ ?container ?containerP ?containerO.
+ }
+} WHERE {
+ # Please note, if you ever copy/paste this migration the next time, there should be 4
+ # operation URI's in total. Since two consumers
+ VALUES ?operation {
+
+
+ }
+
+ GRAPH ?g {
+ ?job a ;
+ task:operation ?operation.
+
+ ?task dct:isPartOf ?job;
+ task:resultsContainer ?container.
+
+ ?container a nfo:DataContainer;
+ ?containerP ?containerO.
+ }
+}
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162925-flush-eredienst-mandatarissen-consumer-tasks.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162925-flush-eredienst-mandatarissen-consumer-tasks.sparql
new file mode 100644
index 0000000..b474ae7
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162925-flush-eredienst-mandatarissen-consumer-tasks.sparql
@@ -0,0 +1,33 @@
+PREFIX mu:
+PREFIX task:
+PREFIX dct:
+PREFIX prov:
+PREFIX nie:
+PREFIX ext:
+PREFIX oslc:
+PREFIX cogs:
+PREFIX adms:
+PREFIX nfo:
+PREFIX dbpedia:
+
+
+DELETE {
+ GRAPH ?g {
+ ?task ?taskP ?taskO.
+ }
+} WHERE {
+ # Please note, if you ever copy/paste this migration the next time, there should be 4
+ # operation URI's in total. Since two consumers
+ VALUES ?operation {
+
+
+ }
+
+ GRAPH ?g {
+ ?job a ;
+ task:operation ?operation.
+
+ ?task dct:isPartOf ?job;
+ ?taskP ?taskO.
+ }
+}
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162926-flush-eredienst-mandatarissen-consumer-jobs.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162926-flush-eredienst-mandatarissen-consumer-jobs.sparql
new file mode 100644
index 0000000..76a13b2
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162926-flush-eredienst-mandatarissen-consumer-jobs.sparql
@@ -0,0 +1,31 @@
+ PREFIX mu:
+PREFIX task:
+PREFIX dct:
+PREFIX prov:
+PREFIX nie:
+PREFIX ext:
+PREFIX oslc:
+PREFIX cogs:
+PREFIX adms:
+PREFIX nfo:
+PREFIX dbpedia:
+
+
+DELETE {
+ GRAPH ?g {
+ ?job ?jobP ?jobO.
+ }
+} WHERE {
+ # Please note, if you ever copy/paste this migration the next time, there should be 4
+ # operation URI's in total. Since two consumers
+ VALUES ?operation {
+
+
+ }
+
+ GRAPH ?g {
+ ?job a ;
+ task:operation ?operation;
+ ?jobP ?jobO.
+ }
+}
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162927-remove-no-types.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162927-remove-no-types.sparql
new file mode 100644
index 0000000..aa04c3a
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162927-remove-no-types.sparql
@@ -0,0 +1,17 @@
+DELETE {
+ GRAPH ?g {
+ ?s ?p ?o
+ }
+}
+WHERE {
+ VALUES ?g {
+
+
+ }
+ GRAPH ?g {
+ ?s ?p ?o.
+ FILTER NOT EXISTS {
+ ?s a ?type.
+ }
+ }
+}
\ No newline at end of file
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162928-remove-loket-persons-birthdates.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162928-remove-loket-persons-birthdates.sparql
new file mode 100644
index 0000000..0353903
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162928-remove-loket-persons-birthdates.sparql
@@ -0,0 +1,33 @@
+PREFIX ch:
+PREFIX mandaat:
+PREFIX org:
+PREFIX person:
+PREFIX persoon:
+
+# Delete birthdates
+DELETE {
+ GRAPH {
+ ?birthDate ?p ?o .
+ }
+} WHERE {
+ VALUES ?g {
+
+
+ }
+
+ GRAPH ?g {
+ ?person a person:Person .
+ }
+
+ FILTER EXISTS {
+ GRAPH {
+ ?agent a ch:AgentInPositie ;
+ mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
+ }
+ }
+
+ GRAPH {
+ ?person persoon:heeftGeboorte ?birthDate .
+ ?birthDate ?p ?o .
+ }
+}
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162929-remove-loket-persons-identifiers.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162929-remove-loket-persons-identifiers.sparql
new file mode 100644
index 0000000..e8e19bc
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162929-remove-loket-persons-identifiers.sparql
@@ -0,0 +1,54 @@
+PREFIX ch:
+PREFIX generiek:
+PREFIX mandaat:
+PREFIX org:
+PREFIX person:
+PREFIX persoon:
+
+# Delete identifiers
+DELETE {
+ GRAPH {
+ ?identifier ?pidentifier ?oidentifier .
+ ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
+ }
+} WHERE {
+ VALUES ?g {
+
+
+ }
+
+ GRAPH ?g {
+ ?person a person:Person .
+ }
+
+ FILTER EXISTS {
+ GRAPH {
+ ?agent a ch:AgentInPositie ;
+ mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
+ }
+ }
+
+ GRAPH {
+ ?person persoon:registratie ?identifier .
+
+ ?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier ;
+ ?pidentifier ?oidentifier .
+
+ ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
+ }
+}
+
+;
+
+DELETE {
+ GRAPH {
+ ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
+ }
+} WHERE {
+
+ GRAPH {
+ ?structuredIdentifier a ;
+ ?pstructuredIdentifier ?ostructuredIdentifier.
+ }
+
+}
\ No newline at end of file
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162930-remove-loket-persons.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162930-remove-loket-persons.sparql
new file mode 100644
index 0000000..9c876b6
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162930-remove-loket-persons.sparql
@@ -0,0 +1,38 @@
+PREFIX ch:
+PREFIX mandaat:
+PREFIX org:
+PREFIX person:
+
+# Delete persons
+DELETE {
+ GRAPH ?g {
+ ?person ?p ?o .
+ }
+
+ GRAPH {
+ ?person ?pprivate ?oprivate .
+ }
+} WHERE {
+ VALUES ?g {
+
+
+ }
+
+ GRAPH ?g {
+ ?person a person:Person ;
+ ?p ?o .
+ }
+
+ FILTER EXISTS {
+ GRAPH {
+ ?agent a ch:AgentInPositie ;
+ mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
+ }
+ }
+
+ OPTIONAL {
+ GRAPH {
+ ?person ?pprivate ?oprivate .
+ }
+ }
+}
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162931-contact-points.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162931-contact-points.sparql
new file mode 100644
index 0000000..83d8a6f
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162931-contact-points.sparql
@@ -0,0 +1,22 @@
+DELETE {
+ GRAPH {
+ ?contactPoint ?pcontactPoint ?ocontactPoint .
+ ?address ?paddress ?oaddress .
+ }
+} WHERE {
+ GRAPH {
+ ?agent a .
+
+ OPTIONAL {
+ ?agent ?contactPoint .
+
+ ?contactPoint ?pcontactPoint ?ocontactPoint .
+
+ OPTIONAL {
+ ?contactPoint ?address .
+
+ ?address ?paddress ?oaddress .
+ }
+ }
+ }
+}
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162932-remove-agents-in-position.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162932-remove-agents-in-position.sparql
new file mode 100644
index 0000000..31d2d87
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162932-remove-agents-in-position.sparql
@@ -0,0 +1,10 @@
+DELETE {
+ GRAPH {
+ ?agent ?pagent ?oagent .
+ }
+} WHERE {
+ GRAPH {
+ ?agent a ;
+ ?pagent ?oagent .
+ }
+}
diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162933-flush-landing-zone.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162933-flush-landing-zone.sparql
new file mode 100644
index 0000000..8588ee0
--- /dev/null
+++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162933-flush-landing-zone.sparql
@@ -0,0 +1,2 @@
+CLEAR GRAPH ;
+CLEAR GRAPH ;
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 6f87138..7847126 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -370,7 +370,7 @@ services:
# LOKET MANDATARISSEN CONSUMERS
################################################################################
mandatarissen-consumer:
- image: lblod/delta-consumer:0.1.0
+ image: lblod/delta-consumer:0.1.1
environment:
DCR_SERVICE_NAME: "mandatarissen-consumer"
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -403,7 +403,7 @@ services:
# LOKET LEIDINGGEVENDEN CONSUMERS
################################################################################
leidinggevenden-consumer:
- image: lblod/delta-consumer:0.1.0
+ image: lblod/delta-consumer:0.1.1
environment:
DCR_SERVICE_NAME: "leidinggevenden-consumer"
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -436,13 +436,13 @@ services:
# LOKET WORSHIP SENSITIVE CONSUMERS
################################################################################
worship-services-main-info-consumer:
- image: lblod/delta-consumer:0.1.0
+ image: lblod/delta-consumer:0.1.1
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
DCR_SERVICE_NAME: "worship-services-main-info-consumer"
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
- DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
- DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
+ DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive/main"
+ DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing/main"
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-main-info-consumer"
DCR_DISABLE_INITIAL_SYNC: "true"
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
@@ -469,13 +469,13 @@ services:
logging: *default-logging
worship-services-private-info-consumer:
- image: lblod/delta-consumer:0.1.0
+ image: lblod/delta-consumer:0.1.1
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
DCR_SERVICE_NAME: "worship-services-private-info-consumer"
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
- DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
- DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
+ DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive/private"
+ DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing/private"
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-private-info-consumer"
DCR_DISABLE_INITIAL_SYNC: "true"
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"