diff --git a/CHANGELOG.md b/CHANGELOG.md index eb68860..c811e39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,64 @@ # Changelog +## 1.29.1 (2025-02-14) +### Backend + - re-init `worship-services-sensitive-consumer` [OP-3483] +### Deploy notes +:warning: Ensure first loket did a full re-sync again of this delta-stream. +``` +drc stop worship-services-main-info-consumer worship-services-private-info-consumer +drc retart migrations +``` +Update `docker-compose.override.yml` to remove the config of the consumers and replace it by: +``` + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_DISABLE_INITIAL_SYNC: "false" + DCR_DISABLE_DELTA_INGEST: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_DISABLE_INITIAL_SYNC: "false" + DCR_DISABLE_DELTA_INGEST: "false" +``` +Then +``` +drc up -d db mandatarissen-consumer leidinggevenden-consumer worship-services-main-info-consumer worship-services-private-info-consumer +# Wait until success of the previous step +``` +Then, update `docker-compose.override.yml` to: +``` + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "db" + DCR_REMAPPING_DATABASE: "db" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "database" + DCR_REMAPPING_DATABASE: "database" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +``` +drc up -d +sh scripts/reset-elastic.sh +``` ## 1.29.0 (2025-02-07) ### Backend diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162924-flush-eredienst-mandatarissen-consumer-containers.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162924-flush-eredienst-mandatarissen-consumer-containers.sparql new file mode 100644 index 0000000..7ad2301 --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162924-flush-eredienst-mandatarissen-consumer-containers.sparql @@ -0,0 +1,36 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?container ?containerP ?containerO. + } +} WHERE { + # Please note, if you ever copy/paste this migration the next time, there should be 4 + # operation URI's in total. Since two consumers + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation. + + ?task dct:isPartOf ?job; + task:resultsContainer ?container. + + ?container a nfo:DataContainer; + ?containerP ?containerO. + } +} diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162925-flush-eredienst-mandatarissen-consumer-tasks.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162925-flush-eredienst-mandatarissen-consumer-tasks.sparql new file mode 100644 index 0000000..b474ae7 --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162925-flush-eredienst-mandatarissen-consumer-tasks.sparql @@ -0,0 +1,33 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?task ?taskP ?taskO. + } +} WHERE { + # Please note, if you ever copy/paste this migration the next time, there should be 4 + # operation URI's in total. Since two consumers + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation. + + ?task dct:isPartOf ?job; + ?taskP ?taskO. + } +} diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162926-flush-eredienst-mandatarissen-consumer-jobs.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162926-flush-eredienst-mandatarissen-consumer-jobs.sparql new file mode 100644 index 0000000..76a13b2 --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162926-flush-eredienst-mandatarissen-consumer-jobs.sparql @@ -0,0 +1,31 @@ + PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?job ?jobP ?jobO. + } +} WHERE { + # Please note, if you ever copy/paste this migration the next time, there should be 4 + # operation URI's in total. Since two consumers + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation; + ?jobP ?jobO. + } +} diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162927-remove-no-types.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162927-remove-no-types.sparql new file mode 100644 index 0000000..aa04c3a --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162927-remove-no-types.sparql @@ -0,0 +1,17 @@ +DELETE { + GRAPH ?g { + ?s ?p ?o + } +} +WHERE { + VALUES ?g { + + + } + GRAPH ?g { + ?s ?p ?o. + FILTER NOT EXISTS { + ?s a ?type. + } + } +} \ No newline at end of file diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162928-remove-loket-persons-birthdates.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162928-remove-loket-persons-birthdates.sparql new file mode 100644 index 0000000..0353903 --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162928-remove-loket-persons-birthdates.sparql @@ -0,0 +1,33 @@ +PREFIX ch: +PREFIX mandaat: +PREFIX org: +PREFIX person: +PREFIX persoon: + +# Delete birthdates +DELETE { + GRAPH { + ?birthDate ?p ?o . + } +} WHERE { + VALUES ?g { + + + } + + GRAPH ?g { + ?person a person:Person . + } + + FILTER EXISTS { + GRAPH { + ?agent a ch:AgentInPositie ; + mandaat:isBestuurlijkeAliasVan|org:heldBy ?person . + } + } + + GRAPH { + ?person persoon:heeftGeboorte ?birthDate . + ?birthDate ?p ?o . + } +} diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162929-remove-loket-persons-identifiers.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162929-remove-loket-persons-identifiers.sparql new file mode 100644 index 0000000..e8e19bc --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162929-remove-loket-persons-identifiers.sparql @@ -0,0 +1,54 @@ +PREFIX ch: +PREFIX generiek: +PREFIX mandaat: +PREFIX org: +PREFIX person: +PREFIX persoon: + +# Delete identifiers +DELETE { + GRAPH { + ?identifier ?pidentifier ?oidentifier . + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} WHERE { + VALUES ?g { + + + } + + GRAPH ?g { + ?person a person:Person . + } + + FILTER EXISTS { + GRAPH { + ?agent a ch:AgentInPositie ; + mandaat:isBestuurlijkeAliasVan|org:heldBy ?person . + } + } + + GRAPH { + ?person persoon:registratie ?identifier . + + ?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier ; + ?pidentifier ?oidentifier . + + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} + +; + +DELETE { + GRAPH { + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} WHERE { + + GRAPH { + ?structuredIdentifier a ; + ?pstructuredIdentifier ?ostructuredIdentifier. + } + +} \ No newline at end of file diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162930-remove-loket-persons.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162930-remove-loket-persons.sparql new file mode 100644 index 0000000..9c876b6 --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162930-remove-loket-persons.sparql @@ -0,0 +1,38 @@ +PREFIX ch: +PREFIX mandaat: +PREFIX org: +PREFIX person: + +# Delete persons +DELETE { + GRAPH ?g { + ?person ?p ?o . + } + + GRAPH { + ?person ?pprivate ?oprivate . + } +} WHERE { + VALUES ?g { + + + } + + GRAPH ?g { + ?person a person:Person ; + ?p ?o . + } + + FILTER EXISTS { + GRAPH { + ?agent a ch:AgentInPositie ; + mandaat:isBestuurlijkeAliasVan|org:heldBy ?person . + } + } + + OPTIONAL { + GRAPH { + ?person ?pprivate ?oprivate . + } + } +} diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162931-contact-points.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162931-contact-points.sparql new file mode 100644 index 0000000..83d8a6f --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162931-contact-points.sparql @@ -0,0 +1,22 @@ +DELETE { + GRAPH { + ?contactPoint ?pcontactPoint ?ocontactPoint . + ?address ?paddress ?oaddress . + } +} WHERE { + GRAPH { + ?agent a . + + OPTIONAL { + ?agent ?contactPoint . + + ?contactPoint ?pcontactPoint ?ocontactPoint . + + OPTIONAL { + ?contactPoint ?address . + + ?address ?paddress ?oaddress . + } + } + } +} diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162932-remove-agents-in-position.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162932-remove-agents-in-position.sparql new file mode 100644 index 0000000..31d2d87 --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162932-remove-agents-in-position.sparql @@ -0,0 +1,10 @@ +DELETE { + GRAPH { + ?agent ?pagent ?oagent . + } +} WHERE { + GRAPH { + ?agent a ; + ?pagent ?oagent . + } +} diff --git a/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162933-flush-landing-zone.sparql b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162933-flush-landing-zone.sparql new file mode 100644 index 0000000..8588ee0 --- /dev/null +++ b/config/migrations/2025/20250213162849-update-worship-consumer-again/20250213162933-flush-landing-zone.sparql @@ -0,0 +1,2 @@ +CLEAR GRAPH ; +CLEAR GRAPH ; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6f87138..7847126 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -370,7 +370,7 @@ services: # LOKET MANDATARISSEN CONSUMERS ################################################################################ mandatarissen-consumer: - image: lblod/delta-consumer:0.1.0 + image: lblod/delta-consumer:0.1.1 environment: DCR_SERVICE_NAME: "mandatarissen-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API @@ -403,7 +403,7 @@ services: # LOKET LEIDINGGEVENDEN CONSUMERS ################################################################################ leidinggevenden-consumer: - image: lblod/delta-consumer:0.1.0 + image: lblod/delta-consumer:0.1.1 environment: DCR_SERVICE_NAME: "leidinggevenden-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API @@ -436,13 +436,13 @@ services: # LOKET WORSHIP SENSITIVE CONSUMERS ################################################################################ worship-services-main-info-consumer: - image: lblod/delta-consumer:0.1.0 + image: lblod/delta-consumer:0.1.1 environment: DCR_SYNC_BASE_URL: "https://loket.lblod.info/" DCR_SERVICE_NAME: "worship-services-main-info-consumer" DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump" - DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive" - DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive/main" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing/main" DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-main-info-consumer" DCR_DISABLE_INITIAL_SYNC: "true" BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" @@ -469,13 +469,13 @@ services: logging: *default-logging worship-services-private-info-consumer: - image: lblod/delta-consumer:0.1.0 + image: lblod/delta-consumer:0.1.1 environment: DCR_SYNC_BASE_URL: "https://loket.lblod.info/" DCR_SERVICE_NAME: "worship-services-private-info-consumer" DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump" - DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive" - DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive/private" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing/private" DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-private-info-consumer" DCR_DISABLE_INITIAL_SYNC: "true" BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"