diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d799aa..5701996 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,14 @@ ### Backend #### Consumer - Upgraded `leidinggevenden-consumer` [OP-3533], [OP-3511] +- Upgraded `worship-services-sensitive-consumer` [OP-3337] + ### Deploy notes ``` drc down; ``` -Update `docker-compose.override.yml` to: + +Update `docker-compose.override.yml` to remove the config of `worship-services-sensitive-consumer` and replace it by: ``` leidinggevenden-consumer: environment: @@ -17,11 +20,30 @@ Update `docker-compose.override.yml` to: DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso DCR_DISABLE_DELTA_INGEST: "false" DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_DISABLE_INITIAL_SYNC: "false" + DCR_DISABLE_DELTA_INGEST: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_DISABLE_INITIAL_SYNC: "false" + DCR_DISABLE_DELTA_INGEST: "false" ``` Then: ``` -drc up -d migrations; drc logs -ft --tail=200 migrations -drc up -d db leidinggevenden-consumer +drc up -d triplestore migrations +drc up -d db leidinggevenden-consumer worship-services-main-info-consumer worship-services-private-info-consumer + # Wait until success of the previous step ``` Then, update `docker-compose.override.yml` to: @@ -33,19 +55,40 @@ Then, update `docker-compose.override.yml` to: DCR_REMAPPING_DATABASE: "db" DCR_DISABLE_DELTA_INGEST: "false" DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "db" + DCR_REMAPPING_DATABASE: "db" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "database" + DCR_REMAPPING_DATABASE: "database" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" ``` ``` drc up -d sh scripts/reset-elastic.sh ``` -## 1.28.5 (TODO) + +## 1.28.5 (2025-01-23) ### Frontend -- Bump to [v1.28.2](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.2) +- Bump to [v1.28.3](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.3) ### Backend - Datafix: correct merger change events for worship organisations (OP-3534) +- Avoid municipality filter to include partial matches (OP-3509) ### Deploy notes - `drc restart migrations-triggering-indexing; drc logs -ft --tail=200 migrations-triggering-indexing` - `drc pull frontend; drc up -d frontend` +- Perform a full re-index by executing `./scripts/reset-elastic.sh` ## 1.28.4 (2025-01-16) ### Backend diff --git a/README.md b/README.md index 0b9011f..7f3ba6e 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ The application depends on external data. You'll have to perform extra steps if These are multiple data sets, we'll try to be exhaustive on how to ingest these. Even though the procedures are similar. #### Setup http://data.lblod.info/vocabularies/leidinggevenden/Functionaris + Note: this is assuming you have never ran this before. ``` drc down; @@ -46,6 +47,67 @@ Then, update `docker-compose.override.yml` to: DCR_DISABLE_DELTA_INGEST: "false" DCR_DISABLE_INITIAL_SYNC: "false" ``` +Finally, put the stack back up and reindex elasticsearch +``` +drc up -d +sh scripts/reset-elastic.sh +``` + +#### Setup worship mandatees sync + +Note: this is assuming you have never ran this before. +``` +drc down; +``` +Update `docker-compose.override.yml` to: +``` + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lokaalbestuur.vlaanderen.be/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_REMAPPING_DATABASE: "triplestore" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lokaalbestuur.vlaanderen.be/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_REMAPPING_DATABASE: "triplestore" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +Then: +``` +drc up -d migrations +drc up -d db worship-services-main-info-consumer worship-services-private-info-consumer +# Wait until success of the previous step +``` +Then, update `docker-compose.override.yml` to: +``` +worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lokaalbestuur.vlaanderen.be/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "db" + DCR_REMAPPING_DATABASE: "db" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lokaalbestuur.vlaanderen.be/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "db" + DCR_REMAPPING_DATABASE: "db" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +Finally, put the stack back up and reindex elasticsearch ``` drc up -d sh scripts/reset-elastic.sh diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/data-cleanup/phone.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/data-cleanup/phone.rq new file mode 100644 index 0000000..7e761c1 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/data-cleanup/phone.rq @@ -0,0 +1,21 @@ +PREFIX schema: + +CONSTRUCT { + ?s schema:telephone ?cleanedPhoneNumber . +} WHERE { + ?s schema:telephone ?phoneNumber . + + # Remove all non digits + BIND(REPLACE(?phoneNumber, """[^+\\d]""", "") AS ?scrapeResult) + + # Replace leading 00 with + + BIND(REPLACE(?scrapeResult, """^00""", "+") AS ?replacedZeroZero) + + BIND(REPLACE(?replacedZeroZero, """^0""", "+32") AS ?defaultBe) + + BIND(REPLACE(?defaultBe, """\\+320""", "+32") AS ?extendedDefaultBe) + + BIND(REPLACE(?extendedDefaultBe, """^320""", "+32") AS ?finalResult) + + BIND(IF(REGEX(?finalResult, """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$"""), CONCAT("tel:", ?finalResult), ?phoneNumber) AS ?cleanedPhoneNumber) +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/contacthub-eindedatum.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/contacthub-eindedatum.rq new file mode 100644 index 0000000..d8be57a --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/contacthub-eindedatum.rq @@ -0,0 +1,10 @@ +PREFIX ch: +PREFIX xsd: + +CONSTRUCT { + ?s ch:eindedatum ?endDateTime . +} WHERE { + ?s ch:eindedatum ?endDate . + + BIND(STRDT(CONCAT(STR(?endDate), 'T02:00:00'), xsd:dateTime) as ?endDateTime) +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/contacthub-startdatum.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/contacthub-startdatum.rq new file mode 100644 index 0000000..d9f285b --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/contacthub-startdatum.rq @@ -0,0 +1,10 @@ +PREFIX ch: +PREFIX xsd: + +CONSTRUCT { + ?s ch:startdatum ?startDateTime . +} WHERE { + ?s ch:startdatum ?startDate . + + BIND(STRDT(CONCAT(STR(?startDate), 'T02:00:00'), xsd:dateTime) as ?startDateTime) +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/ere-geplande-einddatum-aanstelling.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/ere-geplande-einddatum-aanstelling.rq new file mode 100644 index 0000000..8c98cd8 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/date-to-datetime/ere-geplande-einddatum-aanstelling.rq @@ -0,0 +1,10 @@ +PREFIX ere: +PREFIX xsd: + +CONSTRUCT { + ?s ere:geplandeEinddatumAanstelling ?dateTime . +} WHERE { + ?s ere:geplandeEinddatumAanstelling ?date . + + BIND(STRDT(CONCAT(STR(?date), 'T02:00:00'), xsd:dateTime) as ?dateTime) +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inheritance/Agent.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inheritance/Agent.rq new file mode 100644 index 0000000..e875898 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inheritance/Agent.rq @@ -0,0 +1,20 @@ +PREFIX ch: +PREFIX dc_terms: +PREFIX ere: +PREFIX lblodlg: +PREFIX mandaat: +PREFIX person: + +CONSTRUCT { + ?s a dc_terms:Agent . +} WHERE { + ?s a ?type . + + VALUES ?type { + lblodlg:Functionaris + mandaat:Mandataris + ere:EredienstMandataris + ere:RolBedienaar + person:Person + } +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inheritance/AgentInPositie.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inheritance/AgentInPositie.rq new file mode 100644 index 0000000..dc787ef --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inheritance/AgentInPositie.rq @@ -0,0 +1,17 @@ +PREFIX ch: +PREFIX ere: +PREFIX lblodlg: +PREFIX mandaat: + +CONSTRUCT { + ?s a ch:AgentInPositie . +} WHERE { + ?s a ?type . + + VALUES ?type { + lblodlg:Functionaris + mandaat:Mandataris + ere:EredienstMandataris + ere:RolBedienaar + } +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inverse/held-by-to-holds.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inverse/held-by-to-holds.rq new file mode 100644 index 0000000..f51be45 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inverse/held-by-to-holds.rq @@ -0,0 +1,8 @@ +PREFIX org: + +CONSTRUCT { + ?o org:holds ?s. +} +WHERE { + ?s org:heldBy ?o. +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inverse/holds-to-held-by.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inverse/holds-to-held-by.rq new file mode 100644 index 0000000..0b373ed --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/inverse/holds-to-held-by.rq @@ -0,0 +1,8 @@ +PREFIX org: + +CONSTRUCT { + ?o org:heldBy ?s. +} +WHERE { + ?s org:holds ?o. +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/address.sparql b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/address.sparql new file mode 100644 index 0000000..1637ed2 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/address.sparql @@ -0,0 +1,11 @@ + +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + + FILTER NOT EXISTS { + ?s ?p "" . + } +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/bestuursfunctie.sparql b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/bestuursfunctie.sparql new file mode 100644 index 0000000..fd84a65 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/bestuursfunctie.sparql @@ -0,0 +1,7 @@ +CONSTRUCT { + ?s ?contactpoint. +} +WHERE { + ?s a . + ?s ?contactpoint. +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/contactpoint.sparql b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/contactpoint.sparql new file mode 100644 index 0000000..0301a8a --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/contactpoint.sparql @@ -0,0 +1,12 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + + FILTER (?p NOT IN ()) + + FILTER NOT EXISTS { + ?s ?p "" . + } +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/eredienst-mandataris.sparql b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/eredienst-mandataris.sparql new file mode 100644 index 0000000..188de7d --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/eredienst-mandataris.sparql @@ -0,0 +1,15 @@ +PREFIX ch: +PREFIX ere: + +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + + FILTER (?p NOT IN (ch:startdatum, ch:eindedatum, ere:geplandeEinddatumAanstelling)) + + FILTER NOT EXISTS { + ?s ?p "" . + } +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/person.sparql b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/person.sparql new file mode 100644 index 0000000..1276ece --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/person.sparql @@ -0,0 +1,16 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + + FILTER (?p IN ( + , + , + + )) + + FILTER NOT EXISTS { + ?s ?p "" . + } +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/rol-bedienaar.sparql b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/rol-bedienaar.sparql new file mode 100644 index 0000000..7e42662 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/pass/rol-bedienaar.sparql @@ -0,0 +1,10 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + + FILTER NOT EXISTS { + ?s ?p "" . + } +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/predicate/gebruikte-voornaam.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/predicate/gebruikte-voornaam.rq new file mode 100644 index 0000000..5a21395 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/predicate/gebruikte-voornaam.rq @@ -0,0 +1,14 @@ +PREFIX foaf: +PREFIX persoon: +PREFIX person: + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) + +# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam + +CONSTRUCT { + ?s persoon:gebruikteVoornaam ?name . +} WHERE { + ?s a person:Person ; + foaf:name ?name . +} diff --git a/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/predicate/given-name.rq b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/predicate/given-name.rq new file mode 100644 index 0000000..7b00b14 --- /dev/null +++ b/config/loket-consumer/worship-services-main-info/mapping/queries/dl2op/predicate/given-name.rq @@ -0,0 +1,14 @@ +PREFIX foaf: +PREFIX persoon_loket: +PREFIX person: + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) + +# First name: persoon:gebruikteVoornaam => foaf:givenName + +CONSTRUCT { + ?s foaf:givenName ?gebruikteVoornaam . +} WHERE { + ?s a person:Person ; + persoon_loket:gebruikteVoornaam ?gebruikteVoornaam . +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/pass/geboorte.sparql b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/pass/geboorte.sparql new file mode 100644 index 0000000..fc48f01 --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/pass/geboorte.sparql @@ -0,0 +1,10 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + + FILTER NOT EXISTS { + ?s ?p "" . + } +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/Geboorte.rq b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/Geboorte.rq new file mode 100644 index 0000000..92e544f --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/Geboorte.rq @@ -0,0 +1,10 @@ +PREFIX persoon: +PREFIX persoon_loket: + +# Map from http persoon to https persoon namespace + +CONSTRUCT { + ?s a persoon:Geboorte +} WHERE { + ?s a persoon_loket:Geboorte . +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/datum.rq b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/datum.rq new file mode 100644 index 0000000..87fea54 --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/datum.rq @@ -0,0 +1,10 @@ +PREFIX persoon: +PREFIX persoon_loket: + +# Map from http persoon to https persoon namespace + +CONSTRUCT { + ?s persoon:datum ?o . +} WHERE { + ?s persoon_loket:datum ?o . +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/geslacht.rq b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/geslacht.rq new file mode 100644 index 0000000..c7da92a --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/geslacht.rq @@ -0,0 +1,10 @@ +PREFIX persoon: +PREFIX persoon_loket: + +# Map from http persoon to https persoon namespace + +CONSTRUCT { + ?s persoon:geslacht ?o . +} WHERE { + ?s persoon_loket:geslacht ?o . +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/heeftGeboorte.rq b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/heeftGeboorte.rq new file mode 100644 index 0000000..4816186 --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/heeftGeboorte.rq @@ -0,0 +1,10 @@ +PREFIX persoon: +PREFIX persoon_loket: + +# Map from http persoon to https persoon namespace + +CONSTRUCT { + ?s persoon:heeftGeboorte ?o . +} WHERE { + ?s persoon_loket:heeftGeboorte ?o . +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/heeftNationaliteit.rq b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/heeftNationaliteit.rq new file mode 100644 index 0000000..1546294 --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/heeftNationaliteit.rq @@ -0,0 +1,10 @@ +PREFIX persoon: +PREFIX persoon_loket: + +# Map from http persoon to https persoon namespace + +CONSTRUCT { + ?s persoon:heeftNationaliteit ?o . +} WHERE { + ?s persoon_loket:heeftNationaliteit ?o . +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/registratie.rq b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/registratie.rq new file mode 100644 index 0000000..a2a3e5c --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/persoon-http-to-https/registratie.rq @@ -0,0 +1,10 @@ +PREFIX persoon: +PREFIX persoon_loket: + +# Map from http persoon to https persoon namespace + +CONSTRUCT { + ?s persoon:registratie ?o . +} WHERE { + ?s persoon_loket:registratie ?o . +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/rrn/identifier.rq b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/rrn/identifier.rq new file mode 100644 index 0000000..ebb6f9f --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/rrn/identifier.rq @@ -0,0 +1,19 @@ +PREFIX generiek: +PREFIX skos: +PREFIX mu: + +CONSTRUCT { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation . +} +WHERE { + ?identifier skos:notation ?notation. + + BIND(MD5(CONCAT(STR(?identifier),"STRUCTURED_IDENTIFIER")) as ?uuid) + BIND(IRI(CONCAT("http://data.lblod.info/id/gestructureerdeIdentificatoren/", ?uuid)) AS ?structuredIdentifier) +} diff --git a/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/rrn/registratie.rq b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/rrn/registratie.rq new file mode 100644 index 0000000..495a0ed --- /dev/null +++ b/config/loket-consumer/worship-services-private-info/mapping/queries/dl2op/rrn/registratie.rq @@ -0,0 +1,10 @@ +PREFIX adms: +PREFIX persoon: + +# identifier property in the export is only used for RRN. +CONSTRUCT { + ?person persoon:registratie ?rrn . +} +WHERE { + ?person adms:identifier ?rrn . +} diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/config.js b/config/loket-consumer/worship-services-sensitive-dispatching/config.js deleted file mode 100644 index e89bca3..0000000 --- a/config/loket-consumer/worship-services-sensitive-dispatching/config.js +++ /dev/null @@ -1,40 +0,0 @@ -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; -const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; -const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; -const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql'; -const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); -const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); -const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); -const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); -const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); - -// Graphs -const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/worship-service`; -const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/worship-privacy-centric-graph`; -const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`; - -// Mappings -const MAIN_INFO_MAPPING = "main-info"; -const PRIVATE_INFO_MAPPING = "private-info"; - -// ENDPOINTS -const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db'; -const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; - -module.exports = { - BATCH_SIZE, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MAX_DB_RETRY_ATTEMPTS, - MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - LANDING_ZONE_GRAPH, - LANDING_ZONE_DATABASE_ENDPOINT, - TARGET_GRAPH, - PRIVACY_SENSITIVE_GRAPH, - MAIN_INFO_MAPPING, - PRIVATE_INFO_MAPPING -}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js b/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js deleted file mode 100644 index a8df81c..0000000 --- a/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js +++ /dev/null @@ -1,11 +0,0 @@ -const contextConfig = { - addTypes: { - scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both - exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta - }, - contextQueries: [] -} - -module.exports = { - contextConfig -}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js b/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js deleted file mode 100644 index 70c510e..0000000 --- a/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js +++ /dev/null @@ -1,114 +0,0 @@ -const { - transformStatements, - deleteFromTargetGraph, - insertIntoTargetGraph, - deleteFromPrivateGraph, - insertIntoPrivateGraph -} = require('./util'); - -const { - MAIN_INFO_MAPPING, - PRIVATE_INFO_MAPPING -} = require('./config'); - -/** -* Dispatch the fetched information to a target graph. -* Note: will be ADDED to it's own graph. -* We take only care of adding them, not updating triples, this is a TODO -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, fetch } = lib; - let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; - - // Both arrays are the same length, so we can zip them together for easier processing - zippedChangeSets = termObjectChangeSets.map((o, i) => ({ - original: o, - withContext: termObjectChangeSetsWithContext[i] - })); - - console.log(`Received ${zippedChangeSets.length} change sets`) - - for (let { original, withContext } of zippedChangeSets) { - let originalDeletes, originalInserts, insertsWithContext, deletesWithContext; - try { - originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - } catch (e) { - console.log(`Error while processing change set: ${e}`); - console.log(`Original change set: ${JSON.stringify(original)}`); - throw e; - } - - // Extra context needed for mapping from DL to OP model and filtering based on type. - try { - insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - } catch (e) { - console.log(`Error while processing change set with context: ${e}`); - console.log(`Change set with context: ${JSON.stringify(withContext)}`); - throw e; - } - - if (originalDeletes.length) { - // Map main deletes from DL to OP model - const transformedMainDeletes = await transformStatements(fetch, deletesWithContext, MAIN_INFO_MAPPING); - - if (!transformedMainDeletes.length) { - console.log(`Warn: Delete main statements mapped to empty result.`); - console.log(`Input: ${deletesWithContext}`); - console.log(`Output: ${transformedMainDeletes}`); - } else { - await deleteFromTargetGraph(lib, transformedMainDeletes); - } - - // Map private deletes from DL to OP model - const transformedPrivateDeletes = await transformStatements(fetch, deletesWithContext, PRIVATE_INFO_MAPPING); - - if (!transformedPrivateDeletes.length) { - console.log(`Warn: Delete private statements mapped to empty result.`); - console.log(`Input: ${deletesWithContext}`); - console.log(`Output: ${transformedPrivateDeletes}`); - } else { - await deleteFromPrivateGraph(lib, transformedPrivateDeletes); - } - } - - if (originalInserts.length) { - // Map main inserts from DL to OP model - const transformedMainInserts = await transformStatements(fetch, insertsWithContext, MAIN_INFO_MAPPING); - - if (!transformedMainInserts.length) { - console.log(`Warn: Insert statements mapped to empty result.`); - console.log(`Input: ${insertsWithContext}`); - console.log(`Output: ${transformedMainInserts}`); - } else { - await insertIntoTargetGraph(lib, transformedMainInserts); - } - - // Map private inserts from DL to OP model - const transformedPrivateInserts = await transformStatements(fetch, insertsWithContext, PRIVATE_INFO_MAPPING); - - if (!transformedPrivateInserts.length) { - console.log(`Warn: Insert statements mapped to empty result.`); - console.log(`Input: ${insertsWithContext}`); - console.log(`Output: ${transformedPrivateInserts}`); - } else { - await insertIntoPrivateGraph(lib, transformedPrivateInserts); - } - } - } -} - -module.exports = { - dispatch -}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js b/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js deleted file mode 100644 index 559250f..0000000 --- a/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js +++ /dev/null @@ -1,106 +0,0 @@ -const { batchedDbUpdate, transformLandingZoneGraph } = require('./util'); -const { - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - TARGET_GRAPH, - PRIVACY_SENSITIVE_GRAPH, - LANDING_ZONE_GRAPH, - LANDING_ZONE_DATABASE_ENDPOINT, - MAIN_INFO_MAPPING, - PRIVATE_INFO_MAPPING -} = require('./config'); - -const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; - -/** -* Dispatch the fetched information to a target graph. -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjects } data - The fetched quad information, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, muAuthSudo } = lib; - const { termObjects } = data; - - // Steps: - // 1. Write the triples to the landing zone graph - withouth any mapping or filtering - // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) - - if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { - console.warn(`Service configured to skip MU_AUTH!`); - } - console.log(`Using ${endpoint} to insert triples`); - - if (termObjects.length) { - const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - - await batchedDbUpdate( - muAuthSudo.updateSudo, - LANDING_ZONE_GRAPH, - originalInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - LANDING_ZONE_DATABASE_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } -} - -async function onFinishInitialIngest(lib) { - const { muAuthSudo, fetch } = lib; - - console.log(`!! On-finish triggered !!`); - - const transformedMainInsertTriples = await transformLandingZoneGraph(fetch, endpoint, MAIN_INFO_MAPPING); - console.log(`Transformed ${transformedMainInsertTriples.length} main triples`); - - if (transformedMainInsertTriples.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - TARGET_GRAPH, - transformedMainInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } - - const transformedPrivateInsertTriples = await transformLandingZoneGraph(fetch, endpoint, PRIVATE_INFO_MAPPING); - console.log(`Transformed ${transformedPrivateInsertTriples.length} private triples`); - - if (transformedPrivateInsertTriples.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - PRIVACY_SENSITIVE_GRAPH, - transformedPrivateInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } -} - -module.exports = { - dispatch, - onFinishInitialIngest -}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/util.js b/config/loket-consumer/worship-services-sensitive-dispatching/util.js deleted file mode 100644 index 4b17b75..0000000 --- a/config/loket-consumer/worship-services-sensitive-dispatching/util.js +++ /dev/null @@ -1,204 +0,0 @@ -const { - MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - TARGET_GRAPH, - PRIVACY_SENSITIVE_GRAPH, - LANDING_ZONE_GRAPH, - BATCH_SIZE -} = require('./config'); - -async function batchedDbUpdate( - muUpdate, - graph, - triples, - extraHeaders, - endpoint, - batchSize, - maxAttempts, - sleepBetweenBatches = 1000, - sleepTimeOnFail = 1000, - operation = 'INSERT' -) { - - for (let i = 0; i < triples.length; i += batchSize) { - console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); - - const batch = triples.slice(i, i + batchSize).join('\n'); - - const insertCall = async () => { - await muUpdate(` -${operation} DATA { -GRAPH <${graph}> { -${batch} -} -} -`, extraHeaders, endpoint); - }; - - await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); - - console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); - await new Promise(r => setTimeout(r, sleepBetweenBatches)); - } -} - -async function operationWithRetry(callback, - attempt, - maxAttempts, - sleepTimeOnFail) { - try { - if (typeof callback === "function") - return await callback(); - else // Catch error from promise - not how I would do it normally, but allows re use of existing code. - return await callback; - } - catch (e) { - console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); - console.log(`Error: ${e}`); - console.log(`Sleeping ${sleepTimeOnFail} ms`); - - if (attempt >= maxAttempts) { - console.log(`Max attempts reached for ${callback.toString()}, giving up`); - throw e; - } - - await new Promise(r => setTimeout(r, sleepTimeOnFail)); - return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); - } -} - -/** - * Splits an array into two parts, a part that passes and a part that fails a predicate function. - * Credits: https://github.com/benjay10 - * @public - * @function partition - * @param {Array} arr - Array to be partitioned - * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. - * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively - */ -function partition(arr, fn) { - let passes = [], fails = []; - arr.forEach((item) => (fn(item) ? passes : fails).push(item)); - return { passes, fails }; -} - -/** - * Send triples to reasoning service for conversion - * - */ -async function transformTriples(fetch, triples, mapping) { - return await operationWithRetry(mainConversion(fetch, triples, mapping), 0, - MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); -} - -async function mainConversion(fetch, triples, mapping) { - let formdata = new URLSearchParams(); - formdata.append("data", triples); - - let requestOptions = { - method: 'POST', - body: formdata, - redirect: 'follow' - }; - - const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) - return await response.text(); -} - -async function transformStatements(fetch, triples, mapping = 'main') { - console.log(`Received ${JSON.stringify(triples)} to transform`); - const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping); - statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : []; - console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`); - return statements; -} - -async function deleteFromTargetGraph(lib, statements) { - console.log(`Deleting ${statements.length} statements from target graph`); - console.log(`Statements: ${JSON.stringify(statements)}`) - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - TARGET_GRAPH, - statements, - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'DELETE'); -} - -async function deleteFromPrivateGraph(lib, statements) { - console.log(`Deleting ${statements.length} statements from target graph`); - console.log(`Statements: ${JSON.stringify(statements)}`) - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - PRIVACY_SENSITIVE_GRAPH, - statements, - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'DELETE'); -} - -async function insertIntoTargetGraph(lib, statements) { - console.log(`Inserting ${statements.length} statements into target graph`); - console.log(`Statements: ${JSON.stringify(statements)}`) - - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - TARGET_GRAPH, - statements, - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'INSERT'); -} - -async function insertIntoPrivateGraph(lib, statements) { - console.log(`Inserting ${statements.length} statements into target graph`); - console.log(`Statements: ${JSON.stringify(statements)}`) - - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - PRIVACY_SENSITIVE_GRAPH, - statements, - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'INSERT'); -} - -async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') { - console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`); - - const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`); - const text = await response.text(); - const statements = text.replace(/\n{2,}/g, '').split('\n'); - - return statements; -} - -module.exports = { - batchedDbUpdate, - partition, - transformStatements, - transformLandingZoneGraph, - deleteFromTargetGraph, - insertIntoTargetGraph, - deleteFromPrivateGraph, - insertIntoPrivateGraph -}; diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095500-flush-eredienst-mandatarissen-consumer-containers.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095500-flush-eredienst-mandatarissen-consumer-containers.sparql new file mode 100644 index 0000000..8e07b83 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095500-flush-eredienst-mandatarissen-consumer-containers.sparql @@ -0,0 +1,37 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?container ?containerP ?containerO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation; + dct:created ?created. + + FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) + + ?task dct:isPartOf ?job; + task:resultsContainer ?container. + + ?container a nfo:DataContainer; + ?containerP ?containerO. + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095600-flush-eredienst-mandatarissen-consumer-tasks.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095600-flush-eredienst-mandatarissen-consumer-tasks.sparql new file mode 100644 index 0000000..1f7f105 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095600-flush-eredienst-mandatarissen-consumer-tasks.sparql @@ -0,0 +1,34 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?task ?taskP ?taskO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + dct:created ?created; + task:operation ?operation. + + FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) + + ?task dct:isPartOf ?job; + ?taskP ?taskO. + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095700-flush-eredienst-mandatarissen-consumer-jobs.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095700-flush-eredienst-mandatarissen-consumer-jobs.sparql new file mode 100644 index 0000000..ddada53 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2023/20250115095700-flush-eredienst-mandatarissen-consumer-jobs.sparql @@ -0,0 +1,32 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?job ?jobP ?jobO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + dct:created ?created; + task:operation ?operation; + ?jobP ?jobO. + + FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115095800-flush-eredienst-mandatarissen-consumer-containers.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115095800-flush-eredienst-mandatarissen-consumer-containers.sparql new file mode 100644 index 0000000..5823e06 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115095800-flush-eredienst-mandatarissen-consumer-containers.sparql @@ -0,0 +1,37 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?container ?containerP ?containerO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation; + dct:created ?created. + + FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) + + ?task dct:isPartOf ?job; + task:resultsContainer ?container. + + ?container a nfo:DataContainer; + ?containerP ?containerO. + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115095900-flush-eredienst-mandatarissen-consumer-tasks.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115095900-flush-eredienst-mandatarissen-consumer-tasks.sparql new file mode 100644 index 0000000..86bc693 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115095900-flush-eredienst-mandatarissen-consumer-tasks.sparql @@ -0,0 +1,34 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?task ?taskP ?taskO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + dct:created ?created; + task:operation ?operation. + + FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) + + ?task dct:isPartOf ?job; + ?taskP ?taskO. + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115096000-flush-eredienst-mandatarissen-consumer-jobs.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115096000-flush-eredienst-mandatarissen-consumer-jobs.sparql new file mode 100644 index 0000000..a508dbf --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2024/20250115096000-flush-eredienst-mandatarissen-consumer-jobs.sparql @@ -0,0 +1,32 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?job ?jobP ?jobO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + dct:created ?created; + task:operation ?operation; + ?jobP ?jobO. + + FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096100-flush-eredienst-mandatarissen-consumer-containers.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096100-flush-eredienst-mandatarissen-consumer-containers.sparql new file mode 100644 index 0000000..da32084 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096100-flush-eredienst-mandatarissen-consumer-containers.sparql @@ -0,0 +1,34 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?container ?containerP ?containerO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation. + + ?task dct:isPartOf ?job; + task:resultsContainer ?container. + + ?container a nfo:DataContainer; + ?containerP ?containerO. + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096200-flush-eredienst-mandatarissen-consumer-tasks.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096200-flush-eredienst-mandatarissen-consumer-tasks.sparql new file mode 100644 index 0000000..b783b14 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096200-flush-eredienst-mandatarissen-consumer-tasks.sparql @@ -0,0 +1,31 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?task ?taskP ?taskO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation. + + ?task dct:isPartOf ?job; + ?taskP ?taskO. + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096300-flush-eredienst-mandatarissen-consumer-jobs.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096300-flush-eredienst-mandatarissen-consumer-jobs.sparql new file mode 100644 index 0000000..ea8795f --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250115095500-flush-eredienst-mandatarissen-consumer-jobs/2025/20250115096300-flush-eredienst-mandatarissen-consumer-jobs.sparql @@ -0,0 +1,29 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?job ?jobP ?jobO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation; + ?jobP ?jobO. + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085100-remove-loket-persons-birthdates.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085100-remove-loket-persons-birthdates.sparql new file mode 100644 index 0000000..0353903 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085100-remove-loket-persons-birthdates.sparql @@ -0,0 +1,33 @@ +PREFIX ch: +PREFIX mandaat: +PREFIX org: +PREFIX person: +PREFIX persoon: + +# Delete birthdates +DELETE { + GRAPH { + ?birthDate ?p ?o . + } +} WHERE { + VALUES ?g { + + + } + + GRAPH ?g { + ?person a person:Person . + } + + FILTER EXISTS { + GRAPH { + ?agent a ch:AgentInPositie ; + mandaat:isBestuurlijkeAliasVan|org:heldBy ?person . + } + } + + GRAPH { + ?person persoon:heeftGeboorte ?birthDate . + ?birthDate ?p ?o . + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085200-remove-loket-persons-identifiers.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085200-remove-loket-persons-identifiers.sparql new file mode 100644 index 0000000..1a2c892 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085200-remove-loket-persons-identifiers.sparql @@ -0,0 +1,39 @@ +PREFIX ch: +PREFIX generiek: +PREFIX mandaat: +PREFIX org: +PREFIX person: +PREFIX persoon: + +# Delete identifiers +DELETE { + GRAPH { + ?identifier ?pidentifier ?oidentifier . + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} WHERE { + VALUES ?g { + + + } + + GRAPH ?g { + ?person a person:Person . + } + + FILTER EXISTS { + GRAPH { + ?agent a ch:AgentInPositie ; + mandaat:isBestuurlijkeAliasVan|org:heldBy ?person . + } + } + + GRAPH { + ?person persoon:registratie ?identifier . + + ?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier ; + ?pidentifier ?oidentifier . + + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085300-remove-loket-persons.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085300-remove-loket-persons.sparql new file mode 100644 index 0000000..668e85c --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085300-remove-loket-persons.sparql @@ -0,0 +1,36 @@ +PREFIX ch: +PREFIX mandaat: +PREFIX org: +PREFIX person: + +# Delete persons +DELETE { + GRAPH ?g { + ?person ?p ?o . + } + + GRAPH { + ?person ?pprivate ?oprivate . + } +} WHERE { + VALUES ?g { + + + } + + GRAPH ?g { + ?person a person:Person ; + ?p ?o . + } + + FILTER EXISTS { + GRAPH { + ?agent a ch:AgentInPositie ; + mandaat:isBestuurlijkeAliasVan|org:heldBy ?person . + } + } + + GRAPH { + ?person ?pprivate ?oprivate . + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085400-contact-points.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085400-contact-points.sparql new file mode 100644 index 0000000..83d8a6f --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085400-contact-points.sparql @@ -0,0 +1,22 @@ +DELETE { + GRAPH { + ?contactPoint ?pcontactPoint ?ocontactPoint . + ?address ?paddress ?oaddress . + } +} WHERE { + GRAPH { + ?agent a . + + OPTIONAL { + ?agent ?contactPoint . + + ?contactPoint ?pcontactPoint ?ocontactPoint . + + OPTIONAL { + ?contactPoint ?address . + + ?address ?paddress ?oaddress . + } + } + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085600-remove-agents-in-position.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085600-remove-agents-in-position.sparql new file mode 100644 index 0000000..31d2d87 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116085600-remove-agents-in-position.sparql @@ -0,0 +1,10 @@ +DELETE { + GRAPH { + ?agent ?pagent ?oagent . + } +} WHERE { + GRAPH { + ?agent a ; + ?pagent ?oagent . + } +} diff --git a/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116134600-flush-landing-zone.sparql b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116134600-flush-landing-zone.sparql new file mode 100644 index 0000000..b99b2d7 --- /dev/null +++ b/config/migrations/2025/20250116085000-sync-from-loket-consumers-bump/20250116134600-flush-landing-zone.sparql @@ -0,0 +1 @@ +CLEAR GRAPH \ No newline at end of file diff --git a/config/search/config.json b/config/search/config.json index 643458d..3a5d67e 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -223,7 +223,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -385,7 +385,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -540,7 +540,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -727,7 +727,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -880,7 +880,7 @@ } }, "municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", diff --git a/config/search/dev/config.json b/config/search/dev/config.json index bdb3394..ab1902c 100644 --- a/config/search/dev/config.json +++ b/config/search/dev/config.json @@ -195,7 +195,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -357,7 +357,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -512,7 +512,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -700,7 +700,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -853,7 +853,7 @@ } }, "municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a474a7f..deeebea 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -78,7 +78,9 @@ services: restart: "no" leidinggevenden-consumer: restart: "no" - worship-services-sensitive-consumer: + worship-services-main-info-consumer: + restart: "no" + worship-services-private-info-consumer: restart: "no" ldes-backend: environment: diff --git a/docker-compose.yml b/docker-compose.yml index 5914f08..138ead4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.28.2 + image: lblod/frontend-organization-portal:1.28.3 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: @@ -433,27 +433,64 @@ services: ################################################################################ # LOKET WORSHIP SENSITIVE CONSUMERS ################################################################################ - worship-services-sensitive-consumer: - image: lblod/delta-consumer:0.0.24 + worship-services-main-info-consumer: + image: lblod/delta-consumer:0.1.0 environment: DCR_SYNC_BASE_URL: "https://loket.lblod.info/" - DCR_SERVICE_NAME: "worship-services-sensitive-consumer" + DCR_SERVICE_NAME: "worship-services-main-info-consumer" DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump" DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive" DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing" - DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-main-info-consumer" DCR_DISABLE_INITIAL_SYNC: "true" BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files" - BATCH_SIZE: 50 SLEEP_BETWEEN_BATCHES: 10 DCR_KEEP_DELTA_FILES: "true" - DCR_ENABLE_DELTA_CONTEXT: "true" DCR_LANDING_ZONE_DATABASE: "triplestore" - DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/worship-services-sensitive" + DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/worship-services-main-info" + DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/worship-service" + DCR_DELTA_JOBS_RETENTION_PERIOD: "30" + DCR_ENABLE_TRIPLE_REMAPPING: "true" + DCR_BATCH_SIZE: 1000 + SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" + SUDO_QUERY_RETRY: "true" volumes: - - ./config/loket-consumer/worship-services-sensitive-dispatching:/config/triples-dispatching/custom-dispatching - - ./data/files/consumer-files/worship-services-sensitive:/consumer-files/ + - ./config/loket-consumer/worship-services-main-info/mapping:/config/mapping + - ./data/files/consumer-files/worship-services-main-info:/consumer-files/ + restart: always + labels: + - "logging=true" + links: + - db:database + - triplestore:virtuoso + logging: *default-logging + + worship-services-private-info-consumer: + image: lblod/delta-consumer:0.1.0 + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info/" + DCR_SERVICE_NAME: "worship-services-private-info-consumer" + DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-private-info-consumer" + DCR_DISABLE_INITIAL_SYNC: "true" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" + DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files" + SLEEP_BETWEEN_BATCHES: 10 + DCR_KEEP_DELTA_FILES: "true" + DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/worship-services-private-info" + DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/worship-privacy-centric-graph" + DCR_DELTA_JOBS_RETENTION_PERIOD: "30" + DCR_ENABLE_TRIPLE_REMAPPING: "true" + DCR_BATCH_SIZE: 1000 + SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" + SUDO_QUERY_RETRY: "true" + volumes: + - ./config/loket-consumer/worship-services-private-info/mapping:/config/mapping + - ./data/files/consumer-files/worship-services-private-info:/consumer-files/ restart: always labels: - "logging=true"