Compare commits

...

14 Commits

Author SHA1 Message Date
Claire Lovisa
20f010bd42 Release v1.29.2 2025-02-17 16:47:32 +01:00
Claire Lovisa
7bb08363e2 Merge pull request #517 from lblod/hotfix/restore-betrokken-lokale-besturen
Restore betrokken lokale besturen
2025-02-17 16:45:29 +01:00
Claire Lovisa
b66278aec5 Restore betrokken lokale besturen 2025-02-17 16:15:17 +01:00
cecemel
b273a77923 Merge pull request #515 from lblod/hotfix/flush-eredienst-mandatarissen
Hotfix/flush eredienst mandatarissen
2025-02-14 16:45:59 +01:00
felix
f689f90e37 Update CHANGELOG.md 2025-02-14 14:31:17 +01:00
felix
20bc051103 Refine migrations 2025-02-13 17:37:52 +01:00
felix
8f57a160e0 update migration name 2025-02-13 17:37:41 +01:00
felix
6f731813bb Ensure different operation jobs for both consumers 2025-02-13 17:05:00 +01:00
felix
03635e4b12 bump consumers 2025-02-13 17:04:52 +01:00
felix
741a7cae2c added migrations to flush again 2025-02-13 17:04:23 +01:00
Claire Lovisa
9ea9d90751 Release v1.29.0 2025-02-07 20:12:22 +01:00
Claire Lovisa
d8114bec4e Release v1.29.0-1 2025-02-05 14:22:40 +01:00
Claire Lovisa
a352683c6f Merge pull request #512 from lblod/fix/mkissing-formatting-dates-consumers
Add dates mapping rules for mandatarissen and leidinggevenden consumers
2025-02-05 14:21:50 +01:00
Claire Lovisa
d601d7f34e Add dates mapping rules for consumers
The mandatarissen and leidinggevenden consumers were missing a date
formatting mapping that is making sure that the data we display to the
user is of the same date as the one shown in the original app. This
transcription was made by the reasoner and we forgot to include it in
the mapping configuration.

I also updated the timestamp of migrations that need to run again when
I'll re-deploy to QA and DEV. It will have no impact on the upcoming
PROD deploy.
2025-02-05 10:05:39 +01:00
69 changed files with 3395 additions and 18 deletions

View File

@@ -1,6 +1,69 @@
# Changelog
## 1.29.2 (2025-02-17)
### Data fix
- restore bestrokken lokale besturen that have been accidentally flushed [OP-3546]
## 1.29.1 (2025-02-14)
### Backend
- re-init `worship-services-sensitive-consumer` [OP-3483]
### Deploy notes
:warning: Ensure first loket did a full re-sync again of this delta-stream.
```
drc stop worship-services-main-info-consumer worship-services-private-info-consumer
drc retart migrations
```
Update `docker-compose.override.yml` to remove the config of the consumers and replace it by:
```
worship-services-main-info-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
DCR_SECRET_KEY: "key-of-the-producer"
DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
DCR_DISABLE_INITIAL_SYNC: "false"
DCR_DISABLE_DELTA_INGEST: "false"
worship-services-private-info-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
DCR_SECRET_KEY: "key-of-the-producer"
DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
DCR_DISABLE_INITIAL_SYNC: "false"
DCR_DISABLE_DELTA_INGEST: "false"
```
Then
```
drc up -d db mandatarissen-consumer leidinggevenden-consumer worship-services-main-info-consumer worship-services-private-info-consumer
# Wait until success of the previous step
```
Then, update `docker-compose.override.yml` to:
```
worship-services-main-info-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
DCR_SECRET_KEY: "key-of-the-producer"
DCR_LANDING_ZONE_DATABASE: "db"
DCR_REMAPPING_DATABASE: "db"
DCR_DISABLE_DELTA_INGEST: "false"
DCR_DISABLE_INITIAL_SYNC: "false"
worship-services-private-info-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint
DCR_SECRET_KEY: "key-of-the-producer"
DCR_LANDING_ZONE_DATABASE: "database"
DCR_REMAPPING_DATABASE: "database"
DCR_DISABLE_DELTA_INGEST: "false"
DCR_DISABLE_INITIAL_SYNC: "false"
```
```
drc up -d
sh scripts/reset-elastic.sh
```
## 1.29.0 (prerelease v1.29.0-0: 2025-02-03)
## 1.29.0 (2025-02-07)
### Backend
#### Consumer
- Upgraded `mandatarissen-consumer` [OP-3510]

View File

@@ -0,0 +1,11 @@
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
CONSTRUCT {
?s mandaat:einde ?endDateTime .
} WHERE {
?s mandaat:einde ?endDate .
BIND(SUBSTR(STR(?endDate), 1, 10) as ?endDateWithoutTimezone)
BIND(STRDT(CONCAT(?endDateWithoutTimezone, 'T02:00:00'), xsd:dateTime) as ?endDateTime)
}

View File

@@ -0,0 +1,11 @@
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
CONSTRUCT {
?s mandaat:start ?startDateTime .
} WHERE {
?s mandaat:start ?startDate .
BIND(SUBSTR(STR(?startDate), 1, 10) as ?startDateWithoutTimezone)
BIND(STRDT(CONCAT(?startDateWithoutTimezone, 'T02:00:00'), xsd:dateTime) as ?startDateTime)
}

View File

@@ -11,5 +11,4 @@ CONSTRUCT {
lblodlg:Functionaris
person:Person
}
}

View File

@@ -5,6 +5,7 @@ CONSTRUCT {
?s a ch:AgentInPositie .
} WHERE {
?s a ?type .
VALUES ?type {
lblodlg:Functionaris
}

View File

@@ -7,4 +7,4 @@ CONSTRUCT {
FILTER NOT EXISTS {
?s ?p "" .
}
}
}

View File

@@ -4,4 +4,4 @@ CONSTRUCT {
WHERE {
?s a <http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie>.
?s <http://schema.org/contactPoint> ?contactpoint.
}
}

View File

@@ -8,4 +8,4 @@ CONSTRUCT {
FILTER NOT EXISTS {
?s ?p "" .
}
}
}

View File

@@ -1,10 +1,14 @@
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
CONSTRUCT {
?s ?p ?o.
} WHERE {
?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris>;
?p ?o.
FILTER NOT EXISTS {
?s ?p "" .
}
}
FILTER (?p NOT IN (mandaat:start, mandaat:einde))
FILTER NOT EXISTS {
?s ?p "" .
}
}

View File

@@ -13,4 +13,4 @@ CONSTRUCT {
FILTER NOT EXISTS {
?s ?p "" .
}
}
}

View File

@@ -0,0 +1,11 @@
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
CONSTRUCT {
?s mandaat:einde ?endDateTime .
} WHERE {
?s mandaat:einde ?endDate .
BIND(SUBSTR(STR(?endDate), 1, 10) as ?endDateWithoutTimezone)
BIND(STRDT(CONCAT(?endDateWithoutTimezone, 'T02:00:00'), xsd:dateTime) as ?endDateTime)
}

View File

@@ -0,0 +1,11 @@
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
CONSTRUCT {
?s mandaat:start ?startDateTime .
} WHERE {
?s mandaat:start ?startDate .
BIND(SUBSTR(STR(?startDate), 1, 10) as ?startDateWithoutTimezone)
BIND(STRDT(CONCAT(?startDateWithoutTimezone, 'T02:00:00'), xsd:dateTime) as ?startDateTime)
}

View File

@@ -1,3 +1,5 @@
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
CONSTRUCT {
?s ?p ?o.
} WHERE {
@@ -19,6 +21,8 @@ CONSTRUCT {
<http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000003>
))
FILTER (?p NOT IN (mandaat:start, mandaat:einde))
FILTER NOT EXISTS {
?s ?p "" .
}

View File

@@ -0,0 +1,36 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?container ?containerP ?containerO.
}
} WHERE {
# Please note, if you ever copy/paste this migration the next time, there should be 4
# operation URI's in total. Since two consumers
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
task:operation ?operation.
?task dct:isPartOf ?job;
task:resultsContainer ?container.
?container a nfo:DataContainer;
?containerP ?containerO.
}
}

View File

@@ -0,0 +1,33 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?task ?taskP ?taskO.
}
} WHERE {
# Please note, if you ever copy/paste this migration the next time, there should be 4
# operation URI's in total. Since two consumers
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
task:operation ?operation.
?task dct:isPartOf ?job;
?taskP ?taskO.
}
}

View File

@@ -0,0 +1,31 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?job ?jobP ?jobO.
}
} WHERE {
# Please note, if you ever copy/paste this migration the next time, there should be 4
# operation URI's in total. Since two consumers
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
task:operation ?operation;
?jobP ?jobO.
}
}

View File

@@ -0,0 +1,17 @@
DELETE {
GRAPH ?g {
?s ?p ?o
}
}
WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/worship-privacy-centric-graph>
<http://mu.semte.ch/graphs/worship-service>
}
GRAPH ?g {
?s ?p ?o.
FILTER NOT EXISTS {
?s a ?type.
}
}
}

View File

@@ -0,0 +1,33 @@
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX person: <http://www.w3.org/ns/person#>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
# Delete birthdates
DELETE {
GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
?birthDate ?p ?o .
}
} WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/shared>
<http://mu.semte.ch/graphs/worship-service>
}
GRAPH ?g {
?person a person:Person .
}
FILTER EXISTS {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?agent a ch:AgentInPositie ;
mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
}
}
GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
?person persoon:heeftGeboorte ?birthDate .
?birthDate ?p ?o .
}
}

View File

@@ -0,0 +1,54 @@
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX person: <http://www.w3.org/ns/person#>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
# Delete identifiers
DELETE {
GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
?identifier ?pidentifier ?oidentifier .
?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
}
} WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/shared>
<http://mu.semte.ch/graphs/worship-service>
}
GRAPH ?g {
?person a person:Person .
}
FILTER EXISTS {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?agent a ch:AgentInPositie ;
mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
}
}
GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
?person persoon:registratie ?identifier .
?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier ;
?pidentifier ?oidentifier .
?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
}
}
;
DELETE {
GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier .
}
} WHERE {
GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
?structuredIdentifier a <https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator>;
?pstructuredIdentifier ?ostructuredIdentifier.
}
}

View File

@@ -0,0 +1,38 @@
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX person: <http://www.w3.org/ns/person#>
# Delete persons
DELETE {
GRAPH ?g {
?person ?p ?o .
}
GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
?person ?pprivate ?oprivate .
}
} WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/shared>
<http://mu.semte.ch/graphs/worship-service>
}
GRAPH ?g {
?person a person:Person ;
?p ?o .
}
FILTER EXISTS {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?agent a ch:AgentInPositie ;
mandaat:isBestuurlijkeAliasVan|org:heldBy ?person .
}
}
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/worship-privacy-centric-graph> {
?person ?pprivate ?oprivate .
}
}
}

View File

@@ -0,0 +1,22 @@
DELETE {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?contactPoint ?pcontactPoint ?ocontactPoint .
?address ?paddress ?oaddress .
}
} WHERE {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?agent a <http://data.lblod.info/vocabularies/contacthub/AgentInPositie> .
OPTIONAL {
?agent <http://schema.org/contactPoint> ?contactPoint .
?contactPoint ?pcontactPoint ?ocontactPoint .
OPTIONAL {
?contactPoint <http://www.w3.org/ns/locn#address> ?address .
?address ?paddress ?oaddress .
}
}
}
}

View File

@@ -0,0 +1,10 @@
DELETE {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?agent ?pagent ?oagent .
}
} WHERE {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?agent a <http://data.lblod.info/vocabularies/contacthub/AgentInPositie> ;
?pagent ?oagent .
}
}

View File

@@ -0,0 +1,2 @@
CLEAR GRAPH <http://mu.semte.ch/graphs/landing-zone/worship-services-private-info>;
CLEAR GRAPH <http://mu.semte.ch/graphs/landing-zone/worship-services-main-info>;

View File

@@ -0,0 +1 @@
http://mu.semte.ch/graphs/worship-service

File diff suppressed because it is too large Load Diff

View File

@@ -370,7 +370,7 @@ services:
# LOKET MANDATARISSEN CONSUMERS
################################################################################
mandatarissen-consumer:
image: lblod/delta-consumer:0.1.0
image: lblod/delta-consumer:0.1.1
environment:
DCR_SERVICE_NAME: "mandatarissen-consumer"
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -403,7 +403,7 @@ services:
# LOKET LEIDINGGEVENDEN CONSUMERS
################################################################################
leidinggevenden-consumer:
image: lblod/delta-consumer:0.1.0
image: lblod/delta-consumer:0.1.1
environment:
DCR_SERVICE_NAME: "leidinggevenden-consumer"
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -436,13 +436,13 @@ services:
# LOKET WORSHIP SENSITIVE CONSUMERS
################################################################################
worship-services-main-info-consumer:
image: lblod/delta-consumer:0.1.0
image: lblod/delta-consumer:0.1.1
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
DCR_SERVICE_NAME: "worship-services-main-info-consumer"
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive/main"
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing/main"
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-main-info-consumer"
DCR_DISABLE_INITIAL_SYNC: "true"
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
@@ -469,13 +469,13 @@ services:
logging: *default-logging
worship-services-private-info-consumer:
image: lblod/delta-consumer:0.1.0
image: lblod/delta-consumer:0.1.1
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
DCR_SERVICE_NAME: "worship-services-private-info-consumer"
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive/private"
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing/private"
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-private-info-consumer"
DCR_DISABLE_INITIAL_SYNC: "true"
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"