Merge pull request #407 from lblod/hotfix/1.23.1-1

Update adressenregister to use basisregisters adressen V2 API
fix: Update adressenregister to use basisregisters adressen V2 API
2024-04-04 11:51:39 +02:00 · 2024-04-04 11:50:08 +02:00 · 2024-04-02 10:16:08 +02:00 · 2024-04-02 08:10:18 +00:00 · 2024-04-02 08:08:48 +00:00 · 2024-04-02 10:01:10 +02:00
11 changed files with 269 additions and 49 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -19,6 +19,7 @@ docker-compose.override.yml
 .env

 /config/migrations/private-data/*
+/config/migrations/local/*

 /config/proxy/nginx.htpasswd

--- a/config/delta-producer/organizations-public-info/export.json
+++ b/config/delta-producer/organizations-public-info/export.json
@@ -15,6 +15,8 @@
        "http://www.w3.org/2004/02/skos/core#altLabel",
        "http://www.w3.org/ns/adms#identifier",
        "http://www.w3.org/ns/regorg#orgStatus",
+        "http://www.w3.org/ns/org#hasPrimarySite",
+        "http://www.w3.org/ns/org#hasSite",
        "http://www.w3.org/ns/org#changedBy",
        "http://www.w3.org/ns/org#resultedFrom",
        "http://www.w3.org/ns/org#hasPost",
@@ -40,6 +42,8 @@
        "http://www.w3.org/2004/02/skos/core#altLabel",
        "http://www.w3.org/ns/adms#identifier",
        "http://www.w3.org/ns/regorg#orgStatus",
+        "http://www.w3.org/ns/org#hasPrimarySite",
+        "http://www.w3.org/ns/org#hasSite",
        "http://www.w3.org/ns/org#changedBy",
        "http://www.w3.org/ns/org#resultedFrom",
        "http://www.w3.org/ns/org#hasPost",
@@ -73,6 +77,8 @@
        "http://www.w3.org/ns/org#changedBy",
        "http://www.w3.org/ns/org#classification",
        "http://www.w3.org/ns/org#hasPost",
+        "http://www.w3.org/ns/org#hasPrimarySite",
+        "http://www.w3.org/ns/org#hasSite",
        "http://www.w3.org/ns/org#hasSubOrganization",
        "http://www.w3.org/ns/org#linkedTo",
        "http://www.w3.org/ns/org#resultedFrom",
@@ -120,6 +126,59 @@
        "http://data.lblod.info/vocabularies/erediensten/functie"
      ]
    },
+    {
+      "type": "http://www.w3.org/ns/org#Site",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/shared",
+        "http://mu.semte.ch/graphs/worship-service",
+        "http://mu.semte.ch/graphs/administrative-unit"
+      ],
+      "properties": [
+        "http://mu.semte.ch/vocabularies/core/uuid",
+        "http://www.w3.org/ns/org#siteAddress",
+        "https://data.vlaanderen.be/ns/organisatie#bestaatUit",
+        "http://data.lblod.info/vocabularies/erediensten/vestigingstype"
+      ]
+    },
+    {
+      "type": "http://www.w3.org/ns/locn#Address",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/shared",
+        "http://mu.semte.ch/graphs/worship-service",
+        "http://mu.semte.ch/graphs/administrative-unit"
+      ],
+      "properties": [
+        "http://mu.semte.ch/vocabularies/core/uuid",
+        "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer",
+        "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer",
+        "http://www.w3.org/ns/locn#thoroughfare",
+        "http://www.w3.org/ns/locn#postCode",
+        "https://data.vlaanderen.be/ns/adres#gemeentenaam",
+        "http://www.w3.org/ns/locn#adminUnitL2",
+        "https://data.vlaanderen.be/ns/adres#land",
+        "http://www.w3.org/ns/locn#fullAddress",
+        "https://data.vlaanderen.be/ns/adres#verwijstNaar",
+        "http://purl.org/dc/terms/source"
+      ]
+    },
+    {
+      "type": "http://schema.org/ContactPoint",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/shared",
+        "http://mu.semte.ch/graphs/worship-service",
+        "http://mu.semte.ch/graphs/administrative-unit"
+      ],
+      "properties": [
+        "http://mu.semte.ch/vocabularies/core/uuid",
+        "http://www.w3.org/ns/locn#address",
+        "http://xmlns.com/foaf/0.1/page",
+        "http://schema.org/faxNumber",
+        "http://schema.org/contactType",
+        "http://xmlns.com/foaf/0.1/page",
+        "http://schema.org/email",
+        "http://schema.org/telephone"
+      ]
+    },
    {
      "type": "http://www.w3.org/ns/adms#Identifier",
      "graphsFilter": [
--- a/config/migrations-sample/README.md
+++ b/config/migrations-sample/README.md
@@ -0,0 +1,15 @@
+# Motivation
+
+This folder contains migration files that are typically executed on an as-needed, or ad hoc, basis.
+In addition, these files can function as templates or provide inspiration for future migrations.
+
+## Warning
+
+- Please keep in mind, when adding these files to the migrations folder, it's important to rename them according to the conventions established by the migration service
+- As the data model or the data location may undergo changes over time, it's imporant to have a clear understanding of the implications when using these files.
+
+## Template files
+
+- `pseudonimyze-data.sparql-template` - the file that contains the sparql for pseudonymizing data in the triplestore. It should run every time the db is "refreshed" with a copy from PROD.
+- `remove-stale-jobs.sparql-template` - removing `busy` and `scheduled` jobs. 
+- `remove-type-in-producer-graph.sparql-template` - the migration to handle the occasional sync issue between OP and Loket. 
--- a/config/migrations-sample/pseudonimyze-data.sparql-template
+++ b/config/migrations-sample/pseudonimyze-data.sparql-template
@@ -0,0 +1,149 @@
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#> 
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX schema: <http://schema.org/>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX locn: <http://www.w3.org/ns/locn#>
+PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+
+DELETE {
+    GRAPH ?g {
+      ?person foaf:familyName ?fn.
+      ?person foaf:givenName ?gn.
+      ?person persoon:gebruikteVoornaam ?gvn.
+    }
+} INSERT {
+    GRAPH ?g {
+      ?person foaf:givenName ?gn_s.
+      ?person foaf:familyName ?fn_s.
+      ?person persoon:gebruikteVoornaam ?gvn_s.
+    }
+} WHERE {
+  VALUES ?g { <http://mu.semte.ch/graphs/organisatieportaal> <http://redpencil.data.gift/id/deltas/producer/organizations>}
+  GRAPH ?g {
+    BIND("njbyevfeRTE8651B" AS ?salt).
+    ?person a person:Person.
+    OPTIONAL {
+      ?person foaf:givenName ?gn.
+      BIND(SHA1(CONCAT(?salt, ?gn)) AS ?gn_s).
+    }
+    OPTIONAL {
+      ?person foaf:familyName ?fn.
+      BIND(SHA1(CONCAT(?salt, ?fn)) AS ?fn_s).
+    }
+    OPTIONAL {
+      ?person persoon:gebruikteVoornaam ?gvn.
+      BIND(SHA1(CONCAT(?salt, ?gvn)) AS ?gvn_s).
+    }
+  }
+}
+
+;
+
+DELETE {
+    GRAPH ?h {
+      ?birth persoon:datum ?dob.
+      ?identificator generiek:lokaleIdentificator ?rrn .
+      ?person persoon:heeftNationaliteit ?nationality .
+      ?person persoon:geslacht ?gender .
+    }
+} INSERT {
+    GRAPH ?h {
+      ?birth persoon:datum "1977-01-01T00:00:00Z"^^xsd:dateTime .
+      ?identificator generiek:lokaleIdentificator "42012205181" .
+    }
+} WHERE {
+  VALUES ?g { <http://mu.semte.ch/graphs/organisatieportaal> <http://redpencil.data.gift/id/deltas/producer/organizations>}
+  VALUES ?h { <http://mu.semte.ch/graphs/privacy-centric-graph> }
+  GRAPH ?g {
+    ?person a person:Person.
+  }
+  GRAPH ?h {
+    OPTIONAL {
+      ?person persoon:heeftGeboorte ?birth .
+      ?birth persoon:datum ?dob.
+    }
+    OPTIONAL {
+      ?person persoon:registratie/generiek:gestructureerdeIdentificator ?identificator .
+      ?identificator generiek:lokaleIdentificator ?rrn .
+    }
+    OPTIONAL {
+      ?person persoon:geslacht ?gender .
+    }
+    OPTIONAL {
+      ?person persoon:heeftNationaliteit ?nationality .
+    }
+  }
+}
+
+;
+
+DELETE {
+    GRAPH ?g {
+      ?cp schema:telephone ?phone.
+      ?cp schema:email ?mail.
+      ?address locn:fullAddress ?fa.
+      ?address adres:Adresvoorstelling.huisnummer ?huisnr.
+      ?address adres:Adresvoorstelling.busnummer ?busnr.
+      ?address locn:thoroughfare ?street.
+      ?address locn:postCode ?postCode.
+      ?address adres:gemeentenaam ?gemeente.
+    }
+} INSERT {
+    GRAPH ?g {
+      ?cp schema:telephone ?phone_s.
+      ?cp schema:email ?mail_s.
+      ?address locn:fullAddress ?fa_s.
+      ?address adres:Adresvoorstelling.huisnummer ?huisnr_s.
+      ?address adres:Adresvoorstelling.busnummer ?busnr_s.
+      ?address locn:thoroughfare ?street_s.
+      ?address locn:postCode ?postCode_s.
+      ?address adres:gemeentenaam ?gemeente_s.
+    }
+} WHERE {
+  VALUES ?g { <http://mu.semte.ch/graphs/organisatieportaal> <http://redpencil.data.gift/id/deltas/producer/organizations>}
+  GRAPH ?g {
+    BIND("njbyevfeRTE8651B" AS ?salt).
+    ?cp a <http://schema.org/ContactPoint> .
+
+    OPTIONAL {
+      ?cp schema:telephone ?phone.
+      BIND(SHA1(CONCAT(?salt, ?phone)) AS ?phone_s).
+    }
+    OPTIONAL {
+      ?cp schema:email ?mail.
+      BIND(CONCAT(SHA1(CONCAT(?salt, ?mail)), "@email.com") AS ?mail_s).
+    }
+    OPTIONAL {
+      ?cp locn:address ?address.
+      OPTIONAL {
+        ?address locn:fullAddress ?fa.
+        BIND(SHA1(CONCAT(?salt, ?fa)) AS ?fa_s).
+      }
+      OPTIONAL {
+        ?address adres:Adresvoorstelling.huisnummer ?huisnr.
+        BIND(SHA1(CONCAT(?salt, ?huisnr)) AS ?huisnr_s).
+      }
+      OPTIONAL {
+        ?address adres:Adresvoorstelling.busnummer ?busnr.
+        BIND(SHA1(CONCAT(?salt, ?busnr)) AS ?busnr_s).
+      }
+      OPTIONAL {
+        ?address locn:thoroughfare ?street.
+        BIND(SHA1(CONCAT(?salt, ?street)) AS ?street_s).
+      }
+      OPTIONAL {
+        ?address locn:postCode ?postCode.
+        BIND(SHA1(CONCAT(?salt, ?postCode)) AS ?postCode_s).
+      }
+      OPTIONAL {
+        ?address adres:gemeentenaam ?gemeente.
+        BIND(SHA1(CONCAT(?salt, ?gemeente)) AS ?gemeente_s).
+      }
+    }
+  }
+}
--- a/config/migrations-sample/remove-stale-jobs.sparql-template
+++ b/config/migrations-sample/remove-stale-jobs.sparql-template
@@ -0,0 +1,25 @@
+PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX oslc: <http://open-services.net/ns/core#>
+PREFIX cogs: <http://vocab.deri.ie/cogs#>
+PREFIX adms: <http://www.w3.org/ns/adms#>
+
+DELETE {
+    GRAPH ?g {
+    ?jobUri a <http://vocab.deri.ie/cogs#Job>.
+    ?jobUri adms:status ?status.
+    ?jobUri ?p ?o
+  }
+}
+WHERE {
+  GRAPH ?g {
+    ?jobUri a <http://vocab.deri.ie/cogs#Job>.
+    ?jobUri adms:status ?status.
+    ?jobUri ?p ?o
+   FILTER(?status IN (<http://redpencil.data.gift/id/concept/JobStatus/busy>, <http://redpencil.data.gift/id/concept/JobStatus/scheduled>))
+  }
+}
--- a/config/migrations-sample/remove-type-in-producer-graph.sparql-template
+++ b/config/migrations-sample/remove-type-in-producer-graph.sparql-template
@@ -0,0 +1,9 @@
+DELETE WHERE {
+  GRAPH <http://redpencil.data.gift/id/deltas/producer/public> {
+    <http://data.lblod.info/id/besturenVanDeEredienst/4ea0cc455e37195c10acf457a8902ffb> a ?firsttype.
+    <http://data.lblod.info/id/besturenVanDeEredienst/66ce16dcb455dc95588c05916736d511> a ?secondtype.
+    <http://data.lblod.info/id/besturenVanDeEredienst/69dca6d7c6bf9d6073771788358c74e5> a ?thirdtype.
+    <http://data.lblod.info/id/besturenVanDeEredienst/c9ce0539d719b63b60598148b084a7e8> a ?fourthtype.
+    <http://data.lblod.info/id/besturenVanDeEredienst/560d86a055eddb83fd60e0ea3043a708> a ?fifthtype.
+  }
+}
--- a/config/migrations/2024/20240318142001-add-new-recognized-worship-types.sparql
+++ b/config/migrations/2024/20240318142001-add-new-recognized-worship-types.sparql
@@ -1,27 +0,0 @@
-PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
-PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
-PREFIX core: <http://mu.semte.ch/vocabularies/core/>
-
-INSERT {
-  GRAPH <http://mu.semte.ch/graphs/public> {
-    <http://lblod.data.gift/concepts/cbd4c767ea184b41bf51b65d406754d4> a code:TypeEredienst ;
-      core:uuid "cbd4c767ea184b41bf51b65d406754d4" ;
-      skos:prefLabel "Niet-confessioneel" ;
-      skos:inScheme <http://lblod.data.gift/concept-schemes/5be1fce008d73105d9bc6de9e488b0b9> ;
-      skos:inScheme <http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b> ;
-      skos:topConceptOf <http://lblod.data.gift/concept-schemes/5be1fce008d73105d9bc6de9e488b0b9> .
-  }
-}
-
-;
-
-INSERT {
-  GRAPH <http://mu.semte.ch/graphs/public> {
-    <http://lblod.data.gift/concepts/f076c85e72814278814776b038ac33f7> a code:TypeEredienst ;
-      core:uuid "f076c85e72814278814776b038ac33f7" ;
-      skos:prefLabel "Boeddhistisch" ;
-      skos:inScheme <http://lblod.data.gift/concept-schemes/5be1fce008d73105d9bc6de9e488b0b9> ;
-      skos:inScheme <http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b> ;
-      skos:topConceptOf <http://lblod.data.gift/concept-schemes/5be1fce008d73105d9bc6de9e488b0b9> .
-  }
-}
--- a/config/migrations/2024/20240318144801-update-missing-types-representative-bodies.sparql
+++ b/config/migrations/2024/20240318144801-update-missing-types-representative-bodies.sparql
@@ -1,15 +0,0 @@
-PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
-
-INSERT {
-  GRAPH <http://mu.semte.ch/graphs/worship-service> {
-   <http://data.lblod.info/id/representatieveOrganen/45a0c64b0f2ee4ac0fa8c151e6f2209c> a ere:RepresentatiefOrgaan ;
-         ere:typeEredienst <http://lblod.data.gift/concepts/cbd4c767ea184b41bf51b65d406754d4>
-  }
-}
-;
-INSERT {
-  GRAPH <http://mu.semte.ch/graphs/worship-service> {
-   <http://data.lblod.info/id/representatieveOrganen/5871503aa86b44cd470f97228a2ce413> a ere:RepresentatiefOrgaan ;
-         ere:typeEredienst <http://lblod.data.gift/concepts/f076c85e72814278814776b038ac33f7>
-  }
-}
--- a/config/migrations/local/.gitkeep
+++ b/config/migrations/local/.gitkeep
--- a/config/migrations/local/README.md
+++ b/config/migrations/local/README.md
@@ -0,0 +1,3 @@
+# Motivation
+
+This folder is designed to contain migration files specific to a specific deployed instance of the app. These files will be ignored by git. The typical use case for this folder includes migrations containing sensitive data or those related to a re-sync of consumers. Migrations that should be applied to all instances of the app should be stored outside of this folder, but within the 'migrations' folder.
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -175,13 +175,13 @@ services:
    links:
      - db:database
  adressenregister:
-    image: lblod/adressenregister-fuzzy-search-service:0.7.0
+    image: lblod/adressenregister-fuzzy-search-service:0.8.0
    labels:
      - "logging=true"
    restart: always
    logging: *default-logging
  frontend:
-    image: lblod/frontend-organization-portal:1.23.0-0
+    image: lblod/frontend-organization-portal:1.23.0
    volumes:
      - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf
    environment:
@@ -535,7 +535,7 @@ services:
  # LOKET MANDATARISSEN CONSUMERS
  ################################################################################
  mandatarissen-consumer:
-    image: lblod/delta-consumer:0.0.17-rc.7
+    image: lblod/delta-consumer:0.0.24
    environment:
      DCR_SERVICE_NAME: "mandatarissen-consumer"
      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -566,7 +566,7 @@ services:
  # LOKET LEIDINGGEVENDEN CONSUMERS
  ################################################################################
  leidinggevenden-consumer:
-    image: lblod/delta-consumer:0.0.18
+    image: lblod/delta-consumer:0.0.24
    environment:
      DCR_SERVICE_NAME: "leidinggevenden-consumer"
      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -597,7 +597,7 @@ services:
  # LOKET WORSHIP SENSITIVE CONSUMERS
  ################################################################################
  worship-services-sensitive-consumer:
-    image: lblod/delta-consumer:0.0.17-rc.7
+    image: lblod/delta-consumer:0.0.24
    environment:
      DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
      DCR_SERVICE_NAME: "worship-services-sensitive-consumer"
@@ -608,7 +608,8 @@ services:
      DCR_DISABLE_INITIAL_SYNC: "true"
      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
      DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files"
-      BATCH_SIZE: 1000
+      BATCH_SIZE: 50
+      SLEEP_BETWEEN_BATCHES: 10
      DCR_KEEP_DELTA_FILES: "true"
      DCR_ENABLE_DELTA_CONTEXT: "true"
      DCR_LANDING_ZONE_DATABASE: "triplestore"
@@ -662,7 +663,7 @@ services:
  # MU SEARCH
  ################################################################################
  search:
-    image: semtech/mu-search:0.9.1
+    image: semtech/mu-search:0.10.0-beta.5
    links:
      - db:database
    restart: always
Author	SHA1	Message	Date
Laurent Deketelaere	e507c31a3d	Merge pull request #407 from lblod/hotfix/1.23.1-1 Update adressenregister to use basisregisters adressen V2 API	2024-04-04 11:51:39 +02:00
Laurent Deketelaere	a966e25832	fix: Update adressenregister to use basisregisters adressen V2 API	2024-04-04 11:50:08 +02:00
Laurent Deketelaere	ec749c81dd	Merge pull request #403 from lblod/hotfix/ws-consumer-tweaks bump + tweaked ingestion blobs to smaller amounts	2024-04-02 10:16:08 +02:00
mirdono	aa7ce23a85	chore: bumped consumers to latest version	2024-04-02 08:10:18 +00:00
mirdono	0592e10a8c	chore: bump mu-search version	2024-04-02 08:08:48 +00:00
felix	c24f56c7cb	bump + tweaked ingestion blobs to smaller amounts to make it more digestable for OP (+ search)	2024-04-02 10:01:10 +02:00
Laurent Deketelaere	69fa0b5d8b	chore: Bump frontend to v1.23.0	2024-04-02 09:43:35 +02:00
Laurent Deketelaere	f07c1a561b	Merge pull request #402 from lblod/revert-add-recognized-worship-types Revert "Merge pull request #397 from lblod/fix/add-recognized-worship-types"	2024-04-02 09:24:20 +02:00
Laurent Deketelaere	954f72ef7e	Revert "Merge pull request #397 from lblod/fix/add-recognized-worship-types" This reverts commit `9ed2cfa080`, reversing changes made to `37f071c065`.	2024-03-29 16:00:01 +01:00
Laurent Deketelaere	23e7ee7ff2	chore: Bump frontend to v1.23.0-1	2024-03-29 13:59:04 +01:00
Laurent Deketelaere	6e0e3cd8ea	Merge pull request #399 from lblod/revert-exclude-clb-data-from-producer Revert "Merge pull request #387 from lblod/feat/exclude-clb-data-from-producer"	2024-03-28 09:47:35 +01:00
Laurent Deketelaere	6c572992be	Revert "Merge pull request #387 from lblod/feat/exclude-clb-data-from-producer" This reverts commit `25a5adaf5e`, reversing changes made to `c6bb94816d`.	2024-03-27 16:17:41 +01:00
Laurent Deketelaere	51d7c53970	Merge pull request #384 from lblod/chore/add-migration-template-samples Chore: Add migration template samples	2024-03-27 09:55:14 +01:00
Laurent Deketelaere	54f1d8932c	Add `remove-stale-jobs.sparql-template` migration	2024-03-27 09:52:28 +01:00
Laurent Deketelaere	4dc4bf84fa	Add `remove-type-in-producer-graph.sparql-template`	2024-02-28 12:16:35 +01:00
Laurent Deketelaere	60ba1df934	chore: gitignore migrations/local	2024-02-27 15:27:08 +01:00
Laurent Deketelaere	1fc6a43e9c	chore: Add migration samples	2024-02-27 15:21:39 +01:00