From 1fc6a43e9c9183a91042c1375fd97f63c5aed5d7 Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Tue, 27 Feb 2024 15:21:39 +0100 Subject: [PATCH 1/4] chore: Add migration samples --- config/migrations-sample/README.md | 13 ++ .../pseudonimyze-data.sparql-template | 149 ++++++++++++++++++ config/migrations/local/.gitkeep | 0 config/migrations/local/README.md | 3 + 4 files changed, 165 insertions(+) create mode 100644 config/migrations-sample/README.md create mode 100644 config/migrations-sample/pseudonimyze-data.sparql-template create mode 100644 config/migrations/local/.gitkeep create mode 100644 config/migrations/local/README.md diff --git a/config/migrations-sample/README.md b/config/migrations-sample/README.md new file mode 100644 index 0000000..d2e79fa --- /dev/null +++ b/config/migrations-sample/README.md @@ -0,0 +1,13 @@ +# Motivation + +This folder contains migration files that are typically executed on an as-needed, or ad hoc, basis. +In addition, these files can function as templates or provide inspiration for future migrations. + +## Warning + +- Please keep in mind, when adding these files to the migrations folder, it's important to rename them according to the conventions established by the migration service +- As the data model or the data location may undergo changes over time, it's imporant to have a clear understanding of the implications when using these files. + +## Template files + +- `pseudonimyze-data.sparql-template` - the file that contains the sparql for pseudonymizing data in the triplestore. It should run every time the db is "refreshed" with a copy from PROD. diff --git a/config/migrations-sample/pseudonimyze-data.sparql-template b/config/migrations-sample/pseudonimyze-data.sparql-template new file mode 100644 index 0000000..c3d49da --- /dev/null +++ b/config/migrations-sample/pseudonimyze-data.sparql-template @@ -0,0 +1,149 @@ +PREFIX rdf: +PREFIX rdfs: +PREFIX person: +PREFIX persoon: +PREFIX foaf: +PREFIX schema: +PREFIX mandaat: +PREFIX locn: +PREFIX adres: +PREFIX xsd: +PREFIX generiek: + +DELETE { + GRAPH ?g { + ?person foaf:familyName ?fn. + ?person foaf:givenName ?gn. + ?person persoon:gebruikteVoornaam ?gvn. + } +} INSERT { + GRAPH ?g { + ?person foaf:givenName ?gn_s. + ?person foaf:familyName ?fn_s. + ?person persoon:gebruikteVoornaam ?gvn_s. + } +} WHERE { + VALUES ?g { } + GRAPH ?g { + BIND("njbyevfeRTE8651B" AS ?salt). + ?person a person:Person. + OPTIONAL { + ?person foaf:givenName ?gn. + BIND(SHA1(CONCAT(?salt, ?gn)) AS ?gn_s). + } + OPTIONAL { + ?person foaf:familyName ?fn. + BIND(SHA1(CONCAT(?salt, ?fn)) AS ?fn_s). + } + OPTIONAL { + ?person persoon:gebruikteVoornaam ?gvn. + BIND(SHA1(CONCAT(?salt, ?gvn)) AS ?gvn_s). + } + } +} + +; + +DELETE { + GRAPH ?h { + ?birth persoon:datum ?dob. + ?identificator generiek:lokaleIdentificator ?rrn . + ?person persoon:heeftNationaliteit ?nationality . + ?person persoon:geslacht ?gender . + } +} INSERT { + GRAPH ?h { + ?birth persoon:datum "1977-01-01T00:00:00Z"^^xsd:dateTime . + ?identificator generiek:lokaleIdentificator "42012205181" . + } +} WHERE { + VALUES ?g { } + VALUES ?h { } + GRAPH ?g { + ?person a person:Person. + } + GRAPH ?h { + OPTIONAL { + ?person persoon:heeftGeboorte ?birth . + ?birth persoon:datum ?dob. + } + OPTIONAL { + ?person persoon:registratie/generiek:gestructureerdeIdentificator ?identificator . + ?identificator generiek:lokaleIdentificator ?rrn . + } + OPTIONAL { + ?person persoon:geslacht ?gender . + } + OPTIONAL { + ?person persoon:heeftNationaliteit ?nationality . + } + } +} + +; + +DELETE { + GRAPH ?g { + ?cp schema:telephone ?phone. + ?cp schema:email ?mail. + ?address locn:fullAddress ?fa. + ?address adres:Adresvoorstelling.huisnummer ?huisnr. + ?address adres:Adresvoorstelling.busnummer ?busnr. + ?address locn:thoroughfare ?street. + ?address locn:postCode ?postCode. + ?address adres:gemeentenaam ?gemeente. + } +} INSERT { + GRAPH ?g { + ?cp schema:telephone ?phone_s. + ?cp schema:email ?mail_s. + ?address locn:fullAddress ?fa_s. + ?address adres:Adresvoorstelling.huisnummer ?huisnr_s. + ?address adres:Adresvoorstelling.busnummer ?busnr_s. + ?address locn:thoroughfare ?street_s. + ?address locn:postCode ?postCode_s. + ?address adres:gemeentenaam ?gemeente_s. + } +} WHERE { + VALUES ?g { } + GRAPH ?g { + BIND("njbyevfeRTE8651B" AS ?salt). + ?cp a . + + OPTIONAL { + ?cp schema:telephone ?phone. + BIND(SHA1(CONCAT(?salt, ?phone)) AS ?phone_s). + } + OPTIONAL { + ?cp schema:email ?mail. + BIND(CONCAT(SHA1(CONCAT(?salt, ?mail)), "@email.com") AS ?mail_s). + } + OPTIONAL { + ?cp locn:address ?address. + OPTIONAL { + ?address locn:fullAddress ?fa. + BIND(SHA1(CONCAT(?salt, ?fa)) AS ?fa_s). + } + OPTIONAL { + ?address adres:Adresvoorstelling.huisnummer ?huisnr. + BIND(SHA1(CONCAT(?salt, ?huisnr)) AS ?huisnr_s). + } + OPTIONAL { + ?address adres:Adresvoorstelling.busnummer ?busnr. + BIND(SHA1(CONCAT(?salt, ?busnr)) AS ?busnr_s). + } + OPTIONAL { + ?address locn:thoroughfare ?street. + BIND(SHA1(CONCAT(?salt, ?street)) AS ?street_s). + } + OPTIONAL { + ?address locn:postCode ?postCode. + BIND(SHA1(CONCAT(?salt, ?postCode)) AS ?postCode_s). + } + OPTIONAL { + ?address adres:gemeentenaam ?gemeente. + BIND(SHA1(CONCAT(?salt, ?gemeente)) AS ?gemeente_s). + } + } + } +} \ No newline at end of file diff --git a/config/migrations/local/.gitkeep b/config/migrations/local/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/config/migrations/local/README.md b/config/migrations/local/README.md new file mode 100644 index 0000000..518aaf9 --- /dev/null +++ b/config/migrations/local/README.md @@ -0,0 +1,3 @@ +# Motivation + +This folder is designed to contain migration files specific to a specific deployed instance of the app. These files will be ignored by git. The typical use case for this folder includes migrations containing sensitive data or those related to a re-sync of consumers. Migrations that should be applied to all instances of the app should be stored outside of this folder, but within the 'migrations' folder. From 60ba1df934c6eaca1b20c8019f77e7ddde39b89a Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Tue, 27 Feb 2024 15:22:30 +0100 Subject: [PATCH 2/4] chore: gitignore migrations/local --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4b8e33a..894c430 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ docker-compose.override.yml .env /config/migrations/private-data/* +/config/migrations/local/* /config/proxy/nginx.htpasswd From 4dc4bf84fa2d3e2ab836501070f3df3cf81492c7 Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Wed, 28 Feb 2024 12:16:35 +0100 Subject: [PATCH 3/4] Add `remove-type-in-producer-graph.sparql-template` --- config/migrations-sample/README.md | 1 + .../remove-type-in-producer-graph.sparql-template | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 config/migrations-sample/remove-type-in-producer-graph.sparql-template diff --git a/config/migrations-sample/README.md b/config/migrations-sample/README.md index d2e79fa..e506f8b 100644 --- a/config/migrations-sample/README.md +++ b/config/migrations-sample/README.md @@ -11,3 +11,4 @@ In addition, these files can function as templates or provide inspiration for fu ## Template files - `pseudonimyze-data.sparql-template` - the file that contains the sparql for pseudonymizing data in the triplestore. It should run every time the db is "refreshed" with a copy from PROD. +- `remove-type-in-producer-graph.sparql-template` - the migration to handle the occasional sync issue between OP and Loket. diff --git a/config/migrations-sample/remove-type-in-producer-graph.sparql-template b/config/migrations-sample/remove-type-in-producer-graph.sparql-template new file mode 100644 index 0000000..3f16e82 --- /dev/null +++ b/config/migrations-sample/remove-type-in-producer-graph.sparql-template @@ -0,0 +1,9 @@ +DELETE WHERE { + GRAPH { + a ?firsttype. + a ?secondtype. + a ?thirdtype. + a ?fourthtype. + a ?fifthtype. + } +} \ No newline at end of file From 54f1d8932c88ae8eeb648646a47651a654249297 Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Wed, 27 Mar 2024 09:52:28 +0100 Subject: [PATCH 4/4] Add `remove-stale-jobs.sparql-template` migration --- config/migrations-sample/README.md | 1 + .../remove-stale-jobs.sparql-template | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 config/migrations-sample/remove-stale-jobs.sparql-template diff --git a/config/migrations-sample/README.md b/config/migrations-sample/README.md index e506f8b..c5aa12e 100644 --- a/config/migrations-sample/README.md +++ b/config/migrations-sample/README.md @@ -11,4 +11,5 @@ In addition, these files can function as templates or provide inspiration for fu ## Template files - `pseudonimyze-data.sparql-template` - the file that contains the sparql for pseudonymizing data in the triplestore. It should run every time the db is "refreshed" with a copy from PROD. +- `remove-stale-jobs.sparql-template` - removing `busy` and `scheduled` jobs. - `remove-type-in-producer-graph.sparql-template` - the migration to handle the occasional sync issue between OP and Loket. diff --git a/config/migrations-sample/remove-stale-jobs.sparql-template b/config/migrations-sample/remove-stale-jobs.sparql-template new file mode 100644 index 0000000..a0120a4 --- /dev/null +++ b/config/migrations-sample/remove-stale-jobs.sparql-template @@ -0,0 +1,25 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: + +DELETE { + GRAPH ?g { + ?jobUri a . + ?jobUri adms:status ?status. + ?jobUri ?p ?o + } +} +WHERE { + GRAPH ?g { + ?jobUri a . + ?jobUri adms:status ?status. + ?jobUri ?p ?o + FILTER(?status IN (, )) + } +}