diff --git a/.gitignore b/.gitignore index 4b8e33a..894c430 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ docker-compose.override.yml .env /config/migrations/private-data/* +/config/migrations/local/* /config/proxy/nginx.htpasswd diff --git a/config/migrations-sample/README.md b/config/migrations-sample/README.md new file mode 100644 index 0000000..c5aa12e --- /dev/null +++ b/config/migrations-sample/README.md @@ -0,0 +1,15 @@ +# Motivation + +This folder contains migration files that are typically executed on an as-needed, or ad hoc, basis. +In addition, these files can function as templates or provide inspiration for future migrations. + +## Warning + +- Please keep in mind, when adding these files to the migrations folder, it's important to rename them according to the conventions established by the migration service +- As the data model or the data location may undergo changes over time, it's imporant to have a clear understanding of the implications when using these files. + +## Template files + +- `pseudonimyze-data.sparql-template` - the file that contains the sparql for pseudonymizing data in the triplestore. It should run every time the db is "refreshed" with a copy from PROD. +- `remove-stale-jobs.sparql-template` - removing `busy` and `scheduled` jobs. +- `remove-type-in-producer-graph.sparql-template` - the migration to handle the occasional sync issue between OP and Loket. diff --git a/config/migrations-sample/pseudonimyze-data.sparql-template b/config/migrations-sample/pseudonimyze-data.sparql-template new file mode 100644 index 0000000..c3d49da --- /dev/null +++ b/config/migrations-sample/pseudonimyze-data.sparql-template @@ -0,0 +1,149 @@ +PREFIX rdf: +PREFIX rdfs: +PREFIX person: +PREFIX persoon: +PREFIX foaf: +PREFIX schema: +PREFIX mandaat: +PREFIX locn: +PREFIX adres: +PREFIX xsd: +PREFIX generiek: + +DELETE { + GRAPH ?g { + ?person foaf:familyName ?fn. + ?person foaf:givenName ?gn. + ?person persoon:gebruikteVoornaam ?gvn. + } +} INSERT { + GRAPH ?g { + ?person foaf:givenName ?gn_s. + ?person foaf:familyName ?fn_s. + ?person persoon:gebruikteVoornaam ?gvn_s. + } +} WHERE { + VALUES ?g { } + GRAPH ?g { + BIND("njbyevfeRTE8651B" AS ?salt). + ?person a person:Person. + OPTIONAL { + ?person foaf:givenName ?gn. + BIND(SHA1(CONCAT(?salt, ?gn)) AS ?gn_s). + } + OPTIONAL { + ?person foaf:familyName ?fn. + BIND(SHA1(CONCAT(?salt, ?fn)) AS ?fn_s). + } + OPTIONAL { + ?person persoon:gebruikteVoornaam ?gvn. + BIND(SHA1(CONCAT(?salt, ?gvn)) AS ?gvn_s). + } + } +} + +; + +DELETE { + GRAPH ?h { + ?birth persoon:datum ?dob. + ?identificator generiek:lokaleIdentificator ?rrn . + ?person persoon:heeftNationaliteit ?nationality . + ?person persoon:geslacht ?gender . + } +} INSERT { + GRAPH ?h { + ?birth persoon:datum "1977-01-01T00:00:00Z"^^xsd:dateTime . + ?identificator generiek:lokaleIdentificator "42012205181" . + } +} WHERE { + VALUES ?g { } + VALUES ?h { } + GRAPH ?g { + ?person a person:Person. + } + GRAPH ?h { + OPTIONAL { + ?person persoon:heeftGeboorte ?birth . + ?birth persoon:datum ?dob. + } + OPTIONAL { + ?person persoon:registratie/generiek:gestructureerdeIdentificator ?identificator . + ?identificator generiek:lokaleIdentificator ?rrn . + } + OPTIONAL { + ?person persoon:geslacht ?gender . + } + OPTIONAL { + ?person persoon:heeftNationaliteit ?nationality . + } + } +} + +; + +DELETE { + GRAPH ?g { + ?cp schema:telephone ?phone. + ?cp schema:email ?mail. + ?address locn:fullAddress ?fa. + ?address adres:Adresvoorstelling.huisnummer ?huisnr. + ?address adres:Adresvoorstelling.busnummer ?busnr. + ?address locn:thoroughfare ?street. + ?address locn:postCode ?postCode. + ?address adres:gemeentenaam ?gemeente. + } +} INSERT { + GRAPH ?g { + ?cp schema:telephone ?phone_s. + ?cp schema:email ?mail_s. + ?address locn:fullAddress ?fa_s. + ?address adres:Adresvoorstelling.huisnummer ?huisnr_s. + ?address adres:Adresvoorstelling.busnummer ?busnr_s. + ?address locn:thoroughfare ?street_s. + ?address locn:postCode ?postCode_s. + ?address adres:gemeentenaam ?gemeente_s. + } +} WHERE { + VALUES ?g { } + GRAPH ?g { + BIND("njbyevfeRTE8651B" AS ?salt). + ?cp a . + + OPTIONAL { + ?cp schema:telephone ?phone. + BIND(SHA1(CONCAT(?salt, ?phone)) AS ?phone_s). + } + OPTIONAL { + ?cp schema:email ?mail. + BIND(CONCAT(SHA1(CONCAT(?salt, ?mail)), "@email.com") AS ?mail_s). + } + OPTIONAL { + ?cp locn:address ?address. + OPTIONAL { + ?address locn:fullAddress ?fa. + BIND(SHA1(CONCAT(?salt, ?fa)) AS ?fa_s). + } + OPTIONAL { + ?address adres:Adresvoorstelling.huisnummer ?huisnr. + BIND(SHA1(CONCAT(?salt, ?huisnr)) AS ?huisnr_s). + } + OPTIONAL { + ?address adres:Adresvoorstelling.busnummer ?busnr. + BIND(SHA1(CONCAT(?salt, ?busnr)) AS ?busnr_s). + } + OPTIONAL { + ?address locn:thoroughfare ?street. + BIND(SHA1(CONCAT(?salt, ?street)) AS ?street_s). + } + OPTIONAL { + ?address locn:postCode ?postCode. + BIND(SHA1(CONCAT(?salt, ?postCode)) AS ?postCode_s). + } + OPTIONAL { + ?address adres:gemeentenaam ?gemeente. + BIND(SHA1(CONCAT(?salt, ?gemeente)) AS ?gemeente_s). + } + } + } +} \ No newline at end of file diff --git a/config/migrations-sample/remove-stale-jobs.sparql-template b/config/migrations-sample/remove-stale-jobs.sparql-template new file mode 100644 index 0000000..a0120a4 --- /dev/null +++ b/config/migrations-sample/remove-stale-jobs.sparql-template @@ -0,0 +1,25 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: + +DELETE { + GRAPH ?g { + ?jobUri a . + ?jobUri adms:status ?status. + ?jobUri ?p ?o + } +} +WHERE { + GRAPH ?g { + ?jobUri a . + ?jobUri adms:status ?status. + ?jobUri ?p ?o + FILTER(?status IN (, )) + } +} diff --git a/config/migrations-sample/remove-type-in-producer-graph.sparql-template b/config/migrations-sample/remove-type-in-producer-graph.sparql-template new file mode 100644 index 0000000..3f16e82 --- /dev/null +++ b/config/migrations-sample/remove-type-in-producer-graph.sparql-template @@ -0,0 +1,9 @@ +DELETE WHERE { + GRAPH { + a ?firsttype. + a ?secondtype. + a ?thirdtype. + a ?fourthtype. + a ?fifthtype. + } +} \ No newline at end of file diff --git a/config/migrations/local/.gitkeep b/config/migrations/local/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/config/migrations/local/README.md b/config/migrations/local/README.md new file mode 100644 index 0000000..518aaf9 --- /dev/null +++ b/config/migrations/local/README.md @@ -0,0 +1,3 @@ +# Motivation + +This folder is designed to contain migration files specific to a specific deployed instance of the app. These files will be ignored by git. The typical use case for this folder includes migrations containing sensitive data or those related to a re-sync of consumers. Migrations that should be applied to all instances of the app should be stored outside of this folder, but within the 'migrations' folder.