Merge pull request #384 from lblod/chore/add-migration-template-samples

Chore: Add migration template samples

.gitignore

@@ -19,6 +19,7 @@ docker-compose.override.yml
 .env
 
 /config/migrations/private-data/*
+/config/migrations/local/*
 
 /config/proxy/nginx.htpasswd
 

config/migrations-sample/README.md

@@ -0,0 +1,15 @@
+# Motivation
+
+This folder contains migration files that are typically executed on an as-needed, or ad hoc, basis.
+In addition, these files can function as templates or provide inspiration for future migrations.
+
+## Warning
+
+- Please keep in mind, when adding these files to the migrations folder, it's important to rename them according to the conventions established by the migration service
+- As the data model or the data location may undergo changes over time, it's imporant to have a clear understanding of the implications when using these files.
+
+## Template files
+
+- `pseudonimyze-data.sparql-template` - the file that contains the sparql for pseudonymizing data in the triplestore. It should run every time the db is "refreshed" with a copy from PROD.
+- `remove-stale-jobs.sparql-template` - removing `busy` and `scheduled` jobs. 
+- `remove-type-in-producer-graph.sparql-template` - the migration to handle the occasional sync issue between OP and Loket. 

config/migrations-sample/pseudonimyze-data.sparql-template

@@ -0,0 +1,149 @@
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#> 
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX schema: <http://schema.org/>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX locn: <http://www.w3.org/ns/locn#>
+PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+
+DELETE {
+    GRAPH ?g {
+      ?person foaf:familyName ?fn.
+      ?person foaf:givenName ?gn.
+      ?person persoon:gebruikteVoornaam ?gvn.
+    }
+} INSERT {
+    GRAPH ?g {
+      ?person foaf:givenName ?gn_s.
+      ?person foaf:familyName ?fn_s.
+      ?person persoon:gebruikteVoornaam ?gvn_s.
+    }
+} WHERE {
+  VALUES ?g { <http://mu.semte.ch/graphs/organisatieportaal> <http://redpencil.data.gift/id/deltas/producer/organizations>}
+  GRAPH ?g {
+    BIND("njbyevfeRTE8651B" AS ?salt).
+    ?person a person:Person.
+    OPTIONAL {
+      ?person foaf:givenName ?gn.
+      BIND(SHA1(CONCAT(?salt, ?gn)) AS ?gn_s).
+    }
+    OPTIONAL {
+      ?person foaf:familyName ?fn.
+      BIND(SHA1(CONCAT(?salt, ?fn)) AS ?fn_s).
+    }
+    OPTIONAL {
+      ?person persoon:gebruikteVoornaam ?gvn.
+      BIND(SHA1(CONCAT(?salt, ?gvn)) AS ?gvn_s).
+    }
+  }
+}
+
+;
+
+DELETE {
+    GRAPH ?h {
+      ?birth persoon:datum ?dob.
+      ?identificator generiek:lokaleIdentificator ?rrn .
+      ?person persoon:heeftNationaliteit ?nationality .
+      ?person persoon:geslacht ?gender .
+    }
+} INSERT {
+    GRAPH ?h {
+      ?birth persoon:datum "1977-01-01T00:00:00Z"^^xsd:dateTime .
+      ?identificator generiek:lokaleIdentificator "42012205181" .
+    }
+} WHERE {
+  VALUES ?g { <http://mu.semte.ch/graphs/organisatieportaal> <http://redpencil.data.gift/id/deltas/producer/organizations>}
+  VALUES ?h { <http://mu.semte.ch/graphs/privacy-centric-graph> }
+  GRAPH ?g {
+    ?person a person:Person.
+  }
+  GRAPH ?h {
+    OPTIONAL {
+      ?person persoon:heeftGeboorte ?birth .
+      ?birth persoon:datum ?dob.
+    }
+    OPTIONAL {
+      ?person persoon:registratie/generiek:gestructureerdeIdentificator ?identificator .
+      ?identificator generiek:lokaleIdentificator ?rrn .
+    }
+    OPTIONAL {
+      ?person persoon:geslacht ?gender .
+    }
+    OPTIONAL {
+      ?person persoon:heeftNationaliteit ?nationality .
+    }
+  }
+}
+
+;
+
+DELETE {
+    GRAPH ?g {
+      ?cp schema:telephone ?phone.
+      ?cp schema:email ?mail.
+      ?address locn:fullAddress ?fa.
+      ?address adres:Adresvoorstelling.huisnummer ?huisnr.
+      ?address adres:Adresvoorstelling.busnummer ?busnr.
+      ?address locn:thoroughfare ?street.
+      ?address locn:postCode ?postCode.
+      ?address adres:gemeentenaam ?gemeente.
+    }
+} INSERT {
+    GRAPH ?g {
+      ?cp schema:telephone ?phone_s.
+      ?cp schema:email ?mail_s.
+      ?address locn:fullAddress ?fa_s.
+      ?address adres:Adresvoorstelling.huisnummer ?huisnr_s.
+      ?address adres:Adresvoorstelling.busnummer ?busnr_s.
+      ?address locn:thoroughfare ?street_s.
+      ?address locn:postCode ?postCode_s.
+      ?address adres:gemeentenaam ?gemeente_s.
+    }
+} WHERE {
+  VALUES ?g { <http://mu.semte.ch/graphs/organisatieportaal> <http://redpencil.data.gift/id/deltas/producer/organizations>}
+  GRAPH ?g {
+    BIND("njbyevfeRTE8651B" AS ?salt).
+    ?cp a <http://schema.org/ContactPoint> .
+
+    OPTIONAL {
+      ?cp schema:telephone ?phone.
+      BIND(SHA1(CONCAT(?salt, ?phone)) AS ?phone_s).
+    }
+    OPTIONAL {
+      ?cp schema:email ?mail.
+      BIND(CONCAT(SHA1(CONCAT(?salt, ?mail)), "@email.com") AS ?mail_s).
+    }
+    OPTIONAL {
+      ?cp locn:address ?address.
+      OPTIONAL {
+        ?address locn:fullAddress ?fa.
+        BIND(SHA1(CONCAT(?salt, ?fa)) AS ?fa_s).
+      }
+      OPTIONAL {
+        ?address adres:Adresvoorstelling.huisnummer ?huisnr.
+        BIND(SHA1(CONCAT(?salt, ?huisnr)) AS ?huisnr_s).
+      }
+      OPTIONAL {
+        ?address adres:Adresvoorstelling.busnummer ?busnr.
+        BIND(SHA1(CONCAT(?salt, ?busnr)) AS ?busnr_s).
+      }
+      OPTIONAL {
+        ?address locn:thoroughfare ?street.
+        BIND(SHA1(CONCAT(?salt, ?street)) AS ?street_s).
+      }
+      OPTIONAL {
+        ?address locn:postCode ?postCode.
+        BIND(SHA1(CONCAT(?salt, ?postCode)) AS ?postCode_s).
+      }
+      OPTIONAL {
+        ?address adres:gemeentenaam ?gemeente.
+        BIND(SHA1(CONCAT(?salt, ?gemeente)) AS ?gemeente_s).
+      }
+    }
+  }
+}

config/migrations-sample/remove-stale-jobs.sparql-template

@@ -0,0 +1,25 @@
+PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX oslc: <http://open-services.net/ns/core#>
+PREFIX cogs: <http://vocab.deri.ie/cogs#>
+PREFIX adms: <http://www.w3.org/ns/adms#>
+
+DELETE {
+    GRAPH ?g {
+    ?jobUri a <http://vocab.deri.ie/cogs#Job>.
+    ?jobUri adms:status ?status.
+    ?jobUri ?p ?o
+  }
+}
+WHERE {
+  GRAPH ?g {
+    ?jobUri a <http://vocab.deri.ie/cogs#Job>.
+    ?jobUri adms:status ?status.
+    ?jobUri ?p ?o
+   FILTER(?status IN (<http://redpencil.data.gift/id/concept/JobStatus/busy>, <http://redpencil.data.gift/id/concept/JobStatus/scheduled>))
+  }
+}

config/migrations-sample/remove-type-in-producer-graph.sparql-template

@@ -0,0 +1,9 @@
+DELETE WHERE {
+  GRAPH <http://redpencil.data.gift/id/deltas/producer/public> {
+    <http://data.lblod.info/id/besturenVanDeEredienst/4ea0cc455e37195c10acf457a8902ffb> a ?firsttype.
+    <http://data.lblod.info/id/besturenVanDeEredienst/66ce16dcb455dc95588c05916736d511> a ?secondtype.
+    <http://data.lblod.info/id/besturenVanDeEredienst/69dca6d7c6bf9d6073771788358c74e5> a ?thirdtype.
+    <http://data.lblod.info/id/besturenVanDeEredienst/c9ce0539d719b63b60598148b084a7e8> a ?fourthtype.
+    <http://data.lblod.info/id/besturenVanDeEredienst/560d86a055eddb83fd60e0ea3043a708> a ?fifthtype.
+  }
+}

config/migrations/local/README.md

@@ -0,0 +1,3 @@
+# Motivation
+
+This folder is designed to contain migration files specific to a specific deployed instance of the app. These files will be ignored by git. The typical use case for this folder includes migrations containing sensitive data or those related to a re-sync of consumers. Migrations that should be applied to all instances of the app should be stored outside of this folder, but within the 'migrations' folder.