From 73e6c2c60c4e416b5bf31d15a950e36fe65fdeda Mon Sep 17 00:00:00 2001 From: Panos Alexopoulos Date: Wed, 7 May 2025 16:38:26 +0200 Subject: [PATCH] Added two migrations, one for adding province URIs to address entities, and one for adding a concept scheme for provinces Updated add province uri migration to avoid duplicates Added a cleanup migration for adding province URIs to addresses Updated the export,json config to expose the province URI of addresses Fixing errors after review Updated CHANGELOG Refactored and corrected migrations, changed the timestamps refactoring cleanup migrations Refactored the cleanup add province migrations Updated changelog to ensure db-cleanup-service picks up the new jobs. --- CHANGELOG.md | 2 + .../delta-producer/organizations/export.json | 1 + config/delta-producer/public/export.json | 2 + ...051-create-provinces-concept-scheme.sparql | 26 ++++++++++++ ...3102116-add-province_uri-to-address.sparql | 40 ++++++++++++++++++ ...e_uri-to-adress-administrative_units.graph | 1 + ...nce_uri-to-adress-administrative_units.ttl | 41 +++++++++++++++++++ ...vince_uri-to-adress-worship_services.graph | 1 + ...rovince_uri-to-adress-worship_services.ttl | 39 ++++++++++++++++++ 9 files changed, 153 insertions(+) create mode 100644 config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102051-create-provinces-concept-scheme.sparql create mode 100644 config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102116-add-province_uri-to-address.sparql create mode 100644 config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102140-cleanup-add_province_uri-to-adress-administrative_units.graph create mode 100644 config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102140-cleanup-add_province_uri-to-adress-administrative_units.ttl create mode 100644 config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102212-cleanup-add_province_uri-to-adress-worship_services.graph create mode 100644 config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102212-cleanup-add_province_uri-to-adress-worship_services.ttl diff --git a/CHANGELOG.md b/CHANGELOG.md index c706de0..1ccb333 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ + Move werkingsgebieden to public graph + Link district werkingsgebieden to Antwerp municipality werkingsgebied + Add missing werkingsgebied for Borsbeek district +- Write migration to extract from existing primary sites the province URI [OP-3597] ### Deploy Notes ``` drc restart migrations; drc logs -ft --tail=200 migrations @@ -18,6 +19,7 @@ drc exec kbo-data-sync curl -X POST http://localhost/sync-all-kbo-data drc restart delta-producer-publication-graph-maintainer drc exec delta-producer-background-jobs-initiator curl -X POST http://localhost/public/healing-jobs # or wait until nightly healing kicks in drc exec delta-producer-background-jobs-initiator curl -X POST http://localhost/organizations/healing-jobs # or wait until nightly healing kicks in +drc exec db-cleanup curl -X POST "http://localhost/cleanup" ``` ## v1.31.2 (2025-04-02) diff --git a/config/delta-producer/organizations/export.json b/config/delta-producer/organizations/export.json index 5ad0419..d59150e 100644 --- a/config/delta-producer/organizations/export.json +++ b/config/delta-producer/organizations/export.json @@ -336,6 +336,7 @@ "http://www.w3.org/ns/locn#postCode", "https://data.vlaanderen.be/ns/adres#gemeentenaam", "http://www.w3.org/ns/locn#adminUnitL2", + "https://data.vlaanderen.be/ns/adres#adresIsGelegenIn", "https://data.vlaanderen.be/ns/adres#land", "http://www.w3.org/ns/locn#fullAddress", "https://data.vlaanderen.be/ns/adres#verwijstNaar", diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index 67a79de..cdbff5f 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -252,6 +252,7 @@ "http://www.w3.org/ns/locn#postCode", "https://data.vlaanderen.be/ns/adres#gemeentenaam", "http://www.w3.org/ns/locn#adminUnitL2", + "https://data.vlaanderen.be/ns/adres#adresIsGelegenIn", "https://data.vlaanderen.be/ns/adres#land", "http://www.w3.org/ns/locn#fullAddress", "https://data.vlaanderen.be/ns/adres#verwijstNaar", @@ -279,6 +280,7 @@ "http://www.w3.org/ns/locn#postCode", "https://data.vlaanderen.be/ns/adres#gemeentenaam", "http://www.w3.org/ns/locn#adminUnitL2", + "https://data.vlaanderen.be/ns/adres#adresIsGelegenIn", "https://data.vlaanderen.be/ns/adres#land", "http://www.w3.org/ns/locn#fullAddress", "https://data.vlaanderen.be/ns/adres#verwijstNaar", diff --git a/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102051-create-provinces-concept-scheme.sparql b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102051-create-provinces-concept-scheme.sparql new file mode 100644 index 0000000..52900ca --- /dev/null +++ b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102051-create-provinces-concept-scheme.sparql @@ -0,0 +1,26 @@ +PREFIX skos: +PREFIX concept_schemes: +PREFIX ext: +PREFIX dcterms: + +INSERT DATA { + GRAPH { + concept_schemes:ProvincesConceptScheme a skos:ConceptScheme ; + skos:prefLabel "Provinces Concept Scheme" ; + dcterms:description "A concept scheme containing all the location entities that are provinces". + } +} + +; + +INSERT +{ + GRAPH { + ?province_uri rdf:type skos:Concept ; + skos:inScheme concept_schemes:ProvincesConceptScheme . + } +} WHERE { + GRAPH { + ?province_uri ext:werkingsgebiedNiveau "Provincie". + } +} diff --git a/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102116-add-province_uri-to-address.sparql b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102116-add-province_uri-to-address.sparql new file mode 100644 index 0000000..d613449 --- /dev/null +++ b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102116-add-province_uri-to-address.sparql @@ -0,0 +1,40 @@ +PREFIX locn: +PREFIX ext: +PREFIX adres: + + +INSERT { + GRAPH { + ?address adres:adresIsGelegenIn ?province_uri . + } +} WHERE { + GRAPH { + ?address rdf:type locn:Address ; + locn:adminUnitL2 ?province_literal . + } + GRAPH { + ?province_uri rdfs:label ?province_literal ; + ext:werkingsgebiedNiveau "Provincie" . + } +} + +; + +INSERT { + GRAPH { + ?address adres:adresIsGelegenIn ?province_uri . + } +} WHERE { + GRAPH { + ?address rdf:type locn:Address ; + locn:adminUnitL2 ?province_literal . + } + GRAPH { + ?province_uri rdfs:label ?province_literal ; + ext:werkingsgebiedNiveau "Provincie" . + } +} + + + + diff --git a/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102140-cleanup-add_province_uri-to-adress-administrative_units.graph b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102140-cleanup-add_province_uri-to-adress-administrative_units.graph new file mode 100644 index 0000000..092c3a9 --- /dev/null +++ b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102140-cleanup-add_province_uri-to-adress-administrative_units.graph @@ -0,0 +1 @@ +http://mu.semte.ch/graphs/public diff --git a/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102140-cleanup-add_province_uri-to-adress-administrative_units.ttl b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102140-cleanup-add_province_uri-to-adress-administrative_units.ttl new file mode 100644 index 0000000..6418af7 --- /dev/null +++ b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102140-cleanup-add_province_uri-to-adress-administrative_units.ttl @@ -0,0 +1,41 @@ +@prefix cleanup: . +@prefix mu: . +@prefix dcterms: . + + a cleanup:Job ; + mu:uuid "42245582-ea60-4d44-80ef-c686901eccb5" ; + dcterms:title "Insert missing province uris for addresses of administrative units" ; + cleanup:randomQuery """ + + PREFIX locn: + PREFIX ext: + PREFIX adres: + + INSERT { + GRAPH { + ?address adres:adresIsGelegenIn ?province_uri . + } + } + WHERE { + GRAPH { + ?address rdf:type locn:Address ; + locn:adminUnitL2 ?province_literal . + FILTER NOT EXISTS { + ?address adres:adresIsGelegenIn ?province_uri2 . + } + } + GRAPH { + ?province_uri rdfs:label ?province_literal ; + ext:werkingsgebiedNiveau "Provincie" . + BIND (?province_uri as ?province_uri2) + + } + + } + + """ . + + + + + diff --git a/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102212-cleanup-add_province_uri-to-adress-worship_services.graph b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102212-cleanup-add_province_uri-to-adress-worship_services.graph new file mode 100644 index 0000000..092c3a9 --- /dev/null +++ b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102212-cleanup-add_province_uri-to-adress-worship_services.graph @@ -0,0 +1 @@ +http://mu.semte.ch/graphs/public diff --git a/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102212-cleanup-add_province_uri-to-adress-worship_services.ttl b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102212-cleanup-add_province_uri-to-adress-worship_services.ttl new file mode 100644 index 0000000..e2d7dfe --- /dev/null +++ b/config/migrations/2025/20250513102026-add-province_uris-to_adress/20250513102212-cleanup-add_province_uri-to-adress-worship_services.ttl @@ -0,0 +1,39 @@ +@prefix cleanup: . +@prefix mu: . +@prefix dcterms: . + + a cleanup:Job ; + mu:uuid "cdbb9f4e-90cc-40f6-a0a9-bd0bd8c9105f" ; + dcterms:title "Insert missing province uris for addresses of worship services" ; + cleanup:randomQuery """ + + PREFIX locn: + PREFIX ext: + PREFIX adres: + + INSERT { + GRAPH { + ?address adres:adresIsGelegenIn ?province_uri . + } + } + WHERE { + GRAPH { + ?address rdf:type locn:Address ; + locn:adminUnitL2 ?province_literal . + FILTER NOT EXISTS { + ?address adres:adresIsGelegenIn ?province_uri2 . + } + } + GRAPH { + ?province_uri rdfs:label ?province_literal ; + ext:werkingsgebiedNiveau "Provincie" . + BIND (?province_uri as ?province_uri2) + } + + } + + """ . + + + +