From 3d7d56d9da308cb759e4d0ea9e46bf2dfe27a657 Mon Sep 17 00:00:00 2001 From: Panos Alexopoulos Date: Fri, 24 Oct 2025 09:57:15 +0200 Subject: [PATCH 1/3] Added a cleanup query to find the triples that involve entities of type Bestuurseenheid either as subject or object, and are shared between the shared and administrative unit graphs, and then remove them from the administrative unit graph --- ...plicate-municipalities-and-provinces.graph | 1 + ...duplicate-municipalities-and-provinces.ttl | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.graph create mode 100644 config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl diff --git a/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.graph b/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.graph new file mode 100644 index 0000000..d5c0765 --- /dev/null +++ b/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.graph @@ -0,0 +1 @@ +http://mu.semte.ch/graphs/public \ No newline at end of file diff --git a/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl b/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl new file mode 100644 index 0000000..e541d45 --- /dev/null +++ b/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl @@ -0,0 +1,58 @@ +@prefix cleanup: . +@prefix mu: . +@prefix dcterms: . + + a cleanup:Job ; + mu:uuid "8ffd9f11-db25-430d-a66a-31fc8b393d5f" ; + dcterms:title "Insert missing province uris for addresses of administrative units" ; + cleanup:randomQuery """ + + PREFIX besluit: + + DELETE { + GRAPH { + ?s ?p ?o . + } + } + WHERE { + VALUES (?g1 ?g2) { + ( ) + } + + { + # Subject is a Bestuurseenheid + GRAPH ?g1 { + ?s a besluit:Bestuurseenheid ; + ?p ?o . + FILTER(?p != rdf:type) + } + GRAPH ?g2 { + ?s a besluit:Bestuurseenheid ; + ?p ?o . + FILTER(?p != rdf:type) + } + } + UNION + { + # Object is a Bestuurseenheid + GRAPH ?g1 { + ?s ?p ?o . + ?o a besluit:Bestuurseenheid . + FILTER(?p != rdf:type) + } + GRAPH ?g2 { + ?s ?p ?o . + ?o a besluit:Bestuurseenheid . + FILTER(?p != rdf:type) + } + } + } + ORDER BY ?s ?p ?o + + + """ . + + + + + From 2eea02673be496e5bff4f07f6ba9c776d9054043 Mon Sep 17 00:00:00 2001 From: Panos Alexopoulos Date: Fri, 24 Oct 2025 14:04:32 +0200 Subject: [PATCH 2/3] Debugged cleanup query and added entry to changelog --- CHANGELOG.md | 8 ++++++++ ...7-cleanup-deduplicate-municipalities-and-provinces.ttl | 5 +---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c12c199..a96cf6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 1.37.0 (2025-10-24) +- Added cleanup job to clean duplicated data of municipalities and provinces [OP-3676] + +#### CLI +``` +drc exec db-cleanup curl -X GET "http://localhost/runCronJob?cronJobID=8ffd9f11-db25-430d-a66a-31fc8b393d5f" +``` + ## 1.36.0 (2025-10-03) ### Dashboard - Enable ACM/IDM for the dashboard [OP-3674] diff --git a/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl b/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl index e541d45..27f9b14 100644 --- a/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl +++ b/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl @@ -4,7 +4,7 @@ a cleanup:Job ; mu:uuid "8ffd9f11-db25-430d-a66a-31fc8b393d5f" ; - dcterms:title "Insert missing province uris for addresses of administrative units" ; + dcterms:title "Cleaning duplicated data of municipalities and provinces" ; cleanup:randomQuery """ PREFIX besluit: @@ -20,7 +20,6 @@ } { - # Subject is a Bestuurseenheid GRAPH ?g1 { ?s a besluit:Bestuurseenheid ; ?p ?o . @@ -34,7 +33,6 @@ } UNION { - # Object is a Bestuurseenheid GRAPH ?g1 { ?s ?p ?o . ?o a besluit:Bestuurseenheid . @@ -47,7 +45,6 @@ } } } - ORDER BY ?s ?p ?o """ . From 7374c861fbd9bd28d766dd407d8c3153804967b7 Mon Sep 17 00:00:00 2001 From: Panos Alexopoulos Date: Fri, 24 Oct 2025 14:13:47 +0200 Subject: [PATCH 3/3] Updated changelog --- CHANGELOG.md | 1 + ...-cleanup-deduplicate-municipalities-and-provinces.ttl | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a96cf6d..e1803d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ #### CLI ``` +drc restart migrations drc exec db-cleanup curl -X GET "http://localhost/runCronJob?cronJobID=8ffd9f11-db25-430d-a66a-31fc8b393d5f" ``` diff --git a/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl b/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl index 27f9b14..2f7222e 100644 --- a/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl +++ b/config/migrations/2025/20251024093007-cleanup-deduplicate-municipalities-and-provinces.ttl @@ -45,11 +45,4 @@ } } } - - - """ . - - - - - + """ . \ No newline at end of file