diff --git a/CHANGELOG.md b/CHANGELOG.md index 529e961..0f2f0e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog ## Unreleased +### Frontend +- Bump to TODO: link to release with scope of operation functionality +### Backend +- Add a service to manage organisation's their scope of operation [OP-3205] +- datamodel: Allow a location to be within multiple other locations [OP-3205] +- datafix: set scope of operation for worship administrative units [OP-3626] +- Clean addresses linked to both sites and contact points of mandatarissen and addresses that are in the wrong graph (admin unit graph but linked to a contact point in the worship graph) [DL-6701] +- datafix: cleanup of general memberships if specific ones exist [OP-3640] +### Deploy notes +``` +drc pull scope-of-operation; drc up -d scope-of-operation +drc restart db resource dispatcher +drc pull frontend; drc up -d frontend +drc restart migrations; drc logs -ft --tail=200 migrations +``` + +The labels for new location resources created when setting the scope of operation for worship services are not necessarily alphabetically sorted. To correct this run the `correct-location-labels` project script using [mu-cli](https://github.com/mu-semtech/mu-cli) and restart the migrations service once more to execute the generated local migrations. + +``` +mu script project-scripts correct-location-labels +drc restart migrations; drc logs -ft --tail=200 migrations +``` ## v1.34.0 (2025-07-11) ### Frontend diff --git a/config/authorization/config.ex b/config/authorization/config.ex index fcb6d7e..0390610 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -53,7 +53,8 @@ defmodule Acl.UserGroups.Config do "http://lblod.data.gift/vocabularies/organisatie/VoorwaardenBedienaarCriterium", "http://lblod.data.gift/vocabularies/organisatie/BedienaarCriteriumBewijsstuk", "http://lblod.data.gift/vocabularies/organisatie/EredienstBeroepen", - "http://lblod.data.gift/vocabularies/organisatie/Rechtsvormtype" + "http://lblod.data.gift/vocabularies/organisatie/Rechtsvormtype", + "http://www.w3.org/ns/prov#Location" ] @org_type [ @@ -80,11 +81,10 @@ defmodule Acl.UserGroups.Config do "http://data.lblod.info/vocabularies/contacthub/AgentInPositie", "http://mu.semte.ch/vocabularies/ext/KboOrganisatie", "http://www.w3.org/ns/org#Membership", - "http://www.w3.org/2006/time#ProperInterval" + "http://www.w3.org/2006/time#ProperInterval", ] @shared_protected_type [ - "http://www.w3.org/ns/prov#Location", "http://xmlns.com/foaf/0.1/Image", "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject" ] @@ -117,7 +117,7 @@ defmodule Acl.UserGroups.Config do ?account ?session_role. FILTER( ?session_role = \"dashboard-user\" ) }" - } + } end def user_groups do @@ -256,7 +256,7 @@ defmodule Acl.UserGroups.Config do "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject" ] } - }, + } ] }, diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 55093c1..5100511 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -393,6 +393,10 @@ defmodule Dispatcher do forward(conn, path, "http://kbo-data-sync/sync-kbo-data/") end + match "/scope-of-operation/*path", %{layer: api_services, accept: %{json: true}} do + forward(conn, path, "http://scope-of-operation/") + end + get "/ldes/*path", %{layer: :api_services} do forward(conn, path, "http://ldes-backend/") end diff --git a/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250707132349-chore--extract-scopes-for-worship-organisations.sparql b/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250707132349-chore--extract-scopes-for-worship-organisations.sparql new file mode 100644 index 0000000..07a5da9 --- /dev/null +++ b/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250707132349-chore--extract-scopes-for-worship-organisations.sparql @@ -0,0 +1,112 @@ +PREFIX ere: +PREFIX org: +PREFIX besluit: + +# This migration extracts the relations between (central) worship services and +# their locations according to the rules provided by business. These links are +# inserted into a temporary graph for further processing. + +# Worship services +## Municipality organised +INSERT { + GRAPH { + ?ws besluit:werkingsgebied ?scope . + } +} WHERE { + GRAPH { + ?ws a ere:BestuurVanDeEredienst ; + ere:typeEredienst ?worshipType . + VALUES ?worshipType { + # Anglicaans + # Israëlitisch + # Protestants + # Rooms-Katholiek + } + + ?involvement a ere:BetrokkenLokaleBesturen ; + org:organization ?ws ; + ere:typebetrokkenheid ?involvementType . + VALUES ?involvementType { + # Mede-financierend + # Toezichthoudend + } + ?org ere:betrokkenBestuur ?involvement . + } + GRAPH { + ?org besluit:werkingsgebied ?scope . + } +} +; +## Province organised +INSERT { + GRAPH { + ?ws besluit:werkingsgebied ?scope . + } +} WHERE { + GRAPH { + ?ws a ere:BestuurVanDeEredienst ; + ere:typeEredienst ?worshipType . + VALUES ?worshipType { + # Islamitisch + # Orthodox + } + + ?involvement a ere:BetrokkenLokaleBesturen ; + org:organization ?ws ; + ere:typebetrokkenheid ?involvementType . + VALUES ?involvementType { + # Adviserend + } + ?org ere:betrokkenBestuur ?involvement . + } + GRAPH { + ?org besluit:werkingsgebied ?scope . + } +} +; +# Central worship services +## Municipality organised +INSERT { + GRAPH { + ?ws besluit:werkingsgebied ?scope . + } +} WHERE { + GRAPH { + ?ws a ere:CentraalBestuurVanDeEredienst ; + ere:typeEredienst ?worshipType . + VALUES ?worshipType { + # Rooms-Katholiek + } + + ?involvement a ere:BetrokkenLokaleBesturen ; + org:organization ?ws ; + ere:typebetrokkenheid ?involvementType . + VALUES ?involvementType { + # Toezichthoudend + } + ?org ere:betrokkenBestuur ?involvement . + } + GRAPH { + ?org besluit:werkingsgebied ?scope . + } +} +; +## Province organised +## Note: Business requested to use the location for the municipality of Ghent as +## default value. They will update it manually afterwards. +INSERT { + GRAPH { + ?ws besluit:werkingsgebied ?locationGhent . + } +} WHERE { + GRAPH { + ?ws a ere:CentraalBestuurVanDeEredienst ; + ere:typeEredienst ?worshipType . + VALUES ?worshipType { + # Islamitisch + # Orthodox + } + + BIND(URI("http://data.lblod.info/id/werkingsgebieden/ce94eeae827cdc5c00f3f2f4276ff628580edefb3aa5861e4669b0c5d93adc57") AS ?locationGhent) + } +} diff --git a/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250709140704-feat--set-scope-for-worship-organisations-with-single-location.sparql b/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250709140704-feat--set-scope-for-worship-organisations-with-single-location.sparql new file mode 100644 index 0000000..c1355dd --- /dev/null +++ b/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250709140704-feat--set-scope-for-worship-organisations-with-single-location.sparql @@ -0,0 +1,24 @@ +PREFIX besluit: + +# This migration sets the scope of operation for worship organisations that have +# single, already existing, location as scope. This migration depends on +# `20250707132349-chore--extract-scopes-for-worship-organisations` being +# executed first, as it uses the data created by the migration. +INSERT { + GRAPH { + ?ws besluit:werkingsgebied ?scope . + } +} WHERE { + { + SELECT DISTINCT ?ws (count(DISTINCT ?scope) as ?count) + WHERE { + GRAPH { + ?ws besluit:werkingsgebied ?scope . + } + } + } + FILTER (?count = 1) + GRAPH { + ?ws besluit:werkingsgebied ?scope . + } +} diff --git a/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250709160017-feat--set-scope-for-worship-organisations-with-multiple-locations.sparql b/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250709160017-feat--set-scope-for-worship-organisations-with-multiple-locations.sparql new file mode 100644 index 0000000..c8d2754 --- /dev/null +++ b/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250709160017-feat--set-scope-for-worship-organisations-with-multiple-locations.sparql @@ -0,0 +1,49 @@ +PREFIX prov: +PREFIX mu: +PREFIX ext: +PREFIX geo: +PREFIX besluit: +PREFIX rdfs: + +# This migration sets the scope of operation for worship organisations that have +# multiple location as scope. It creates new aggregate locations to contain the +# already existing municipality/province level ones. This migration depends on +# `20250707132349-chore--extract-scopes-for-worship-organisations` being +# executed first, as it uses the data created by the migration. +INSERT { + GRAPH { + ?location a prov:Location ; + mu:uuid ?locationUuid ; + rdfs:label ?scopeLabel ; + ext:werkingsgebiedNiveau """Samengesteld werkingsgebied""" . + ?scope geo:sfWithin ?location . + } + GRAPH { + ?ws besluit:werkingsgebied ?location . + } +} WHERE { + { + SELECT DISTINCT ?ws (count(DISTINCT ?scope) AS ?count) (GROUP_CONCAT(?uuid," ") AS ?scopeUuids) (GROUP_CONCAT(?label,", ") AS ?scopeLabel) + WHERE { + GRAPH { + ?ws besluit:werkingsgebied ?scope . + } + GRAPH { + ?scope mu:uuid ?uuid ; + rdfs:label ?label . + } + } + } + FILTER (?count > 1) + GRAPH { + ?ws besluit:werkingsgebied ?scope . + } + # NOTE (10/07/2025): The SPARQL specification does not specify any order for + # `GROUP_CONCAT`. But virtuoso does seem to concatenate elements always in the + # same order. Therefore, this way of generating a UUID is a bit of a hack to + # ensure that new locations that contain the same locations will receive the + # same UUUID, which results in the same location resource being inserted + # twice. + BIND (STR(SHA256(?scopeUuids)) AS ?locationUuid) + BIND (URI(CONCAT("http://data.lblod.info/id/werkingsgebieden/", ?locationUuid)) AS ?location) +} diff --git a/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250710095207-chore--clear-temp-graph-worship-scopes.sparql b/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250710095207-chore--clear-temp-graph-worship-scopes.sparql new file mode 100644 index 0000000..0ec8f78 --- /dev/null +++ b/config/migrations/2025/20250707132349-feat--add-scope-for-operation-worship-organisations/20250710095207-chore--clear-temp-graph-worship-scopes.sparql @@ -0,0 +1 @@ +CLEAR GRAPH diff --git a/config/migrations/2025/20250708142100-fix-same-address-linked-to-both-site-and-contact-point.sparql b/config/migrations/2025/20250708142100-fix-same-address-linked-to-both-site-and-contact-point.sparql new file mode 100644 index 0000000..bbd4003 --- /dev/null +++ b/config/migrations/2025/20250708142100-fix-same-address-linked-to-both-site-and-contact-point.sparql @@ -0,0 +1,30 @@ +DELETE { + GRAPH ?h { + ?site ?oldAddressUri . + } +} INSERT { + GRAPH ?g { + ?newAddressUri a ; + ?newAddressUuid ; + ?p ?o . + + ?site ?newAddressUri . + } +} WHERE { + GRAPH ?g { + ?oldAddressUri a ; + ?p ?o . + + FILTER (?p NOT IN ()) + } + + ?mandatarisContact ?oldAddressUri . + + GRAPH ?h { + ?site a ; + ?oldAddressUri . + } + + BIND(MD5(CONCAT(?oldAddressUri,"NEW_ADDRESS")) AS ?newAddressUuid) . + BIND(IRI(CONCAT("http://data.lblod.info/id/adressen/", STR(?newAddressUuid))) AS ?newAddressUri) . +} diff --git a/config/migrations/2025/20250718132455-remove-general-memberships-if-specifics-exists.sparql b/config/migrations/2025/20250718132455-remove-general-memberships-if-specifics-exists.sparql new file mode 100644 index 0000000..ecf8e78 --- /dev/null +++ b/config/migrations/2025/20250718132455-remove-general-memberships-if-specifics-exists.sparql @@ -0,0 +1,33 @@ +PREFIX org: +PREFIX mu: +PREFIX skos: + +DELETE { + GRAPH ?g { + ?membership ?p ?o . + } +} +WHERE { + GRAPH ?g { + # Find memberships that use the general 'gerelateerd' role + ?membership a org:Membership ; + org:role ; + org:member ?orgMember ; + org:organization ?organization . + + # Only delete if there's a specific role for the inverse membership + FILTER EXISTS { + ?specific_membership a org:Membership ; + org:member ?organization ; + org:organization ?orgMember ; + org:role ?specificRole . + + VALUES ?specificRole { + # participant + # stichtend lid + } + } + + ?membership ?p ?o . + } +} diff --git a/config/migrations/2025/20250722114200-fix-contact-addresses-being-in-wrong-graph.sparql b/config/migrations/2025/20250722114200-fix-contact-addresses-being-in-wrong-graph.sparql new file mode 100644 index 0000000..d91d279 --- /dev/null +++ b/config/migrations/2025/20250722114200-fix-contact-addresses-being-in-wrong-graph.sparql @@ -0,0 +1,18 @@ +DELETE { + GRAPH { + ?address ?p ?o . + } +} INSERT { + GRAPH { + ?address ?p ?o . + } +} WHERE { + GRAPH { + ?contactPoint a ; + ?address . + } + + GRAPH { + ?address ?p ?o . + } +} diff --git a/config/resources/domain.json b/config/resources/domain.json index 76a02a7..f335d76 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -945,7 +945,7 @@ "located-within": { "predicate": "geo:sfWithin", "target": "locations", - "cardinality": "one" + "cardinality": "many" }, "locations": { "predicate": "geo:sfWithin", diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 617f243..6ff8a35 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -74,6 +74,8 @@ services: restart: "no" construct-organization-relationships: restart: "no" + scope-of-operation: + restart: "no" mandatarissen-consumer: restart: "no" leidinggevenden-consumer: diff --git a/docker-compose.yml b/docker-compose.yml index edf2825..c502d4e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -285,6 +285,14 @@ services: - "logging=true" restart: always logging: *default-logging + scope-of-operation: + image: lblod/scope-of-operation-service:0.1.0 + links: + - db:database + labels: + - "logging=true" + restart: always + logging: *default-logging report-generation: image: lblod/loket-report-generation-service:0.8.3 environment: diff --git a/scripts/project/config.json b/scripts/project/config.json index 1ef0d69..66a03a0 100644 --- a/scripts/project/config.json +++ b/scripts/project/config.json @@ -17,6 +17,22 @@ "app": "/app/" } }, + { + "documentation": { + "command": "correct-location-labels", + "description": "generate migrations to correct the labels of aggregated locations", + "arguments": [] + }, + "environment": { + "image": "nvdk/ruby-semantic-works-cli", + "interactive": true, + "script": "correct-location-labels/run.rb", + "join_networks": true + }, + "mounts": { + "app": "/app/" + } + }, { "documentation": { "command": "mu-search-delta", diff --git a/scripts/project/correct-location-labels/run.rb b/scripts/project/correct-location-labels/run.rb new file mode 100755 index 0000000..e86626f --- /dev/null +++ b/scripts/project/correct-location-labels/run.rb @@ -0,0 +1,106 @@ +#!/usr/bin/env ruby +require 'date' +require 'rdf/turtle' +require 'fileutils' + +$stdout.sync = true +ENV['LOG_SPARQL_ALL']='false' +ENV['MU_SPARQL_ENDPOINT']='http://triplestore:8890/sparql' +ENV['MU_SPARQL_TIMEOUT']='180' +require 'mu/auth-sudo' + +def ensure_dir(path) + unless Dir.exist?(path) + FileUtils.mkdir_p(path) + end +end + +def sparql_escape_uri(value) + '<' + value.to_s.gsub(/[\\"<>]/) { |s| '\\' + s } + '>' +end + +def sparql_escape_string(str) + '"""' + str.gsub(/[\\"]/) { |s| '\\' + s } + '"""' +end + +def get_locations() + query = <<~EOF +PREFIX prov: +PREFIX rdfs: +PREFIX ext: + +SELECT DISTINCT ?location ?label +WHERE { + GRAPH { + ?location a prov:Location ; + rdfs:label ?label ; + ext:werkingsgebiedNiveau """Samengesteld werkingsgebied""" . + } +} +EOF + Mu::AuthSudo.query(query) +end + +def sort_literal(literal) + literal + .humanize + .split(", ") + .sort + .join(", ") +end + +# Create output directory +timestamp=`date +%Y%0m%0d%0H%0M%0S`.strip.to_i +output_dir = "/app/config/migrations/local/2025/#{timestamp}-correct-labels-aggregate-locations/" +ensure_dir(output_dir) + + +# Generate SPARQL migration to remove existing labels +path = File.join(output_dir, "#{timestamp}-remove-labels-aggregate-locations.sparql") +File.open(path, 'w') do |file| + file.write < +PREFIX rdfs: +PREFIX ext: + +DELETE { + GRAPH { + ?location rdfs:label ?label + } +} WHERE { + GRAPH { + ?location a prov:Location ; + rdfs:label ?label ; + ext:werkingsgebiedNiveau """Samengesteld werkingsgebied""" . + } +} +EOF +end + + +# Generate TTL migration to insert alphabetically sorted labels +timestamp += 1 +filename = "#{timestamp}-insert-labels-aggregate-locations" + +path = File.join(output_dir, "#{filename}.graph") +File.open(path, 'w') do |file| + file.write < . +EOF +end + +locations = get_locations() +locations.each do |location| + File.open(path, 'a') do |file| + file.write <