From e2479f626a62e985757ca48ef53923c53de212fd Mon Sep 17 00:00:00 2001 From: Niels Vandekeybus Date: Wed, 9 Oct 2024 17:12:11 +0200 Subject: [PATCH] feature: add script to generate exports Script to export the data for administrative units and their governing bodies. --- scripts/project/config.json | 16 ++ scripts/project/export/enrichment.owl | 22 ++ scripts/project/export/run.rb | 358 ++++++++++++++++++++++++++ 3 files changed, 396 insertions(+) create mode 100644 scripts/project/export/enrichment.owl create mode 100755 scripts/project/export/run.rb diff --git a/scripts/project/config.json b/scripts/project/config.json index 471f975..1ef0d69 100644 --- a/scripts/project/config.json +++ b/scripts/project/config.json @@ -1,6 +1,22 @@ { "version": "0.1", "scripts": [ + { + "documentation": { + "command": "create-export", + "description": "create export of administrative units and their bodies", + "arguments": [] + }, + "environment": { + "image": "nvdk/ruby-semantic-works-cli", + "interactive": true, + "script": "export/run.rb", + "join_networks": true + }, + "mounts": { + "app": "/app/" + } + }, { "documentation": { "command": "mu-search-delta", diff --git a/scripts/project/export/enrichment.owl b/scripts/project/export/enrichment.owl new file mode 100644 index 0000000..04fe317 --- /dev/null +++ b/scripts/project/export/enrichment.owl @@ -0,0 +1,22 @@ +@prefix rdf: . +@prefix rdfs: . +@prefix owl: . +@prefix generiek: . +@prefix mandaat: . +@prefix org: . +@prefix besluit: . +@prefix ere: . + + a owl:Ontology . + +# Equivalent Properties +generiek:isTijdspecialisatieVan owl:equivalentProperty + , + mandaat:isTijdspecialisatieVan . +org:classification owl:equivalentProperty besluit:classificatie . +generiek:gestructureerdeIdentificator owl:equivalentProperty . + +# Subclass mappings +ere:BestuurVanDeEredienst rdfs:subClassOf besluit:Bestuurseenheid . +ere:CentraalBestuurVanDeEredienst rdfs:subClassOf besluit:Bestuurseenheid . +ere:RepresentatiefOrgaan rdfs:subClassOf besluit:Bestuurseenheid. diff --git a/scripts/project/export/run.rb b/scripts/project/export/run.rb new file mode 100755 index 0000000..461d836 --- /dev/null +++ b/scripts/project/export/run.rb @@ -0,0 +1,358 @@ +#!/usr/bin/env ruby + +require 'bundler/inline' +require 'yaml' +require 'securerandom' +require 'date' +require 'rdf/vocab' +require 'tty-prompt' +require 'rdf/turtle' +require 'rdf/reasoner' + +$stdout.sync = true +ENV['LOG_SPARQL_ALL']='false' +ENV['MU_SPARQL_ENDPOINT']='http://triplestore:8890/sparql' +ENV['MU_SPARQL_TIMEOUT']='180' +require 'mu/auth-sudo' + + + +def sparql_escape_uri(value) + '<' + value.to_s.gsub(/[\\"<>]/) { |s| '\\' + s } + '>' +end + +def sparql_escape_string(str) + '"""' + str.gsub(/[\\"]/) { |s| '\\' + s } + '"""' +end + +def search_classifications + query = <<~EOF +PREFIX org: +PREFIX skos: +PREFIX besluit: +SELECT DISTINCT ?type ?name ?classification WHERE { + ?org a ?type; + org:classification ?classification. + ?classification skos:prefLabel ?name. + VALUES ?type { besluit:Bestuurseenheid } +} ORDER BY ?name +EOF + Mu::AuthSudo.query(query) +end + +def get_admin_units(classifications) + query = <<~EOF +PREFIX org: +PREFIX skos: +PREFIX besluit: +SELECT DISTINCT ?org ?name WHERE { + ?org a ?type; + skos:prefLabel ?name; + org:classification ?classification. + VALUES ?type { besluit:Bestuurseenheid } + VALUES ?classification { #{classifications.map{ |c| sparql_escape_uri(c) }.join(" ")}} +} +EOF + Mu::AuthSudo.query(query) +end + +def export_admin_unit(unit) + repository = RDF::Repository.new + query = <<~EOF +PREFIX org: +PREFIX skos: +PREFIX besluit: +PREFIX generiek: +CONSTRUCT { +?org ?p ?o +} +WHERE { +VALUES ?org { #{sparql_escape_uri(unit)}} +?org ?p ?o. +VALUES ?p { + rdf:type + org:classification + besluit:classificatie + skos:prefLabel + + + +} +} +EOF + repository << Mu::AuthSudo.query(query) + query = <<~EOF +PREFIX org: +PREFIX skos: +PREFIX besluit: +PREFIX generiek: +CONSTRUCT { +?org ?id. +?id ?p ?o. +?id skos:notation ?localId. +?id ?source. +} +WHERE { +VALUES ?org { #{sparql_escape_uri(unit)}} +?org ?id. +?id ?p ?o. +FILTER (?p NOT IN(, )) +?id ?source. +FILTER(?source != "SharePoint identificator") +?id ?gestructureerdeIdentificator. +?gestructureerdeIdentificator ?localId +} +EOF + repository << Mu::AuthSudo.query(query) + query = <<~EOF +PREFIX org: +PREFIX skos: +PREFIX besluit: +PREFIX generiek: +CONSTRUCT { +?org besluit:werkingsgebied ?gebied. +?gebied ?p ?o. +} +WHERE { +VALUES ?org { #{sparql_escape_uri(unit)}} +?org besluit:werkingsgebied ?gebied. +?gebied ?p ?o +} +EOF + repository << Mu::AuthSudo.query(query) + + query = <<~EOF +PREFIX org: +PREFIX skos: +PREFIX besluit: +PREFIX generiek: +PREFIX mandaat: +CONSTRUCT { +?bestuursorgaan ?bp ?bo. +?bestuursorgaanIntijd generiek:isTijdspecialisatieVan ?bestuursorgaan. +?bestuursorgaanIntijd ?bop ?boo. +} +WHERE { +VALUES ?org { #{sparql_escape_uri(unit)}} +?bestuursorgaan besluit:bestuurt ?org. +?bestuursorgaan ?bp ?bo. +values ?tijdspecialisatie { +generiek:isTijdspecialisatieVan +mandaat:isTijdspecialisatieVan +} +?bestuursorgaanIntijd ?tijdspecialisatie ?bestuursorgaan. +?bestuursorgaanIntijd ?bop ?boo. +} + +EOF + repository << Mu::AuthSudo.query(query) + query = <<~EOF +PREFIX org: +PREFIX skos: +PREFIX besluit: +PREFIX generiek: +CONSTRUCT { +?mandaat ?p ?o. +?role ?roleP ?roleO. +} +WHERE { +VALUES ?org { #{sparql_escape_uri(unit)}} +?bestuursorgaan besluit:bestuurt ?org. +?bestuursorgaanIntijd generiek:isTijdspecialisatieVan ?bestuursorgaan. +VALUES ?post { org:hasPost } +?bestuursorgaanIntijd ?post ?mandaat. +?mandaat ?p ?o. +?mandaat org:role ?role. +OPTIONAL { + ?role ?roleP ?roleO. + FILTER(!STRSTARTS(STR(?role), "http://data.vlaanderen.be")) +} +} +EOF + repository << Mu::AuthSudo.query(query) + repository +end + +def ensure_dir(path) + unless Dir.exist?(path) + Dir.mkdir(path) + end +end + +def sanitize_unix_path(path) + # Regular expression to match characters not allowed in Unix paths + sanitized_path = path.gsub(/[^a-zA-Z0-9._\-\/]/, '-') + sanitized_path +end + + +def enrich_graph(data) + RDF::Reasoner.apply(:owl,:rdfs) + graph = RDF::Graph.load("./enrichment.owl") + graph << clean_up_dates(data) + graph.entail!(:subClassOf) + graph.entail!(:equivalentProperty) + graph +end + +DATE_PREDICATES=[ + RDF::URI("http://data.vlaanderen.be/ns/mandaat#bindingStart"), + RDF::URI("http://data.vlaanderen.be/ns/mandaat#bindingEinde"), + RDF::URI("https://data.vlaanderen.be/ns/generiek#bindingStart"), + RDF::URI("https://data.vlaanderen.be/ns/generiek#bindingEinde") +] + +def convert_date_to_datetime(date_literal) + begin + date = Date.parse(date_literal.to_s) + datetime = DateTime.new(date.year, date.month, date.day) + RDF::Literal.new(datetime.strftime("%Y-%m-%dT%T"), datatype: RDF::XSD.dateTime) + rescue ArgumentError => e + puts "Invalid date format for #{date_literal}: #{e.message}" + nil + end +end + +def clean_up_dates(data) + clean_data = RDF::Graph.new + data.each_statement do |statement| + if DATE_PREDICATES.include?(statement.predicate) + obj = statement.object + if obj.is_a?(RDF::Literal) && obj.datatype == RDF::XSD.date + new_object = convert_date_to_datetime(obj) + if new_object + clean_data << [statement.subject, statement.predicate, new_object] # Add the new statement + end + elsif obj.is_a?(RDF::Literal) && obj.datatype == RDF::XSD.dateTime + clean_data << statement + else + puts "unexpected value #{obj.inspect}for statement:\n #{statement.inspect}" + end + else + clean_data << statement + end + end + clean_data +end + +prompt = TTY::Prompt.new +prompt.say("\n\n") +prompt.say("This script will help you export administrative units and their respective bodies") +options = search_classifications.map do |binding| + {name: "#{binding[:name]}", value: binding[:classification]} +end + +if options.size > 0 + classifications = prompt.multi_select("Please select the matching admin unit classifications", options) + admin_units = get_admin_units(classifications) + prompt.say("found #{admin_units.size} admin units") + output_dir = prompt.ask("Provide the directory the exports should be written to?", default: "/app/exports/") + ensure_dir(output_dir) + admin_units.each do |unit| + repo = export_admin_unit(unit[:org]) + repo = enrich_graph(repo) + file_name = sanitize_unix_path(unit[:name].to_s) + timestamp=`date +%Y%0m%0d%0H%0M%0S`.strip.to_i + path = File.join(output_dir, "#{timestamp}-remove-data-#{file_name}-bestuurseenheid.sparql") + File.open(path, 'w') do |file| + file.write < +PREFIX mandaat: +PREFIX generiek: +PREFIX org: +PREFIX prov: +DELETE { + GRAPH { + ?id ?p ?o. +} +} WHERE { + GRAPH { +<#{unit[:org]}> ?id. + ?id ?p ?o. +} +}; +DELETE { + GRAPH { + ?gebied ?p ?o. +} +} WHERE { + GRAPH { +<#{unit[:org]}> besluit:werkingsgebied ?gebied. + ?gebied ?p ?o. +} +}; +DELETE WHERE { + GRAPH { +<#{unit[:org]}> ?p ?o +} +}; +EOF + end + path = File.join(output_dir, "#{timestamp}-remove-data-#{file_name}-bestuursorganen.sparql") + File.open(path, 'w') do |file| + file.write < +PREFIX mandaat: +PREFIX generiek: +PREFIX org: +PREFIX prov: +DELETE { + GRAPH { +?mandaat ?p ?o +} +} WHERE { + GRAPH { +?bestuursorgaan besluit:bestuurt <#{unit[:org]}> . +?bestuursorgaanIntijd mandaat:isTijdspecialisatieVan ?bestuursorgaan. +VALUES ?post { org:hasPost } +?bestuursorgaanIntijd ?post ?mandaat. +?mandaat ?p ?o. +} +}; +DELETE { + GRAPH { +?mandaat ?p ?o +} +} WHERE { + GRAPH { +?bestuursorgaan besluit:bestuurt <#{unit[:org]}> . +?bestuursorgaanIntijd mandaat:isTijdspecialisatieVan ?bestuursorgaan. +VALUES ?post { org:hasPost } +?bestuursorgaanIntijd ?post ?mandaat. +?mandaat ?p ?o. +} +}; +DELETE { + GRAPH { +?bestuursorgaanIntijd ?p ?o +} +} WHERE { + GRAPH { +?bestuursorgaan besluit:bestuurt <#{unit[:org]}> . +?bestuursorgaanIntijd mandaat:isTijdspecialisatieVan ?bestuursorgaan. +VALUES ?post { org:hasPost } +?bestuursorgaanIntijd ?p ?o +} +}; +DELETE { + GRAPH { +?bestuursorgaan ?p ?o +} +} WHERE { + GRAPH { +?bestuursorgaan besluit:bestuurt <#{unit[:org]}>; ?p ?o +} +} +EOF + end + timestamp+=1 + path = File.join(output_dir, "#{timestamp}-#{SecureRandom.uuid}-#{file_name}.ttl") + File.open(path, 'w') do |file| + file.write repo.dump(:ttl) + end + prompt.say("Exported #{repo.size} statements to #{path}") + end +else + abort("No classifications found?") +end +