diff --git a/.gitignore b/.gitignore index caed09e..ccfbba6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ !/data/db/toLoad !/data/db/toLoad/* /data/files/* +!/data/files/exports/.gitkeep /data/temp/* docker-compose.override.yml .idea diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 4ef9420..cce1b56 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -373,6 +373,14 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-worship-posts/files/" end + ################################################################# + # Exports + ################################################################# + + get "/download-exports/*path" do + Proxy.forward conn, path, "http://download-exports/" + end + ############################################################### # frontend layer ############################################################### diff --git a/config/reports/index.js b/config/reports/index.js new file mode 100644 index 0000000..3a124aa --- /dev/null +++ b/config/reports/index.js @@ -0,0 +1,5 @@ +import OrganizationsReport from './organizations-report' + +export default [ + OrganizationsReport, +]; diff --git a/config/reports/organizations-report.js b/config/reports/organizations-report.js new file mode 100644 index 0000000..d8e666e --- /dev/null +++ b/config/reports/organizations-report.js @@ -0,0 +1,115 @@ +import { generateReportFromData, batchedQuery } from '../helpers.js'; +import { PREFIXES } from './utils' + +export default { + cronPattern: '0 00 23 * * *', + name: 'organizations', + execute: async () => { + const reportData = { + title: 'organizations', + description: 'List of the Organizations', + filePrefix: 'exports/organizations' + }; + + console.log('Generating organizations report'); + + const queryStringPart1 = ` + ${PREFIXES} + + SELECT DISTINCT ?bestuur ?label ?classification ?typeEredienst + WHERE { + VALUES ?type { + org:Organization + besluit:Bestuurseenheid + } + + GRAPH ?g { + ?bestuur a ?type . + OPTIONAL { ?bestuur skos:prefLabel ?label. } + } + + OPTIONAL { + ?bestuur org:classification ?classificationUri . + ?classificationUri skos:prefLabel ?classification. + } + + OPTIONAL { + ?bestuur ere:typeEredienst ?typeEredienstUri. + ?typeEredienstUri skos:prefLabel ?typeEredienst. + } + + FILTER (?g IN ( , , )) + } + `; + + const queryResponsePart1 = await batchedQuery(queryStringPart1); + + const dataPart1 = queryResponsePart1.results.bindings.reduce( (acc, row) => { + acc[getSafeValue(row, 'bestuur')] = { + bestuur: getSafeValue(row, 'bestuur'), + label: getSafeValue(row, 'label'), + classification: getSafeValue(row, 'classification'), + typeEredienst: getSafeValue(row, 'typeEredienst'), + }; + return acc; + }, {}); + + const queryStringPart2 = ` + ${PREFIXES} + + SELECT DISTINCT ?bestuur ?provincie + WHERE { + VALUES ?type { + org:Organization + besluit:Bestuurseenheid + } + + GRAPH ?g { + ?bestuur a ?type . + } + + GRAPH ?h { + OPTIONAL { + ?bestuur org:hasPrimarySite ?primarySite. + + OPTIONAL { + ?primarySite organisatie:bestaatUit ?sa. + OPTIONAL { ?sa locn:adminUnitL2 ?provincie. } + } + } + } + + FILTER (?g IN ( , , )) + FILTER (?h IN ( , , )) + } + `; + + const queryResponsePart2 = await batchedQuery(queryStringPart2); + const dataPart2 = queryResponsePart2.results.bindings.reduce( (acc, row) => { + let dataPart = { + bestuur: getSafeValue(row, 'bestuur'), + provincie: getSafeValue(row, 'provincie') + }; + + acc[getSafeValue(row, 'bestuur')] = Object.assign(dataPart, dataPart1[getSafeValue(row, 'bestuur')]); + return acc; + }, {}); + + await generateReportFromData(Object.values(dataPart2), [ + 'bestuur', + 'label', + 'classification', + 'typeEredienst', + 'provincie' + ], reportData); + } +}; + +function getSafeValue(entry, property){ + return entry[property] ? wrapInQuote(entry[property].value) : null; +} + +// Some values might contain commas; wrapping them in escapes quotes doesn't disrupt the columns. +function wrapInQuote(value) { + return `\"${value}\"`; +} diff --git a/config/reports/utils.js b/config/reports/utils.js new file mode 100644 index 0000000..fdda6e4 --- /dev/null +++ b/config/reports/utils.js @@ -0,0 +1,29 @@ +export const PREFIXES = ` + PREFIX adms: + PREFIX adres: + PREFIX besluit: + PREFIX ch: + PREFIX code: + PREFIX dbpedia: + PREFIX dc_terms: + PREFIX ere: + PREFIX euro: + PREFIX euvoc: + PREFIX ext: + PREFIX foaf: + PREFIX generiek: + PREFIX lblodlg: + PREFIX locn: + PREFIX mandaat: + PREFIX org: + PREFIX organisatie: + PREFIX person: + PREFIX persoon: + PREFIX prov: + PREFIX rdfs: + PREFIX regorg: + PREFIX schema: + PREFIX skos: + PREFIX vcard: + PREFIX xsd: +`; diff --git a/docker-compose.yml b/docker-compose.yml index 882655b..e6b5e9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -270,6 +270,30 @@ services: - "logging=true" restart: always logging: *default-logging + report-generation: + image: lblod/loket-report-generation-service:0.7.0 + environment: + DEFAULT_GRAPH: "http://mu.semte.ch/graphs/shared" + ONLY_KEEP_LATEST_REPORT: "true" + links: + - db:database + volumes: + - ./data/files:/share + - ./config/reports:/config + labels: + - "logging=true" + restart: always + logging: *default-logging + download-exports: + image: lblod/download-exports-service:0.1.0 + links: + - db:database + volumes: + - ./data/files:/share + labels: + - "logging=true" + restart: always + logging: *default-logging sync-ovo-numbers: image: lblod/sync-ovo-numbers-service:0.1.0 links: