Add report generation service + orgs report (#331)

* Add report generation service + orgs report

* Adapt configuration

* Add download-exports service to the stack

* Update gitignore

* Bump report generation service

* Remove unnecessary import

* Add dispatcher rule for download exports

.gitignore

@@ -5,6 +5,7 @@
 !/data/db/toLoad
 !/data/db/toLoad/*
 /data/files/*
+!/data/files/exports/.gitkeep
 /data/temp/*
 docker-compose.override.yml
 .idea

config/dispatcher/dispatcher.ex

@@ -373,6 +373,14 @@ defmodule Dispatcher do
     Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-worship-posts/files/"
   end
 
+  #################################################################
+  #  Exports
+  #################################################################
+
+  get "/download-exports/*path" do
+    Proxy.forward conn, path, "http://download-exports/"
+  end
+
   ###############################################################
   # frontend layer
   ###############################################################

config/reports/index.js

@@ -0,0 +1,5 @@
+import OrganizationsReport from './organizations-report'
+
+export default [
+  OrganizationsReport,
+];

config/reports/organizations-report.js

@@ -0,0 +1,115 @@
+import { generateReportFromData, batchedQuery } from '../helpers.js';
+import { PREFIXES } from './utils'
+
+export default {
+  cronPattern: '0 00 23 * * *',
+  name: 'organizations',
+  execute: async () => {
+    const reportData = {
+      title: 'organizations',
+      description: 'List of the Organizations',
+      filePrefix: 'exports/organizations'
+    };
+
+    console.log('Generating organizations report');
+
+    const queryStringPart1 = `
+    ${PREFIXES}
+
+    SELECT DISTINCT ?bestuur ?label ?classification ?typeEredienst
+    WHERE {
+      VALUES ?type {
+        org:Organization
+        besluit:Bestuurseenheid
+      }
+
+      GRAPH ?g {
+        ?bestuur a ?type .
+        OPTIONAL { ?bestuur skos:prefLabel ?label. }
+      }
+
+      OPTIONAL {
+        ?bestuur org:classification ?classificationUri .
+        ?classificationUri skos:prefLabel ?classification.
+      }
+
+      OPTIONAL {
+        ?bestuur ere:typeEredienst ?typeEredienstUri.
+        ?typeEredienstUri skos:prefLabel ?typeEredienst.
+      }
+
+      FILTER (?g IN ( <http://mu.semte.ch/graphs/administrative-unit>, <http://mu.semte.ch/graphs/worship-service>, <http://mu.semte.ch/graphs/shared>))
+    }
+    `;
+
+    const queryResponsePart1 = await batchedQuery(queryStringPart1);
+
+    const dataPart1 = queryResponsePart1.results.bindings.reduce( (acc, row) => {
+      acc[getSafeValue(row, 'bestuur')] = {
+        bestuur: getSafeValue(row, 'bestuur'),
+        label: getSafeValue(row, 'label'),
+        classification: getSafeValue(row, 'classification'),
+        typeEredienst: getSafeValue(row, 'typeEredienst'),
+      };
+      return acc;
+    }, {});
+
+    const queryStringPart2 = `
+    ${PREFIXES}
+
+    SELECT DISTINCT ?bestuur ?provincie
+    WHERE {
+      VALUES ?type {
+        org:Organization
+        besluit:Bestuurseenheid
+      }
+
+      GRAPH ?g {
+        ?bestuur a ?type .
+      }
+
+      GRAPH ?h {
+        OPTIONAL {
+          ?bestuur org:hasPrimarySite ?primarySite.
+
+          OPTIONAL {
+            ?primarySite organisatie:bestaatUit ?sa.
+            OPTIONAL { ?sa locn:adminUnitL2 ?provincie. }
+          }
+        }
+      }
+
+      FILTER (?g IN ( <http://mu.semte.ch/graphs/administrative-unit>, <http://mu.semte.ch/graphs/worship-service>, <http://mu.semte.ch/graphs/shared>))
+      FILTER (?h IN ( <http://mu.semte.ch/graphs/administrative-unit>, <http://mu.semte.ch/graphs/worship-service>, <http://mu.semte.ch/graphs/shared>))
+    }
+    `;
+
+    const queryResponsePart2 = await batchedQuery(queryStringPart2);
+    const dataPart2 = queryResponsePart2.results.bindings.reduce( (acc, row) => {
+      let dataPart = {
+        bestuur: getSafeValue(row, 'bestuur'),
+        provincie: getSafeValue(row, 'provincie')
+      };
+
+      acc[getSafeValue(row, 'bestuur')] = Object.assign(dataPart, dataPart1[getSafeValue(row, 'bestuur')]);
+      return acc;
+    }, {});
+
+    await generateReportFromData(Object.values(dataPart2), [
+      'bestuur',
+      'label',
+      'classification',
+      'typeEredienst',
+      'provincie'
+    ], reportData);
+  }
+};
+
+function getSafeValue(entry, property){
+  return entry[property] ? wrapInQuote(entry[property].value) : null;
+}
+
+// Some values might contain commas; wrapping them in escapes quotes doesn't disrupt the columns.
+function wrapInQuote(value) {
+  return `\"${value}\"`;
+}

config/reports/utils.js

@@ -0,0 +1,29 @@
+export const PREFIXES = `
+  PREFIX adms: <http://www.w3.org/ns/adms#>
+  PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
+  PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+  PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
+  PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+  PREFIX dbpedia: <http://dbpedia.org/ontology/>
+  PREFIX dc_terms: <http://purl.org/dc/terms/>
+  PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+  PREFIX euro: <http://data.europa.eu/m8g/>
+  PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
+  PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+  PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+  PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+  PREFIX lblodlg: <https://data.lblod.info/vocabularies/leidinggevenden/>
+  PREFIX locn: <http://www.w3.org/ns/locn#>
+  PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+  PREFIX org: <http://www.w3.org/ns/org#>
+  PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
+  PREFIX person: <http://www.w3.org/ns/person#>
+  PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
+  PREFIX prov: <http://www.w3.org/ns/prov#>
+  PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+  PREFIX regorg: <http://www.w3.org/ns/regorg#>
+  PREFIX schema: <http://schema.org/>
+  PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+  PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
+  PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+`;

docker-compose.yml

@@ -270,6 +270,30 @@ services:
       - "logging=true"
     restart: always
     logging: *default-logging
+  report-generation:
+    image: lblod/loket-report-generation-service:0.7.0
+    environment:
+      DEFAULT_GRAPH: "http://mu.semte.ch/graphs/shared"
+      ONLY_KEEP_LATEST_REPORT: "true"
+    links:
+      - db:database
+    volumes:
+      - ./data/files:/share
+      - ./config/reports:/config
+    labels:
+      - "logging=true"
+    restart: always
+    logging: *default-logging
+  download-exports:
+    image: lblod/download-exports-service:0.1.0
+    links:
+      - db:database
+    volumes:
+      - ./data/files:/share
+    labels:
+      - "logging=true"
+    restart: always
+    logging: *default-logging
   sync-ovo-numbers:
     image: lblod/sync-ovo-numbers-service:0.1.0
     links: