commit 3de5d8f18402da9ab815555d24333ed1e0b6dfb0 Author: Claire Lovisa Date: Thu Sep 21 10:31:34 2023 +0200 Initial commit diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d2cbd29 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true +indent_style = space +indent_size = 2 + + +[*.{diff,md}] +trim_trailing_whitespace = false \ No newline at end of file diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000..f4e9851 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,17 @@ +module.exports = { + root: true, + parserOptions: { + ecmaVersion: 2018, + sourceType: 'module' + }, + extends: [ + 'eslint:recommended', + ], + env: { + node: true + }, + rules: { + semi: 'warn', + "no-console": "off" + }, +}; diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e0d2b93 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*~ +node_modules/ +package-lock.json +.log/ diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..43c97e7 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/.woodpecker/.latest.yml b/.woodpecker/.latest.yml new file mode 100644 index 0000000..b378726 --- /dev/null +++ b/.woodpecker/.latest.yml @@ -0,0 +1,10 @@ +pipeline: + build: + image: plugins/docker + settings: + repo: ${CI_REPO} + tags: latest + secrets: [ docker_username, docker_password ] +when: + branch: master + event: push diff --git a/.woodpecker/.release.yml b/.woodpecker/.release.yml new file mode 100644 index 0000000..4e41117 --- /dev/null +++ b/.woodpecker/.release.yml @@ -0,0 +1,10 @@ +pipeline: + build: + image: plugins/docker + settings: + repo: ${CI_REPO} + tags: "${CI_COMMIT_TAG##v}" + secrets: [ docker_username, docker_password ] +when: + event: tag + tag: v* diff --git a/.woodpecker/.test-build.yml b/.woodpecker/.test-build.yml new file mode 100644 index 0000000..1a0eb9a --- /dev/null +++ b/.woodpecker/.test-build.yml @@ -0,0 +1,9 @@ +pipeline: + build: + image: plugins/docker + settings: + repo: ${CI_REPO} + dry_run: true +when: + event: + - pull_request diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f7be412 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,3 @@ +FROM semtech/mu-javascript-template:1.6.0 +LABEL maintainer="redpencil.io " + diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..b7afb6d --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,9 @@ +MIT License + +Copyright 2023 Vlaamse Overheid + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e17de75 --- /dev/null +++ b/README.md @@ -0,0 +1,51 @@ + + +
+
+

sync-ovo-numbers-service

+

+ Service that syncs OVO numbers of OP to match with those in Wegwijs. +
+ Report Bug + ยท + Open PR +

+
+ + +## ๐Ÿ“– Description + +This service maintains OP's OVO numbers in sync together with [Wegwijs](https://wegwijs.vlaanderen.be/#/organisations), which is the official source of OVO numbers. + +It gets triggered when a KBO is updated in OP as well as on a regular basis via a cron job to ensure our dataset is in sync with theirs. + +

(back to top)

+ +## โฉ Quick setup + +### ๐Ÿ‹ Docker-compose.yml +```yaml + sync-ovo-numbers: + image: lblod/sync-ovo-numbers-service + links: + - db:database + labels: + - "logging=true" + restart: always + logging: *default-logging +``` + +### ๐Ÿ—’๏ธ Config + +// TODO probably no delta config if we go for the endpoint option right ? + +

(back to top)

+ +## ๐Ÿ”‘ Environment variables + +| ENV | description | default | required | +|---|---|---|---| +| xxx | xxx | | X | + + +

(back to top)

diff --git a/app.js b/app.js new file mode 100644 index 0000000..7cd3d2e --- /dev/null +++ b/app.js @@ -0,0 +1,69 @@ +import { app, errorHandler } from 'mu'; +import bodyParser from 'body-parser'; +import fetch from 'node-fetch'; +import { Delta } from "./lib/delta"; +import { isKbo, getIdentifiers, constructOvoStructure, updateOvoNumberAndUri } from "./lib/queries"; + +app.use(bodyParser.json({ + type: function (req) { + return /^application\/json/.test(req.get('content-type')); + } +})); + +// TODO update readme with env variables etc at the end + +// TODO current issue: when the KBO get updated, it doesn't refresh in the frontend. +// Alternative: endpoint to call in the frontend directly, that way we can trigger a refresh on the OVO ? +app.post('/delta', async function (req, res, next) { + try { + const entries = new Delta(req.body).getInsertsFor('https://data.vlaanderen.be/ns/generiek#lokaleIdentificator'); + + if (!entries.length) { + console.log('Delta dit not contain potential KBOs, awaiting the next batch!'); + return res.status(204).send(); + } + + for (let entry of entries) { + const isEntryKbo = await isKbo(entry) + + if (isEntryKbo) { + const identifiers = await getIdentifiers(entry); + + const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:${identifiers.kbo}&fields=ovoNumber,kboNumber)`); + const data = await response.json(); + + let wegwijsOvo = null; + if (data.length) { + // We got a match on the KBO, getting the associated OVO back + const wegwijsInfo = data[0]; // Wegwijs should only have only one entry per KBO + if (wegwijsInfo.ovoNumber) + wegwijsOvo = wegwijsInfo.ovoNumber; + } + + if (wegwijsOvo != identifiers.ovo) { + let ovoStructuredIdUri = identifiers.ovoStructuredId; + + if (!ovoStructuredIdUri) { + ovoStructuredIdUri = await constructOvoStructure(entry); + } + + await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo); + } + } + } + + return res.status(200).send().end(); + } catch (e) { + console.log(`Something unexpected went wrong while handling delta task!`); + console.error(e); + return next(e); + } +}); + +// TODO CRON JOB (+ endpoint to easily test it?) +// 1x per day. Query all admin units that have a KBO in our DB, with OVO numbers (optional) +// Then we compare with wegwijs, if possible via one big query to get all the codes at once, +// but if not possible multiple smaller ones +// Then we compare the KBO / OVO couples and as soon as we have a mismatch, we correct it + +app.use(errorHandler); diff --git a/lib/delta.js b/lib/delta.js new file mode 100644 index 0000000..390a69b --- /dev/null +++ b/lib/delta.js @@ -0,0 +1,28 @@ +import { flatten } from 'lodash'; + +export class Delta { + + constructor(delta) { + this.delta = delta; + } + + get inserts() { + return flatten(this.delta.map(changeSet => changeSet.inserts)); + } + + get deletes() { + return flatten(this.delta.map(changeSet => changeSet.deletes)); + } + + getInsertsFor(predicate) { + return this.inserts + .filter(t => t.predicate.value === predicate) + .map(t => t.subject.value); + } + + getDeletesFor(predicate) { + return this.deletes + .filter(t => t.predicate.value === predicate) + .map(t => t.subject.value); + } +} diff --git a/lib/queries.js b/lib/queries.js new file mode 100644 index 0000000..a56fc8b --- /dev/null +++ b/lib/queries.js @@ -0,0 +1,160 @@ +import { uuid, sparqlEscapeUri, sparqlEscapeString } from "mu"; +import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; + +export async function isKbo(subject) { + const queryStr = ` + ASK { + VALUES ?g { + + + } + GRAPH ?g { + ${sparqlEscapeUri(subject)} a ; + ?kbo . + + ?id ${sparqlEscapeUri(subject)} ; + ?notation . + + FILTER (?notation IN ("KBO nummer"@nl, "KBO nummer")) + } + } + `; + + const result = await query(queryStr); + return result.boolean; +} + +// Assumptions on the config : in the path, ?s is the subject of the delta and ?matchingUuid the id +export async function getIdentifiers(kboStructuredId) { + const queryStr = ` + SELECT DISTINCT ?kbo ?ovo ?ovoStructuredId WHERE { + VALUES ?g { + + + } + GRAPH ?g { + ${sparqlEscapeUri(kboStructuredId)} a ; + ?kbo . + + ?kboId ${sparqlEscapeUri(kboStructuredId)} ; + ?kboNotation . + + ?bestuurseeheid ?kboId . + + FILTER (?kboNotation IN ("KBO nummer"@nl, "KBO nummer")) + + OPTIONAL { + ?bestuurseeheid ?ovoId . + + ?ovoId ?ovoStructuredId ; + ?ovoNotation . + + OPTIONAL { ?ovoStructuredId ?ovo . } + + FILTER (?ovoNotation IN ("OVO-nummer"@nl, "OVO-nummer")) + } + } + } + `; + + const result = await query(queryStr); + + if (result.results.bindings.length) { + const binding = result.results.bindings[0]; // We should only have one KBO/OVO couple + return { + kbo: binding.kbo.value, + ovo: binding.ovo?.value, + ovoStructuredId: binding.ovoStructuredId?.value, + } + } + return null; +} + +export async function constructOvoStructure(kboStructuredId) { + const idUuid = uuid(); + const idUri = `http://data.lblod.info/id/identificatoren/${idUuid}` + const structuredIdUuid = uuid(); + const structuredIdUri = `http://data.lblod.info/id/gestructureerdeIdentificatoren/${structuredIdUuid}` + + const updateStr = ` + INSERT { + GRAPH ?g { + ?bestuurseenheid ${sparqlEscapeUri(idUri)} . + + ${sparqlEscapeUri(idUri)} a ; + ${sparqlEscapeString(idUuid)} ; + "OVO-nummer" ; + ${sparqlEscapeUri(structuredIdUri)} . + + ${sparqlEscapeUri(structuredIdUri)} a ; + ${sparqlEscapeString(structuredIdUuid)} . + } + } WHERE { + VALUES ?g { + + + } + GRAPH ?g { + ${sparqlEscapeUri(kboStructuredId)} a ; + ?kbo . + + ?kboId ${sparqlEscapeUri(kboStructuredId)} ; + ?notation . + + ?bestuurseenheid ?kboId . + + FILTER (?notation IN ("KBO nummer"@nl, "KBO nummer")) + } + } + `; + + await update(updateStr); + return structuredIdUri; +} + +export async function updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo) { + let updateStr = ` + DELETE { + GRAPH ?g { + ?bestuureseenheid ?ovoUri . + ${sparqlEscapeUri(ovoStructuredIdUri)} ?ovo . + } + } + `; + + if (wegwijsOvo) { + updateStr += ` + INSERT { + GRAPH ?g { + ?bestuureseenheid ${sparqlEscapeUri(`http://data.vlaanderen.be/id/organisatie/${wegwijsOvo}`)} . + ${sparqlEscapeUri(ovoStructuredIdUri)} ${sparqlEscapeString(wegwijsOvo)} . + } + } + `; + } + + updateStr += ` + WHERE { + VALUES ?g { + + + } + GRAPH ?g { + + ${sparqlEscapeUri(ovoStructuredIdUri)} a . + + OPTIONAL { + ?bestuureseenheid + / ${sparqlEscapeUri(ovoStructuredIdUri)} ; + ?ovoUri . + } + + OPTIONAL { ${sparqlEscapeUri(ovoStructuredIdUri)} ?ovo . } + } + } + `; + + await update(updateStr); +} + +// Update queries: via mu-auth if it comes from the frontend ? diff --git a/package.json b/package.json new file mode 100644 index 0000000..66fa090 --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "name": "sync-ovo-numbers-service", + "version": "0.0.0", + "description": "Syncs OVO numbers with Wegwijs based on KBOs", + "author": "redpencil.io", + "license": "MIT", + "dependencies": { + "@lblod/mu-auth-sudo": "^0.6.1", + "lodash.flatten": "4.4.0", + "node-fetch": "^2.6.1" + } +}