diff --git a/app.js b/app.js index 7cd3d2e..490a361 100644 --- a/app.js +++ b/app.js @@ -1,8 +1,16 @@ import { app, errorHandler } from 'mu'; import bodyParser from 'body-parser'; import fetch from 'node-fetch'; +import { CronJob } from 'cron'; import { Delta } from "./lib/delta"; -import { isKbo, getIdentifiers, constructOvoStructure, updateOvoNumberAndUri } from "./lib/queries"; +import { + isKbo, + getIdentifiers, + constructOvoStructure, + updateOvoNumberAndUri, + getAllOvoAndKboCouples +} from "./lib/queries"; +import { CRON_PATTERN } from './config'; app.use(bodyParser.json({ type: function (req) { @@ -40,7 +48,7 @@ app.post('/delta', async function (req, res, next) { wegwijsOvo = wegwijsInfo.ovoNumber; } - if (wegwijsOvo != identifiers.ovo) { + if (wegwijsOvo && (wegwijsOvo != identifiers.ovo)) { let ovoStructuredIdUri = identifiers.ovoStructuredId; if (!ovoStructuredIdUri) { @@ -60,10 +68,59 @@ app.post('/delta', async function (req, res, next) { } }); -// TODO CRON JOB (+ endpoint to easily test it?) -// 1x per day. Query all admin units that have a KBO in our DB, with OVO numbers (optional) -// Then we compare with wegwijs, if possible via one big query to get all the codes at once, -// but if not possible multiple smaller ones -// Then we compare the KBO / OVO couples and as soon as we have a mismatch, we correct it +new CronJob(CRON_PATTERN, async function() { + const now = new Date().toISOString(); + console.log(`OVO numbers healing triggered by cron job at ${now}`); + try { + await healOvoNumbers(); + } catch (err) { + console.log(`An error occurred during OVO numbers healing at ${now}: ${err}`) + } +}, null, true); + +async function healOvoNumbers() { + try { + console.log('Healing starting...'); + const identifiersCouplesOP = await getAllOvoAndKboCouples(); + const identifiersCouplesWegwijs = await getAllOvoAndKboCouplesWegwijs(); + + for (const identifiersCoupleOP of identifiersCouplesOP) { + const wegwijsOvo = identifiersCouplesWegwijs[identifiersCoupleOP.kbo]; + // If a KBO can't be found in wegwijs but we already have an OVO for it in OP, we keep that OVO. + // It happens especially a lot for worship services that sometimes lack data in Wegwijs + if (wegwijsOvo && (identifiersCoupleOP.ovo != wegwijsOvo)) { + // We have a mismatch, let's update the OVO number + let ovoStructuredIdUri = identifiersCoupleOP.ovoStructuredId; + if (!ovoStructuredIdUri) { + ovoStructuredIdUri = await constructOvoStructure(identifiersCoupleOP.kboStructuredId); + } + + await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo); + } + } + console.log('Healing complete!'); + } catch (err) { + console.log(`An error occurred during OVO numbers healing: ${err}`) + } +} + +async function getAllOvoAndKboCouplesWegwijs() { + let couples = {}; + + const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:/.*[0-9].*/&fields=ovoNumber,kboNumber&scroll=true`); + const scrollId = JSON.parse(response.headers.get('x-search-metadata')).scrollId; + let data = await response.json(); + + do { + data.forEach(unit => { + couples[unit.kboNumber] = unit.ovoNumber; + }); + + const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations/scroll?id=${scrollId}`); + data = await response.json(); + } while (data.length) + + return couples; +} app.use(errorHandler); diff --git a/config.js b/config.js new file mode 100644 index 0000000..de6c026 --- /dev/null +++ b/config.js @@ -0,0 +1 @@ +export const CRON_PATTERN = process.env.CRON_PATTERN || '0 0 0 * * *'; // every day at midnight \ No newline at end of file diff --git a/lib/queries.js b/lib/queries.js index a56fc8b..52d55dc 100644 --- a/lib/queries.js +++ b/lib/queries.js @@ -24,7 +24,6 @@ export async function isKbo(subject) { return result.boolean; } -// Assumptions on the config : in the path, ?s is the subject of the delta and ?matchingUuid the id export async function getIdentifiers(kboStructuredId) { const queryStr = ` SELECT DISTINCT ?kbo ?ovo ?ovoStructuredId WHERE { @@ -157,4 +156,52 @@ export async function updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo) { await update(updateStr); } +export async function getAllOvoAndKboCouples() { + const queryStr = ` + SELECT DISTINCT ?kbo ?ovo ?kboStructuredId ?ovoStructuredId WHERE { + VALUES ?g { + + + } + GRAPH ?g { + ?kboId ?kboStructuredId ; + ?kboNotation . + + ?kboStructuredId a ; + ?kbo . + + ?bestuurseeheid ?kboId . + + FILTER (?kboNotation IN ("KBO nummer"@nl, "KBO nummer")) + + OPTIONAL { + ?bestuurseeheid ?ovoId . + + ?ovoId ?ovoStructuredId ; + ?ovoNotation . + + OPTIONAL { ?ovoStructuredId ?ovo . } + + FILTER (?ovoNotation IN ("OVO-nummer"@nl, "OVO-nummer")) + } + } + } + `; + + const result = await query(queryStr); + + if (result.results.bindings.length) { + const bindings = result.results.bindings; + return bindings.map(binding => { + return { + kbo: binding.kbo.value, + kboStructuredId: binding.kboStructuredId.value, + ovo: binding.ovo?.value, + ovoStructuredId: binding.ovoStructuredId?.value, + } + }); + } + return null; +} + // Update queries: via mu-auth if it comes from the frontend ? diff --git a/package.json b/package.json index 66fa090..eed9d0b 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "license": "MIT", "dependencies": { "@lblod/mu-auth-sudo": "^0.6.1", + "cron": "^3.0.0", "lodash.flatten": "4.4.0", "node-fetch": "^2.6.1" }