From 0cb98aed1ed64836f3ecdea46fea21ab339aa033 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 3 Oct 2023 15:46:57 +0200 Subject: [PATCH] Use endpoint instead of deltas for live updates --- app.js | 66 ++++++++++++++++---------------------------------- lib/delta.js | 28 --------------------- lib/queries.js | 33 +++++-------------------- 3 files changed, 27 insertions(+), 100 deletions(-) delete mode 100644 lib/delta.js diff --git a/app.js b/app.js index 490a361..9bee86b 100644 --- a/app.js +++ b/app.js @@ -1,10 +1,7 @@ import { app, errorHandler } from 'mu'; -import bodyParser from 'body-parser'; import fetch from 'node-fetch'; import { CronJob } from 'cron'; -import { Delta } from "./lib/delta"; import { - isKbo, getIdentifiers, constructOvoStructure, updateOvoNumberAndUri, @@ -12,59 +9,38 @@ import { } from "./lib/queries"; import { CRON_PATTERN } from './config'; -app.use(bodyParser.json({ - type: function (req) { - return /^application\/json/.test(req.get('content-type')); - } -})); - // TODO update readme with env variables etc at the end -// TODO current issue: when the KBO get updated, it doesn't refresh in the frontend. -// Alternative: endpoint to call in the frontend directly, that way we can trigger a refresh on the OVO ? -app.post('/delta', async function (req, res, next) { +app.post('/sync-from-kbo/:kboStructuredIdUuid', async function(req, res) { try { - const entries = new Delta(req.body).getInsertsFor('https://data.vlaanderen.be/ns/generiek#lokaleIdentificator'); + const kboStructuredIdUuid = req.params.kboStructuredIdUuid; + const identifiers = await getIdentifiers(kboStructuredIdUuid); - if (!entries.length) { - console.log('Delta dit not contain potential KBOs, awaiting the next batch!'); - return res.status(204).send(); + const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:${identifiers.kbo}&fields=ovoNumber,kboNumber)`); + const data = await response.json(); + + let wegwijsOvo = null; + if (data.length) { + // We got a match on the KBO, getting the associated OVO back + const wegwijsInfo = data[0]; // Wegwijs should only have only one entry per KBO + if (wegwijsInfo.ovoNumber) + wegwijsOvo = wegwijsInfo.ovoNumber; } - for (let entry of entries) { - const isEntryKbo = await isKbo(entry) + if (wegwijsOvo && (wegwijsOvo != identifiers.ovo)) { + let ovoStructuredIdUri = identifiers.ovoStructuredId; - if (isEntryKbo) { - const identifiers = await getIdentifiers(entry); - - const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:${identifiers.kbo}&fields=ovoNumber,kboNumber)`); - const data = await response.json(); - - let wegwijsOvo = null; - if (data.length) { - // We got a match on the KBO, getting the associated OVO back - const wegwijsInfo = data[0]; // Wegwijs should only have only one entry per KBO - if (wegwijsInfo.ovoNumber) - wegwijsOvo = wegwijsInfo.ovoNumber; - } - - if (wegwijsOvo && (wegwijsOvo != identifiers.ovo)) { - let ovoStructuredIdUri = identifiers.ovoStructuredId; - - if (!ovoStructuredIdUri) { - ovoStructuredIdUri = await constructOvoStructure(entry); - } - - await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo); - } + if (!ovoStructuredIdUri) { + ovoStructuredIdUri = await constructOvoStructure(identifiers.kboStructuredId); } + + await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo); } - return res.status(200).send().end(); + return res.status(200).send(); // since we await, it should be 200 } catch (e) { - console.log(`Something unexpected went wrong while handling delta task!`); - console.error(e); - return next(e); + console.log("Something went wrong while calling /sync-from-kbo", e); + return res.status(500).send(); } }); diff --git a/lib/delta.js b/lib/delta.js deleted file mode 100644 index 390a69b..0000000 --- a/lib/delta.js +++ /dev/null @@ -1,28 +0,0 @@ -import { flatten } from 'lodash'; - -export class Delta { - - constructor(delta) { - this.delta = delta; - } - - get inserts() { - return flatten(this.delta.map(changeSet => changeSet.inserts)); - } - - get deletes() { - return flatten(this.delta.map(changeSet => changeSet.deletes)); - } - - getInsertsFor(predicate) { - return this.inserts - .filter(t => t.predicate.value === predicate) - .map(t => t.subject.value); - } - - getDeletesFor(predicate) { - return this.deletes - .filter(t => t.predicate.value === predicate) - .map(t => t.subject.value); - } -} diff --git a/lib/queries.js b/lib/queries.js index 52d55dc..6bdb12c 100644 --- a/lib/queries.js +++ b/lib/queries.js @@ -1,41 +1,19 @@ import { uuid, sparqlEscapeUri, sparqlEscapeString } from "mu"; import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; -export async function isKbo(subject) { +export async function getIdentifiers(kboStructuredIdUuid) { const queryStr = ` - ASK { + SELECT DISTINCT ?kbo ?ovo ?kboStructuredId ?ovoStructuredId WHERE { VALUES ?g { } GRAPH ?g { - ${sparqlEscapeUri(subject)} a ; + ?kboStructuredId a ; + ${sparqlEscapeString(kboStructuredIdUuid)} ; ?kbo . - ?id ${sparqlEscapeUri(subject)} ; - ?notation . - - FILTER (?notation IN ("KBO nummer"@nl, "KBO nummer")) - } - } - `; - - const result = await query(queryStr); - return result.boolean; -} - -export async function getIdentifiers(kboStructuredId) { - const queryStr = ` - SELECT DISTINCT ?kbo ?ovo ?ovoStructuredId WHERE { - VALUES ?g { - - - } - GRAPH ?g { - ${sparqlEscapeUri(kboStructuredId)} a ; - ?kbo . - - ?kboId ${sparqlEscapeUri(kboStructuredId)} ; + ?kboId ?kboStructuredId ; ?kboNotation . ?bestuurseeheid ?kboId . @@ -62,6 +40,7 @@ export async function getIdentifiers(kboStructuredId) { const binding = result.results.bindings[0]; // We should only have one KBO/OVO couple return { kbo: binding.kbo.value, + kboStructuredId: binding.kboStructuredId.value, ovo: binding.ovo?.value, ovoStructuredId: binding.ovoStructuredId?.value, }