felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use endpoint instead of deltas for live updates

Browse Source
This commit is contained in:
Claire Lovisa
2023-10-03 15:46:57 +02:00
parent 4547418ad7
commit 0cb98aed1e
3 changed files with 27 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
app.js
View File

@@ -1,10 +1,7 @@
import { app, errorHandler } from 'mu';
import bodyParser from 'body-parser';
import fetch from 'node-fetch';
import { CronJob } from 'cron';
import { Delta } from "./lib/delta";
import {
isKbo,
getIdentifiers,
constructOvoStructure,
updateOvoNumberAndUri,
@@ -12,59 +9,38 @@ import {
} from "./lib/queries";
import { CRON_PATTERN } from './config';
app.use(bodyParser.json({
type: function (req) {
return /^application\/json/.test(req.get('content-type'));
}
}));
// TODO update readme with env variables etc at the end
// TODO current issue: when the KBO get updated, it doesn't refresh in the frontend.
// Alternative: endpoint to call in the frontend directly, that way we can trigger a refresh on the OVO ?
app.post('/delta', async function (req, res, next) {
app.post('/sync-from-kbo/:kboStructuredIdUuid', async function(req, res) {
try {
const entries = new Delta(req.body).getInsertsFor('https://data.vlaanderen.be/ns/generiek#lokaleIdentificator');
const kboStructuredIdUuid = req.params.kboStructuredIdUuid;
const identifiers = await getIdentifiers(kboStructuredIdUuid);
if (!entries.length) {
console.log('Delta dit not contain potential KBOs, awaiting the next batch!');
return res.status(204).send();
const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:${identifiers.kbo}&fields=ovoNumber,kboNumber)`);
const data = await response.json();
let wegwijsOvo = null;
if (data.length) {
// We got a match on the KBO, getting the associated OVO back
const wegwijsInfo = data[0]; // Wegwijs should only have only one entry per KBO
if (wegwijsInfo.ovoNumber)
wegwijsOvo = wegwijsInfo.ovoNumber;
}
for (let entry of entries) {
const isEntryKbo = await isKbo(entry)
if (wegwijsOvo && (wegwijsOvo != identifiers.ovo)) {
let ovoStructuredIdUri = identifiers.ovoStructuredId;
if (isEntryKbo) {
const identifiers = await getIdentifiers(entry);
const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:${identifiers.kbo}&fields=ovoNumber,kboNumber)`);
const data = await response.json();
let wegwijsOvo = null;
if (data.length) {
// We got a match on the KBO, getting the associated OVO back
const wegwijsInfo = data[0]; // Wegwijs should only have only one entry per KBO
if (wegwijsInfo.ovoNumber)
wegwijsOvo = wegwijsInfo.ovoNumber;
}
if (wegwijsOvo && (wegwijsOvo != identifiers.ovo)) {
let ovoStructuredIdUri = identifiers.ovoStructuredId;
if (!ovoStructuredIdUri) {
ovoStructuredIdUri = await constructOvoStructure(entry);
}
await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo);
}
if (!ovoStructuredIdUri) {
ovoStructuredIdUri = await constructOvoStructure(identifiers.kboStructuredId);
}
await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo);
}
return res.status(200).send().end();
return res.status(200).send(); // since we await, it should be 200
} catch (e) {
console.log(`Something unexpected went wrong while handling delta task!`);
console.error(e);
return next(e);
console.log("Something went wrong while calling /sync-from-kbo", e);
return res.status(500).send();
}
});

28
lib/delta.js
View File

@@ -1,28 +0,0 @@
import { flatten } from 'lodash';
export class Delta {
constructor(delta) {
this.delta = delta;
}
get inserts() {
return flatten(this.delta.map(changeSet => changeSet.inserts));
}
get deletes() {
return flatten(this.delta.map(changeSet => changeSet.deletes));
}
getInsertsFor(predicate) {
return this.inserts
.filter(t => t.predicate.value === predicate)
.map(t => t.subject.value);
}
getDeletesFor(predicate) {
return this.deletes
.filter(t => t.predicate.value === predicate)
.map(t => t.subject.value);
}
}

33
lib/queries.js
View File

@@ -1,41 +1,19 @@
import { uuid, sparqlEscapeUri, sparqlEscapeString } from "mu";
import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo';
export async function isKbo(subject) {
export async function getIdentifiers(kboStructuredIdUuid) {
const queryStr = `
ASK {
SELECT DISTINCT ?kbo ?ovo ?kboStructuredId ?ovoStructuredId WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
<http://mu.semte.ch/graphs/worship-service>
}
GRAPH ?g {
${sparqlEscapeUri(subject)} a <https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator> ;
?kboStructuredId a <https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator> ;
<http://mu.semte.ch/vocabularies/core/uuid> ${sparqlEscapeString(kboStructuredIdUuid)} ;
<https://data.vlaanderen.be/ns/generiek#lokaleIdentificator> ?kbo .
?id <https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ${sparqlEscapeUri(subject)} ;
<http://www.w3.org/2004/02/skos/core#notation> ?notation .
FILTER (?notation IN ("KBO nummer"@nl, "KBO nummer"))
}
}
`;
const result = await query(queryStr);
return result.boolean;
}
export async function getIdentifiers(kboStructuredId) {
const queryStr = `
SELECT DISTINCT ?kbo ?ovo ?ovoStructuredId WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
<http://mu.semte.ch/graphs/worship-service>
}
GRAPH ?g {
${sparqlEscapeUri(kboStructuredId)} a <https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator> ;
<https://data.vlaanderen.be/ns/generiek#lokaleIdentificator> ?kbo .
?kboId <https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ${sparqlEscapeUri(kboStructuredId)} ;
?kboId <https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ?kboStructuredId ;
<http://www.w3.org/2004/02/skos/core#notation> ?kboNotation .
?bestuurseeheid <http://www.w3.org/ns/adms#identifier> ?kboId .
@@ -62,6 +40,7 @@ export async function getIdentifiers(kboStructuredId) {
const binding = result.results.bindings[0]; // We should only have one KBO/OVO couple
return {
kbo: binding.kbo.value,
kboStructuredId: binding.kboStructuredId.value,
ovo: binding.ovo?.value,
ovoStructuredId: binding.ovoStructuredId?.value,
}