felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

getting more data to create kbo organisation

Browse Source
This commit is contained in:
Elias Papanikitas
2024-02-07 10:44:14 +01:00
parent a669398de4
commit 2f48d6b1a8
2 changed files with 226 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
app.js
View File

@@ -1,37 +1,56 @@
import { app, errorHandler } from 'mu';
import fetch from 'node-fetch';
import { CronJob } from 'cron';
import { app, errorHandler } from "mu";
import fetch from "node-fetch";
import { CronJob } from "cron";
import {
getIdentifiers,
constructOvoStructure,
updateOvoNumberAndUri,
getAllOvoAndKboCouples
getAllOvoAndKboCouples,
createKboOrganisation,
createNewKboOrg,
linkAbbOrgToKboOrg,
} from "./lib/queries";
import { CRON_PATTERN } from './config';
import { CRON_PATTERN } from "./config";
app.post('/sync-from-kbo/:kboStructuredIdUuid', async function(req, res) {
app.post("/sync-kbo-data/:kboStructuredIdUuid", async function (req, res) {
try {
const kboStructuredIdUuid = req.params.kboStructuredIdUuid;
const identifiers = await getIdentifiers(kboStructuredIdUuid);
const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:${identifiers.kbo}&fields=ovoNumber,kboNumber)`);
const wegwijsUrl = `https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:${identifiers.kbo}&fields=changeTime,name,ovoNumber,kboNumber,labels,contacts,organisationClassifications,locations,parents)`;
const response = await fetch(wegwijsUrl);
const data = await response.json();
let wegwijsOvo = null;
if (data.length) {
// We got a match on the KBO, getting the associated OVO back
const wegwijsInfo = data[0]; // Wegwijs should only have only one entry per KBO
if (wegwijsInfo.ovoNumber)
wegwijsOvo = wegwijsInfo.ovoNumber;
let kboObject = getKboFields(wegwijsInfo);
if (kboObject.ovoNumber) {
wegwijsOvo = kboObject.ovoNumber;
}
let kboOrg = "";
//console.log(kboObject);
if (!identifiers.kboOrg) {
kboOrg = await createNewKboOrg(kboObject, "null");
}
await linkAbbOrgToKboOrg(identifiers.adminUnit, kboOrg, identifiers.kbo);
}
if (wegwijsOvo && (wegwijsOvo != identifiers.ovo)) {
//Update Ovo Number
if (wegwijsOvo && wegwijsOvo != identifiers.ovo) {
console.log(identifiers.ovo);
let ovoStructuredIdUri = identifiers.ovoStructuredId;
if (!ovoStructuredIdUri) {
ovoStructuredIdUri = await constructOvoStructure(identifiers.kboStructuredId);
ovoStructuredIdUri = await constructOvoStructure(
identifiers.kboStructuredId
);
}
await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo);
}
@@ -42,57 +61,160 @@ app.post('/sync-from-kbo/:kboStructuredIdUuid', async function(req, res) {
}
});
new CronJob(CRON_PATTERN, async function() {
const now = new Date().toISOString();
console.log(`OVO numbers healing triggered by cron job at ${now}`);
try {
await healOvoNumbers();
} catch (err) {
console.log(`An error occurred during OVO numbers healing at ${now}: ${err}`)
}
}, null, true);
function getKboFields(data) {
console.log(data);
let changeTime = data.changeTime;
let organisationName = data.name;
let ovoNumber = data.ovoNumber;
let kboNumber = data.kboNumber;
//no lables = undefined
let formalName = data.labels
?.filter((fields) => {
return fields.labelTypeId === "83c1c22a-6776-0ad6-68d2-819e1c6eec66";
})
.pop()?.value;
let email = data.contacts
?.filter((fields) => {
return (
fields.contactTypeId === "f611e91d-a811-4519-afa6-08f0e9dcdd9b" &&
!fields.validity.hasOwnProperty("end")
);
})
.pop()?.value;
let rechtsvorm = data.organisationClassifications
?.filter((fields) => {
return (
fields.organisationClassificationId ===
"3c809ffd-f864-456f-8704-28c16528daa2"
);
})
.pop()?.organisationClassificationName;
let phone = data.contacts
?.filter((fields) => {
return (
fields.contactTypeId === "d792d4ff-8c34-4336-8434-56ecb40f2fa4" &&
!fields.validity.hasOwnProperty("end")
);
})
.pop()?.value;
let website = data.contacts
?.filter((fields) => {
return (
fields.contactTypeId === "6763f372-02c5-478c-b7d7-802dc60a64b9" &&
!fields.validity.hasOwnProperty("end")
);
})
.pop()?.value;
let province = data.parents?.parentOrganisationName;
let formattedAddress = data.locations
?.filter((fields) => {
return (
fields.isMainLocation === true &&
!fields.validity?.hasOwnProperty("end")
);
})
.pop()?.formattedAddress;
let adressComponent = data.locations
?.filter((fields) => {
return (
fields.isMainLocation === true &&
!fields.validity?.hasOwnProperty("end")
);
})
.pop()?.components;
return {
changeTime: (typeof changeTime != 'undefined' ? changeTime : ""),
organisationName: (typeof organisationName != 'undefined' ? organisationName : ""),
ovoNumber: (typeof ovoNumber != 'undefined' ? ovoNumber : ""),
kboNumber: (typeof kboNumber != 'undefined' ? kboNumber : ""),
formalName: (typeof formalName != 'undefined' ? formalName : ""),
rechtsvorm: (typeof rechtsvorm != 'undefined' ? rechtsvorm : ""),
email: (typeof email != 'undefined' ? email : ""),
phone: (typeof phone != 'undefined' ? phone : ""),
website: (typeof website != 'undefined' ? website : ""),
province: (typeof province != 'undefined' ? province : ""),
formattedAddress: (typeof formattedAddress != 'undefined' ? formattedAddress : ""),
adressComponent: (typeof adressComponent != 'undefined' ? adressComponent : ""),
};
}
new CronJob(
CRON_PATTERN,
async function () {
const now = new Date().toISOString();
console.log(`OVO numbers healing triggered by cron job at ${now}`);
try {
await healOvoNumbers();
} catch (err) {
console.log(
`An error occurred during OVO numbers healing at ${now}: ${err}`
);
}
},
null,
true
);
async function healOvoNumbers() {
try {
console.log('Healing starting...');
console.log("Healing starting...");
const identifiersCouplesOP = await getAllOvoAndKboCouples();
const identifiersCouplesWegwijs = await getAllOvoAndKboCouplesWegwijs();
for (const identifiersCoupleOP of identifiersCouplesOP) {
const wegwijsOvo = identifiersCouplesWegwijs[identifiersCoupleOP.kbo];
// If a KBO can't be found in wegwijs but we already have an OVO for it in OP, we keep that OVO.
// It happens especially a lot for worship services that sometimes lack data in Wegwijs
if (wegwijsOvo && (identifiersCoupleOP.ovo != wegwijsOvo)) {
if (wegwijsOvo && identifiersCoupleOP.ovo != wegwijsOvo) {
// We have a mismatch, let's update the OVO number
let ovoStructuredIdUri = identifiersCoupleOP.ovoStructuredId;
if (!ovoStructuredIdUri) {
ovoStructuredIdUri = await constructOvoStructure(identifiersCoupleOP.kboStructuredId);
ovoStructuredIdUri = await constructOvoStructure(
identifiersCoupleOP.kboStructuredId
);
}
await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo);
}
}
console.log('Healing complete!');
console.log("Healing complete!");
} catch (err) {
console.log(`An error occurred during OVO numbers healing: ${err}`)
console.log(`An error occurred during OVO numbers healing: ${err}`);
}
}
async function getAllOvoAndKboCouplesWegwijs() {
let couples = {};
const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:/.*[0-9].*/&fields=ovoNumber,kboNumber&scroll=true`);
const scrollId = JSON.parse(response.headers.get('x-search-metadata')).scrollId;
const response = await fetch(
`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:/.*[0-9].*/&fields=ovoNumber,kboNumber&scroll=true`
);
const scrollId = JSON.parse(
response.headers.get("x-search-metadata")
).scrollId;
let data = await response.json();
do {
data.forEach(unit => {
data.forEach((unit) => {
couples[unit.kboNumber] = unit.ovoNumber;
});
const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations/scroll?id=${scrollId}`);
const response = await fetch(
`https://api.wegwijs.vlaanderen.be/v1/search/organisations/scroll?id=${scrollId}`
);
data = await response.json();
} while (data.length)
} while (data.length);
return couples;
}

75
lib/queries.js
View File

@@ -3,7 +3,7 @@ import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo';
export async function getIdentifiers(kboStructuredIdUuid) {
const queryStr = `
SELECT DISTINCT ?kbo ?ovo ?kboStructuredId ?ovoStructuredId WHERE {
SELECT DISTINCT ?kbo ?ovo ?kboStructuredId ?ovoStructuredId ?bestuurseenheid ?kboOrganisation WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
<http://mu.semte.ch/graphs/worship-service>
@@ -16,7 +16,7 @@ export async function getIdentifiers(kboStructuredIdUuid) {
?kboId <https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ?kboStructuredId ;
<http://www.w3.org/2004/02/skos/core#notation> ?kboNotation .
?bestuurseeheid <http://www.w3.org/ns/adms#identifier> ?kboId .
?bestuurseenheid <http://www.w3.org/ns/adms#identifier> ?kboId .
FILTER (?kboNotation IN ("KBO nummer"@nl, "KBO nummer"))
@@ -27,6 +27,8 @@ export async function getIdentifiers(kboStructuredIdUuid) {
<http://www.w3.org/2004/02/skos/core#notation> ?ovoNotation .
OPTIONAL { ?ovoStructuredId <https://data.vlaanderen.be/ns/generiek#lokaleIdentificator> ?ovo . }
?bestuurseenheid owl:sameAs ?kboOrganisation .
FILTER (?ovoNotation IN ("OVO-nummer"@nl, "OVO-nummer"))
}
@@ -43,6 +45,8 @@ export async function getIdentifiers(kboStructuredIdUuid) {
kboStructuredId: binding.kboStructuredId.value,
ovo: binding.ovo?.value,
ovoStructuredId: binding.ovoStructuredId?.value,
adminUnit: binding.bestuurseenheid.value,
kboUnit: binding.kboOrganisation?.value
}
}
return null;
@@ -131,7 +135,7 @@ export async function updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo) {
}
}
`;
await update(updateStr);
}
@@ -183,4 +187,69 @@ export async function getAllOvoAndKboCouples() {
return null;
}
export async function createRechtsVormStructure(){
}
export async function createNewKboOrg(kboOrg ,kboNummer){
const newKboOrgIdUuId = uuid();
const newKboIdUri = `http://data.lblod.info/id/kboBestuurseenheden/${newKboOrgIdUuId}`;
const contactPointUuidId = uuid();
const contactPointUri = `http://data.lblod.info/id/contact-punten/${contactPointUuidId}`;
let updateStr = `
INSERT {
GRAPH ?g {
${sparqlEscapeUri(contactPointUri)} a <http://schema.org/ContactPoint> ;
<http://mu.semte.ch/vocabularies/core/uuid> ${sparqlEscapeString(contactPointUuidId)} ;
<http://schema.org/email> ${sparqlEscapeString(kboOrg.email)} ;
<http://schema.org/telephone> ${sparqlEscapeString(kboOrg.phone)} ;
<http://xmlns.com/foaf/0.1/page> ${sparqlEscapeString(kboOrg.website)}.
${sparqlEscapeUri(newKboIdUri)} a <http://mu.semte.ch/vocabularies/ext/KboEenheid> ;
<http://mu.semte.ch/vocabularies/core/uuid> ${sparqlEscapeString(newKboOrgIdUuId)} ;
<http://mu.semte.ch/vocabularies/ext/rechtsvorm> ${sparqlEscapeString(kboOrg.rechtsvorm)} ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ${sparqlEscapeString(kboOrg.organisationName)} ;
<http://www.w3.org/2004/02/skos/core#altLabel> ${sparqlEscapeString(kboOrg.formalName)} ;
<https://data.lblod.info/vocabularies/generiek/geplandeEindDatum> ${sparqlEscapeString(kboOrg.changeTime)} ;
<http://schema.org/contactPoint> ${sparqlEscapeUri(contactPointUri)} .
}
}WHERE{
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
}
}
`
await update(updateStr);
return newKboIdUri;
}
export async function linkAbbOrgToKboOrg(abbOrgUri ,kboOrgUri){
let updateStr = `
INSERT {
GRAPH ?g {
${sparqlEscapeUri(kboOrgUri)} <http://www.w3.org/2002/07/owl#sameAs> ${sparqlEscapeUri(abbOrgUri)} .
}
}WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
}
}
`
await update(updateStr);
}
// Update queries: via mu-auth if it comes from the frontend ?