From c0f99d7621f66e9a21cf8e0cba2b70c217d384eb Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Fri, 23 Feb 2024 08:30:56 +0100 Subject: [PATCH] updating/cron job fixed --- app.js | 129 ++++++++++++++---- lib/queries.js | 352 +++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 396 insertions(+), 85 deletions(-) diff --git a/app.js b/app.js index c45a92c..9c040f3 100644 --- a/app.js +++ b/app.js @@ -6,10 +6,11 @@ import { constructOvoStructure, updateOvoNumberAndUri, getAllOvoAndKboCouples, - createKboOrganisation, createNewKboOrg, linkAbbOrgToKboOrg, getKboIdentifiers, + updateKboOrg, + getAllAbbKboOrganizations, } from "./lib/queries"; import { CRON_PATTERN, ORGANIZATION_STATUS } from "./config"; @@ -40,19 +41,34 @@ app.post( if (createKbo && kboObject) { //create kbo organisation - let kboOrg = null; + let kboOrg = null; const kboIdentifiers = await getKboIdentifiers(identifiers.adminUnit); let update = false; if (kboIdentifiers) { - update = new Date(kboObject.changeTime).getTime() < new Date(kboIdentifiers.changeTime).getTime() ? true : false; - } - if(!kboIdentifiers || update){ + update = + new Date(kboObject.changeTime).getTime() > + new Date(kboIdentifiers.changeTime).getTime() + ? true + : false; + } + + if (!kboIdentifiers) { kboOrg = await createNewKboOrg(kboObject, identifiers.kboId); await linkAbbOrgToKboOrg(identifiers.adminUnit, kboOrg); } + + if (update) { + updateKboOrg(kboObject, kboIdentifiers); + } + return res.status(200).send(); } + + console.log("testing healing of kbo organizations"); + await healKboUnits(); + console.log("end of test"); + //Update Ovo Number if (wegwijsOvo && wegwijsOvo != identifiers.ovo) { console.log(identifiers.ovo); @@ -95,10 +111,14 @@ function getKboFields(data) { return fields.labelTypeId === "83c1c22a-6776-0ad6-68d2-819e1c6eec66"; }) .pop()?.validity?.start; - let activeState = (data.labels - ?.filter((fields) => { - return fields.labelTypeId === "83c1c22a-6776-0ad6-68d2-819e1c6eec66" && !fields.validity.hasOwnProperty("end"); - }) ? ORGANIZATION_STATUS.ACTIVE : ORGANIZATION_STATUS.INACTIVE) + let activeState = data.labels?.filter((fields) => { + return ( + fields.labelTypeId === "83c1c22a-6776-0ad6-68d2-819e1c6eec66" && + !fields.validity.hasOwnProperty("end") + ); + }) + ? ORGANIZATION_STATUS.ACTIVE + : ORGANIZATION_STATUS.INACTIVE; let email = data.contacts ?.filter((fields) => { return ( @@ -147,28 +167,34 @@ function getKboFields(data) { let adressComponent = data.locations ?.filter((fields) => { return ( - fields.locationTypeId === '537c0b5b-8ab8-fc3d-0b37-f8249cbdd3ba' && + fields.locationTypeId === "537c0b5b-8ab8-fc3d-0b37-f8249cbdd3ba" && !fields.validity?.hasOwnProperty("end") ); }) .pop()?.components; return { - changeTime: (changeTime ? changeTime : " "), - organisationName: (organisationName ? organisationName : " "), - shortName: (shortName ? shortName : organisationName), - ovoNumber: (ovoNumber ? ovoNumber : " "), - kboNumber: (kboNumber ? kboNumber : " "), - formalName: (formalName ? formalName : " "), - startDate: (startDate ? startDate : " "), + changeTime: changeTime ? changeTime : " ", + organisationName: organisationName + ? organisationName + : "Geen gegevens opgenomen in KBO", + shortName: shortName ? shortName : organisationName, + ovoNumber: ovoNumber ? ovoNumber : "Geen gegevens opgenomen in KBO", + kboNumber: kboNumber ? kboNumber : "Geen gegevens opgenomen in KBO", + formalName: formalName ? formalName : "Geen gegevens opgenomen in KBO", + startDate: startDate ? startDate : "Geen gegevens opgenomen in KBO", activeState: activeState, - rechtsvorm: (rechtsvorm ? rechtsvorm : " "), - email: (email ? email : " "), - phone: (phone ? phone : " "), - website: (website ? website : " "), - province: (province ? province : " "), - formattedAddress: (formattedAddress ? formattedAddress : " "), - adressComponent: (adressComponent ? adressComponent : " "), + rechtsvorm: rechtsvorm ? rechtsvorm : "Geen gegevens opgenomen in KBO", + email: email ? email : "Geen gegevens opgenomen in KBO", + phone: phone ? phone : "Geen gegevens opgenomen in KBO", + website: website ? website : "Geen gegevens opgenomen in KBO", + province: province ? province : "Geen gegevens opgenomen in KBO", + formattedAddress: formattedAddress + ? formattedAddress + : "Geen gegevens opgenomen in KBO", + adressComponent: adressComponent + ? adressComponent + : "Geen gegevens opgenomen in KBO", }; } @@ -179,6 +205,7 @@ new CronJob( console.log(`OVO numbers healing triggered by cron job at ${now}`); try { await healOvoNumbers(); + await healKboUnits(); } catch (err) { console.log( `An error occurred during OVO numbers healing at ${now}: ${err}` @@ -191,9 +218,11 @@ new CronJob( async function healOvoNumbers() { try { - console.log("Healing starting..."); + console.log("Healing ovo starting..."); const identifiersCouplesOP = await getAllOvoAndKboCouples(); - const identifiersCouplesWegwijs = await getAllOvoAndKboCouplesWegwijs(); + const identifiersCouplesWegwijs = await getAllOvoAndKboCouplesWegwijs( + false + ); for (const identifiersCoupleOP of identifiersCouplesOP) { const wegwijsOvo = identifiersCouplesWegwijs[identifiersCoupleOP.kbo]; @@ -217,11 +246,48 @@ async function healOvoNumbers() { } } -async function getAllOvoAndKboCouplesWegwijs() { +async function healKboUnits() { + try { + console.log("Healing kbo starting..."); + const kboIdentifiersOP = await getAllAbbKboOrganizations(); + const kboIdentifiersWegwijs = await getAllOvoAndKboCouplesWegwijs(true); + + let update = false; + + for (const kboIdentifierOP of kboIdentifiersOP) { + const wegwijsKboOrg = kboIdentifiersWegwijs[kboIdentifierOP.kbo]; + + if (wegwijsKboOrg) { + if (!kboIdentifierOP.kboOrg) { + console.log("creating new kbo unit"); + + let newKboOrgUri = await createNewKboOrg(wegwijsKboOrg, kboIdentifierOP.kboId); + + console.log("linking kbo to abb unit"); + await linkAbbOrgToKboOrg(kboIdentifierOP.abbOrg, newKboOrgUri); + } + + if (kboIdentifierOP.changeTime) { + update = new Date(kboObject.changeTime).getTime() > new Date(kboIdentifiers.changeTime).getTime() ? true : false; + } + + if (update) { + const kboIdentifiers = await getKboIdentifiers(identifiers.adminUnit); + updateKboOrg(wegwijsKboOrg, kboIdentifiers); + } + } + } + console.log("Healing complete!"); + } catch (err) { + console.log(`An error occurred during KBO organization healing: ${err}`); + } +} + +async function getAllOvoAndKboCouplesWegwijs(healKboUnit) { let couples = {}; const response = await fetch( - `https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:/.*[0-9].*/&fields=ovoNumber,kboNumber&scroll=true` + `https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:/.*[0-9].*/&changeTime,name,shortName,ovoNumber,kboNumber,labels,contacts,organisationClassifications,locations,parents&scroll=true` ); const scrollId = JSON.parse( response.headers.get("x-search-metadata") @@ -230,7 +296,12 @@ async function getAllOvoAndKboCouplesWegwijs() { do { data.forEach((unit) => { - couples[unit.kboNumber] = unit.ovoNumber; + const wegwijsUnit = getKboFields(unit); + if (healKboUnit) { + couples[wegwijsUnit.kboNumber] = wegwijsUnit; + } else { + couples[wegwijsUnit.kboNumber] = wegwijsUnit.ovoNumber; + } }); const response = await fetch( diff --git a/lib/queries.js b/lib/queries.js index 6173ad9..f3990d5 100644 --- a/lib/queries.js +++ b/lib/queries.js @@ -1,5 +1,5 @@ import { uuid, sparqlEscapeUri, sparqlEscapeString } from "mu"; -import { querySudo as query, updateSudo as update } from '@lblod/mu-auth-sudo'; +import { querySudo as query, updateSudo as update } from "@lblod/mu-auth-sudo"; export async function getIdentifiers(kboStructuredIdUuid) { const queryStr = ` @@ -10,7 +10,9 @@ export async function getIdentifiers(kboStructuredIdUuid) { } GRAPH ?g { ?kboStructuredId a ; - ${sparqlEscapeString(kboStructuredIdUuid)} ; + ${sparqlEscapeString( + kboStructuredIdUuid + )} ; ?kbo . ?kboId ?kboStructuredId ; @@ -46,13 +48,13 @@ export async function getIdentifiers(kboStructuredIdUuid) { ovoStructuredId: binding.ovoStructuredId?.value, kboId: binding.kboId?.value, adminUnit: binding.bestuurseenheid.value, - kboUnit: binding.kboOrganisation?.value - } + kboUnit: binding.kboOrganisation?.value, + }; } return null; } -export async function getKboIdentifiers(administrativeUnit, changeTime) { +export async function getKboIdentifiers(administrativeUnit) { const queryStr = ` SELECT DISTINCT ?kboUnit ?abbUnit ?changeTime ?contactPointUri ?addressUri WHERE { VALUES ?g { @@ -67,7 +69,7 @@ export async function getKboIdentifiers(administrativeUnit, changeTime) { ?contactPointUri ?addressUri . - ?kboUnit owl:sameAs ?abbUnit + ?kboUnit ?abbUnit FILTER(?abbUnit = ${sparqlEscapeUri(administrativeUnit)}) } @@ -84,29 +86,39 @@ export async function getKboIdentifiers(administrativeUnit, changeTime) { changeTime: binding.changeTime?.value, contactPointUri: binding.contactPointUri?.value, addressUri: binding.addressUri?.value, - } + }; } return null; } export async function constructOvoStructure(kboStructuredId) { const idUuid = uuid(); - const idUri = `http://data.lblod.info/id/identificatoren/${idUuid}` + const idUri = `http://data.lblod.info/id/identificatoren/${idUuid}`; const structuredIdUuid = uuid(); - const structuredIdUri = `http://data.lblod.info/id/gestructureerdeIdentificatoren/${structuredIdUuid}` + const structuredIdUri = `http://data.lblod.info/id/gestructureerdeIdentificatoren/${structuredIdUuid}`; const updateStr = ` INSERT { GRAPH ?g { - ?bestuurseenheid ${sparqlEscapeUri(idUri)} . + ?bestuurseenheid ${sparqlEscapeUri( + idUri + )} . ${sparqlEscapeUri(idUri)} a ; - ${sparqlEscapeString(idUuid)} ; + ${sparqlEscapeString( + idUuid + )} ; "OVO-nummer" ; - ${sparqlEscapeUri(structuredIdUri)} . + ${sparqlEscapeUri( + structuredIdUri + )} . - ${sparqlEscapeUri(structuredIdUri)} a ; - ${sparqlEscapeString(structuredIdUuid)} . + ${sparqlEscapeUri( + structuredIdUri + )} a ; + ${sparqlEscapeString( + structuredIdUuid + )} . } } WHERE { VALUES ?g { @@ -114,10 +126,14 @@ export async function constructOvoStructure(kboStructuredId) { } GRAPH ?g { - ${sparqlEscapeUri(kboStructuredId)} a ; + ${sparqlEscapeUri( + kboStructuredId + )} a ; ?kbo . - ?kboId ${sparqlEscapeUri(kboStructuredId)} ; + ?kboId ${sparqlEscapeUri( + kboStructuredId + )} ; ?notation . ?bestuurseenheid ?kboId . @@ -136,7 +152,9 @@ export async function updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo) { DELETE { GRAPH ?g { ?bestuureseenheid ?ovoUri . - ${sparqlEscapeUri(ovoStructuredIdUri)} ?ovo . + ${sparqlEscapeUri( + ovoStructuredIdUri + )} ?ovo . } } `; @@ -145,8 +163,14 @@ export async function updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo) { updateStr += ` INSERT { GRAPH ?g { - ?bestuureseenheid ${sparqlEscapeUri(`http://data.vlaanderen.be/id/organisatie/${wegwijsOvo}`)} . - ${sparqlEscapeUri(ovoStructuredIdUri)} ${sparqlEscapeString(wegwijsOvo)} . + ?bestuureseenheid ${sparqlEscapeUri( + `http://data.vlaanderen.be/id/organisatie/${wegwijsOvo}` + )} . + ${sparqlEscapeUri( + ovoStructuredIdUri + )} ${sparqlEscapeString( + wegwijsOvo + )} . } } `; @@ -160,19 +184,25 @@ export async function updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo) { } GRAPH ?g { - ${sparqlEscapeUri(ovoStructuredIdUri)} a . + ${sparqlEscapeUri( + ovoStructuredIdUri + )} a . OPTIONAL { ?bestuureseenheid - / ${sparqlEscapeUri(ovoStructuredIdUri)} ; + / ${sparqlEscapeUri( + ovoStructuredIdUri + )} ; ?ovoUri . } - OPTIONAL { ${sparqlEscapeUri(ovoStructuredIdUri)} ?ovo . } + OPTIONAL { ${sparqlEscapeUri( + ovoStructuredIdUri + )} ?ovo . } } } `; - + await update(updateStr); } @@ -212,19 +242,60 @@ export async function getAllOvoAndKboCouples() { if (result.results.bindings.length) { const bindings = result.results.bindings; - return bindings.map(binding => { + return bindings.map((binding) => { return { kbo: binding.kbo.value, kboStructuredId: binding.kboStructuredId.value, ovo: binding.ovo?.value, ovoStructuredId: binding.ovoStructuredId?.value, - } + }; }); } return null; } -export async function createNewKboOrg(kboOrg, kboId, isUpdate, oldKbo){ +export async function getAllAbbKboOrganizations(){ + let queryStr = ` + SELECT ?kbo ?abbOrg ?kboOrg ?changeTime ?kboId where { + GRAPH ?g { + ?kboId ?kboStructuredId ; + ?kboNotation . + + ?kboStructuredId a ; + ?kbo . + + ?abbOrg ?kboId . + + FILTER (?kboNotation IN ("KBO nummer"@nl, "KBO nummer")) + + OPTIONAL{ + ?kboOrg owl:sameAs ?abbOrg . + ?kboOrg ?changeTime . + } + } + VALUES ?g { + + } + } + ` + const result = await query(queryStr); + + if (result.results.bindings.length) { + const bindings = result.results.bindings; + return bindings.map((binding) => { + return { + kbo: binding.kbo.value, + abbOrg: binding.abbOrg.value, + kboOrg: binding.kboOrg?.value, + kboId: binding.kboId?.value, + changeTime: binding.changeTime?.value, + }; + }); + } + return null; +} + +export async function createNewKboOrg(kboOrg, kboId) { const newKboOrgIdUuId = uuid(); const newKboIdUri = `http://data.lblod.info/id/kboBestuurseenheden/${newKboOrgIdUuId}`; @@ -234,69 +305,238 @@ export async function createNewKboOrg(kboOrg, kboId, isUpdate, oldKbo){ const adressUuidId = uuid(); const adressUri = `http://data.lblod.info/id/adressen/${adressUuidId}`; - let updateStr = ""; - - updateStr += ` + console.log("creating:" + kboOrg); + console.log("with:" + kboId); + + let updateStr = ` INSERT { GRAPH ?g { ${sparqlEscapeUri(adressUri)} a ; - ${sparqlEscapeString(adressUuidId)} ; - ${sparqlEscapeString(kboOrg.formattedAddress)} . + ${sparqlEscapeString( + adressUuidId + )} ; + ${sparqlEscapeString( + kboOrg.formattedAddress + )} . ${sparqlEscapeUri(contactPointUri)} a ; - ${sparqlEscapeString(contactPointUuidId)} ; - ${sparqlEscapeString(kboOrg.email)} ; - ${sparqlEscapeString(kboOrg.phone)} ; - ${sparqlEscapeString("Kbocontact")} ; - ${sparqlEscapeString(kboOrg.website)} ; - ${sparqlEscapeUri(adressUri)}. + ${sparqlEscapeString( + contactPointUuidId + )} ; + ${sparqlEscapeString( + kboOrg.email + )} ; + ${sparqlEscapeString( + kboOrg.phone + )} ; + ${sparqlEscapeString( + "Kbocontact" + )} ; + ${sparqlEscapeString( + kboOrg.website + )} ; + ${sparqlEscapeUri( + adressUri + )}. - ${sparqlEscapeUri(newKboIdUri)} a ; - ${sparqlEscapeString(newKboOrgIdUuId)} ; - ${sparqlEscapeString(kboOrg.rechtsvorm)} ; - ${sparqlEscapeString(kboOrg.shortName)} ; - ${sparqlEscapeString(kboOrg.startDate)} ; - ${sparqlEscapeString(kboOrg.organisationName)} ; - ${sparqlEscapeString(kboOrg.formalName)} ; - ${sparqlEscapeString(kboOrg.changeTime)} ; - ${sparqlEscapeUri(kboId)} ; - ${sparqlEscapeUri(kboOrg.activeState)} ; - ${sparqlEscapeUri(contactPointUri)} . + ${sparqlEscapeUri( + newKboIdUri + )} a ; + ${sparqlEscapeString( + newKboOrgIdUuId + )} ; + ${sparqlEscapeString( + kboOrg.rechtsvorm + )} ; + ${sparqlEscapeString( + kboOrg.shortName + )} ; + ${sparqlEscapeString( + kboOrg.startDate + )} ; + ${sparqlEscapeString( + kboOrg.organisationName + )} ; + ${sparqlEscapeString( + kboOrg.formalName + )} ; + ${sparqlEscapeString( + kboOrg.changeTime + )} ; + ${sparqlEscapeUri( + kboId + )} ; + ${sparqlEscapeUri( + kboOrg.activeState + )} ; + ${sparqlEscapeUri( + contactPointUri + )} . } }WHERE{ VALUES ?g { } } - ` + `; await update(updateStr); return newKboIdUri; } -export async function linkAbbOrgToKboOrg(abbOrgUri ,kboOrgUri){ +export async function updateKboOrg(newKboInfo, identifiers) { + //update Adress + let updateStr = ` + DELETE{ + GRAPH ?g{ + ?adressUri a ; + ?fulladdress . + } + } + INSERT{ + graph ?g{ + ?adressUri a ; + ${sparqlEscapeString(newKboInfo.formattedAddress)} . + } + } + WHERE{ + VALUES ?g { + + } + GRAPH ?g { + ?adressUri a ; + ?fulladdress . + + FILTER(?adressUri = ${sparqlEscapeUri(identifiers.addressUri)}) + } + } + `; + await update(updateStr); + + //update ContactPoint + updateStr = ` + DELETE{ + GRAPH ?g{ + ?contact a ; + ?email ; + ?phone ; + ?website . +} + } + INSERT{ + graph ?g{ + ?contact a ; + ${sparqlEscapeString( + newKboInfo.email + )} ; + ${sparqlEscapeString( + newKboInfo.phone + )} ; + ${sparqlEscapeString( + newKboInfo.website + )} . + } + } + WHERE{ + VALUES ?g { + + } + GRAPH ?g { + ?contact a ; + ?email ; + ?phone ; + ?website . + + FILTER(?contact = ${sparqlEscapeUri(identifiers.contactPointUri)}) + } + } + `; + await update(updateStr); + + //update kboUnit + updateStr = ` + DELETE{ + GRAPH ?g{ + ?kboUnit a ; + ?rechtsVorm ; + ?shortName ; + ?startDate ; + ?orgName ; + ?commercialName ; + ?changeTime ; + ?activeState . + } + }INSERT{ + graph ?g{ + ?kboUnit a ; + ${sparqlEscapeString( + newKboInfo.rechtsvorm + )} ; + ${sparqlEscapeString( + newKboInfo.shortName + )} ; + ${sparqlEscapeString( + newKboInfo.startDate + )} ; + ${sparqlEscapeString( + newKboInfo.organisationName + )} ; + ${sparqlEscapeString( + newKboInfo.formalName + )} ; + ${sparqlEscapeString( + newKboInfo.changeTime + )} ; + ${sparqlEscapeUri( + newKboInfo.activeState + )} . + } +} +WHERE{ + VALUES ?g { + + } + GRAPH ?g { + ?kboUnit a ; + ?rechtsVorm ; + ?shortName ; + ?startDate ; + ?orgName ; + ?commercialName ; + ?changeTime ; + ?activeState . + + FILTER(?kboUnit = ${sparqlEscapeUri(identifiers.kboUnit)}) + } +} + `; + await update(updateStr); +} + +export async function linkAbbOrgToKboOrg(abbOrgUri, kboOrgUri) { let updateStr = ` INSERT { GRAPH ?g { - ${sparqlEscapeUri(kboOrgUri)} ${sparqlEscapeUri(abbOrgUri)} . + ${sparqlEscapeUri( + kboOrgUri + )} ${sparqlEscapeUri(abbOrgUri)} . } }WHERE { VALUES ?g { } - ${sparqlEscapeUri(kboOrgUri)} a . + ${sparqlEscapeUri( + kboOrgUri + )} a . } - ` + `; await update(updateStr); } - - - // Update queries: via mu-auth if it comes from the frontend ?