fixed some pr comments

This commit is contained in:
Elias Papanikitas
2024-03-13 11:21:08 +01:00
parent 9c5a6ac4f6
commit 6b559889d7
5 changed files with 280 additions and 264 deletions

49
app.js
View File

@@ -1,4 +1,4 @@
import { app, errorHandler, uuid } from "mu";
import { app, errorHandler } from "mu";
import fetch from "node-fetch";
import { CronJob } from "cron";
import {
@@ -14,53 +14,54 @@ import {
import {
CRON_PATTERN,
ORGANIZATION_STATUS,
WEGWIJSAPI,
WEGWIJSAPIFIELDS,
WEGWIJS_API,
WEGWIJS_API_FIELDS,
} from "./config";
import { API_STATUS_CODES } from "./apiErrorHandler";
import { WEGWIJS_DATA_OBJECT_IDS } from "./wegwijObjectDataIds";
import { API_STATUS_CODES } from "./api-error-handler";
import { WEGWIJS_DATA_OBJECT_IDS } from "./wegwijs-object-data-ids";
app.post("/sync-kbo-data/:kboStructuredIdUuid", async function (req, res) {
try {
const kboStructuredIdUuid = req.params.kboStructuredIdUuid;
const identifiers = await getAbbOrganizationInfo(kboStructuredIdUuid);
const AbbOrganizationInfo = await getAbbOrganizationInfo(kboStructuredIdUuid);
if (!identifiers?.kbo) {
if (!AbbOrganizationInfo?.kbo) {
return throwServerError(API_STATUS_CODES.STATUS_403, res);
}
const wegwijsUrl = `${WEGWIJSAPI}?q=kboNumber:${identifiers.kbo}&fields=${WEGWIJSAPIFIELDS}`;
const wegwijsUrl = `${WEGWIJS_API}?q=kboNumber:${AbbOrganizationInfo.kbo}&fields=${WEGWIJS_API_FIELDS}`;
console.log("url: " + wegwijsUrl);
const response = await fetch(wegwijsUrl);
const data = await response.json();
let kboObject = null;
if (!data.length) {
return throwServerError(API_STATUS_CODES.STATUS_402, res);
}
// We got a match on the KBO, getting the associated OVO back
const wegwijsInfo = data[0]; // Wegwijs should only have only one entry per KBO
kboObject = getKboFields(wegwijsInfo);
const kboIdentifiers = await getKboOrgnizationInfo(identifiers.adminUnit);
const kboObject = getKboFields(wegwijsInfo);
const kboIdentifiers = await getKboOrgnizationInfo(AbbOrganizationInfo.adminUnit);
if (!kboIdentifiers && kboObject) {
await createKbo(kboObject, identifiers.kboId, identifiers.adminUnit);
await createKbo(kboObject, AbbOrganizationInfo.kboId, AbbOrganizationInfo.adminUnit);
}
if (isUpdate(kboObject, kboIdentifiers)) {
updateKboOrg(kboObject, kboIdentifiers);
}
let wegwijsOvo = kboObject.ovoNumber ?? null;
//Update Ovo Number
if (wegwijsOvo && wegwijsOvo != identifiers.ovo) {
console.log(identifiers.ovo);
await healAbbWithWegWijsData();
let ovoStructuredIdUri = identifiers.ovoStructuredId;
let wegwijsOvo = kboObject.ovoNumber ?? null;
//Update Ovo Number
if (wegwijsOvo && wegwijsOvo != AbbOrganizationInfo.ovo) {
let ovoStructuredIdUri = AbbOrganizationInfo.ovoStructuredId;
if (!ovoStructuredIdUri) {
ovoStructuredIdUri = await constructOvoStructure(
identifiers.kboStructuredId
AbbOrganizationInfo.kboStructuredId
);
}
await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo);
@@ -167,9 +168,13 @@ async function healAbbWithWegWijsData() {
const wegwijsOvo = wegwijsKboOrg.ovoNumber;
// If a KBO can't be found in wegwijs but we already have an OVO for it in OP, we keep that OVO.
// It happens especially a lot for worship services that sometimes lack data in Wegwijs
if (wegwijsOvo && kboIdentifiersOP.ovo != wegwijsOvo) {
if (wegwijsOvo && kboIdentifierOP.ovo != wegwijsOvo) {
// We have a mismatch, let's update the OVO number
let ovoStructuredIdUri = kboIdentifiersOP.ovoStructuredId;
let ovoStructuredIdUri = kboIdentifierOP.ovoStructuredId;
console.log(ovoStructuredIdUri);
if (!ovoStructuredIdUri) {
ovoStructuredIdUri = await constructOvoStructure(
kboIdentifierOP.kboStructuredId
@@ -203,7 +208,7 @@ async function getAllOvoAndKboCouplesWegwijs() {
let couples = {};
const response = await fetch(
`${WEGWIJSAPI}?q=kboNumber:/.*[0-9].*/&fields=${WEGWIJSAPIFIELDS},parents&scroll=true`
`${WEGWIJS_API}?q=kboNumber:/.*[0-9].*/&fields=${WEGWIJS_API_FIELDS},parents&scroll=true`
);
const scrollId = JSON.parse(
response.headers.get("x-search-metadata")
@@ -216,7 +221,7 @@ async function getAllOvoAndKboCouplesWegwijs() {
couples[wegwijsUnit.kboNumber] = wegwijsUnit;
});
const response = await fetch(`${WEGWIJSAPI}/scroll?id=${scrollId}`);
const response = await fetch(`${WEGWIJS_API}/scroll?id=${scrollId}`);
data = await response.json();
} while (data.length);

View File

@@ -5,6 +5,6 @@ export const ORGANIZATION_STATUS = {
INACTIVE: "http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311"
}
export const WEGWIJSAPI = "https://api.wegwijs.vlaanderen.be/v1/search/organisations";
export const WEGWIJSAPIFIELDS = "changeTime,name,shortName,ovoNumber,kboNumber,labels,contacts,organisationClassifications,locations"
export const WEGWIJS_API = "https://api.wegwijs.vlaanderen.be/v1/search/organisations";
export const WEGWIJS_API_FIELDS = "changeTime,name,shortName,ovoNumber,kboNumber,labels,contacts,organisationClassifications,locations"

View File

@@ -23,13 +23,12 @@ export async function getAbbOrganizationInfo(kboStructuredIdUuid) {
FILTER (?kboNotation IN ("KBO nummer"@nl, "KBO nummer"))
OPTIONAL {
?bestuurseeheid <http://www.w3.org/ns/adms#identifier> ?ovoId .
?bestuurseenheid <http://www.w3.org/ns/adms#identifier> ?ovoId .
?ovoId <https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ?ovoStructuredId ;
<http://www.w3.org/2004/02/skos/core#notation> ?ovoNotation .
OPTIONAL { ?ovoStructuredId <https://data.vlaanderen.be/ns/generiek#lokaleIdentificator> ?ovo .
}
OPTIONAL { ?ovoStructuredId <https://data.vlaanderen.be/ns/generiek#lokaleIdentificator> ?ovo . }
FILTER (?ovoNotation IN ("OVO-nummer"@nl, "OVO-nummer"))
}
@@ -294,96 +293,97 @@ export async function createNewKboOrg(kboOrg, kboId) {
${sparqlEscapeUri(adressUri)} a <http://www.w3.org/ns/locn#Address> ;
<http://mu.semte.ch/vocabularies/core/uuid> ${sparqlEscapeString(
adressUuidId
)} `
if(kboOrg.formattedAddress){
)} `;
if (kboOrg.formattedAddress) {
updateStr += `;
<http://www.w3.org/ns/locn#fullAddress> ${sparqlEscapeString(
kboOrg.formattedAddress
)}`
kboOrg.formattedAddress
)}`;
}
updateStr += " ."
updateStr += " .";
updateStr += `
${sparqlEscapeUri(contactPointUri)} a <http://schema.org/ContactPoint> ;
<http://mu.semte.ch/vocabularies/core/uuid> ${sparqlEscapeString(
contactPointUuidId)} `
if(kboOrg.email){
contactPointUuidId
)} `;
if (kboOrg.email) {
updateStr += `; <http://schema.org/email> ${sparqlEscapeString(
kboOrg.email
)}`
)}`;
}
if(kboOrg.phone){
if (kboOrg.phone) {
updateStr += `; <http://schema.org/telephone> ${sparqlEscapeString(
kboOrg.phone
)}`
)}`;
}
if(kboOrg.website){
if (kboOrg.website) {
updateStr += `; <http://xmlns.com/foaf/0.1/page> ${sparqlEscapeString(
kboOrg.website
)}`
}
)}`;
}
updateStr += `
; <http://schema.org/contactType> ${sparqlEscapeString("Kbocontact")} ;
<http://www.w3.org/ns/locn#address> ${sparqlEscapeUri(adressUri)}.`
<http://www.w3.org/ns/locn#address> ${sparqlEscapeUri(adressUri)}.`;
updateStr +=` ${sparqlEscapeUri(
newKboIdUri
)} a <http://mu.semte.ch/vocabularies/ext/KboEenheid> ;
updateStr += ` ${sparqlEscapeUri(
newKboIdUri
)} a <http://mu.semte.ch/vocabularies/ext/KboEenheid> ;
<http://mu.semte.ch/vocabularies/core/uuid> ${sparqlEscapeString(
newKboOrgIdUuId
)}`
)}`;
if(kboOrg.rechtsvorm){
if (kboOrg.rechtsvorm) {
updateStr += `; <http://mu.semte.ch/vocabularies/ext/rechtsvorm> ${sparqlEscapeString(
kboOrg.rechtsvorm
)}`
}
if(kboOrg.shortName){
updateStr += `; <http://mu.semte.ch/vocabularies/ext/shortName> ${sparqlEscapeString(
)}`;
}
if (kboOrg.shortName) {
updateStr += `; <http://www.w3.org/2004/02/skos/core#altLabel> ${sparqlEscapeString(
kboOrg.shortName
)}`
}
if(kboOrg.startDate){
)}`;
}
if (kboOrg.startDate) {
updateStr += `; <http://mu.semte.ch/vocabularies/ext/startDate> ${sparqlEscapeString(
kboOrg.startDate
)}`
}
if(kboOrg.formalName){
updateStr += `; <http://www.w3.org/2004/02/skos/core#altLabel> ${sparqlEscapeString(
)}`;
}
if (kboOrg.formalName) {
updateStr += `; <http://www.w3.org/2004/02/skos/core#prefLabel> ${sparqlEscapeString(
kboOrg.formalName
)}`
}
if(kboOrg.organisationName){
updateStr += `; <http://www.w3.org/2004/02/skos/core#prefLabel> ${sparqlEscapeString(
kboOrg.organisationName
)}`
}
if(kboOrg.changeTime){
)}`;
}
// if (kboOrg.organisationName) {
// updateStr += `; <http://www.w3.org/2004/02/skos/core#prefLabel> ${sparqlEscapeString(
// kboOrg.organisationName
// )}`;
//}
if (kboOrg.changeTime) {
updateStr += `; <https://data.lblod.info/vocabularies/generiek/geplandeEindDatum> ${sparqlEscapeString(
kboOrg.changeTime
)}`
}
if(kboId){
)}`;
}
if (kboId) {
updateStr += `; <http://www.w3.org/ns/adms#identifier> ${sparqlEscapeUri(
kboId
)}`
}
if(kboOrg.activeState){
)}`;
}
if (kboOrg.activeState) {
updateStr += `; <http://www.w3.org/ns/regorg#orgStatus> ${sparqlEscapeUri(
kboOrg.activeState
)}`
}
)}`;
}
updateStr += `; <http://schema.org/contactPoint> ${sparqlEscapeUri(
contactPointUri
)} .
contactPointUri
)} .
}
}WHERE{
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
<http://mu.semte.ch/graphs/worship-service>
}
}
`;
@@ -392,197 +392,14 @@ export async function createNewKboOrg(kboOrg, kboId) {
}
export async function updateKboOrg(newKboInfo, identifiers) {
//update address
let updateStr = `
DELETE{
GRAPH ?g{
?adressUri a <http://www.w3.org/ns/locn#Address> ;
<http://www.w3.org/ns/locn#fullAddress> ?fulladdress .
}
INSERT{
graph ?g{
?adressUri a <http://www.w3.org/ns/locn#Address>
`;
if(newKboInfo.formattedAddress){
updateStr += `
;
<http://www.w3.org/ns/locn#fullAddress> ${sparqlEscapeString(
newKboInfo.formattedAddress
)}
`;
}
updateStr += `
.
}
}
`;
updateStr += `
WHERE{
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
}
GRAPH ?g {
?adressUri a <http://www.w3.org/ns/locn#Address> ;
<http://www.w3.org/ns/locn#fullAddress> ?fulladdress .
FILTER(?adressUri = ${sparqlEscapeUri(identifiers.addressUri)})
}
}
`;
await update(updateStr);
updateKboAddress(newKboInfo, identifiers);
//update ContactPoint
updateStr = `
DELETE{
GRAPH ?g{
?contact a <http://schema.org/ContactPoint> ;
<http://schema.org/email> ?email ;
<http://schema.org/telephone> ?phone ;
<http://xmlns.com/foaf/0.1/page> ?website .
}
}
INSERT{
graph ?g{
?contact a <http://schema.org/ContactPoint>
`;
if(newKboInfo.email){
updateStr += `; <http://schema.org/email> ${sparqlEscapeString(
newKboInfo.email
)}`
}
if(newKboInfo.phone){
updateStr += `; <http://schema.org/telephone> ${sparqlEscapeString(
newKboInfo.phone
)}`
}
if(newKboInfo.phone){
updateStr += `; <http://xmlns.com/foaf/0.1/page> ${sparqlEscapeString(
newKboInfo.website
)}`
}
updateStr += `
.
}
}
`;
updateStr += `
WHERE{
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
}
GRAPH ?g {
?contact a <http://schema.org/ContactPoint> ;
<http://schema.org/email> ?email ;
<http://schema.org/telephone> ?phone ;
<http://xmlns.com/foaf/0.1/page> ?website .
FILTER(?contact = ${sparqlEscapeUri(identifiers.contactPointUri)})
}
}
`;
await update(updateStr);
updateKboContact(newKboInfo, identifiers);
//update kboUnit
updateStr = `
DELETE{
GRAPH ?g{
?kboUnit a <http://mu.semte.ch/vocabularies/ext/KboEenheid> ;
<http://mu.semte.ch/vocabularies/ext/rechtsvorm> ?rechtsVorm ;
<http://mu.semte.ch/vocabularies/ext/shortName> ?shortName ;
<http://mu.semte.ch/vocabularies/ext/startDate> ?startDate ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?orgName ;
<http://www.w3.org/2004/02/skos/core#altLabel> ?commercialName ;
<https://data.lblod.info/vocabularies/generiek/geplandeEindDatum> ?changeTime ;
<http://www.w3.org/ns/regorg#orgStatus> ?activeState .
}
}
INSERT{
graph ?g{
?kboUnit a <http://mu.semte.ch/vocabularies/ext/KboEenheid>
`;
if(newKboInfo.rechtsvorm){
updateStr += `
;
<http://mu.semte.ch/vocabularies/ext/rechtsvorm> ${sparqlEscapeString(
newKboInfo.rechtsvorm
)}
`;
}
if(newKboInfo.shortName){
updateStr += `
;
<http://mu.semte.ch/vocabularies/ext/shortName> ${sparqlEscapeString(
newKboInfo.shortName
)}
`;
}
if(newKboInfo.startDate){
updateStr += `
;
<http://mu.semte.ch/vocabularies/ext/startDate> ${sparqlEscapeString(
newKboInfo.startDate
)}
`;
}
if(newKboInfo.organisationName){
updateStr += `
;
<http://www.w3.org/2004/02/skos/core#prefLabel> ${sparqlEscapeString(
newKboInfo.organisationName
)}
`;
}
if(newKboInfo.formalName){
updateStr += `
;
<http://www.w3.org/2004/02/skos/core#altLabel> ${sparqlEscapeString(
newKboInfo.formalName
)}
`;
}
if(newKboInfo.changeTime){
updateStr += `
;
<https://data.lblod.info/vocabularies/generiek/geplandeEindDatum> ${sparqlEscapeString(
newKboInfo.changeTime
)}
`;
}
if(newKboInfo.activeState){
updateStr += `
;
<http://www.w3.org/ns/regorg#orgStatus> ${sparqlEscapeUri(
newKboInfo.activeState
)}
`;
}
updateStr += `
.
}
}
`;
updateStr += `
WHERE{
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
}
GRAPH ?g {
?kboUnit a <http://mu.semte.ch/vocabularies/ext/KboEenheid> ;
<http://mu.semte.ch/vocabularies/ext/rechtsvorm> ?rechtsVorm ;
<http://mu.semte.ch/vocabularies/ext/shortName> ?shortName ;
<http://mu.semte.ch/vocabularies/ext/startDate> ?startDate ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?orgName ;
<http://www.w3.org/2004/02/skos/core#altLabel> ?commercialName ;
<https://data.lblod.info/vocabularies/generiek/geplandeEindDatum> ?changeTime ;
<http://www.w3.org/ns/regorg#orgStatus> ?activeState .
FILTER(?kboUnit = ${sparqlEscapeUri(identifiers.kboUnit)})
}
}
`;
await update(updateStr);
updateKboAdministrativeUnit(newKboInfo, identifiers);
}
export async function linkAbbOrgToKboOrg(abbOrgUri, kboOrgUri) {
@@ -597,6 +414,7 @@ export async function linkAbbOrgToKboOrg(abbOrgUri, kboOrgUri) {
}WHERE {
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
<http://mu.semte.ch/graphs/worship-service>
}
${sparqlEscapeUri(
@@ -610,4 +428,197 @@ export async function linkAbbOrgToKboOrg(abbOrgUri, kboOrgUri) {
}
}
async function updateKboAddress(newKboInfo, identifiers) {
let updateStr = `
DELETE{
GRAPH ?g{
?adressUri a <http://www.w3.org/ns/locn#Address> ;
<http://www.w3.org/ns/locn#fullAddress> ?fulladdress .
}
INSERT{
graph ?g{
?adressUri a <http://www.w3.org/ns/locn#Address>
`;
if (newKboInfo.formattedAddress) {
updateStr += `
;
<http://www.w3.org/ns/locn#fullAddress> ${sparqlEscapeString(
newKboInfo.formattedAddress
)}
`;
}
updateStr += `
.
}
}
`;
updateStr += `
WHERE{
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
}
GRAPH ?g {
?adressUri a <http://www.w3.org/ns/locn#Address> ;
<http://www.w3.org/ns/locn#fullAddress> ?fulladdress .
FILTER(?adressUri = ${sparqlEscapeUri(identifiers.addressUri)})
}
}
`;
await update(updateStr);
}
async function updateKboContact(newKboInfo, identifiers) {
let updateStr = `
DELETE{
GRAPH ?g{
?contact a <http://schema.org/ContactPoint> ;
<http://schema.org/email> ?email ;
<http://schema.org/telephone> ?phone ;
<http://xmlns.com/foaf/0.1/page> ?website .
}
}
INSERT{
graph ?g{
?contact a <http://schema.org/ContactPoint>
`;
if (newKboInfo.email) {
updateStr += `; <http://schema.org/email> ${sparqlEscapeString(
newKboInfo.email
)}`;
}
if (newKboInfo.phone) {
updateStr += `; <http://schema.org/telephone> ${sparqlEscapeString(
newKboInfo.phone
)}`;
}
if (newKboInfo.phone) {
updateStr += `; <http://xmlns.com/foaf/0.1/page> ${sparqlEscapeString(
newKboInfo.website
)}`;
}
updateStr += `
.
}
}
`;
updateStr += `
WHERE{
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
}
GRAPH ?g {
?contact a <http://schema.org/ContactPoint> ;
<http://schema.org/email> ?email ;
<http://schema.org/telephone> ?phone ;
<http://xmlns.com/foaf/0.1/page> ?website .
FILTER(?contact = ${sparqlEscapeUri(identifiers.contactPointUri)})
}
}
`;
await update(updateStr);
}
async function updateKboAdministrativeUnit(newKboInfo, identifiers) {
let updateStr = `
DELETE{
GRAPH ?g{
?kboUnit a <http://mu.semte.ch/vocabularies/ext/KboEenheid> ;
<http://mu.semte.ch/vocabularies/ext/rechtsvorm> ?rechtsVorm ;
<http://mu.semte.ch/vocabularies/ext/shortName> ?shortName ;
<http://mu.semte.ch/vocabularies/ext/startDate> ?startDate ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?orgName ;
<http://www.w3.org/2004/02/skos/core#altLabel> ?commercialName ;
<https://data.lblod.info/vocabularies/generiek/geplandeEindDatum> ?changeTime ;
<http://www.w3.org/ns/regorg#orgStatus> ?activeState .
}
}
INSERT{
graph ?g{
?kboUnit a <http://mu.semte.ch/vocabularies/ext/KboEenheid>
`;
if (newKboInfo.rechtsvorm) {
updateStr += `
;
<http://mu.semte.ch/vocabularies/ext/rechtsvorm> ${sparqlEscapeString(
newKboInfo.rechtsvorm
)}
`;
}
if (newKboInfo.shortName) {
updateStr += `
;
<http://mu.semte.ch/vocabularies/ext/shortName> ${sparqlEscapeString(
newKboInfo.shortName
)}
`;
}
if (newKboInfo.startDate) {
updateStr += `
;
<http://mu.semte.ch/vocabularies/ext/startDate> ${sparqlEscapeString(
newKboInfo.startDate
)}
`;
}
if (newKboInfo.organisationName) {
updateStr += `
;
<http://www.w3.org/2004/02/skos/core#prefLabel> ${sparqlEscapeString(
newKboInfo.organisationName
)}
`;
}
if (newKboInfo.formalName) {
updateStr += `
;
<http://www.w3.org/2004/02/skos/core#altLabel> ${sparqlEscapeString(
newKboInfo.formalName
)}
`;
}
if (newKboInfo.changeTime) {
updateStr += `
;
<https://data.lblod.info/vocabularies/generiek/geplandeEindDatum> ${sparqlEscapeString(
newKboInfo.changeTime
)}
`;
}
if (newKboInfo.activeState) {
updateStr += `
;
<http://www.w3.org/ns/regorg#orgStatus> ${sparqlEscapeUri(
newKboInfo.activeState
)}
`;
}
updateStr += `
.
}
}
`;
updateStr += `
WHERE{
VALUES ?g {
<http://mu.semte.ch/graphs/administrative-unit>
}
GRAPH ?g {
?kboUnit a <http://mu.semte.ch/vocabularies/ext/KboEenheid> ;
<http://mu.semte.ch/vocabularies/ext/rechtsvorm> ?rechtsVorm ;
<http://mu.semte.ch/vocabularies/ext/shortName> ?shortName ;
<http://mu.semte.ch/vocabularies/ext/startDate> ?startDate ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?orgName ;
<http://www.w3.org/2004/02/skos/core#altLabel> ?commercialName ;
<https://data.lblod.info/vocabularies/generiek/geplandeEindDatum> ?changeTime ;
<http://www.w3.org/ns/regorg#orgStatus> ?activeState .
FILTER(?kboUnit = ${sparqlEscapeUri(identifiers.kboUnit)})
}
}
`;
await update(updateStr);
}
// Update queries: via mu-auth if it comes from the frontend ?