Add healing

app.js

@@ -1,8 +1,16 @@
 import { app, errorHandler } from 'mu';
 import bodyParser from 'body-parser';
 import fetch from 'node-fetch';
+import { CronJob } from 'cron';
 import { Delta } from "./lib/delta";
-import { isKbo, getIdentifiers, constructOvoStructure, updateOvoNumberAndUri } from "./lib/queries";
+import {
+  isKbo,
+  getIdentifiers,
+  constructOvoStructure,
+  updateOvoNumberAndUri,
+  getAllOvoAndKboCouples
+} from "./lib/queries";
+import { CRON_PATTERN } from './config';
 
 app.use(bodyParser.json({
   type: function (req) {
@@ -40,7 +48,7 @@ app.post('/delta', async function (req, res, next) {
             wegwijsOvo = wegwijsInfo.ovoNumber;
         }
 
-        if (wegwijsOvo != identifiers.ovo) {
+        if (wegwijsOvo && (wegwijsOvo != identifiers.ovo)) {
           let ovoStructuredIdUri = identifiers.ovoStructuredId;
 
           if (!ovoStructuredIdUri) {
@@ -60,10 +68,59 @@ app.post('/delta', async function (req, res, next) {
   }
 });
 
-// TODO CRON JOB (+ endpoint to easily test it?)
-// 1x per day. Query all admin units that have a KBO in our DB, with OVO numbers (optional)
-// Then we compare with wegwijs, if possible via one big query to get all the codes at once,
-// but if not possible multiple smaller ones
-// Then we compare the KBO / OVO couples and as soon as we have a mismatch, we correct it
+new CronJob(CRON_PATTERN, async function() {
+  const now = new Date().toISOString();
+  console.log(`OVO numbers healing triggered by cron job at ${now}`);
+  try {
+    await healOvoNumbers();
+  } catch (err) {
+    console.log(`An error occurred during OVO numbers healing at ${now}: ${err}`)
+  }
+}, null, true);
+
+async function healOvoNumbers() {
+  try {
+    console.log('Healing starting...');
+    const identifiersCouplesOP = await getAllOvoAndKboCouples();
+    const identifiersCouplesWegwijs = await getAllOvoAndKboCouplesWegwijs();
+  
+    for (const identifiersCoupleOP of identifiersCouplesOP) {
+      const wegwijsOvo = identifiersCouplesWegwijs[identifiersCoupleOP.kbo];
+      // If a KBO can't be found in wegwijs but we already have an OVO for it in OP, we keep that OVO.
+      // It happens especially a lot for worship services that sometimes lack data in Wegwijs
+      if (wegwijsOvo && (identifiersCoupleOP.ovo != wegwijsOvo)) {
+        // We have a mismatch, let's update the OVO number
+        let ovoStructuredIdUri = identifiersCoupleOP.ovoStructuredId;
+        if (!ovoStructuredIdUri) {
+          ovoStructuredIdUri = await constructOvoStructure(identifiersCoupleOP.kboStructuredId);
+        }
+
+        await updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo);
+      }
+    }
+    console.log('Healing complete!');
+  } catch (err) {
+    console.log(`An error occurred during OVO numbers healing: ${err}`)
+  }
+}
+
+async function getAllOvoAndKboCouplesWegwijs() {
+  let couples = {};
+
+  const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations?q=kboNumber:/.*[0-9].*/&fields=ovoNumber,kboNumber&scroll=true`);
+  const scrollId = JSON.parse(response.headers.get('x-search-metadata')).scrollId;
+  let data = await response.json();
+
+  do {
+    data.forEach(unit => {
+      couples[unit.kboNumber] = unit.ovoNumber;
+    });
+
+    const response = await fetch(`https://api.wegwijs.vlaanderen.be/v1/search/organisations/scroll?id=${scrollId}`);
+    data = await response.json();
+  } while (data.length)
+
+  return couples;
+}
 
 app.use(errorHandler);

config.js

@@ -0,0 +1 @@
+export const CRON_PATTERN = process.env.CRON_PATTERN || '0 0 0 * * *'; // every day at midnight

lib/queries.js

@@ -24,7 +24,6 @@ export async function isKbo(subject) {
   return result.boolean;
 }
 
-// Assumptions on the config : in the path, ?s is the subject of the delta and ?matchingUuid the id
 export async function getIdentifiers(kboStructuredId) {
   const queryStr = `
     SELECT DISTINCT ?kbo ?ovo ?ovoStructuredId WHERE {
@@ -157,4 +156,52 @@ export async function updateOvoNumberAndUri(ovoStructuredIdUri, wegwijsOvo) {
   await update(updateStr);
 }
 
+export async function getAllOvoAndKboCouples() {
+  const queryStr = `
+    SELECT DISTINCT ?kbo ?ovo ?kboStructuredId ?ovoStructuredId WHERE {
+      VALUES ?g {
+        <http://mu.semte.ch/graphs/administrative-unit>
+        <http://mu.semte.ch/graphs/worship-service>
+      }
+      GRAPH ?g {
+        ?kboId <https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ?kboStructuredId ;
+          <http://www.w3.org/2004/02/skos/core#notation> ?kboNotation .
+
+        ?kboStructuredId a <https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator> ;
+          <https://data.vlaanderen.be/ns/generiek#lokaleIdentificator> ?kbo .
+
+        ?bestuurseeheid <http://www.w3.org/ns/adms#identifier> ?kboId .
+
+        FILTER (?kboNotation IN ("KBO nummer"@nl, "KBO nummer"))
+
+        OPTIONAL {
+          ?bestuurseeheid <http://www.w3.org/ns/adms#identifier> ?ovoId .
+
+          ?ovoId <https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ?ovoStructuredId ;
+            <http://www.w3.org/2004/02/skos/core#notation> ?ovoNotation .
+
+          OPTIONAL { ?ovoStructuredId <https://data.vlaanderen.be/ns/generiek#lokaleIdentificator> ?ovo . }
+
+          FILTER (?ovoNotation IN ("OVO-nummer"@nl, "OVO-nummer"))
+        }
+      }
+    }
+  `;
+
+  const result = await query(queryStr);
+
+  if (result.results.bindings.length) {
+    const bindings = result.results.bindings;
+    return bindings.map(binding => {
+      return {
+        kbo: binding.kbo.value,
+        kboStructuredId: binding.kboStructuredId.value,
+        ovo: binding.ovo?.value,
+        ovoStructuredId: binding.ovoStructuredId?.value,
+      }
+    });
+  }
+  return null;
+}
+
 // Update queries: via mu-auth if it comes from the frontend ?

package.json

@@ -6,6 +6,7 @@
   "license": "MIT",
   "dependencies": {
     "@lblod/mu-auth-sudo": "^0.6.1",
+    "cron": "^3.0.0",
     "lodash.flatten": "4.4.0",
     "node-fetch": "^2.6.1"
   }