chore: Bump frontend to per-release v1.24.0-0

Fix kbo-data-sync rewrite all `fullAddress` and override `sameAs`

config/authorization/config.ex

@@ -76,7 +76,8 @@ defmodule Acl.UserGroups.Config do
     "http://lblod.data.gift/vocabularies/organisatie/VeranderingsgebeurtenisResultaat",
     "http://www.w3.org/ns/org#Post",
     "http://www.w3.org/ns/org#ChangeEvent",
-    "http://data.lblod.info/vocabularies/contacthub/AgentInPositie"
+    "http://data.lblod.info/vocabularies/contacthub/AgentInPositie",
+    "http://mu.semte.ch/vocabularies/ext/KboOrganisatie",
   ]
 
   @shared_protected_type [

config/clb-consumer/contact-data/config.js

@@ -0,0 +1,30 @@
+const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 1000;
+const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
+const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
+const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
+const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
+const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
+const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 100);
+const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
+const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
+
+// Graphs
+const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/clb-contact-data`;
+
+// ENDPOINTS
+const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'database';
+const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
+
+module.exports = {
+  BATCH_SIZE,
+  MU_CALL_SCOPE_ID_INITIAL_SYNC,
+  BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+  DIRECT_DATABASE_ENDPOINT,
+  MAX_DB_RETRY_ATTEMPTS,
+  MAX_REASONING_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
+  LANDING_ZONE_GRAPH,
+  LANDING_ZONE_DATABASE_ENDPOINT
+};

config/clb-consumer/contact-data/delta-context-config.js

@@ -0,0 +1,268 @@
+
+const PREFIXES = `
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+  PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+  PREFIX lblodgeneriek: <https://data.lblod.info/vocabularies/generiek/>
+  PREFIX org: <http://www.w3.org/ns/org#>
+  PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+  PREFIX adms: <http://www.w3.org/ns/adms#>
+  PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+  PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+  PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
+  PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+  PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
+  PREFIX prov: <http://www.w3.org/ns/prov#>
+  PREFIX schema: <http://schema.org/>
+  PREFIX locn: <http://www.w3.org/ns/locn#>
+  PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>`
+
+const contextConfig = {
+  addTypes: {
+    scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
+    exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
+  },
+  contextQueries: [
+    {
+      trigger: { // subjectType or predicateValue
+        subjectType: "org:Site"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ${subject} ext:goesInGraph ?graphOne.
+          ${subject} ext:goesInGraph ?graphTwo.
+          ${subject} ext:goesInGraph ?graphThree.
+          ${subject} ext:goesInGraph ?graphFour.
+        } WHERE {
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+              ?adminUnitOne org:hasPrimarySite ${subject}.
+            }
+            BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+              ?adminUnitTwo org:hasSite ${subject}.
+            }
+            BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphTwo)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/worship-service> {
+              ?adminUnitThree org:hasPrimarySite ${subject}.
+            }
+            BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphThree)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/worship-service> {
+              ?adminUnitFour org:hasSite ${subject}.
+            }
+            BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphFour)
+          }
+        }`
+    },
+    {
+      trigger: { // subjectType or predicateValue
+        subjectType: "schema:ContactPoint"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ${subject} ext:goesInGraph ?graphOne.
+          ${subject} ext:goesInGraph ?graphTwo.
+        } WHERE {
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+              ?siteOne org:siteAddress ${subject}.
+            }
+            BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/worship-service> {
+              ?siteTwo org:siteAddress ${subject}.
+            }
+            BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphTwo)
+          }
+        }`
+    },
+    {
+      trigger: { // subjectType or predicateValue
+        subjectType: "locn:Address"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ${subject} ext:goesInGraph ?graphOne.
+          ${subject} ext:goesInGraph ?graphTwo.
+        } WHERE {
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+              ?siteOne organisatie:bestaatUit ${subject}.
+            }
+            BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/worship-service> {
+              ?siteTwo organisatie:bestaatUit ${subject}.
+            }
+            BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphTwo)
+          }
+        }`
+    },
+    //Predicate triggers to solve out of order deltas
+    {
+      trigger: { // subjectType or predicateValue
+        predicateValue: "org:hasSite"
+      },
+      //We assume that only administrative units are in the shared graph of OP due to the need of sharing the prefLabel with worships services
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ${subject} ext:goesInGraph ?graphOne.
+          ${subject} ext:goesInGraph ?graphTwo.
+          ?site ?pSite ?oSite.
+          ?contact ?pContact ?oContact.
+          ?address ?pAddress ?oAddress.
+          ?addressContact ?pAddressContact ?oAddressContact.
+        } WHERE {
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+              ${subject} a ?type.
+            }
+            BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/shared> {
+              ${subject} a ?type.
+            }
+            BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/worship-service> {
+              ${subject} a ?type.
+            }
+            BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphTwo)
+          }
+          ${subject} org:hasSite ?site.
+          ?site ?pSite ?oSite.
+          {
+            ?site org:siteAddress ?contact.
+            ?contact ?pContact ?oContact.
+            ?contact locn:address ?addressContact.
+            ?addressContact ?pAddressContact ?oAddressContact.
+          }
+          UNION
+          {
+            ?site org:siteAddress ?contact.
+            ?contact ?pContact ?oContact.
+          }
+          UNION
+          {
+            ?site organisatie:bestaatUit ?address.
+            ?address ?pAddress ?oAddress.
+          }        
+        }`
+    },
+    {
+      trigger: { // subjectType or predicateValue
+        predicateValue: "org:hasPrimarySite"
+      },
+      //We assume that only administrative units are in the shared graph of OP due to the need of sharing the prefLabel with worships services
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ${subject} ext:goesInGraph ?graphOne.
+          ${subject} ext:goesInGraph ?graphTwo.
+          ?site ?pSite ?oSite.
+          ?contact ?pContact ?oContact.
+          ?address ?pAddress ?oAddress.
+          ?addressContact ?pAddressContact ?oAddressContact.
+        } WHERE {
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+              ${subject} a ?type.
+            }
+            BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/shared> {
+              ${subject} a ?type.
+            }
+            BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
+          }
+          OPTIONAL {
+            GRAPH <http://mu.semte.ch/graphs/worship-service> {
+              ${subject} a ?type.
+            }
+            BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphTwo)
+          }
+          ${subject} org:hasPrimarySite ?site.
+          ?site ?pSite ?oSite.
+          {
+            ?site org:siteAddress ?contact.
+            ?contact ?pContact ?oContact.
+            ?contact locn:address ?addressContact.
+            ?addressContact ?pAddressContact ?oAddressContact.
+          }
+          UNION
+          {
+            ?site org:siteAddress ?contact.
+            ?contact ?pContact ?oContact.
+          }
+          UNION
+          {
+            ?site organisatie:bestaatUit ?address.
+            ?address ?pAddress ?oAddress.
+          }        
+        }`
+    },
+    {
+      trigger: { // subjectType or predicateValue
+        predicateValue: "org:siteAddress"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ?contact ?pContact ?oContact.
+          ?addressContact ?pAddressContact ?oAddressContact.
+        } WHERE {
+          ${subject} org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+          OPTIONAL {
+            ?contact locn:address ?addressContact.
+            ?addressContact ?pAddressContact ?oAddressContact.
+          }
+        }`
+    },
+    {
+      trigger: { // subjectType or predicateValue
+        predicateValue: "locn:address"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ?addressContact ?pAddressContact ?oAddressContact.
+        } WHERE {
+          ${subject} locn:address ?addressContact.
+          ?addressContact ?pAddressContact ?oAddressContact.
+        }`
+    },
+    {
+      trigger: { // subjectType or predicateValue
+        predicateValue: "organisatie:bestaatUit"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ?address ?pAddress ?oAddress.
+        } WHERE {
+          ${subject} organisatie:bestaatUit ?address.
+          ?address ?pAddress ?oAddress.
+        }`
+    }
+  ]
+}
+
+module.exports = {
+  contextConfig,
+  PREFIXES
+};

config/clb-consumer/contact-data/delta-sync-dispatching.js

@@ -0,0 +1,104 @@
+const {
+  deleteFromPublicGraph,
+  insertIntoPublicGraph,
+  deleteFromSpecificGraphs,
+  insertIntoSpecificGraphs
+} = require('./util');
+
+const {
+  LANDING_ZONE_GRAPH
+} = require('./config')
+
+const publicTypes = [
+  '<http://lblod.data.gift/vocabularies/organisatie/TypeVestiging>'
+]
+
+/**
+* Dispatch the fetched information to a target graph.
+* Note: <share://file/data> will be ADDED to it's own graph.
+*   We take only care of adding them, not updating triples, this is a TODO
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
+*          [ {
+*              graph: "<http://foo>",
+*              subject: "<http://bar>",
+*              predicate: "<http://baz>",
+*              object: "<http://boom>^^<http://datatype>"
+*            }
+*         ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+  let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
+
+  // Both arrays are the same length, so we can zip them together for easier processing
+  zippedChangeSets = termObjectChangeSets.map((o, i) => ({
+    original: o,
+    withContext: termObjectChangeSetsWithContext && termObjectChangeSetsWithContext[i]
+  }));
+
+  console.log(`Received ${zippedChangeSets.length} change sets`)
+
+  for (let { original, withContext } of zippedChangeSets) {
+    
+    const {public: insertsOnPublic, target: insertsOnGraphs} = separateBetweenTargetAndPublic(original.inserts, withContext.inserts)
+    const {public: deletesOnPublic, target: deletesOnGraphs} = separateBetweenTargetAndPublic(original.deletes, withContext.deletes)
+
+    await deleteFromPublicGraph(lib, deletesOnPublic);
+    await deleteFromSpecificGraphs(lib, deletesOnGraphs);
+    await insertIntoPublicGraph(lib, insertsOnPublic);
+    await insertIntoSpecificGraphs(lib, insertsOnGraphs);
+
+  }
+}
+
+function separateBetweenTargetAndPublic(changes, context) {
+  const changesOnPublic = new Set();
+  const changesOnGraphs = {};
+  for(let change of changes) {
+    const subject = change.subject;
+    const contextTriples = context.filter((context) => context.subject === subject);
+    const graphTriple = contextTriples.find(
+      (context) => context.predicate === '<http://mu.semte.ch/vocabularies/ext/goesInGraph>' 
+        && context.object !== `<${LANDING_ZONE_GRAPH}>`
+    )
+    const otherContextTriples = context.filter(
+      (context) => context.predicate !== '<http://mu.semte.ch/vocabularies/ext/goesInGraph>' 
+        && context.predicate !== '<http://mu.semte.ch/vocabularies/ext/contextDataGoesInGraph>' 
+        && !(context.subject === subject && context.predicate === '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>')
+    );
+    if(graphTriple) {
+      const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">"
+      if(change.object !== '') {
+        if(!changesOnGraphs[graph]) {
+          changesOnGraphs[graph] = new Set();
+        }  
+        changesOnGraphs[graph].add(`${change.subject} ${change.predicate} ${change.object}.`)
+      }                    
+      for(let triple of otherContextTriples) {
+        if(triple.object === '') continue;
+        if(!changesOnGraphs[graph]) {
+          changesOnGraphs[graph] = new Set();
+        }  
+        changesOnGraphs[graph].add(`${triple.subject} ${triple.predicate} ${triple.object}.`)                                       
+      }
+    }
+    const typeTriple = contextTriples.find(
+      (context) => context.subject === subject 
+        && context.predicate === '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>'
+    );
+    if(!typeTriple) continue;
+    const type = typeTriple.object;
+    if(publicTypes.includes(type)) {
+      changesOnPublic.add(`${change.subject} ${change.predicate} ${change.object}.`)
+      for(let triple of otherContextTriples) {
+        changesOnPublic.add(`${triple.subject} ${triple.predicate} ${triple.object}.`)
+      }
+    }
+  }
+  return {public: changesOnPublic, target: changesOnGraphs}
+}
+
+module.exports = {
+  dispatch
+};

config/clb-consumer/contact-data/initial-sync-dispatching.js

@@ -0,0 +1,69 @@
+const { batchedDbUpdate, moveDataAfterInitialSync } = require('./util');
+const {
+  BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+  DIRECT_DATABASE_ENDPOINT,
+  MU_CALL_SCOPE_ID_INITIAL_SYNC,
+  BATCH_SIZE,
+  MAX_DB_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  LANDING_ZONE_GRAPH,
+} = require('./config');
+
+const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
+
+/**
+* Dispatch the fetched information to a target graph.
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
+*          [ {
+*              graph: "<http://foo>",
+*              subject: "<http://bar>",
+*              predicate: "<http://baz>",
+*              object: "<http://boom>^^<http://datatype>"
+*            }
+*         ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+  const { mu, muAuthSudo } = lib;
+  const { termObjects } = data;
+
+  // Steps:
+  // 1. Write the triples to the landing zone graph - without any mapping or filtering
+  // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
+
+  if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
+    console.warn(`Service configured to skip MU_AUTH!`);
+  }
+  console.log(`Using ${endpoint} to insert triples`);
+
+  if (termObjects.length) {
+    const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+
+    await batchedDbUpdate(
+      muAuthSudo.updateSudo,
+      LANDING_ZONE_GRAPH,
+      originalInsertTriples,
+      { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+      endpoint,
+      BATCH_SIZE,
+      MAX_DB_RETRY_ATTEMPTS,
+      SLEEP_BETWEEN_BATCHES,
+      SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+    );
+  }
+}
+
+async function onFinishInitialIngest(lib,data) {
+  const { muAuthSudo } = lib;
+
+  console.log(`!! On-finish triggered !!`);
+
+  await moveDataAfterInitialSync(muAuthSudo.updateSudo, endpoint)
+}
+
+module.exports = {
+  dispatch,
+  onFinishInitialIngest
+};

config/clb-consumer/contact-data/util.js

@@ -0,0 +1,337 @@
+const {
+  MAX_DB_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  LANDING_ZONE_GRAPH,
+  BATCH_SIZE,
+} = require('./config');
+
+const prefixes = `
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX lblodgeneriek: <https://data.lblod.info/vocabularies/generiek/>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
+PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX schema: <http://schema.org/>
+PREFIX locn: <http://www.w3.org/ns/locn#>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX ext:<http://mu.semte.ch/vocabularies/ext/>
+`
+
+async function batchedDbUpdate(
+  muUpdate,
+  graph,
+  triples,
+  extraHeaders,
+  endpoint,
+  batchSize,
+  maxAttempts,
+  sleepBetweenBatches = 1000,
+  sleepTimeOnFail = 1000,
+  operation = 'INSERT'
+) {
+  for (let i = 0; i < triples.length; i += batchSize) {
+    console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
+
+    const batch = triples.slice(i, i + batchSize).join('\n');
+
+    const insertCall = async () => {
+      await muUpdate(`
+${operation} DATA {
+GRAPH <${graph}> {
+${batch}
+}
+}
+`, extraHeaders, endpoint);
+    };
+
+    await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
+
+    console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
+    await new Promise(r => setTimeout(r, sleepBetweenBatches));
+  }
+}
+
+async function operationWithRetry(callback,
+  attempt,
+  maxAttempts,
+  sleepTimeOnFail) {
+  try {
+    if (typeof callback === "function")
+      return await callback();
+    else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
+      return await callback;
+  }
+  catch (e) {
+    console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
+    console.log(`Error: ${e}`);
+    console.log(`Sleeping ${sleepTimeOnFail} ms`);
+
+    if (attempt >= maxAttempts) {
+      console.log(`Max attempts reached for ${callback.toString()}, giving up`);
+      throw e;
+    }
+
+    await new Promise(r => setTimeout(r, sleepTimeOnFail));
+    return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
+  }
+}
+
+
+async function insertIntoPublicGraph(lib, statements) {
+  console.log(`Inserting ${statements.length} statements into public graph`);
+
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    'http://mu.semte.ch/graphs/public',
+    Array.from(statements),
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'INSERT');
+}
+
+async function insertIntoSpecificGraphs(lib, statementsWithGraphs) {
+
+  for( let graph in statementsWithGraphs) {
+    console.log(`Inserting ${statementsWithGraphs[graph].length} statements into ${graph} graph`);
+    await batchedDbUpdate(
+      lib.muAuthSudo.updateSudo,
+      graph,
+      Array.from(statementsWithGraphs[graph]),
+      {},
+      process.env.MU_SPARQL_ENDPOINT,
+      BATCH_SIZE,
+      MAX_DB_RETRY_ATTEMPTS,
+      SLEEP_BETWEEN_BATCHES,
+      SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+      'INSERT');
+  }
+  
+}
+
+async function deleteFromPublicGraph(lib, statements) {
+  console.log(`Deleting ${statements.length} statements from public graph`);
+
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    'http://mu.semte.ch/graphs/public',
+    Array.from(statements),
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'DELETE');
+}
+
+async function deleteFromSpecificGraphs(lib, statementsWithGraphs) {
+  
+
+  for( let graph in statementsWithGraphs) {
+    console.log(`Deleting ${statementsWithGraphs[graph].length} statements from ${graph} graph`);
+    await batchedDbUpdate(
+      lib.muAuthSudo.updateSudo,
+      graph,
+      Array.from(statementsWithGraphs[graph]),
+      {},
+      process.env.MU_SPARQL_ENDPOINT,
+      BATCH_SIZE,
+      MAX_DB_RETRY_ATTEMPTS,
+      SLEEP_BETWEEN_BATCHES,
+      SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+      'DELETE');
+  }
+  
+}
+
+
+async function moveDataAfterInitialSync(muUpdate, endpoint) {
+  // Move primary sites admin units
+  await muUpdate(`
+    ${prefixes}
+    INSERT {
+      GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+        ?adminUnit org:hasPrimarySite ?site.
+        ?site ?pSite ?oSite.
+        ?contact ?pContact ?oContact.
+        ?address ?pAddress ?oAddress.
+        ?addressContact ?pAddressContact ?oAddressContact.
+      }
+    } WHERE {
+      GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+        ?adminUnit a ?type.
+      }
+      GRAPH <${LANDING_ZONE_GRAPH}> {
+        ?adminUnit org:hasPrimarySite ?site.
+        ?site ?pSite ?oSite.
+        {
+          ?site org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+          ?contact locn:address ?addressContact.
+          ?addressContact ?pAddressContact ?oAddressContact.
+        }
+        UNION
+        {
+          ?site org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+        }
+        UNION
+        {
+          ?site organisatie:bestaatUit ?address.
+          ?address ?pAddress ?oAddress.
+        }
+      }        
+    }
+  `, undefined, endpoint)
+
+  // Move sites administrative units
+  await muUpdate(`
+    ${prefixes}
+    INSERT {
+      GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+        ?adminUnit org:hasSite ?site.
+        ?site ?pSite ?oSite.
+        ?contact ?pContact ?oContact.
+        ?address ?pAddress ?oAddress.
+        ?addressContact ?pAddressContact ?oAddressContact.
+      }
+    } WHERE {
+      GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+        ?adminUnit a ?type.
+      }
+      GRAPH <${LANDING_ZONE_GRAPH}> {
+        ?adminUnit org:hasSite ?site.
+        ?site ?pSite ?oSite.
+        {
+          ?site org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+          ?contact locn:address ?addressContact.
+          ?addressContact ?pAddressContact ?oAddressContact.
+        }
+        UNION
+        {
+          ?site org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+        }
+        UNION
+        {
+          ?site organisatie:bestaatUit ?address.
+          ?address ?pAddress ?oAddress.
+        }     
+      }   
+    }
+  `, undefined, endpoint)
+
+  // Move primary sites worships
+  await muUpdate(`
+    ${prefixes}
+    INSERT {
+      GRAPH <http://mu.semte.ch/graphs/worship-service> {
+        ?adminUnit org:hasPrimarySite ?site.
+        ?site ?pSite ?oSite.
+        ?contact ?pContact ?oContact.
+        ?address ?pAddress ?oAddress.
+        ?addressContact ?pAddressContact ?oAddressContact.
+      }
+    } WHERE {
+      GRAPH <http://mu.semte.ch/graphs/worship-service> {
+        ?adminUnit a ?type.
+      }
+      GRAPH <${LANDING_ZONE_GRAPH}> {
+        ?adminUnit org:hasPrimarySite ?site.
+        ?site ?pSite ?oSite.
+        {
+          ?site org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+          ?contact locn:address ?addressContact.
+          ?addressContact ?pAddressContact ?oAddressContact.
+        }
+        UNION
+        {
+          ?site org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+        }
+        UNION
+        {
+          ?site organisatie:bestaatUit ?address.
+          ?address ?pAddress ?oAddress.
+        }        
+      }
+    }
+  `, undefined, endpoint)
+
+  // Move sites worships
+  await muUpdate(`
+    ${prefixes}
+    INSERT {
+      GRAPH <http://mu.semte.ch/graphs/worship-service> {
+        ?adminUnit org:hasSite ?site.
+        ?site ?pSite ?oSite.
+        ?contact ?pContact ?oContact.
+        ?address ?pAddress ?oAddress.
+        ?addressContact ?pAddressContact ?oAddressContact.
+      }
+    } WHERE {
+      GRAPH <http://mu.semte.ch/graphs/worship-service> {
+        ?adminUnit a ?type.
+      }
+      GRAPH <${LANDING_ZONE_GRAPH}> {
+        ?adminUnit org:hasSite ?site.
+        ?site ?pSite ?oSite.
+        {
+          ?site org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+          ?contact locn:address ?addressContact.
+          ?addressContact ?pAddressContact ?oAddressContact.
+        }
+        UNION
+        {
+          ?site org:siteAddress ?contact.
+          ?contact ?pContact ?oContact.
+        }
+        UNION
+        {
+          ?site organisatie:bestaatUit ?address.
+          ?address ?pAddress ?oAddress.
+        }        
+      }
+    }
+  `, undefined, endpoint)
+
+  // Move sites worships
+  await muUpdate(`
+    ${prefixes}
+    INSERT {
+      GRAPH <http://mu.semte.ch/graphs/public> {
+        ?type a <http://lblod.data.gift/vocabularies/organisatie/TypeVestiging>.
+        ?type ?a ?b.
+      }
+    } WHERE {
+      GRAPH <${LANDING_ZONE_GRAPH}> {
+        ?type a <http://lblod.data.gift/vocabularies/organisatie/TypeVestiging>.
+        ?type ?a ?b.
+      }
+    }
+  `, undefined, endpoint)
+}
+
+module.exports = {
+  batchedDbUpdate,
+  moveDataAfterInitialSync,
+  insertIntoPublicGraph,
+  deleteFromPublicGraph,
+  insertIntoSpecificGraphs,
+  deleteFromSpecificGraphs,
+};

config/delta-producer/background-job-initiator/config.json

@@ -0,0 +1,12 @@
+[
+    {
+        "name": "organizations",
+        "jobsGraph": "http://mu.semte.ch/graphs/jobs",
+        "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations",
+        "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations",
+        "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations",
+        "cronPatternDumpJob": "42 * * * *",
+        "cronPatternHealingJob": "0 0 4 * * *",
+        "startInitialSync": true
+    }
+]

config/delta-producer/dump-file-publisher/config.json

@@ -3,7 +3,7 @@
     "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
     "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
     "fileBaseName": "dump-organizations",
-    "publicationGraphEndpoint": "http://triplestore:8890/sparql",
+    "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql",
     "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
     "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
     "cleanupOldDumps": true
@@ -12,7 +12,7 @@
     "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
     "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
     "fileBaseName": "dump-organizations",
-    "publicationGraphEndpoint": "http://triplestore:8890/sparql",
+    "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql",
     "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
     "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
     "cleanupOldDumps": true

config/delta-producer/organizations-public-info/export.json

@@ -1,7 +1,6 @@
 {
   "conceptScheme": "http://lblod.data.gift/concept-schemes/0d101f02-609d-4cde-bb07-639d874470c4",
   "export": [
-
     {
       "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
       "strictTypeExport": "true",
@@ -13,6 +12,7 @@
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
         "http://www.w3.org/2004/02/skos/core#altLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/regorg#orgStatus",
         "http://www.w3.org/ns/org#hasPrimarySite",
@@ -40,6 +40,7 @@
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
         "http://www.w3.org/2004/02/skos/core#altLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/regorg#orgStatus",
         "http://www.w3.org/ns/org#hasPrimarySite",
@@ -72,6 +73,7 @@
         "http://www.opengis.net/ont/geosparql#sfWithin",
         "http://www.w3.org/2002/07/owl#sameAs",
         "http://www.w3.org/2004/02/skos/core#altLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/org#changedBy",
@@ -98,7 +100,11 @@
         "http://www.w3.org/2004/02/skos/core#prefLabel",
         "http://www.w3.org/ns/org#linkedTo",
         "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
-        "http://www.w3.org/ns/adms#identifier"
+        "http://www.w3.org/ns/adms#identifier",
+        "http://www.w3.org/ns/regorg#legalName",
+        "http://www.w3.org/2004/02/skos/core#altLabel",
+        "http://www.w3.org/ns/org#hasPrimarySite",
+        "http://www.w3.org/ns/org#hasSite"
       ]
     },
     {
@@ -371,4 +377,4 @@
       ]
     }
   ]
-}
+}

config/delta-producer/organizations/export.json

@@ -65,6 +65,7 @@
         "http://schema.org/email",
         "http://schema.org/faxNumber",
         "http://schema.org/telephone",
+        "http://xmlns.com/foaf/0.1/page",
         "http://www.w3.org/ns/locn#address"
       ]
     },
@@ -126,6 +127,7 @@
       ],
       "properties": [
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/regorg#orgStatus",
@@ -154,6 +156,7 @@
       ],
       "properties": [
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/regorg#orgStatus",
@@ -170,6 +173,41 @@
         "http://data.lblod.info/vocabularies/erediensten/typeEredienst"
       ]
     },
+    {
+      "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
+      "strictTypeExport": "true",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/worship-service",
+        "http://mu.semte.ch/graphs/shared",
+        "http://mu.semte.ch/graphs/administrative-unit"
+      ],
+      "properties": [
+        "http://mu.semte.ch/vocabularies/core/uuid",
+        "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur",
+        "http://data.lblod.info/vocabularies/erediensten/denominatie",
+        "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend",
+        "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
+        "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor",
+        "http://data.vlaanderen.be/ns/besluit#werkingsgebied",
+        "http://www.opengis.net/ont/geosparql#sfWithin",
+        "http://www.w3.org/2002/07/owl#sameAs",
+        "http://www.w3.org/2004/02/skos/core#altLabel",
+        "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
+        "http://www.w3.org/ns/adms#identifier",
+        "http://www.w3.org/ns/org#changedBy",
+        "http://www.w3.org/ns/org#classification",
+        "http://www.w3.org/ns/org#hasPost",
+        "http://www.w3.org/ns/org#hasPrimarySite",
+        "http://www.w3.org/ns/org#hasSite",
+        "http://www.w3.org/ns/org#hasSubOrganization",
+        "http://www.w3.org/ns/org#linkedTo",
+        "http://www.w3.org/ns/org#resultedFrom",
+        "http://www.w3.org/ns/regorg#orgStatus",
+        "https://data.lblod.info/vocabularies/generiek/geplandeEindDatum",
+        "http://www.w3.org/ns/org#purpose"
+      ]
+    },
     {
       "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan",
       "strictTypeExport": "true",
@@ -179,6 +217,7 @@
       ],
       "properties": [
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/regorg#orgStatus",
@@ -317,6 +356,18 @@
         "http://www.w3.org/2004/02/skos/core#prefLabel"
       ]
     },
+    {
+      "type": "http://www.w3.org/ns/prov#Location",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/shared"
+      ],
+      "properties": [
+        "http://www.w3.org/2000/01/rdf-schema#label",
+        "http://www.opengis.net/ont/geosparql#sfWithin",
+        "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau",
+        "http://www.w3.org/2004/02/skos/core#exactMatch"
+      ]
+    },
     {
       "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode",
       "graphsFilter": [

config/delta-producer/public/export.json

@@ -15,6 +15,7 @@
       "properties": [
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/regorg#orgStatus",
@@ -46,6 +47,7 @@
       "properties": [
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/regorg#orgStatus",
@@ -84,6 +86,7 @@
         "http://www.w3.org/2002/07/owl#sameAs",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/ns/adms#identifier",
         "http://www.w3.org/ns/org#changedBy",
         "http://www.w3.org/ns/org#classification",
@@ -105,6 +108,8 @@
       "properties": [
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
+        "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/org#linkedTo",
         "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
         "http://www.w3.org/ns/adms#identifier"

config/delta-producer/publication-graph-maintainer/config.json

@@ -0,0 +1,25 @@
+{
+    "organizations": {
+        "deltaInterval": 10000,
+        "deltaPath": "/organizations/delta",
+        "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations",
+        "exportConfigPath": "/config/organizations/export.json",
+        "filesPath": "/organizations/files",
+        "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations",
+        "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations",
+        "jobsGraph": "http://mu.semte.ch/graphs/jobs",
+        "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
+        "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations",
+        "queuePollInterval": 3000,
+        "relativeFilePath": "/deltas/organizations",
+        "serveDeltaFiles": true,
+        "skipMuAuthInitialSync": true,
+        "skipMuAuthHealing": true,
+        "useFileDiff": true,
+        "healingInitialBatchSizeInsert": 1000,
+        "useVirtuosoForExpensiveSelects": true,
+        "errorGraph": "http://mu.semte.ch/graphs/jobs",
+        "loginPath": "/organizations/login",
+        "logOutgoingDelta": true
+    }
+}

config/delta-producer/worship-posts/export.json

@@ -22,6 +22,7 @@
       "properties": [
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/regorg#orgStatus",
         "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
@@ -42,6 +43,7 @@
       "properties": [
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/regorg#orgStatus",
         "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
@@ -61,6 +63,7 @@
       "properties": [
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/regorg#orgStatus",
         "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
@@ -121,6 +124,7 @@
       "properties": [
         "http://mu.semte.ch/vocabularies/core/uuid",
         "http://www.w3.org/2004/02/skos/core#prefLabel",
+        "http://www.w3.org/ns/regorg#legalName",
         "http://www.w3.org/2004/02/skos/core#altLabel",
         "http://www.w3.org/ns/regorg#orgStatus",
         "http://www.w3.org/ns/org#hasPrimarySite",

config/delta/rules.js

@@ -92,7 +92,7 @@ export default [
     match: {
     },
     callback: {
-      url: 'http://delta-producer-pub-graph-maintainer-organizations/delta',
+      url: 'http://delta-producer-publication-graph-maintainer/organizations/delta',
       method: 'POST'
     },
     options: {

config/dispatcher/dispatcher.ex

@@ -219,6 +219,10 @@ defmodule Dispatcher do
     Proxy.forward conn, path, "http://cache/decision-activities/"
   end
 
+  match "/kbo-organizations/*path", %{ accept: [:any], layer: :api} do
+    Proxy.forward conn, path, "http://resource/kbo-organizations/"
+  end
+
   ###############
   # LOGIN
   ###############
@@ -308,8 +312,13 @@ defmodule Dispatcher do
     forward conn, path, "http://construct-administrative-unit-relationships/create-relationships/"
   end
 
+  post "/kbo-data-sync/*path", %{ layer: :api_services, accept: %{ json: true } } do
+    forward conn, path, "http://kbo-data-sync/sync-kbo-data/"
+  end
+
+  # TODO: Remove when the kbo feature flag is removed on the frontend
   post "/sync-ovo-number/*path", %{ layer: :api_services, accept: %{ json: true } } do
-    forward conn, path, "http://sync-ovo-numbers/sync-from-kbo/"
+    forward conn, path, "http://kbo-data-sync/sync-kbo-data/"
   end
 
   get "/ldes/*path", %{ layer: :api_services } do
@@ -353,7 +362,7 @@ defmodule Dispatcher do
   #################################################################
 
   get "/sync/organizations/files/*path" do
-    Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/"
+    Proxy.forward conn, path, "http://delta-producer-publication-graph-maintainer/files/"
   end
 
   #################################################################

config/migrations/2024/20240327131000-delete-all-empty-literals.sparql

@@ -0,0 +1,20 @@
+DELETE {
+  GRAPH ?g { 
+    ?s ?p ?o .
+  }
+}
+WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/administrative-unit>
+    <http://mu.semte.ch/graphs/worship-service>
+    <http://mu.semte.ch/graphs/public>
+    <http://mu.semte.ch/graphs/privacy-centric-graph>
+    <http://mu.semte.ch/graphs/worship-privacy-centric-graph>
+  }
+  graph ?g {
+    ?s ?p ?o
+
+    FILTER (isLiteral(?o))
+    FILTER regex(str(?o), "^\\s*$")
+  }
+}

config/migrations/2024/20240403104506-remove-empty-alternative-names.sparql

@@ -0,0 +1,14 @@
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?s skos:altLabel ?alt .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?s a besluit:Bestuurseenheid ;
+      skos:altLabel ?alt .
+    FILTER (strLen(?alt) = 0)
+  }
+}

config/migrations/2024/20240415085335-correct-representative-body-classification-code-type.sparql

@@ -0,0 +1,15 @@
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/public> {
+    ?s a ext:RepresentatieveOrgaanClassificatieCode .
+  }
+} INSERT {
+  GRAPH <http://mu.semte.ch/graphs/public> {
+    ?s a ext:RepresentatiefOrgaanClassificatieCode .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/public> {
+    ?s a ext:RepresentatieveOrgaanClassificatieCode .
+  }
+}

config/migrations/2024/20240429105000-clear-producer-organizations-graph.sparql

@@ -0,0 +1,3 @@
+# The producer organizations graph is now in the publication-triplestore
+# Lets remove it from the main triplestore
+CLEAR GRAPH <http://redpencil.data.gift/id/deltas/producer/organizations>

config/migrations/2024/20240429134438-remove-dangling-change-event-results.sparql

@@ -0,0 +1,35 @@
+# The change events related to organisation <http://data.lblod.info/id/besturenVanDeEredienst/a8d9f1a45c11c6761306bdf680349593>
+# contains some data errors:
+# - There is a change event result without any corresponding change event. This
+#   causes an error when visiting the change events page for this organisation.
+# - There are two identical change event results for the same change event. This
+#   causes the change event to be shown twice in the table. (Only on PROD)
+#
+# Note: may require restarting the cache before the changes are noticeable.
+
+PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+
+# Remove the dangling change event result
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    <http://lblod.data.info/id/veranderingsgebeurtenis-resultaten/6346C0C38DE5818A7C9F40D8> ?p ?o .
+  }
+}
+WHERE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    <http://lblod.data.info/id/veranderingsgebeurtenis-resultaten/6346C0C38DE5818A7C9F40D8> ?p ?o .
+  }
+}
+
+# Remove one of the duplicate change event results for the change event
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    <http://data.lblod.info/id/veranderingsgebeurtenissen/63C512F34E1F899807CFFB75> code:veranderingsgebeurtenisResultaat <http://lblod.data.info/id/veranderingsgebeurtenis-resultaten/63C512F74E1F899807CFFB77> .
+    <http://lblod.data.info/id/veranderingsgebeurtenis-resultaten/63C512F74E1F899807CFFB77> ?p ?o .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    <http://data.lblod.info/id/veranderingsgebeurtenissen/63C512F34E1F899807CFFB75> code:veranderingsgebeurtenisResultaat <http://lblod.data.info/id/veranderingsgebeurtenis-resultaten/63C512F74E1F899807CFFB77> .
+    <http://lblod.data.info/id/veranderingsgebeurtenis-resultaten/63C512F74E1F899807CFFB77> ?p ?o .
+  }
+}

config/reasoner/dl2op/private-info/mapping.n3

@@ -17,6 +17,7 @@ PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
 PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
 PREFIX string: <http://www.w3.org/2000/10/swap/string#>
 PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
 
 
 # DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)

config/resources/domain.json

@@ -50,7 +50,9 @@
     },
     "people": {
       "class": "person:Person",
-      "super": ["agents"],
+      "super": [
+        "agents"
+      ],
       "attributes": {
         "given-name": {
           "type": "string",
@@ -89,7 +91,9 @@
     },
     "agents-in-position": {
       "class": "ch:AgentInPositie",
-      "super": ["agents"],
+      "super": [
+        "agents"
+      ],
       "attributes": {
         "agent-start-date": {
           "type": "datetime",
@@ -137,7 +141,9 @@
     },
     "mandatories": {
       "class": "mandaat:Mandataris",
-      "super": ["agents-in-position"],
+      "super": [
+        "agents-in-position"
+      ],
       "attributes": {
         "start-date": {
           "type": "datetime",
@@ -180,12 +186,16 @@
           "predicate": "skos:prefLabel"
         }
       },
-      "features": ["include-uri"],
+      "features": [
+        "include-uri"
+      ],
       "new-resource-base": "http://data.vlaanderen.be/id/concept/MandatarisStatusCode/"
     },
     "worship-mandatories": {
       "class": "ere:EredienstMandataris",
-      "super": ["mandatories"],
+      "super": [
+        "mandatories"
+      ],
       "attributes": {
         "expected-end-date": {
           "type": "datetime",
@@ -250,7 +260,9 @@
     },
     "mandates": {
       "class": "mandaat:Mandaat",
-      "super": ["posts"],
+      "super": [
+        "posts"
+      ],
       "relationships": {
         "role-board": {
           "predicate": "org:role",
@@ -274,7 +286,9 @@
     },
     "functionaries": {
       "class": "lblodlg:Functionaris",
-      "super": ["agents-in-position"],
+      "super": [
+        "agents-in-position"
+      ],
       "attributes": {
         "start-date": {
           "type": "datetime",
@@ -312,12 +326,16 @@
           "predicate": "skos:prefLabel"
         }
       },
-      "features": ["include-uri"],
+      "features": [
+        "include-uri"
+      ],
       "new-resource-base": "http://data.vlaanderen.be/id/concept/functionarisStatusCode/"
     },
     "board-positions": {
       "class": "lblodlg:Bestuursfunctie",
-      "super": ["posts"],
+      "super": [
+        "posts"
+      ],
       "relationships": {
         "role-board": {
           "predicate": "org:role",
@@ -367,7 +385,9 @@
     },
     "organizations": {
       "class": "org:Organization",
-      "super": ["agents"],
+      "super": [
+        "agents"
+      ],
       "attributes": {
         "name": {
           "type": "string",
@@ -487,13 +507,21 @@
           "target": "organizations",
           "cardinality": "many",
           "inverse": true
+        },
+        "kbo-organization": {
+          "predicate": "owl:sameAs",
+          "target": "kbo-organizations",
+          "cardinality": "one",
+          "inverse": true
         }
       },
       "new-resource-base": "http://data.lblod.info/id/organisaties/"
     },
     "administrative-units": {
       "class": "besluit:Bestuurseenheid",
-      "super": ["organizations"],
+      "super": [
+        "organizations"
+      ],
       "relationships": {
         "classification": {
           "predicate": "org:classification",
@@ -541,12 +569,16 @@
     },
     "administrative-unit-classification-codes": {
       "class": "code:BestuurseenheidClassificatieCode",
-      "super": ["organization-classification-codes"],
+      "super": [
+        "organization-classification-codes"
+      ],
       "new-resource-base": "http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/"
     },
     "worship-administrative-units": {
       "class": "ere:EredienstBestuurseenheid",
-      "super": ["administrative-units"],
+      "super": [
+        "administrative-units"
+      ],
       "relationships": {
         "recognized-worship-type": {
           "predicate": "ere:typeEredienst",
@@ -569,7 +601,9 @@
     },
     "worship-services": {
       "class": "ere:BestuurVanDeEredienst",
-      "super": ["worship-administrative-units"],
+      "super": [
+        "worship-administrative-units"
+      ],
       "attributes": {
         "denomination": {
           "type": "string",
@@ -594,12 +628,44 @@
     },
     "central-worship-services": {
       "class": "ere:CentraalBestuurVanDeEredienst",
-      "super": ["worship-administrative-units"],
+      "super": [
+        "worship-administrative-units"
+      ],
       "new-resource-base": "http://data.lblod.info/id/centraleBesturenVanDeEredienst/"
     },
+    "kbo-organizations": {
+      "class": "ext:KboOrganisatie",
+      "super": [
+        "organizations"
+      ],
+      "attributes": {
+        "rechtsvorm": {
+          "type": "string",
+          "predicate": "ext:rechtsvorm"
+        },
+        "start-date": {
+          "type": "date",
+          "predicate": "ext:startDate"
+        },
+        "modified": {
+          "type": "date",
+          "predicate": "dc_terms:modified"
+        }
+      },
+      "relationships": {
+          "organization": {
+            "predicate": "owl:sameAs",
+            "target": "organizations",
+            "cardinality": "one"
+          }
+        },
+        "new-resource-base": "http://data.lblod.info/id/kboOrganisaties/"
+    },
     "representative-bodies": {
       "class": "ere:RepresentatiefOrgaan",
-      "super": ["organizations"],
+      "super": [
+        "organizations"
+      ],
       "relationships": {
         "classification": {
           "predicate": "org:classification",
@@ -622,7 +688,9 @@
     },
     "representative-body-classification-codes": {
       "class": "ext:RepresentatiefOrgaanClassificatieCode",
-      "super": ["organization-classification-codes"],
+      "super": [
+        "organization-classification-codes"
+      ],
       "new-resource-base": "http://data.vlaanderen.be/id/concept/RepresentatiefOrgaanClassificatieCode/"
     },
     "governing-bodies": {
@@ -846,7 +914,9 @@
           "predicate": "skos:prefLabel"
         }
       },
-      "features": ["include-uri"],
+      "features": [
+        "include-uri"
+      ],
       "new-resource-base": "http://lblod.data.gift/concepts/"
     },
     "locations": {
@@ -899,7 +969,9 @@
     },
     "ministers": {
       "class": "ere:RolBedienaar",
-      "super": ["agents-in-position"],
+      "super": [
+        "agents-in-position"
+      ],
       "relationships": {
         "minister-position": {
           "predicate": "org:holds",
@@ -967,7 +1039,9 @@
     },
     "minister-positions": {
       "class": "ere:PositieBedienaar",
-      "super": ["posts"],
+      "super": [
+        "posts"
+      ],
       "relationships": {
         "function": {
           "predicate": "ere:functie",
@@ -1069,4 +1143,4 @@
       "new-resource-base": "http://data.relance.lblod.info/id/nace-bel/"
     }
   }
-}
+}

docker-compose.dev.yml

@@ -51,9 +51,9 @@ services:
     restart: "no"
   uri-info:
     restart: "no"
-  delta-producer-bg-jobs-initiator-organizations:
+  delta-producer-background-jobs-initiator:
     restart: "no"
-  delta-producer-pub-graph-maintainer-organizations:
+  delta-producer-publication-graph-maintainer:
     restart: "no"
   search:
     restart: "no"
@@ -92,7 +92,7 @@ services:
   ldes-backend:
     environment:
       BASE_URL: "http://localhost/ldes/"
-  sync-ovo-numbers:
+  kbo-data-sync:
     restart: "no"
   # use this to test acmidm
   # tlsproxy:

docker-compose.yml

@@ -181,7 +181,7 @@ services:
     restart: always
     logging: *default-logging
   frontend:
-    image: lblod/frontend-organization-portal:1.23.0
+    image: lblod/frontend-organization-portal:1.24.0-0
     volumes:
       - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf
     environment:
@@ -206,10 +206,10 @@ services:
     image: semtech/mu-file-service:3.2.0
     volumes:
       - ./data/files:/share
-    labels:
-      - "logging=true"
     links:
       - db:database
+    labels:
+      - "logging=true"
     restart: always
     logging: *default-logging
   jobs-controller:
@@ -264,6 +264,8 @@ services:
       END_DATE_WORSHIP_GOVERNING_BODY: "2026-03-31T00:00:00"
     links:
       - db:database
+    labels:
+      - "logging=true"
     restart: always
     logging: *default-logging
   reasoner:
@@ -298,8 +300,8 @@ services:
       - "logging=true"
     restart: always
     logging: *default-logging
-  sync-ovo-numbers:
-    image: lblod/sync-ovo-numbers-service:0.1.0
+  kbo-data-sync:
+    image: lblod/sync-wegwijs-organization-service:2.0.2
     links:
       - db:database
     labels:
@@ -318,6 +320,8 @@ services:
     volumes:
       - ./data/publication-triplestore:/data
       - ./config/publication-triplestore/virtuoso.ini:/data/virtuoso.ini
+    labels:
+      - "logging=true"
     restart: always
     logging: *default-logging
   delta-producer-dump-file-publisher:
@@ -327,57 +331,39 @@ services:
       - ./data/files:/share
     links:
       - db:database
-    restart: always
-    logging: *default-logging
-
-  ################################################################################
-  # DELTA ORGANIZATIONS: START
-  ################################################################################
-  delta-producer-bg-jobs-initiator-organizations:
-    image: lblod/delta-producer-background-jobs-initiator:0.3.1
-    environment:
-      JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
-      DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations"
-      INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations"
-      HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations"
-      CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour
-      CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM
-      START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts.
     labels:
       - "logging=true"
-    links:
-      - db:database
     restart: always
     logging: *default-logging
-  delta-producer-pub-graph-maintainer-organizations:
-    image: lblod/delta-producer-publication-graph-maintainer:0.15.4
+  delta-producer-background-jobs-initiator:
+    image: lblod/delta-producer-background-jobs-initiator:1.0.1
+    volumes:
+      - ./config/delta-producer/background-job-initiator:/config
+    links:
+      - db:database
+    labels:
+      - "logging=true"
+    restart: always
+    logging: *default-logging
+  delta-producer-publication-graph-maintainer:
+    image: lblod/delta-producer-publication-graph-maintainer:1.1.1-rc.1
     environment:
       MAX_BODY_SIZE: "50mb"
-      JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
-      RELATIVE_FILE_PATH: "deltas/organizations"
-      PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations"
-      HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations"
-      INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations"
-      QUEUE_POLL_INTERVAL: 3000
-      USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
-      SKIP_MU_AUTH_INITIAL_SYNC: "true"
-      HEALING_PATCH_GRAPH_BATCH_SIZE: 500
-      SERVE_DELTA_FILES: "true"
-      PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations"
-      DELTA_INTERVAL_MS: 10000
       PRETTY_PRINT_DIFF_JSON: "true"
-      ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
+      PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql"
+      PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql"
+      SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503"
+      CONFIG_SERVICES_JSON_PATH: '/config/publication-graph-maintainer/config.json'
     volumes:
-      - ./config/delta-producer/organizations:/config
+      - ./config/delta-producer:/config
       - ./data/files/:/share
     links:
       - db:database
       - triplestore:virtuoso
+    labels:
+      - "logging=true"
     restart: always
     logging: *default-logging
-  ################################################################################
-  # DELTA ORGANIZATIONS: END
-  ################################################################################
 
   ################################################################################
   # DELTA ORGANIZATIONS PUBLIC INFO: START
@@ -425,6 +411,8 @@ services:
       - db:database
       - triplestore:virtuoso
     restart: always
+    labels:
+      - "logging=true"
     logging: *default-logging
   ################################################################################
   # DELTA ORGANIZATIONS PUBLIC INFO: END
@@ -474,6 +462,8 @@ services:
       - db:database
       - triplestore:virtuoso
     restart: always
+    labels:
+      - "logging=true"
     logging: *default-logging
   ################################################################################
   # DELTA PUBLIC INFORMATION: END
@@ -523,6 +513,8 @@ services:
       - db:database
       - triplestore:virtuoso
     restart: always
+    labels:
+      - "logging=true"
     logging: *default-logging
   ################################################################################
   # DELTA WORSHIP POSTS: END
@@ -625,6 +617,37 @@ services:
       - triplestore:virtuoso
     logging: *default-logging
 
+################################################################################
+  # CLB CONSUMERS
+  ################################################################################
+  clb-contact-data-consumer:
+    image: lblod/delta-consumer:0.0.18
+    environment:
+      DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/"
+      DCR_SERVICE_NAME: "clb-contact-data-consumer"
+      DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/ContactGegevensCacheGraphDump"
+      DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactData"
+      DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactDataDeltaFileSyncing"
+      DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/clb-contact-data-consumer"
+      DCR_DISABLE_INITIAL_SYNC: "false"
+      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
+      DCR_SYNC_FILES_PATH: "/sync/contact-data/files"
+      BATCH_SIZE: 1000
+      DCR_KEEP_DELTA_FILES: "true"
+      DCR_ENABLE_DELTA_CONTEXT: "true"
+      DCR_LANDING_ZONE_DATABASE: "triplestore"
+      DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/clb-contact-data"
+    volumes:
+      - ./config/clb-consumer/contact-data:/config/triples-dispatching/custom-dispatching
+      - ./data/files/consumer-files/clb-contact-data:/consumer-files/
+    restart: always
+    labels:
+      - "logging=true"
+    links:
+      - db:database
+      - triplestore:virtuoso
+    logging: *default-logging
+
   ################################################################################
   # DELTA CONSUMERS: END
   ################################################################################