Merge pull request #308 from lblod/fix/variable-name-worship-consumer

Variable name worship consumer

config/delta-producer/public/export.json

@@ -279,48 +279,21 @@
     },
     {
       "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
-      "strictTypeExport": "true",
       "graphsFilter": [
         "http://mu.semte.ch/graphs/shared",
-        "http://mu.semte.ch/graphs/worship-service",
-        "http://mu.semte.ch/graphs/administrative-unit"
-      ],
-      "pathToConceptScheme": [
-        "http://www.w3.org/ns/org#classification",
-        "http://www.w3.org/2004/02/skos/core#inScheme"
+        "http://mu.semte.ch/graphs/administrative-unit",
+        "http://mu.semte.ch/graphs/worship-service"
       ],
       "properties": [
         "http://mu.semte.ch/vocabularies/core/uuid",
-        "http://data.vlaanderen.be/ns/mandaat#bindingEinde",
-        "http://data.vlaanderen.be/ns/mandaat#bindingStart",
         "http://data.vlaanderen.be/ns/besluit#bestuurt",
         "http://www.w3.org/ns/org#classification",
-        "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
-        "http://www.w3.org/ns/org#hasPost"
-      ]
-    },
-    {
-      "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
-      "strictTypeExport": "true",
-      "graphsFilter": [
-        "http://mu.semte.ch/graphs/shared",
-        "http://mu.semte.ch/graphs/worship-service",
-        "http://mu.semte.ch/graphs/administrative-unit"
-      ],
-      "pathToConceptScheme": [
-        "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
-        "http://www.w3.org/ns/org#classification",
-        "http://www.w3.org/2004/02/skos/core#inScheme"
-      ],
-      "properties": [
-        "http://mu.semte.ch/vocabularies/core/uuid",
         "http://data.vlaanderen.be/ns/mandaat#bindingEinde",
         "http://data.vlaanderen.be/ns/mandaat#bindingStart",
-        "http://data.vlaanderen.be/ns/besluit#bestuurt",
-        "http://www.w3.org/ns/org#classification",
         "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
         "http://www.w3.org/ns/org#hasPost"
-      ]
+      ],
+      "additionalFilter": "?subject (<https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan>/<http://www.w3.org/ns/org#classification>/<http://www.w3.org/2004/02/skos/core#inScheme>)|(<http://www.w3.org/ns/org#classification>/<http://www.w3.org/2004/02/skos/core#inScheme>) <http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b> ."
     },
     {
       "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",

config/loket-consumer/leidinggevenden-dispatching/config.js

@@ -1,18 +1,20 @@
 const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
 const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
 const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
-const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
+const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
 const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
 const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
 const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
 const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
 const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
-const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
 
+// Graphs
+const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
+const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
 
-if (!process.env.FILE_SYNC_GRAPH)
-  throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
-const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
+// ENDPOINTS
+const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
+const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
 
 module.exports = {
   BATCH_SIZE,
@@ -24,6 +26,7 @@ module.exports = {
   SLEEP_BETWEEN_BATCHES,
   SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
   SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
-  INGEST_GRAPH,
-  FILE_SYNC_GRAPH
+  LANDING_ZONE_GRAPH,
+  TARGET_GRAPH,
+  LANDING_ZONE_DATABASE_ENDPOINT
 };

config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js

@@ -0,0 +1,72 @@
+
+const PREFIXES = `
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>`
+
+const contextConfig = {
+  addTypes: {
+    scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
+    exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
+  },
+  contextQueries: [
+    // Case Leidnggevenden
+    // They can be triggered when being inserted or when their link to a bestuursfunctie is added
+    {
+      trigger: { // subjectType or predicateValue
+        subjectType: "lblodlg:Functionaris"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ?functionaris org:holds ?bestuursfunctie ;
+            ?p ?o .
+          ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ;
+            mandaat:isTijdspecialisatieVan ?orgaan .
+          ?orgaan besluit:bestuurt ?eenheid .
+          ?eenheid besluit:classificatie ?classification .
+        } WHERE {
+          GRAPH <http://mu.semte.ch/graphs/landing-zone/leidinggevenden> {
+            VALUES ?functionaris { ${subject} }
+            ?functionaris org:holds ?bestuursfunctie ;
+              ?p ?o .
+            ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ;
+              mandaat:isTijdspecialisatieVan ?orgaan .
+            ?orgaan besluit:bestuurt ?eenheid .
+            ?eenheid besluit:classificatie ?classification .
+          }
+        }`
+    },
+    {
+      trigger: {
+        predicateValue: "org:holds"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ?functionaris org:holds ?bestuursfunctie ;
+            ?p ?o .
+          ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ;
+            mandaat:isTijdspecialisatieVan ?orgaan .
+          ?orgaan besluit:bestuurt ?eenheid .
+          ?eenheid besluit:classificatie ?classification .
+        } WHERE {
+          GRAPH <http://mu.semte.ch/graphs/landing-zone/leidinggevenden> {
+            VALUES ?functionaris { ${subject} }
+            ?functionaris org:holds ?bestuursfunctie ;
+              ?p ?o .
+            ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ;
+              mandaat:isTijdspecialisatieVan ?orgaan .
+            ?orgaan besluit:bestuurt ?eenheid .
+            ?eenheid besluit:classificatie ?classification .
+          }
+        }`
+    }
+  ]
+}
+
+module.exports = {
+  contextConfig,
+  PREFIXES
+};

config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js

@@ -1,11 +1,8 @@
-const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util');
 const {
-  BATCH_SIZE,
-  MAX_DB_RETRY_ATTEMPTS,
-  SLEEP_BETWEEN_BATCHES,
-  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-  INGEST_GRAPH
-} = require('./config');
+  transformStatements,
+  deleteFromTargetGraph,
+  insertIntoTargetGraph,
+} = require('./util');
 
 /**
 * Dispatch the fetched information to a target graph.
@@ -23,43 +20,61 @@ const {
 * @return {void} Nothing
 */
 async function dispatch(lib, data) {
-  const { mu, muAuthSudo, fetch } = lib;
-  const { termObjectChangeSets } = data;
+  const { mu, fetch } = lib;
+  let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
 
-  for (let { deletes, inserts } of termObjectChangeSets) {
-    const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
-    const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+  // Both arrays are the same length, so we can zip them together for easier processing
+  zippedChangeSets = termObjectChangeSets.map((o, i) => ({
+    original: o,
+    withContext: termObjectChangeSetsWithContext[i]
+  }));
 
-    if (deleteStatements.length) {
-      const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements);
-      if (transformedStatementsToDelete.length) {
-        await deleteFromAllGraphs(
-          muAuthSudo.updateSudo,
-          transformedStatementsToDelete,
-          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-          MAX_DB_RETRY_ATTEMPTS,
-          SLEEP_BETWEEN_BATCHES,
-          SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-        );
+  console.log(`Received ${zippedChangeSets.length} change sets`)
+
+  for (let { original, withContext } of zippedChangeSets) {
+    let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
+    try {
+      originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+      originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+    } catch (e) {
+      console.log(`Error while processing change set: ${e}`);
+      console.log(`Original change set: ${JSON.stringify(original)}`);
+      throw e;
+    }
+
+    // Extra context needed for mapping from DL to OP model and filtering based on type.
+    try {
+      insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+      deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+    } catch (e) {
+      console.log(`Error while processing change set with context: ${e}`);
+      console.log(`Change set with context: ${JSON.stringify(withContext)}`);
+      throw e;
+    }
+
+    if (originalDeletes.length) {
+      // Map deletes from DL to OP model
+      const transformedDeletes = await transformStatements(fetch, deletesWithContext);
+
+      if (!transformedDeletes.length) {
+        console.log(`Warn: Delete statements mapped to empty result.`);
+        console.log(`Input: ${deletesWithContext}`);
+        console.log(`Output: ${transformedDeletes}`);
+      } else {
+        await deleteFromTargetGraph(lib, transformedDeletes);
       }
     }
 
-    if (insertStatements.length) {
-      const transformedStatementsToInsert = await transformStatements(fetch, insertStatements);
-      if (transformedStatementsToInsert.length) {
-        await batchedDbUpdate(
-          muAuthSudo.updateSudo,
-          INGEST_GRAPH,
-          transformedStatementsToInsert,
-          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-          BATCH_SIZE,
-          MAX_DB_RETRY_ATTEMPTS,
-          SLEEP_BETWEEN_BATCHES,
-          SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-          "INSERT"
-        );
+    if (originalInserts.length) {
+      // Map inserts from DL to OP model
+      const transformedInserts = await transformStatements(fetch, insertsWithContext);
+
+      if (!transformedInserts.length) {
+        console.log(`Warn: Insert statements mapped to empty result.`);
+        console.log(`Input: ${insertsWithContext}`);
+        console.log(`Output: ${transformedInserts}`);
+      } else {
+        await insertIntoTargetGraph(lib, transformedInserts);
       }
     }
   }

config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js

@@ -1,13 +1,17 @@
-const { transformStatements, batchedDbUpdate } = require('./util');
-const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
+const {
+  BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
   DIRECT_DATABASE_ENDPOINT,
   MU_CALL_SCOPE_ID_INITIAL_SYNC,
   BATCH_SIZE,
   MAX_DB_RETRY_ATTEMPTS,
   SLEEP_BETWEEN_BATCHES,
   SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-  INGEST_GRAPH
+  TARGET_GRAPH,
+  LANDING_ZONE_GRAPH,
+  LANDING_ZONE_DATABASE_ENDPOINT
 } = require('./config');
+
 const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
 
 /**
@@ -24,34 +28,58 @@ const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT
 * @return {void} Nothing
 */
 async function dispatch(lib, data) {
-  const { mu, muAuthSudo, fetch } = lib;
+  const { mu, muAuthSudo } = lib;
   const { termObjects } = data;
 
+  // Steps:
+  // 1. Write the triples to the landing zone graph - withouth any mapping or filtering
+  // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
+
   if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
     console.warn(`Service configured to skip MU_AUTH!`);
   }
   console.log(`Using ${endpoint} to insert triples`);
 
   if (termObjects.length) {
-    originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
+    const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
 
-    const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples);
-    if (transformedStatementsToInsert.length) {
-      await batchedDbUpdate(
-        muAuthSudo.updateSudo,
-        INGEST_GRAPH,
-        transformedStatementsToInsert,
-        { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
-        endpoint,
-        BATCH_SIZE,
-        MAX_DB_RETRY_ATTEMPTS,
-        SLEEP_BETWEEN_BATCHES,
-        SLEEP_TIME_AFTER_FAILED_DB_OPERATION
-      );
-    }
+    await batchedDbUpdate(
+      muAuthSudo.updateSudo,
+      LANDING_ZONE_GRAPH,
+      originalInsertTriples,
+      { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+      LANDING_ZONE_DATABASE_ENDPOINT,
+      BATCH_SIZE,
+      MAX_DB_RETRY_ATTEMPTS,
+      SLEEP_BETWEEN_BATCHES,
+      SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+    );
   }
 }
 
+async function onFinishInitialIngest(lib) {
+  const { muAuthSudo, fetch } = lib;
+
+  console.log(`!! On-finish triggered !!`);
+
+  const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint);
+
+  console.log(`Transformed ${transformedInsertTriples.length} triples`);
+
+  await batchedDbUpdate(
+    muAuthSudo.updateSudo,
+    TARGET_GRAPH,
+    transformedInsertTriples,
+    { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+    endpoint,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+  );
+}
+
 module.exports = {
-  dispatch
+  dispatch,
+  onFinishInitialIngest
 };

config/loket-consumer/leidinggevenden-dispatching/util.js

@@ -1,9 +1,16 @@
 const {
   MAX_REASONING_RETRY_ATTEMPTS,
-  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
+  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
+  MAX_DB_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  TARGET_GRAPH,
+  LANDING_ZONE_GRAPH,
+  BATCH_SIZE,
 } = require('./config');
 
-async function batchedDbUpdate(muUpdate,
+async function batchedDbUpdate(
+  muUpdate,
   graph,
   triples,
   extraHeaders,
@@ -37,39 +44,6 @@ ${batch}
   }
 }
 
-async function deleteFromAllGraphs(muUpdate,
-  triples,
-  extraHeaders,
-  endpoint,
-  maxAttempts,
-  sleepBetweenBatches = 1000,
-  sleepTimeOnFail = 1000,
-) {
-
-  for (const triple of triples) {
-
-    console.log(`Deleting a triple from all graphs in triplestore`);
-
-    const deleteCall = async () => {
-      await muUpdate(`
-      DELETE {
-        GRAPH ?g {
-          ${triple}
-        }
-      } WHERE {
-        GRAPH ?g {
-          ${triple}
-        }
-      }
-      `, extraHeaders, endpoint);
-    };
-
-    await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
-    console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
-    await new Promise(r => setTimeout(r, sleepBetweenBatches));
-  }
-}
-
 async function operationWithRetry(callback,
   attempt,
   maxAttempts,
@@ -110,18 +84,16 @@ function partition(arr, fn) {
   return { passes, fails };
 }
 
-
 /**
  * Send triples to reasoning service for conversion
  *
  */
-function transformTriples(fetch, triples) {
-  return operationWithRetry(mainConversion(fetch, triples), 0,
+async function transformTriples(fetch, triples, mapping) {
+  return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
     MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
 }
 
-
-function mainConversion(fetch, triples) {
+async function mainConversion(fetch, triples, mapping) {
   let formdata = new URLSearchParams();
   formdata.append("data", triples);
 
@@ -131,23 +103,66 @@ function mainConversion(fetch, triples) {
     redirect: 'follow'
   };
 
-  return fetch("http://reasoner/reason/dl2op/main", requestOptions)
-    .then(response => response.text());
+  const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
+  return await response.text();
 }
 
-function transformStatements(fetch, triples) {
-  return transformTriples(fetch, triples.join('\n')).then(
-    graph => {
-      statements = graph.replace(/\n{2,}/g, '').split('\n')
-      console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`)
-      return statements
-    }
-  )
+async function transformStatements(fetch, triples, mapping = 'main') {
+  console.log(`Received ${JSON.stringify(triples)} to transform`);
+  const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
+  statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
+  console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
+  return statements;
+}
+
+async function deleteFromTargetGraph(lib, statements) {
+  console.log(`Deleting ${statements.length} statements from target graph`);
+  console.log(`Statements:  ${JSON.stringify(statements)}`)
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    TARGET_GRAPH,
+    statements,
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'DELETE');
+}
+
+async function insertIntoTargetGraph(lib, statements) {
+  console.log(`Inserting ${statements.length} statements into target graph`);
+  console.log(`Statements:  ${JSON.stringify(statements)}`)
+
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    TARGET_GRAPH,
+    statements,
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'INSERT');
+}
+
+async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
+  console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
+
+  const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
+  const text = await response.text();
+  const statements = text.replace(/\n{2,}/g, '').split('\n');
+
+  return statements;
 }
 
 module.exports = {
   batchedDbUpdate,
-  deleteFromAllGraphs,
   partition,
-  transformStatements
+  transformStatements,
+  transformLandingZoneGraph,
+  deleteFromTargetGraph,
+  insertIntoTargetGraph
 };

config/loket-consumer/mandatarissen-dispatching/config.js

@@ -1,18 +1,20 @@
 const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
 const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
 const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
-const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
+const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
 const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
 const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
 const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
 const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
 const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
-const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
 
+// Graphs
+const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
+const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
 
-if (!process.env.FILE_SYNC_GRAPH)
-  throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
-const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
+// ENDPOINTS
+const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
+const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
 
 module.exports = {
   BATCH_SIZE,
@@ -24,6 +26,8 @@ module.exports = {
   SLEEP_BETWEEN_BATCHES,
   SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
   SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
-  INGEST_GRAPH,
-  FILE_SYNC_GRAPH
+  LANDING_ZONE_GRAPH,
+  LANDING_ZONE_DATABASE_ENDPOINT,
+  TARGET_GRAPH
 };
+

config/loket-consumer/mandatarissen-dispatching/delta-context-config.js

@@ -0,0 +1,72 @@
+
+const PREFIXES = `
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>`
+
+const contextConfig = {
+  addTypes: {
+    scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
+    exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
+  },
+  contextQueries: [
+    // Case Mandatarissen
+    // They can be triggered when being inserted or when their link to a mandate is added
+    {
+      trigger: { // subjectType or predicateValue
+        subjectType: "mandaat:Mandataris"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ?mandataris org:holds ?mandate ;
+            ?p ?o .
+          ?orgaanInTime org:hasPost ?mandate ;
+            mandaat:isTijdspecialisatieVan ?orgaan .
+          ?orgaan besluit:bestuurt ?eenheid .
+          ?eenheid besluit:classificatie ?classification .
+        } WHERE {
+          GRAPH <http://mu.semte.ch/graphs/landing-zone/mandatarissen> {
+            VALUES ?mandataris { ${subject} }
+            ?mandataris org:holds ?mandate ;
+              ?p ?o .
+            ?orgaanInTime org:hasPost ?mandate ;
+              mandaat:isTijdspecialisatieVan ?orgaan .
+            ?orgaan besluit:bestuurt ?eenheid .
+            ?eenheid besluit:classificatie ?classification .
+          }
+        }`
+    },
+    {
+      trigger: {
+        predicateValue: "org:holds"
+      },
+      queryTemplate: (subject) => `
+        ${PREFIXES}
+        CONSTRUCT {
+          ?mandataris org:holds ?mandate ;
+            ?p ?o .
+          ?orgaanInTime org:hasPost ?mandate ;
+            mandaat:isTijdspecialisatieVan ?orgaan .
+          ?orgaan besluit:bestuurt ?eenheid .
+          ?eenheid besluit:classificatie ?classification .
+        } WHERE {
+          GRAPH <http://mu.semte.ch/graphs/landing-zone/mandatarissen> {
+            VALUES ?mandataris { ${subject} }
+            ?mandataris org:holds ?mandate ;
+              ?p ?o .
+            ?orgaanInTime org:hasPost ?mandate ;
+              mandaat:isTijdspecialisatieVan ?orgaan .
+            ?orgaan besluit:bestuurt ?eenheid .
+            ?eenheid besluit:classificatie ?classification .
+          }
+        }`
+    }
+  ]
+}
+
+module.exports = {
+  contextConfig,
+  PREFIXES
+};

config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js

@@ -0,0 +1,85 @@
+const {
+  transformStatements,
+  deleteFromTargetGraph,
+  insertIntoTargetGraph,
+} = require('./util');
+
+/**
+* Dispatch the fetched information to a target graph.
+* Note: <share://file/data> will be ADDED to it's own graph.
+*   We take only care of adding them, not updating triples, this is a TODO
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
+*          [ {
+*              graph: "<http://foo>",
+*              subject: "<http://bar>",
+*              predicate: "<http://baz>",
+*              object: "<http://boom>^^<http://datatype>"
+*            }
+*         ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+  const { mu, fetch } = lib;
+  let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
+
+  // Both arrays are the same length, so we can zip them together for easier processing
+  zippedChangeSets = termObjectChangeSets.map((o, i) => ({
+    original: o,
+    withContext: termObjectChangeSetsWithContext[i]
+  }));
+
+  console.log(`Received ${zippedChangeSets.length} change sets`)
+
+  for (let { original, withContext } of zippedChangeSets) {
+    let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
+    try {
+      originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+      originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+    } catch (e) {
+      console.log(`Error while processing change set: ${e}`);
+      console.log(`Original change set: ${JSON.stringify(original)}`);
+      throw e;
+    }
+
+    // Extra context needed for mapping from DL to OP model and filtering based on type.
+    try {
+      insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+      deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+    } catch (e) {
+      console.log(`Error while processing change set with context: ${e}`);
+      console.log(`Change set with context: ${JSON.stringify(withContext)}`);
+      throw e;
+    }
+
+    if (originalDeletes.length) {
+      // Map deletes from DL to OP model
+      const transformedDeletes = await transformStatements(fetch, deletesWithContext);
+
+      if (!transformedDeletes.length) {
+        console.log(`Warn: Delete statements mapped to empty result.`);
+        console.log(`Input: ${deletesWithContext}`);
+        console.log(`Output: ${transformedDeletes}`);
+      } else {
+        await deleteFromTargetGraph(lib, transformedDeletes);
+      }
+    }
+
+    if (originalInserts.length) {
+      // Map inserts from DL to OP model
+      const transformedInserts = await transformStatements(fetch, insertsWithContext);
+
+      if (!transformedInserts.length) {
+        console.log(`Warn: Insert statements mapped to empty result.`);
+        console.log(`Input: ${insertsWithContext}`);
+        console.log(`Output: ${transformedInserts}`);
+      } else {
+        await insertIntoTargetGraph(lib, transformedInserts);
+      }
+    }
+  }
+}
+
+module.exports = {
+  dispatch
+};

config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js

@@ -0,0 +1,85 @@
+const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
+const {
+  BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+  DIRECT_DATABASE_ENDPOINT,
+  MU_CALL_SCOPE_ID_INITIAL_SYNC,
+  BATCH_SIZE,
+  MAX_DB_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  TARGET_GRAPH,
+  LANDING_ZONE_GRAPH,
+  LANDING_ZONE_DATABASE_ENDPOINT
+} = require('./config');
+
+const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
+
+/**
+* Dispatch the fetched information to a target graph.
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
+*          [ {
+*              graph: "<http://foo>",
+*              subject: "<http://bar>",
+*              predicate: "<http://baz>",
+*              object: "<http://boom>^^<http://datatype>"
+*            }
+*         ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+  const { mu, muAuthSudo } = lib;
+  const { termObjects } = data;
+
+  // Steps:
+  // 1. Write the triples to the landing zone graph - withouth any mapping or filtering
+  // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
+
+  if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
+    console.warn(`Service configured to skip MU_AUTH!`);
+  }
+  console.log(`Using ${endpoint} to insert triples`);
+
+  if (termObjects.length) {
+    const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+
+    await batchedDbUpdate(
+      muAuthSudo.updateSudo,
+      LANDING_ZONE_GRAPH,
+      originalInsertTriples,
+      { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+      LANDING_ZONE_DATABASE_ENDPOINT,
+      BATCH_SIZE,
+      MAX_DB_RETRY_ATTEMPTS,
+      SLEEP_BETWEEN_BATCHES,
+      SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+    );
+  }
+}
+
+async function onFinishInitialIngest(lib) {
+  const { muAuthSudo, fetch } = lib;
+
+  console.log(`!! On-finish triggered !!`);
+
+  const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint);
+
+  console.log(`Transformed ${transformedInsertTriples.length} triples`);
+
+  await batchedDbUpdate(
+    muAuthSudo.updateSudo,
+    TARGET_GRAPH,
+    transformedInsertTriples,
+    { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+    endpoint,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+  );
+}
+
+module.exports = {
+  dispatch,
+  onFinishInitialIngest
+};

config/loket-consumer/mandatarissen-dispatching/util.js

@@ -1,9 +1,16 @@
 const {
   MAX_REASONING_RETRY_ATTEMPTS,
-  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
+  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
+  MAX_DB_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  TARGET_GRAPH,
+  LANDING_ZONE_GRAPH,
+  BATCH_SIZE
 } = require('./config');
 
-async function batchedDbUpdate(muUpdate,
+async function batchedDbUpdate(
+  muUpdate,
   graph,
   triples,
   extraHeaders,
@@ -37,39 +44,6 @@ ${batch}
   }
 }
 
-async function deleteFromAllGraphs(muUpdate,
-  triples,
-  extraHeaders,
-  endpoint,
-  maxAttempts,
-  sleepBetweenBatches = 1000,
-  sleepTimeOnFail = 1000,
-) {
-
-  for (const triple of triples) {
-
-    console.log(`Deleting a triple from all graphs in triplestore`);
-
-    const deleteCall = async () => {
-      await muUpdate(`
-      DELETE {
-        GRAPH ?g {
-          ${triple}
-        }
-      } WHERE {
-        GRAPH ?g {
-          ${triple}
-        }
-      }
-      `, extraHeaders, endpoint);
-    };
-
-    await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
-    console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
-    await new Promise(r => setTimeout(r, sleepBetweenBatches));
-  }
-}
-
 async function operationWithRetry(callback,
   attempt,
   maxAttempts,
@@ -110,18 +84,16 @@ function partition(arr, fn) {
   return { passes, fails };
 }
 
-
 /**
  * Send triples to reasoning service for conversion
  *
  */
-function transformTriples(fetch, triples, mapping) {
-  return operationWithRetry(mainConversion(fetch, triples, mapping), 0,
+async function transformTriples(fetch, triples, mapping) {
+  return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
     MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
 }
 
-
-function mainConversion(fetch, triples, mapping) {
+async function mainConversion(fetch, triples, mapping) {
   let formdata = new URLSearchParams();
   formdata.append("data", triples);
 
@@ -131,24 +103,66 @@ function mainConversion(fetch, triples, mapping) {
     redirect: 'follow'
   };
 
-  return fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
-    .then(response => response.text());
+  const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
+  return await response.text();
 }
 
+async function transformStatements(fetch, triples, mapping = 'main') {
+  console.log(`Received ${JSON.stringify(triples)} to transform`);
+  const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
+  statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
+  console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
+  return statements;
+}
 
-function transformStatements(fetch, triples, mapping) {
-  return transformTriples(fetch, triples.join('\n'), mapping).then(
-    graph => {
-      statements = graph ? graph.split('\n') : [];
-      console.log(`CONVERSION ${mapping}: FROM ${triples.length} triples to ${statements.length}`)
-      return statements
-    }
-  )
+async function deleteFromTargetGraph(lib, statements) {
+  console.log(`Deleting ${statements.length} statements from target graph`);
+  console.log(`Statements:  ${JSON.stringify(statements)}`)
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    TARGET_GRAPH,
+    statements,
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'DELETE');
+}
+
+async function insertIntoTargetGraph(lib, statements) {
+  console.log(`Inserting ${statements.length} statements into target graph`);
+  console.log(`Statements:  ${JSON.stringify(statements)}`)
+
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    TARGET_GRAPH,
+    statements,
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'INSERT');
+}
+
+async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
+  console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
+
+  const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
+  const text = await response.text();
+  const statements = text.replace(/\n{2,}/g, '').split('\n');
+
+  return statements;
 }
 
 module.exports = {
   batchedDbUpdate,
-  deleteFromAllGraphs,
   partition,
-  transformStatements
+  transformStatements,
+  transformLandingZoneGraph,
+  deleteFromTargetGraph,
+  insertIntoTargetGraph
 };

config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js

@@ -1,70 +0,0 @@
-const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util');
-const {
-  BATCH_SIZE,
-  MAX_DB_RETRY_ATTEMPTS,
-  SLEEP_BETWEEN_BATCHES,
-  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-  INGEST_GRAPH
-} = require('./config');
-
-/**
-* Dispatch the fetched information to a target graph.
-* Note: <share://file/data> will be ADDED to it's own graph.
-*   We take only care of adding them, not updating triples, this is a TODO
-* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
-* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
-*          [ {
-*              graph: "<http://foo>",
-*              subject: "<http://bar>",
-*              predicate: "<http://baz>",
-*              object: "<http://boom>^^<http://datatype>"
-*            }
-*         ]
-* @return {void} Nothing
-*/
-async function dispatch(lib, data) {
-  const { mu, muAuthSudo, fetch } = lib;
-  const { termObjectChangeSets } = data;
-
-  for (let { deletes, inserts } of termObjectChangeSets) {
-    const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
-    const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
-
-    if (deleteStatements.length) {
-      const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements);
-      if (transformedStatementsToDelete.length) {
-        await deleteFromAllGraphs(
-          muAuthSudo.updateSudo,
-          transformedStatementsToDelete,
-          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-          MAX_DB_RETRY_ATTEMPTS,
-          SLEEP_BETWEEN_BATCHES,
-          SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-        );
-      }
-    }
-
-    if (insertStatements.length) {
-      const transformedStatementsToInsert = await transformStatements(fetch, insertStatements);
-      if (transformedStatementsToInsert.length) {
-        await batchedDbUpdate(
-          muAuthSudo.updateSudo,
-          INGEST_GRAPH,
-          transformedStatementsToInsert,
-          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-          BATCH_SIZE,
-          MAX_DB_RETRY_ATTEMPTS,
-          SLEEP_BETWEEN_BATCHES,
-          SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-          "INSERT"
-        );
-      }
-    }
-  }
-}
-
-module.exports = {
-  dispatch
-};

config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js

@@ -1,57 +0,0 @@
-const { transformStatements, batchedDbUpdate } = require('./util');
-const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
-  DIRECT_DATABASE_ENDPOINT,
-  MU_CALL_SCOPE_ID_INITIAL_SYNC,
-  BATCH_SIZE,
-  MAX_DB_RETRY_ATTEMPTS,
-  SLEEP_BETWEEN_BATCHES,
-  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-  INGEST_GRAPH
-} = require('./config');
-const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
-
-/**
-* Dispatch the fetched information to a target graph.
-* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
-* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
-*          [ {
-*              graph: "<http://foo>",
-*              subject: "<http://bar>",
-*              predicate: "<http://baz>",
-*              object: "<http://boom>^^<http://datatype>"
-*            }
-*         ]
-* @return {void} Nothing
-*/
-async function dispatch(lib, data) {
-  const { mu, muAuthSudo, fetch } = lib;
-  const { termObjects } = data;
-
-  if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
-    console.warn(`Service configured to skip MU_AUTH!`);
-  }
-  console.log(`Using ${endpoint} to insert triples`);
-
-  if (termObjects.length) {
-    originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
-
-    const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples);
-    if (transformedStatementsToInsert.length) {
-      await batchedDbUpdate(
-        muAuthSudo.updateSudo,
-        INGEST_GRAPH,
-        transformedStatementsToInsert,
-        { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
-        endpoint,
-        BATCH_SIZE,
-        MAX_DB_RETRY_ATTEMPTS,
-        SLEEP_BETWEEN_BATCHES,
-        SLEEP_TIME_AFTER_FAILED_DB_OPERATION
-      )
-    }
-  }
-}
-
-module.exports = {
-  dispatch
-};

config/loket-consumer/mandate-dispatching/util.js

@@ -1,153 +0,0 @@
-const {
-  MAX_REASONING_RETRY_ATTEMPTS,
-  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
-} = require('./config');
-
-async function batchedDbUpdate(muUpdate,
-  graph,
-  triples,
-  extraHeaders,
-  endpoint,
-  batchSize,
-  maxAttempts,
-  sleepBetweenBatches = 1000,
-  sleepTimeOnFail = 1000,
-  operation = 'INSERT'
-) {
-
-  for (let i = 0; i < triples.length; i += batchSize) {
-    console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
-
-    const batch = triples.slice(i, i + batchSize).join('\n');
-
-    const insertCall = async () => {
-      await muUpdate(`
-${operation} DATA {
-GRAPH <${graph}> {
-${batch}
-}
-}
-`, extraHeaders, endpoint);
-    };
-
-    await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
-
-    console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
-    await new Promise(r => setTimeout(r, sleepBetweenBatches));
-  }
-}
-
-async function deleteFromAllGraphs(muUpdate,
-  triples,
-  extraHeaders,
-  endpoint,
-  maxAttempts,
-  sleepBetweenBatches = 1000,
-  sleepTimeOnFail = 1000,
-) {
-
-  for (const triple of triples) {
-
-    console.log(`Deleting a triple from all graphs in triplestore`);
-
-    const deleteCall = async () => {
-      await muUpdate(`
-      DELETE {
-        GRAPH ?g {
-          ${triple}
-        }
-      } WHERE {
-        GRAPH ?g {
-          ${triple}
-        }
-      }
-      `, extraHeaders, endpoint);
-    };
-
-    await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
-    console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
-    await new Promise(r => setTimeout(r, sleepBetweenBatches));
-  }
-}
-
-async function operationWithRetry(callback,
-  attempt,
-  maxAttempts,
-  sleepTimeOnFail) {
-  try {
-    if (typeof callback === "function")
-      return await callback();
-    else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
-      return await callback;
-  }
-  catch (e) {
-    console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
-    console.log(`Error: ${e}`);
-    console.log(`Sleeping ${sleepTimeOnFail} ms`);
-
-    if (attempt >= maxAttempts) {
-      console.log(`Max attempts reached for ${callback.toString()}, giving up`);
-      throw e;
-    }
-
-    await new Promise(r => setTimeout(r, sleepTimeOnFail));
-    return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
-  }
-}
-
-/**
- * Splits an array into two parts, a part that passes and a part that fails a predicate function.
- * Credits: https://github.com/benjay10
- * @public
- * @function partition
- * @param {Array} arr - Array to be partitioned
- * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
- * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
- */
-function partition(arr, fn) {
-  let passes = [], fails = [];
-  arr.forEach((item) => (fn(item) ? passes : fails).push(item));
-  return { passes, fails };
-}
-
-
-/**
- * Send triples to reasoning service for conversion
- *
- */
-function transformTriples(fetch, triples) {
-  return operationWithRetry(mainConversion(fetch, triples), 0,
-    MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
-}
-
-
-function mainConversion(fetch, triples) {
-  let formdata = new URLSearchParams();
-  formdata.append("data", triples);
-
-  let requestOptions = {
-    method: 'POST',
-    body: formdata,
-    redirect: 'follow'
-  };
-
-  return fetch("http://reasoner/reason/dl2op/main", requestOptions)
-    .then(response => response.text());
-}
-
-function transformStatements(fetch, triples) {
-  return transformTriples(fetch, triples.join('\n')).then(
-    graph => {
-      statements = graph.replace(/\n{2,}/g, '').split('\n')
-      console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`)
-      return statements
-    }
-  )
-}
-
-module.exports = {
-  batchedDbUpdate,
-  deleteFromAllGraphs,
-  partition,
-  transformStatements
-};

config/loket-consumer/positions-dispatching/config.js

@@ -1,51 +0,0 @@
-const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
-const MU_CALL_SCOPE_ID_INITIAL_SYNC =
-  process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC ||
-  "http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync";
-const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES =
-  process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == "true" ? true : false;
-const DIRECT_DATABASE_ENDPOINT =
-  process.env.DIRECT_DATABASE_ENDPOINT || "http://virtuoso:8890/sparql";
-const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
-const MAX_REASONING_RETRY_ATTEMPTS = parseInt(
-  process.env.MAX_REASONING_RETRY_ATTEMPTS || 5
-);
-const SLEEP_BETWEEN_BATCHES = parseInt(
-  process.env.SLEEP_BETWEEN_BATCHES || 1000
-);
-const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(
-  process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000
-);
-const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(
-  process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000
-);
-const INGEST_GRAPH =
-  process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/worship-service`;
-const PRIVACY_SENSITIVE_GRAPH =
-  process.env.PRIVACY_SENSITIVE_GRAPH ||
-  `http://mu.semte.ch/graphs/worship-privacy-centric-graph`;
-const UNFILTERED_MAPPING = "main";
-const MAIN_INFO_MAPPING = "main-info";
-const PRIVATE_INFO_MAPPING = "private-info";
-
-if (!process.env.FILE_SYNC_GRAPH)
-  throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
-const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
-
-module.exports = {
-  BATCH_SIZE,
-  MU_CALL_SCOPE_ID_INITIAL_SYNC,
-  BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
-  DIRECT_DATABASE_ENDPOINT,
-  MAX_DB_RETRY_ATTEMPTS,
-  MAX_REASONING_RETRY_ATTEMPTS,
-  SLEEP_BETWEEN_BATCHES,
-  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
-  INGEST_GRAPH,
-  PRIVACY_SENSITIVE_GRAPH,
-  UNFILTERED_MAPPING,
-  MAIN_INFO_MAPPING,
-  PRIVATE_INFO_MAPPING,
-  FILE_SYNC_GRAPH,
-};

config/loket-consumer/positions-dispatching/delta-sync-dispatching.js

@@ -1,88 +0,0 @@
-const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util');
-const {
-  BATCH_SIZE,
-  MAX_DB_RETRY_ATTEMPTS,
-  SLEEP_BETWEEN_BATCHES,
-  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-  INGEST_GRAPH,
-  PRIVACY_SENSITIVE_GRAPH,
-  UNFILTERED_MAPPING,
-  MAIN_INFO_MAPPING,
-  PRIVATE_INFO_MAPPING,
-} = require('./config');
-
-/**
-* Dispatch the fetched information to a target graph.
-* Note: <share://file/data> will be ADDED to it's own graph.
-*   We take only care of adding them, not updating triples, this is a TODO
-* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
-* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
-*          [ {
-*              graph: "<http://foo>",
-*              subject: "<http://bar>",
-*              predicate: "<http://baz>",
-*              object: "<http://boom>^^<http://datatype>"
-*            }
-*         ]
-* @return {void} Nothing
-*/
-async function dispatch(lib, data) {
-  const { mu, muAuthSudo, fetch } = lib;
-  const { termObjectChangeSets } = data;
-
-  for (let { deletes, inserts } of termObjectChangeSets) {
-    const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
-    const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
-
-    if (deleteStatements.length) {
-      const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING);
-      if (transformedStatementsToDelete.length) {
-        await deleteFromAllGraphs(
-          muAuthSudo.updateSudo,
-          transformedStatementsToDelete,
-          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-          MAX_DB_RETRY_ATTEMPTS,
-          SLEEP_BETWEEN_BATCHES,
-          SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-        );
-      }
-    }
-
-    if (insertStatements.length) {
-      const transformedStatementsToInsert = await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING);
-      if (transformedStatementsToInsert.length) {
-        await batchedDbUpdate(
-          muAuthSudo.updateSudo,
-          INGEST_GRAPH,
-          transformedStatementsToInsert,
-          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-          BATCH_SIZE,
-          MAX_DB_RETRY_ATTEMPTS,
-          SLEEP_BETWEEN_BATCHES,
-          SLEEP_TIME_AFTER_FAILED_DB_OPERATION
-        );
-      }
-
-      const transformedStatementsToInsertPrivate = await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING);
-      if (transformedStatementsToInsertPrivate.length) {
-        await batchedDbUpdate(
-          muAuthSudo.updateSudo,
-          PRIVACY_SENSITIVE_GRAPH,
-          transformedStatementsToInsertPrivate,
-          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-          process.env.MU_SPARQL_ENDPOINT, // Note: this is the default endpoint through auth
-          BATCH_SIZE,
-          MAX_DB_RETRY_ATTEMPTS,
-          SLEEP_BETWEEN_BATCHES,
-          SLEEP_TIME_AFTER_FAILED_DB_OPERATION
-        );
-      }
-    }
-  }
-}
-
-module.exports = {
-  dispatch
-};

config/loket-consumer/positions-dispatching/initial-sync-dispatching.js

@@ -1,75 +0,0 @@
-const { transformStatements, batchedDbUpdate, } = require('./util');
-const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
-  DIRECT_DATABASE_ENDPOINT,
-  MU_CALL_SCOPE_ID_INITIAL_SYNC,
-  BATCH_SIZE,
-  MAX_DB_RETRY_ATTEMPTS,
-  SLEEP_BETWEEN_BATCHES,
-  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-  INGEST_GRAPH,
-  PRIVACY_SENSITIVE_GRAPH,
-  MAIN_INFO_MAPPING,
-  PRIVATE_INFO_MAPPING
-} = require('./config');
-const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
-
-/**
-* Dispatch the fetched information to a target graph.
-* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
-* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
-*          [ {
-*              graph: "<http://foo>",
-*              subject: "<http://bar>",
-*              predicate: "<http://baz>",
-*              object: "<http://boom>^^<http://datatype>"
-*            }
-*         ]
-* @return {void} Nothing
-*/
-async function dispatch(lib, data) {
-  const { mu, muAuthSudo, fetch } = lib;
-  const { termObjects } = data;
-
-  if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
-    console.warn(`Service configured to skip MU_AUTH!`);
-  }
-  console.log(`Using ${endpoint} to insert triples`);
-
-  if (termObjects.length) {
-    originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
-
-    const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples, MAIN_INFO_MAPPING);
-    if (transformedStatementsToInsert.length) {
-      await batchedDbUpdate(
-        muAuthSudo.updateSudo,
-        INGEST_GRAPH,
-        transformedStatementsToInsert,
-        { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
-        endpoint,
-        BATCH_SIZE,
-        MAX_DB_RETRY_ATTEMPTS,
-        SLEEP_BETWEEN_BATCHES,
-        SLEEP_TIME_AFTER_FAILED_DB_OPERATION
-      );
-    }
-
-    const transformedStatementsToInsertPrivate = await transformStatements(fetch, originalInsertTriples, PRIVATE_INFO_MAPPING);
-    if (transformedStatementsToInsertPrivate.length) {
-      await batchedDbUpdate(
-        muAuthSudo.updateSudo,
-        PRIVACY_SENSITIVE_GRAPH,
-        transformedStatementsToInsertPrivate,
-        { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
-        endpoint,
-        BATCH_SIZE,
-        MAX_DB_RETRY_ATTEMPTS,
-        SLEEP_BETWEEN_BATCHES,
-        SLEEP_TIME_AFTER_FAILED_DB_OPERATION
-      );
-    }
-  }
-}
-
-module.exports = {
-  dispatch
-};

config/loket-consumer/worship-services-sensitive-dispatching/config.js

@@ -0,0 +1,40 @@
+const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
+const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
+const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
+const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
+const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
+const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
+const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
+const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
+const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
+
+// Graphs
+const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/worship-service`;
+const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/worship-privacy-centric-graph`;
+const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
+
+// Mappings
+const MAIN_INFO_MAPPING = "main-info";
+const PRIVATE_INFO_MAPPING = "private-info";
+
+// ENDPOINTS
+const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
+const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
+
+module.exports = {
+  BATCH_SIZE,
+  MU_CALL_SCOPE_ID_INITIAL_SYNC,
+  BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+  DIRECT_DATABASE_ENDPOINT,
+  MAX_DB_RETRY_ATTEMPTS,
+  MAX_REASONING_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
+  LANDING_ZONE_GRAPH,
+  LANDING_ZONE_DATABASE_ENDPOINT,
+  TARGET_GRAPH,
+  PRIVACY_SENSITIVE_GRAPH,
+  MAIN_INFO_MAPPING,
+  PRIVATE_INFO_MAPPING
+};

config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js

@@ -0,0 +1,11 @@
+const contextConfig = {
+  addTypes: {
+    scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
+    exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
+  },
+  contextQueries: []
+}
+
+module.exports = {
+  contextConfig
+};

config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js

@@ -0,0 +1,114 @@
+const {
+  transformStatements,
+  deleteFromTargetGraph,
+  insertIntoTargetGraph,
+  deleteFromPrivateGraph,
+  insertIntoPrivateGraph
+} = require('./util');
+
+const {
+  MAIN_INFO_MAPPING,
+  PRIVATE_INFO_MAPPING
+} = require('./config');
+
+/**
+* Dispatch the fetched information to a target graph.
+* Note: <share://file/data> will be ADDED to it's own graph.
+*   We take only care of adding them, not updating triples, this is a TODO
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
+*          [ {
+*              graph: "<http://foo>",
+*              subject: "<http://bar>",
+*              predicate: "<http://baz>",
+*              object: "<http://boom>^^<http://datatype>"
+*            }
+*         ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+  const { mu, fetch } = lib;
+  let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
+
+  // Both arrays are the same length, so we can zip them together for easier processing
+  zippedChangeSets = termObjectChangeSets.map((o, i) => ({
+    original: o,
+    withContext: termObjectChangeSetsWithContext[i]
+  }));
+
+  console.log(`Received ${zippedChangeSets.length} change sets`)
+
+  for (let { original, withContext } of zippedChangeSets) {
+    let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
+    try {
+      originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+      originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+    } catch (e) {
+      console.log(`Error while processing change set: ${e}`);
+      console.log(`Original change set: ${JSON.stringify(original)}`);
+      throw e;
+    }
+
+    // Extra context needed for mapping from DL to OP model and filtering based on type.
+    try {
+      insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+      deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+    } catch (e) {
+      console.log(`Error while processing change set with context: ${e}`);
+      console.log(`Change set with context: ${JSON.stringify(withContext)}`);
+      throw e;
+    }
+
+    if (originalDeletes.length) {
+      // Map main deletes from DL to OP model
+      const transformedMainDeletes = await transformStatements(fetch, deletesWithContext, MAIN_INFO_MAPPING);
+
+      if (!transformedMainDeletes.length) {
+        console.log(`Warn: Delete main statements mapped to empty result.`);
+        console.log(`Input: ${deletesWithContext}`);
+        console.log(`Output: ${transformedMainDeletes}`);
+      } else {
+        await deleteFromTargetGraph(lib, transformedMainDeletes);
+      }
+
+      // Map private deletes from DL to OP model
+      const transformedPrivateDeletes = await transformStatements(fetch, deletesWithContext, PRIVATE_INFO_MAPPING);
+
+      if (!transformedPrivateDeletes.length) {
+        console.log(`Warn: Delete private statements mapped to empty result.`);
+        console.log(`Input: ${deletesWithContext}`);
+        console.log(`Output: ${transformedPrivateDeletes}`);
+      } else {
+        await deleteFromPrivateGraph(lib, transformedPrivateDeletes);
+      }
+    }
+
+    if (originalInserts.length) {
+      // Map main inserts from DL to OP model
+      const transformedMainInserts = await transformStatements(fetch, insertsWithContext, MAIN_INFO_MAPPING);
+
+      if (!transformedMainInserts.length) {
+        console.log(`Warn: Insert statements mapped to empty result.`);
+        console.log(`Input: ${insertsWithContext}`);
+        console.log(`Output: ${transformedMainInserts}`);
+      } else {
+        await insertIntoTargetGraph(lib, transformedMainInserts);
+      }
+
+      // Map private inserts from DL to OP model
+      const transformedPrivateInserts = await transformStatements(fetch, insertsWithContext, PRIVATE_INFO_MAPPING);
+
+      if (!transformedPrivateInserts.length) {
+        console.log(`Warn: Insert statements mapped to empty result.`);
+        console.log(`Input: ${insertsWithContext}`);
+        console.log(`Output: ${transformedPrivateInserts}`);
+      } else {
+        await insertIntoPrivateGraph(lib, transformedPrivateInserts);
+      }
+    }
+  }
+}
+
+module.exports = {
+  dispatch
+};

config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js

@@ -0,0 +1,106 @@
+const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
+const {
+  BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+  DIRECT_DATABASE_ENDPOINT,
+  MU_CALL_SCOPE_ID_INITIAL_SYNC,
+  BATCH_SIZE,
+  MAX_DB_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  TARGET_GRAPH,
+  PRIVACY_SENSITIVE_GRAPH,
+  LANDING_ZONE_GRAPH,
+  LANDING_ZONE_DATABASE_ENDPOINT,
+  MAIN_INFO_MAPPING,
+  PRIVATE_INFO_MAPPING
+} = require('./config');
+
+const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
+
+/**
+* Dispatch the fetched information to a target graph.
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
+*          [ {
+*              graph: "<http://foo>",
+*              subject: "<http://bar>",
+*              predicate: "<http://baz>",
+*              object: "<http://boom>^^<http://datatype>"
+*            }
+*         ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+  const { mu, muAuthSudo } = lib;
+  const { termObjects } = data;
+
+  // Steps:
+  // 1. Write the triples to the landing zone graph - withouth any mapping or filtering
+  // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
+
+  if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
+    console.warn(`Service configured to skip MU_AUTH!`);
+  }
+  console.log(`Using ${endpoint} to insert triples`);
+
+  if (termObjects.length) {
+    const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+
+    await batchedDbUpdate(
+      muAuthSudo.updateSudo,
+      LANDING_ZONE_GRAPH,
+      originalInsertTriples,
+      { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+      LANDING_ZONE_DATABASE_ENDPOINT,
+      BATCH_SIZE,
+      MAX_DB_RETRY_ATTEMPTS,
+      SLEEP_BETWEEN_BATCHES,
+      SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+    );
+  }
+}
+
+async function onFinishInitialIngest(lib) {
+  const { muAuthSudo, fetch } = lib;
+
+  console.log(`!! On-finish triggered !!`);
+
+  const transformedMainInsertTriples = await transformLandingZoneGraph(fetch, endpoint, MAIN_INFO_MAPPING);
+  console.log(`Transformed ${transformedMainInsertTriples.length} main triples`);
+
+  if (transformedMainInsertTriples.length) {
+    await batchedDbUpdate(
+      muAuthSudo.updateSudo,
+      TARGET_GRAPH,
+      transformedMainInsertTriples,
+      { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+      endpoint,
+      BATCH_SIZE,
+      MAX_DB_RETRY_ATTEMPTS,
+      SLEEP_BETWEEN_BATCHES,
+      SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+    );
+  }
+
+  const transformedPrivateInsertTriples = await transformLandingZoneGraph(fetch, endpoint, PRIVATE_INFO_MAPPING);
+  console.log(`Transformed ${transformedPrivateInsertTriples.length} private triples`);
+
+  if (transformedPrivateInsertTriples.length) {
+    await batchedDbUpdate(
+      muAuthSudo.updateSudo,
+      PRIVACY_SENSITIVE_GRAPH,
+      transformedPrivateInsertTriples,
+      { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+      endpoint,
+      BATCH_SIZE,
+      MAX_DB_RETRY_ATTEMPTS,
+      SLEEP_BETWEEN_BATCHES,
+      SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+    );
+  }
+}
+
+module.exports = {
+  dispatch,
+  onFinishInitialIngest
+};

config/loket-consumer/worship-services-sensitive-dispatching/util.js

@@ -0,0 +1,204 @@
+const {
+  MAX_REASONING_RETRY_ATTEMPTS,
+  SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
+  MAX_DB_RETRY_ATTEMPTS,
+  SLEEP_BETWEEN_BATCHES,
+  SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+  TARGET_GRAPH,
+  PRIVACY_SENSITIVE_GRAPH,
+  LANDING_ZONE_GRAPH,
+  BATCH_SIZE
+} = require('./config');
+
+async function batchedDbUpdate(
+  muUpdate,
+  graph,
+  triples,
+  extraHeaders,
+  endpoint,
+  batchSize,
+  maxAttempts,
+  sleepBetweenBatches = 1000,
+  sleepTimeOnFail = 1000,
+  operation = 'INSERT'
+) {
+
+  for (let i = 0; i < triples.length; i += batchSize) {
+    console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
+
+    const batch = triples.slice(i, i + batchSize).join('\n');
+
+    const insertCall = async () => {
+      await muUpdate(`
+${operation} DATA {
+GRAPH <${graph}> {
+${batch}
+}
+}
+`, extraHeaders, endpoint);
+    };
+
+    await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
+
+    console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
+    await new Promise(r => setTimeout(r, sleepBetweenBatches));
+  }
+}
+
+async function operationWithRetry(callback,
+  attempt,
+  maxAttempts,
+  sleepTimeOnFail) {
+  try {
+    if (typeof callback === "function")
+      return await callback();
+    else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
+      return await callback;
+  }
+  catch (e) {
+    console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
+    console.log(`Error: ${e}`);
+    console.log(`Sleeping ${sleepTimeOnFail} ms`);
+
+    if (attempt >= maxAttempts) {
+      console.log(`Max attempts reached for ${callback.toString()}, giving up`);
+      throw e;
+    }
+
+    await new Promise(r => setTimeout(r, sleepTimeOnFail));
+    return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
+  }
+}
+
+/**
+ * Splits an array into two parts, a part that passes and a part that fails a predicate function.
+ * Credits: https://github.com/benjay10
+ * @public
+ * @function partition
+ * @param {Array} arr - Array to be partitioned
+ * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
+ * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
+ */
+function partition(arr, fn) {
+  let passes = [], fails = [];
+  arr.forEach((item) => (fn(item) ? passes : fails).push(item));
+  return { passes, fails };
+}
+
+/**
+ * Send triples to reasoning service for conversion
+ *
+ */
+async function transformTriples(fetch, triples, mapping) {
+  return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
+    MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
+}
+
+async function mainConversion(fetch, triples, mapping) {
+  let formdata = new URLSearchParams();
+  formdata.append("data", triples);
+
+  let requestOptions = {
+    method: 'POST',
+    body: formdata,
+    redirect: 'follow'
+  };
+
+  const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
+  return await response.text();
+}
+
+async function transformStatements(fetch, triples, mapping = 'main') {
+  console.log(`Received ${JSON.stringify(triples)} to transform`);
+  const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
+  statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
+  console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
+  return statements;
+}
+
+async function deleteFromTargetGraph(lib, statements) {
+  console.log(`Deleting ${statements.length} statements from target graph`);
+  console.log(`Statements:  ${JSON.stringify(statements)}`)
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    TARGET_GRAPH,
+    statements,
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'DELETE');
+}
+
+async function deleteFromPrivateGraph(lib, statements) {
+  console.log(`Deleting ${statements.length} statements from target graph`);
+  console.log(`Statements:  ${JSON.stringify(statements)}`)
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    PRIVACY_SENSITIVE_GRAPH,
+    statements,
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'DELETE');
+}
+
+async function insertIntoTargetGraph(lib, statements) {
+  console.log(`Inserting ${statements.length} statements into target graph`);
+  console.log(`Statements:  ${JSON.stringify(statements)}`)
+
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    TARGET_GRAPH,
+    statements,
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'INSERT');
+}
+
+async function insertIntoPrivateGraph(lib, statements) {
+  console.log(`Inserting ${statements.length} statements into target graph`);
+  console.log(`Statements:  ${JSON.stringify(statements)}`)
+
+  await batchedDbUpdate(
+    lib.muAuthSudo.updateSudo,
+    PRIVACY_SENSITIVE_GRAPH,
+    statements,
+    {},
+    process.env.MU_SPARQL_ENDPOINT,
+    BATCH_SIZE,
+    MAX_DB_RETRY_ATTEMPTS,
+    SLEEP_BETWEEN_BATCHES,
+    SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+    'INSERT');
+}
+
+async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
+  console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
+
+  const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
+  const text = await response.text();
+  const statements = text.replace(/\n{2,}/g, '').split('\n');
+
+  return statements;
+}
+
+module.exports = {
+  batchedDbUpdate,
+  partition,
+  transformStatements,
+  transformLandingZoneGraph,
+  deleteFromTargetGraph,
+  insertIntoTargetGraph,
+  deleteFromPrivateGraph,
+  insertIntoPrivateGraph
+};

config/migrations/2023/20230330164600-sync-loket-to-op/20230330164600-remove-loket-persons-before-sync.sparql

@@ -0,0 +1,54 @@
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>    
+PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+PREFIX dbpedia: <http://dbpedia.org/ontology/>
+PREFIX dc_terms: <http://purl.org/dc/terms/>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+PREFIX euro: <http://data.europa.eu/m8g/>
+PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
+PREFIX locn: <http://www.w3.org/ns/locn#>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX regorg: <http://www.w3.org/ns/regorg#>
+PREFIX schema: <http://schema.org/>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/shared> { 
+    ?person
+      ?p ?o
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/shared> { 
+    ?person a person:Person ;
+      ?p ?o.
+  }
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> { 
+    VALUES ?typePosition {
+      <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris>
+      <http://data.vlaanderen.be/ns/mandaat#Mandataris>
+    }
+
+    ?position mandaat:isBestuurlijkeAliasVan ?person ;
+      a ?typePosition .
+  }
+
+  FILTER NOT EXISTS {
+    GRAPH <http://mu.semte.ch/graphs/worship-service> {
+      ?worshipPosition <http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan> ?person .
+    }
+  }
+}

config/migrations/2023/20230330164600-sync-loket-to-op/20230330164601-remove-loket-functionarissen-before-sync.sparql

@@ -0,0 +1,52 @@
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>    
+PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+PREFIX dbpedia: <http://dbpedia.org/ontology/>
+PREFIX dc_terms: <http://purl.org/dc/terms/>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+PREFIX euro: <http://data.europa.eu/m8g/>
+PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
+PREFIX locn: <http://www.w3.org/ns/locn#>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX regorg: <http://www.w3.org/ns/regorg#>
+PREFIX schema: <http://schema.org/>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?functionaris ?p ?o.
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?functionaris a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+      org:holds ?bestuursFunctie ;
+      ?p ?o .
+  }
+
+  ?bestuursorgaanInTime lblodlg:heeftBestuursfunctie ?bestuursFunctie ;
+    generiek:isTijdspecialisatieVan ?bestuursorgaan .
+
+  ?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
+
+  FILTER NOT EXISTS {
+    VALUES ?typeBestuur {
+      <http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>
+      <http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>
+    }
+    ?bestuurseenheid a ?typeBestuur .
+  }
+}

config/migrations/2023/20230330164600-sync-loket-to-op/20230330164602-remove-loket-mandatarissen-before-sync-part-1.sparql

@@ -0,0 +1,53 @@
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>    
+PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+PREFIX dbpedia: <http://dbpedia.org/ontology/>
+PREFIX dc_terms: <http://purl.org/dc/terms/>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+PREFIX euro: <http://data.europa.eu/m8g/>
+PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
+PREFIX locn: <http://www.w3.org/ns/locn#>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX regorg: <http://www.w3.org/ns/regorg#>
+PREFIX schema: <http://schema.org/>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?mandataris ?p ?o.
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> { 
+    ?mandataris a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+      org:holds ?mandaat ;
+      ?p ?o .
+  }
+
+  ?bestuursorgaanInTime org:hasPost ?mandaat ;
+    generiek:isTijdspecialisatieVan ?bestuursorgaan .
+
+  ?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
+
+  FILTER NOT EXISTS {
+    VALUES ?typeBestuur {
+      <http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>
+      <http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>
+    }
+    ?bestuurseenheid a ?typeBestuur .
+  }
+}
+LIMIT 1000000

config/migrations/2023/20230330164600-sync-loket-to-op/20230330164603-remove-loket-mandatarissen-before-sync-part-2.sparql

@@ -0,0 +1,53 @@
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>    
+PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+PREFIX dbpedia: <http://dbpedia.org/ontology/>
+PREFIX dc_terms: <http://purl.org/dc/terms/>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+PREFIX euro: <http://data.europa.eu/m8g/>
+PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
+PREFIX locn: <http://www.w3.org/ns/locn#>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX regorg: <http://www.w3.org/ns/regorg#>
+PREFIX schema: <http://schema.org/>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?mandataris ?p ?o.
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> { 
+    ?mandataris a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+      org:holds ?mandaat ;
+      ?p ?o .
+  }
+
+  ?bestuursorgaanInTime org:hasPost ?mandaat ;
+    generiek:isTijdspecialisatieVan ?bestuursorgaan .
+
+  ?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
+
+  FILTER NOT EXISTS {
+    VALUES ?typeBestuur {
+      <http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>
+      <http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>
+    }
+    ?bestuurseenheid a ?typeBestuur .
+  }
+}
+LIMIT 1000000

config/migrations/2023/20230330164600-sync-loket-to-op/20230330164604-remove-loket-mandatarissen-before-sync-part-3.sparql

@@ -0,0 +1,52 @@
+PREFIX adms: <http://www.w3.org/ns/adms#>
+PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>    
+PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
+PREFIX dbpedia: <http://dbpedia.org/ontology/>
+PREFIX dc_terms: <http://purl.org/dc/terms/>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+PREFIX euro: <http://data.europa.eu/m8g/>
+PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
+PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
+PREFIX locn: <http://www.w3.org/ns/locn#>
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
+PREFIX person: <http://www.w3.org/ns/person#>
+PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
+PREFIX prov: <http://www.w3.org/ns/prov#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX regorg: <http://www.w3.org/ns/regorg#>
+PREFIX schema: <http://schema.org/>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?mandataris ?p ?o.
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> { 
+    ?mandataris a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+      org:holds ?mandaat ;
+      ?p ?o .
+  }
+
+  ?bestuursorgaanInTime org:hasPost ?mandaat ;
+    generiek:isTijdspecialisatieVan ?bestuursorgaan .
+
+  ?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
+
+  FILTER NOT EXISTS {
+    VALUES ?typeBestuur {
+      <http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>
+      <http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>
+    }
+    ?bestuurseenheid a ?typeBestuur .
+  }
+}

config/migrations/2023/20230330164600-sync-loket-to-op/20230330164606-remove-ministers-that-will-get-synced.sparql

@@ -0,0 +1,22 @@
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    ?minister ?p ?o .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> { 
+    ?minister a ere:RolBedienaar ;
+      org:holds ?ministerPosition ;
+      ?p ?o .
+
+    VALUES ?status {
+      <http://lblod.data.gift/concepts/63cc561de9188d64ba5840a42ae8f0d6> # Actief
+      <http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311> # Niet Actief
+    }
+
+    ?worshipService <http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor> ?ministerPosition ;
+      <http://www.w3.org/ns/regorg#orgStatus> ?status .
+  }
+}

config/migrations/2023/20230418093500-add-org-post-type-to-mandates.sparql

@@ -0,0 +1,19 @@
+INSERT {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?s a <http://www.w3.org/ns/org#Post> .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?s a <http://data.vlaanderen.be/ns/mandaat#Mandaat> .
+  }
+}
+
+INSERT {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    ?s a <http://www.w3.org/ns/org#Post> .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/worship-service> {
+    ?s a <http://data.vlaanderen.be/ns/mandaat#Mandaat> .
+  }
+}

config/migrations/2023/20230418131908-change-date-event.sparql

@@ -0,0 +1,17 @@
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+
+delete {
+graph <http://mu.semte.ch/graphs/worship-service> {
+<http://data.lblod.info/id/veranderingsgebeurtenissen/642E815DE025A7C706F41E5D> <http://purl.org/dc/terms/date> "2023-03-30"^^xsd:date .
+}
+}
+insert {
+graph <http://mu.semte.ch/graphs/worship-service> {
+<http://data.lblod.info/id/veranderingsgebeurtenissen/642E815DE025A7C706F41E5D> <http://purl.org/dc/terms/date> "2023-04-06"^^xsd:date .
+}
+}
+where {
+graph <http://mu.semte.ch/graphs/worship-service> {
+<http://data.lblod.info/id/veranderingsgebeurtenissen/642E815DE025A7C706F41E5D> <http://purl.org/dc/terms/date> "2023-03-30"^^xsd:date .
+}
+}

config/migrations/2023/20230421153021-remove-wrong-organen-labels.sparql

@@ -0,0 +1,23 @@
+DELETE DATA {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    <http://data.lblod.info/id/bestuursorganen/ad933a3247ab634eb1a8c7233e77d6ea8cd2b6e1950d302832d065dcb516bcf5>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"OCMWraad(Rechtstreeksverkiezing) Voeren" .
+    <http://data.lblod.info/id/bestuursorganen/082976d57de0655ac4f123b57d5cb008f612956d0a6de1742c02913dcbfcf125>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"OCMWraad(Rechtstreeksverkiezing) Drogenbos" .
+    <http://data.lblod.info/id/bestuursorganen/cc803c53b2e0b17aac1069f5566fb8e1c8dad2f702845515a662229721803c00>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"OCMWraad(Rechtstreeksverkiezing) Wezembeek-Oppem" .
+    <http://data.lblod.info/id/bestuursorganen/f373ece36dc05b29737c0adc14de95682161bb78a55cb85f786fac3fbd5b1cfc>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"OCMWraad(Rechtstreeksverkiezing) Sint-Genesius-Rode" .
+    <http://data.lblod.info/id/bestuursorganen/317126204426dc7cabb60ecb09cbe4ca4830882652884e1e5800e60231195f5c>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"OCMWraad(Rechtstreeksverkiezing) Linkebeek" .
+    <http://data.lblod.info/id/bestuursorganen/db8c3e2da85d7621120a986adc19fd8d21f501b39c01952238073abfe7d4a7e7>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"OCMWraad(Rechtstreeksverkiezing) Wemmel" .
+    <http://data.lblod.info/id/bestuursorganen/d7fffe5c17d1936d46b92620d1e454f38393736988514b5f339910a17c7134f0>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"OCMWraad(Rechtstreeksverkiezing) Kraainem" .
+  }
+}
+;
+INSERT DATA {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    <http://data.lblod.info/id/bestuursorganen/f373ece36dc05b29737c0adc14de95682161bb78a55cb85f786fac3fbd5b1cfc>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"Raad voor Maatschappelijk Welzijn Sint-Genesius-Rode" .
+    <http://data.lblod.info/id/bestuursorganen/317126204426dc7cabb60ecb09cbe4ca4830882652884e1e5800e60231195f5c>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"Raad voor Maatschappelijk Welzijn Linkebeek" .
+    <http://data.lblod.info/id/bestuursorganen/cc803c53b2e0b17aac1069f5566fb8e1c8dad2f702845515a662229721803c00>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"Raad voor Maatschappelijk Welzijn Wezembeek-Oppem" .
+    <http://data.lblod.info/id/bestuursorganen/ad933a3247ab634eb1a8c7233e77d6ea8cd2b6e1950d302832d065dcb516bcf5>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"Raad voor Maatschappelijk Welzijn Voeren" .
+    <http://data.lblod.info/id/bestuursorganen/082976d57de0655ac4f123b57d5cb008f612956d0a6de1742c02913dcbfcf125>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"Raad voor Maatschappelijk Welzijn Drogenbos" .
+    <http://data.lblod.info/id/bestuursorganen/d7fffe5c17d1936d46b92620d1e454f38393736988514b5f339910a17c7134f0>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"Raad voor Maatschappelijk Welzijn Kraainem" .
+    <http://data.lblod.info/id/bestuursorganen/db8c3e2da85d7621120a986adc19fd8d21f501b39c01952238073abfe7d4a7e7>	<http://www.w3.org/2004/02/skos/core#prefLabel>	"Raad voor Maatschappelijk Welzijn Wemmel" .
+  }
+}

config/migrations/2023/20230427090317-change-date-change-event.sparql

@@ -0,0 +1,15 @@
+delete {
+graph <http://mu.semte.ch/graphs/worship-service> {
+<http://data.lblod.info/id/veranderingsgebeurtenissen/644616FDF08BCC5A4747B5A2> <http://purl.org/dc/terms/date> ?date.
+}
+}
+insert {
+graph <http://mu.semte.ch/graphs/worship-service> {
+<http://data.lblod.info/id/veranderingsgebeurtenissen/644616FDF08BCC5A4747B5A2> <http://purl.org/dc/terms/date> "2022-04-22".
+}
+}
+where {
+graph <http://mu.semte.ch/graphs/worship-service> {
+<http://data.lblod.info/id/veranderingsgebeurtenissen/644616FDF08BCC5A4747B5A2> <http://purl.org/dc/terms/date> ?date.
+}
+}

config/migrations/2023/20230503143253-organen-start-and-end-should-be-datetimes-part-one.sparql

@@ -0,0 +1,58 @@
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+
+DELETE {
+  GRAPH ?g {
+    ?orgaanInTime mandaat:bindingEinde ?end .
+  }
+} INSERT {
+  GRAPH ?g {
+    ?orgaanInTime mandaat:bindingEinde ?endDateTime .
+  }
+} WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/worship-service>
+    <http://mu.semte.ch/graphs/administrative-unit>
+  }
+  GRAPH ?g {
+    ?bestuurseenheid a ?type .
+
+    ?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
+
+    ?orgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
+
+    ?orgaanInTime mandaat:bindingEinde ?end .
+    FILTER (datatype(?end) = xsd:date)
+    BIND(strdt(concat(str(?end), 'T01:00:00'), xsd:dateTime) as ?endDateTime)
+  }
+}
+
+;
+
+DELETE {
+  GRAPH ?g {
+    ?orgaanInTime mandaat:bindingStart ?start .
+  }
+} INSERT {
+  GRAPH ?g {
+    ?orgaanInTime mandaat:bindingStart ?startDateTime .
+  }
+} WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/worship-service>
+    <http://mu.semte.ch/graphs/administrative-unit>
+  }
+  GRAPH ?g {
+    ?bestuurseenheid a ?type .
+
+    ?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
+
+    ?orgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
+
+    ?orgaanInTime mandaat:bindingStart ?start .
+    FILTER (datatype(?start) = xsd:date)
+    BIND(strdt(concat(str(?start), 'T01:00:00'), xsd:dateTime) as ?startDateTime)
+  }
+}

config/migrations/2023/20230503143421-organen-start-and-end-should-be-datetimes-part-two.sparql

@@ -0,0 +1,60 @@
+PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
+
+DELETE {
+  GRAPH ?g {
+    ?orgaanInTime mandaat:bindingEinde ?end .
+  }
+} INSERT {
+  GRAPH ?g {
+    ?orgaanInTime mandaat:bindingEinde ?endDateTime .
+  }
+} WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/worship-service>
+    <http://mu.semte.ch/graphs/administrative-unit>
+  }
+  GRAPH ?g {
+    ?bestuurseenheid a ?type .
+
+    ?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
+
+    ?orgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
+
+    ?orgaanInTime mandaat:bindingEinde ?end .
+
+    FILTER (HOURS(?end) = 1)
+    BIND(strdt(concat(xsd:date(?end), 'T00:00:00'), xsd:dateTime) as ?endDateTime)
+  }
+}
+
+;
+
+DELETE {
+  GRAPH ?g {
+    ?orgaanInTime mandaat:bindingStart ?start .
+  }
+} INSERT {
+  GRAPH ?g {
+    ?orgaanInTime mandaat:bindingStart ?startDateTime .
+  }
+} WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/worship-service>
+    <http://mu.semte.ch/graphs/administrative-unit>
+  }
+  GRAPH ?g {
+    ?bestuurseenheid a ?type .
+
+    ?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
+
+    ?orgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
+
+    ?orgaanInTime mandaat:bindingStart ?start .
+
+    FILTER (HOURS(?start) = 1)
+    BIND(strdt(concat(xsd:date(?start), 'T00:00:00'), xsd:dateTime) as ?startDateTime)
+  }
+}

config/migrations/2023/20230504135910-delete-fracties-from-admin-unit-graph.sparql

@@ -0,0 +1,11 @@
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?s ?p ?o .
+  }
+}
+WHERE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?s a <http://data.vlaanderen.be/ns/mandaat#Fractie> ;
+      ?p ?o .
+  }
+}

config/migrations/2023/20230509142145-move-decisions-change-events.sparql

@@ -0,0 +1,75 @@
+
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX concepts: <http://lblod.data.gift/concepts/>
+PREFIX bestuur: <http://data.lblod.info/id/besturenVanDeEredienst/>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX contacthub: <http://data.lblod.info/vocabularies/contacthub/>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX ce:  <http://data.lblod.info/id/veranderingsgebeurtenissen/>
+PREFIX cer:  <http://data.lblod.info/id/veranderingsgebeurtenis-resultaten/>
+PREFIX m8g: <http://data.europa.eu/m8g/>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX eli: <http://data.europa.eu/eli/ontology#>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+
+
+DELETE {
+  graph <http://mu.semte.ch/graphs/administrative-unit> {
+    ?decision ?decisionP ?decisionO.
+  }
+}
+INSERT {
+  graph <http://mu.semte.ch/graphs/worship-service> {
+    ?decision ?decisionP ?decisionO.
+  }
+}
+WHERE {
+  graph <http://mu.semte.ch/graphs/worship-service> {
+    ?changeEvent a <http://www.w3.org/ns/org#ChangeEvent>;
+              <http://data.europa.eu/m8g/hasFormalFramework> ?decision.
+    }
+  graph <http://mu.semte.ch/graphs/administrative-unit> {
+      ?decision ?decisionP ?decisionO.
+
+  }
+
+
+}
+;
+
+
+DELETE {
+  graph <http://mu.semte.ch/graphs/administrative-unit> {
+    ?decisionActivity ?p ?o.
+  }
+}
+INSERT {
+  graph <http://mu.semte.ch/graphs/worship-service> {
+    ?decisionActivity ?p ?o.
+  }
+}
+WHERE {
+  graph <http://mu.semte.ch/graphs/worship-service> {
+    ?decision a <http://data.vlaanderen.be/ns/besluit#Besluit>.
+  }
+  graph <http://mu.semte.ch/graphs/administrative-unit> {
+      ?decisionActivity a <https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit>; 
+      <http://www.w3.org/ns/prov#generated> ?decision;
+      ?p ?o.
+  }
+
+
+}
+
+
+
+
+

config/migrations/2023/20230509142148-delete-duplicate-change-events.sparql

@@ -0,0 +1,57 @@
+
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX dct: <http://purl.org/dc/terms/>
+PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+PREFIX concepts: <http://lblod.data.gift/concepts/>
+PREFIX bestuur: <http://data.lblod.info/id/besturenVanDeEredienst/>
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX contacthub: <http://data.lblod.info/vocabularies/contacthub/>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX ce:  <http://data.lblod.info/id/veranderingsgebeurtenissen/>
+PREFIX cer:  <http://data.lblod.info/id/veranderingsgebeurtenis-resultaten/>
+PREFIX m8g: <http://data.europa.eu/m8g/>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX eli: <http://data.europa.eu/eli/ontology#>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+
+
+DELETE {
+  graph ?g {
+    ?changeEvent ?p ?o.
+    ?changeEventResult ?cP ?cO.
+    ?decision ?decisionP ?decisionO.
+    ?decisionActivity  ?decisionActivityP ?desitionActivityO.
+  }
+}
+WHERE {
+  graph ?g {
+    ?changeEvent a <http://www.w3.org/ns/org#ChangeEvent>;
+                <http://lblod.data.gift/vocabularies/organisatie/veranderingsgebeurtenisResultaat> ?changeEventResult; 
+                mu:uuid ?uuidChangeEvent;
+                ?p ?o.
+    ?changeEventResult a <http://lblod.data.gift/vocabularies/organisatie/VeranderingsgebeurtenisResultaat>;
+                       mu:uuid ?uuidChangeEventResult;
+                      ?cP ?cO.
+
+    optional {
+     ?changeEvent <http://data.europa.eu/m8g/hasFormalFramework> ?decision.
+     ?decision ?decisionP ?decisionO.
+        optional {
+         ?decisionActivity a <https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit>; 
+                           <http://www.w3.org/ns/prov#generated> ?decision;
+                          ?decisionActivityP ?desitionActivityO.
+        }
+
+    }
+
+    filter(?uuidChangeEvent in ("63343865036B579F0A26FF5C", "63343E99036B579F0A26FF6F", "633446D6036B579F0A26FF87","63345772036B579F0A26FF93"))
+
+  }
+}
+

config/migrations/2023/20230509142503-adjust-change-event.sparql

@@ -0,0 +1,16 @@
+delete {
+  graph ?g {
+    <http://data.lblod.info/id/veranderingsgebeurtenissen/620E674C5747C883CA1D2EE4> <http://data.lblod.info/vocabularies/contacthub/typeWijziging> <http://lblod.data.gift/concepts/9f19f14910245f9b14737f1f1a2067f2>.
+  }
+}
+insert {
+  graph ?g {
+    <http://data.lblod.info/id/veranderingsgebeurtenissen/620E674C5747C883CA1D2EE4> <http://data.lblod.info/vocabularies/contacthub/typeWijziging> <http://lblod.data.gift/concepts/4297bc56c1a874240e4566e2c5b8816f>.
+
+  }
+}
+where {
+  graph ?g {
+    <http://data.lblod.info/id/veranderingsgebeurtenissen/620E674C5747C883CA1D2EE4> <http://data.lblod.info/vocabularies/contacthub/typeWijziging> <http://lblod.data.gift/concepts/9f19f14910245f9b14737f1f1a2067f2>.
+  }
+}

config/migrations/2023/20230515111900-remove-not-yet-onboarded-functionarissen.sparql

@@ -0,0 +1,21 @@
+DELETE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?functionaris ?pfunctionaris ?ofunctionaris .
+  }
+} WHERE {
+  GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
+    ?functionaris a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    <http://www.w3.org/ns/org#holds> ?bestuursfunctie ;
+    ?pfunctionaris ?ofunctionaris .
+  }
+
+  ?orgaan <http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie> ?bestuursfunctie ;
+    <http://data.vlaanderen.be/ns/mandaat#isTijdspecialisatieVan>/<http://data.vlaanderen.be/ns/besluit#bestuurt>/<http://data.vlaanderen.be/ns/besluit#classificatie> ?classification .
+
+  FILTER(?classification NOT IN (
+    <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000000>,
+    <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000001>,
+    <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000002>,
+    <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000003>
+  ))
+}

config/migrations/2023/20230522132503-adjust-change-event.sparql

@@ -0,0 +1,19 @@
+delete {
+  graph ?g {
+    <http://data.lblod.info/id/veranderingsgebeurtenissen/6380DDD34B5FEAF28DEDE982> <http://data.lblod.info/vocabularies/contacthub/typeWijziging>
+    <http://lblod.data.gift/concepts/3dd7550843eaf18e1fa1ca6c6c3f2610>.
+  }
+}
+insert {
+  graph ?g {
+    <http://data.lblod.info/id/veranderingsgebeurtenissen/6380DDD34B5FEAF28DEDE982> <http://data.lblod.info/vocabularies/contacthub/typeWijziging>
+    <http://lblod.data.gift/concepts/9f19f14910245f9b14737f1f1a2067f2>.
+
+  }
+}
+where {
+  graph ?g {
+    <http://data.lblod.info/id/veranderingsgebeurtenissen/6380DDD34B5FEAF28DEDE982> <http://data.lblod.info/vocabularies/contacthub/typeWijziging>
+      <http://lblod.data.gift/concepts/3dd7550843eaf18e1fa1ca6c6c3f2610>.
+  }
+}

config/migrations/2023/20230615161050-remove-municipality-bestrokke.sparql

@@ -0,0 +1,22 @@
+
+delete where {
+ graph ?g {
+    <http://data.lblod.info/id/betrokkenLokaleBesturen/f8979469220360bbfecaf66afe1b94ae> ?p ?o.
+ }
+}
+;
+delete {
+ graph ?g {
+    <http://data.lblod.info/id/betrokkenLokaleBesturen/1de71abc7b02a6702960b403bf724ff8> <http://data.lblod.info/vocabularies/erediensten/financieringspercentage> ?percentage.
+ }
+}
+insert {
+ graph ?g {
+    <http://data.lblod.info/id/betrokkenLokaleBesturen/1de71abc7b02a6702960b403bf724ff8> <http://data.lblod.info/vocabularies/erediensten/financieringspercentage> 100.
+ }
+}
+where {
+ graph ?g {
+    <http://data.lblod.info/id/betrokkenLokaleBesturen/1de71abc7b02a6702960b403bf724ff8> <http://data.lblod.info/vocabularies/erediensten/financieringspercentage> ?percentage.
+ }
+}

config/migrations/2023/20230622140000-moslimraad.graph

@@ -0,0 +1 @@
+http://mu.semte.ch/graphs/worship-service

config/migrations/2023/20230622140000-moslimraad.ttl

@@ -0,0 +1,33 @@
+<http://data.lblod.info/id/representatieveOrganen/3b150a479c9e456a94b76c6a59a2f674>
+  a <http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan> ;
+  <http://mu.semte.ch/vocabularies/core/uuid> "3b150a479c9e456a94b76c6a59a2f674";
+  <http://www.w3.org/2004/02/skos/core#prefLabel> "Moslimraad van België";
+  <http://data.lblod.info/vocabularies/erediensten/typeEredienst> <http://lblod.data.gift/concepts/9d8bd472a00bf0a5c7b7186606365a52> ; # Islamitish
+  <http://www.w3.org/ns/regorg#orgStatus> <http://lblod.data.gift/concepts/63cc561de9188d64ba5840a42ae8f0d6> ; # Actief
+  <http://www.w3.org/ns/adms#identifier> <http://data.lblod.info/id/identificatoren/d116df8f3e1d48fbb846131cc48186b7> ;
+  <http://www.w3.org/ns/org#hasPrimarySite> <http://data.lblod.info/id/vestigingen/1a3cc52f51924d25ad7a2c6f0d4d342b> .
+
+<http://data.lblod.info/id/identificatoren/d116df8f3e1d48fbb846131cc48186b7>
+  a <http://www.w3.org/ns/adms#Identifier> ;
+  <http://mu.semte.ch/vocabularies/core/uuid> "d116df8f3e1d48fbb846131cc48186b7" ;
+  <http://www.w3.org/2004/02/skos/core#notation> "KBO nummer" ;
+  <https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> <http://data.lblod.info/id/gestructureerdeIdentificatoren/be28d9316bfb4038bf557c0ec9a45ac1> .
+
+<http://data.lblod.info/id/gestructureerdeIdentificatoren/be28d9316bfb4038bf557c0ec9a45ac1>
+  a <https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator> ;
+  <http://mu.semte.ch/vocabularies/core/uuid> "be28d9316bfb4038bf557c0ec9a45ac1" ;
+  <https://data.vlaanderen.be/ns/generiek#lokaleIdentificator> "0802469122".
+
+<http://data.lblod.info/id/vestigingen/1a3cc52f51924d25ad7a2c6f0d4d342b>
+  a <http://www.w3.org/ns/org#Site> ;
+  <http://mu.semte.ch/vocabularies/core/uuid> "1a3cc52f51924d25ad7a2c6f0d4d342b" ;
+  <https://data.vlaanderen.be/ns/organisatie#bestaatUit> <http://data.lblod.info/id/adressen/29ced73adcc4479b8e96dd20fd2bdc43> .
+
+<http://data.lblod.info/id/adressen/29ced73adcc4479b8e96dd20fd2bdc43>
+  a <http://www.w3.org/ns/locn#Address> ;
+  <http://mu.semte.ch/vocabularies/core/uuid> "29ced73adcc4479b8e96dd20fd2bdc43" ;
+  <http://www.w3.org/ns/locn#thoroughfare> "Wetstraat" ;
+  <https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer> "23" ;
+  <http://www.w3.org/ns/locn#postCode> "1040" ;
+  <https://data.vlaanderen.be/ns/adres#gemeentenaam> "Brussel" ;
+  <http://www.w3.org/ns/locn#adminUnitL2> "Vlaams-Brabant" .

config/migrations/2023/20230623103000-moslimraad-fulladdress.graph

@@ -0,0 +1 @@
+http://mu.semte.ch/graphs/worship-service

config/migrations/2023/20230623103000-moslimraad-fulladdress.ttl

@@ -0,0 +1,2 @@
+<http://data.lblod.info/id/adressen/29ced73adcc4479b8e96dd20fd2bdc43>
+  <http://www.w3.org/ns/locn#fullAddress> "Wetstraat 23, 1040 Brussel, België".

config/reasoner/dl2op/main-info/persoon_http_to_https.n3

@@ -293,16 +293,6 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
   ?s persoon:datumVanAfstamming ?o
 }.
 
-{
-  {
-    ?s persoon_loket:gebruikteVoornaam ?o
-  } e:becomes {
-    ?s persoon:gebruikteVoornaam ?o
-  }
-} => {
-  ?s persoon:gebruikteVoornaam ?o
-}.
-
 {
   {
     ?s persoon_loket:geslacht ?o

config/reasoner/dl2op/main-info/query.n3q

@@ -49,12 +49,6 @@
   ?s a <http://data.vlaanderen.be/ns/besluit#Bestuursorgaan>
 }.
 
-{
-  ?s a <http://data.vlaanderen.be/ns/mandaat#Fractie>
- } => {
-  ?s a <http://data.vlaanderen.be/ns/mandaat#Fractie>
-}.
-
 {
   ?s a <http://data.vlaanderen.be/ns/mandaat#Mandaat>
  } => {

config/reasoner/dl2op/main/inverse_relations.n3

@@ -18,7 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
 PREFIX string: <http://www.w3.org/2000/10/swap/string#>
 PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
 
-
 # Inverse properties
 {
   ?s org:holds ?o.

config/reasoner/dl2op/main/mapping.n3

@@ -18,7 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
 PREFIX string: <http://www.w3.org/2000/10/swap/string#>
 PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
 
-
 # DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)
 # Hence the mapping is in two steps.
 # The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart
@@ -92,6 +91,31 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
   ?s mandaat:einde ?eindeDate
 }.
 
+{
+  ?s mandaat:bindingStart ?bindingStartDateTime.
+  # use backward arrow rule defined below
+  ?bindingStartDateTime :dateFromDateTime ?bindingStartDate.
+  {
+    ?s mandaat:bindingStart ?bindingStartDateTime
+  } e:becomes {
+    ?s mandaat:bindingStart ?bindingStartDate
+  }.
+} => {
+  ?s mandaat:bindingStart ?bindingStartDate
+}.
+
+{
+  ?s mandaat:bindingEinde ?bindingEindeDateTime.
+  # use backward arrow rule defined below
+  ?bindingEindeDateTime :dateFromDateTime ?bindingEindeDate.
+  {
+    ?s mandaat:bindingEinde ?bindingEindeDateTime
+  } e:becomes {
+    ?s mandaat:bindingEinde ?bindingEindeDate
+  }.
+} => {
+  ?s mandaat:bindingEinde ?bindingEindeDate
+}.
 
 # Backward arrow rule is like a function call.
 # Removes time info in the returned date

config/reasoner/dl2op/main/persoon_http_to_https.n3

@@ -18,7 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
 PREFIX string: <http://www.w3.org/2000/10/swap/string#>
 PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
 
-
 # Map from http persoon to https persoon namespace.
 
 ## classes:
@@ -293,16 +292,6 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
   ?s persoon:datumVanAfstamming ?o
 }.
 
-{
-  {
-    ?s persoon_loket:gebruikteVoornaam ?o
-  } e:becomes {
-    ?s persoon:gebruikteVoornaam ?o
-  }
-} => {
-  ?s persoon:gebruikteVoornaam ?o
-}.
-
 {
   {
     ?s persoon_loket:geslacht ?o

config/reasoner/dl2op/main/query.n3q

@@ -0,0 +1,158 @@
+# Filtering classes to ingest
+
+@prefix log: <http://www.w3.org/2000/10/swap/log#> .
+
+{
+  ?s a <http://www.w3.org/ns/person#Person> ;
+    ?p ?o .
+ } => {
+  ?s a <http://www.w3.org/ns/person#Person> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://www.w3.org/ns/locn#Address> ;
+    ?p ?o .
+ } => {
+  ?s a <http://www.w3.org/ns/locn#Address> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://schema.org/ContactPoint> ;
+    ?p ?o .
+ } => {
+  ?s a <http://schema.org/ContactPoint> ;
+    ?p ?o .
+}.
+
+# Functionarissen should be linked to a gemeente, district, provincie or ocmw only (FOR NOW)
+
+{
+  ?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    <http://www.w3.org/ns/org#holds> ?bestuursfunctie ;
+    ?p ?o .
+
+  ?orgaanInTime <http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie> ?bestuursfunctie ;
+    <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> ?orgaan .
+  ?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
+  ?bestuur <http://www.w3.org/ns/org#classification> <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000000> .
+ } => {
+  ?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    <http://www.w3.org/ns/org#holds> ?bestuursfunctie ;
+    ?p ?o .
+
+  ?orgaanInTime <http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie> ?bestuursfunctie ;
+    <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> ?orgaan .
+  ?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
+  ?bestuur <http://www.w3.org/ns/org#classification> <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000001> .
+ } => {
+  ?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    <http://www.w3.org/ns/org#holds> ?bestuursfunctie ;
+    ?p ?o .
+
+  ?orgaanInTime <http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie> ?bestuursfunctie ;
+    <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> ?orgaan .
+  ?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
+  ?bestuur <http://www.w3.org/ns/org#classification> <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000002> .
+ } => {
+  ?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    <http://www.w3.org/ns/org#holds> ?bestuursfunctie ;
+    ?p ?o .
+
+  ?orgaanInTime <http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie> ?bestuursfunctie ;
+    <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> ?orgaan .
+  ?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
+  ?bestuur <http://www.w3.org/ns/org#classification> <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000003> .
+ } => {
+  ?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
+    ?p ?o .
+}.
+
+# Mandatarissen should be linked to a gemeente, district, provincie or ocmw only (FOR NOW)
+
+{
+  ?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+    <http://www.w3.org/ns/org#holds> ?mandate ;
+    ?p ?o .
+
+  ?orgaanInTime <http://www.w3.org/ns/org#hasPost> ?mandate ;
+    <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> ?orgaan .
+  ?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
+  ?bestuur <http://www.w3.org/ns/org#classification> <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000000> .
+ } => {
+  ?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+    <http://www.w3.org/ns/org#holds> ?mandate ;
+    ?p ?o .
+
+  ?orgaanInTime <http://www.w3.org/ns/org#hasPost> ?mandate ;
+    <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> ?orgaan .
+  ?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
+  ?bestuur <http://www.w3.org/ns/org#classification> <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000001> .
+ } => {
+  ?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+    <http://www.w3.org/ns/org#holds> ?mandate ;
+    ?p ?o .
+
+  ?orgaanInTime <http://www.w3.org/ns/org#hasPost> ?mandate ;
+    <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> ?orgaan .
+  ?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
+  ?bestuur <http://www.w3.org/ns/org#classification> <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000002> .
+ } => {
+  ?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+    <http://www.w3.org/ns/org#holds> ?mandate ;
+    ?p ?o .
+
+  ?orgaanInTime <http://www.w3.org/ns/org#hasPost> ?mandate ;
+    <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> ?orgaan .
+  ?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
+  ?bestuur <http://www.w3.org/ns/org#classification> <http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000003> .
+ } => {
+  ?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
+    ?p ?o .
+}.
+
+{
+  ?s a <http://www.w3.org/ns/org#Membership> ;
+    ?p ?o .
+ } => {
+  ?s a <http://www.w3.org/ns/org#Membership> ;
+    ?p ?o .
+}.
+{
+  ?s a <http://www.w3.org/ns/prov#Location> ;
+    ?p ?o .
+ } => {
+  ?s a <http://www.w3.org/ns/prov#Location> ;
+    ?p ?o .
+}.

config/reasoner/dl2op/main/retract_empty_strings.n3

@@ -18,7 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
 PREFIX string: <http://www.w3.org/2000/10/swap/string#>
 PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
 
-
 # Retract empty strings
 {
   {

config/reasoner/dl2op/main/rrn.n3

@@ -18,8 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
 PREFIX string: <http://www.w3.org/2000/10/swap/string#>
 PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
 
-
-
 # Rijksregisternummer
 # Extend identifiers to match OP's model
 # <http://example.org/tst#subject> skos:notation "00010100105".

config/reasoner/dl2op/private-info/persoon_http_to_https.n3

@@ -293,16 +293,6 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
   ?s persoon:datumVanAfstamming ?o
 }.
 
-{
-  {
-    ?s persoon_loket:gebruikteVoornaam ?o
-  } e:becomes {
-    ?s persoon:gebruikteVoornaam ?o
-  }
-} => {
-  ?s persoon:gebruikteVoornaam ?o
-}.
-
 {
   {
     ?s persoon_loket:geslacht ?o

config/resources/domain.json

@@ -578,11 +578,11 @@
       "class": "besluit:Bestuursorgaan",
       "attributes": {
         "start-date": {
-          "type": "date",
+          "type": "datetime",
           "predicate": "mandaat:bindingStart"
         },
         "end-date": {
-          "type": "date",
+          "type": "datetime",
           "predicate": "mandaat:bindingEinde"
         }
       },

config/resources/domain.lisp

@@ -9,6 +9,7 @@
 (defparameter *cache-model-properties* t)
 (defparameter *cache-count-queries* t)
 (defparameter *supply-cache-headers-p* t)
+(defparameter sparql:*experimental-no-application-graph-for-sudo-select-queries* t)
 
 (read-domain-file "domain.json")
 (read-domain-file "auth.json")

docker-compose.yml

@@ -35,7 +35,7 @@ services:
     restart: always
     logging: *default-logging
   db:
-    image: semtech/mu-authorization:feature-service-roam-r1
+    image: semtech/mu-authorization:feature-service-roam-r1.1
     environment:
       MU_SPARQL_ENDPOINT: "http://triplestore:8890/sparql"
       LOG_OUTGOING_SPARQL_QUERIES: "true"
@@ -43,6 +43,8 @@ services:
       LOG_ACCESS_RIGHTS: "true"
       LOG_OUTGOING_SPARQL_QUERY_ROUNDTRIP: "true"
       ERROR_ON_UNWRITTEN_DATA: "true"
+      QUERY_MAX_PROCESSING_TIME: "300000"
+      QUERY_MAX_EXECUTION_TIME: "300000"
     volumes:
       - ./config/authorization:/config
     labels:
@@ -70,7 +72,7 @@ services:
     restart: always
     logging: *default-logging
   resource:
-    image: semtech/mu-cl-resources:1.21.1
+    image: semtech/mu-cl-resources:1.22.2
     environment:
       CACHE_CLEAR_PATH: "http://cache/.mu/clear-keys"
     links:
@@ -82,7 +84,7 @@ services:
     restart: always
     logging: *default-logging
   cache:
-    image: semtech/mu-cache:2.0.1
+    image: semtech/mu-cache:2.0.2
     links:
       - resource:backend
     labels:
@@ -173,13 +175,13 @@ services:
     links:
       - db:database
   adressenregister:
-    image: lblod/adressenregister-fuzzy-search-service:0.6.2
+    image: lblod/adressenregister-fuzzy-search-service:0.6.3
     labels:
       - "logging=true"
     restart: always
     logging: *default-logging
   frontend:
-    image: lblod/frontend-organization-portal:1.16.0
+    image: lblod/frontend-organization-portal:1.17.2
     volumes:
       - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf
     environment:
@@ -226,7 +228,7 @@ services:
     restart: always
     logging: *default-logging
   reasoner:
-    image: eyereasoner/reasoning-service:1.0.1
+    image: eyereasoner/reasoning-service:increased-stack-limit
     volumes:
       - ./config/reasoner:/config
   ################################################################################
@@ -323,7 +325,7 @@ services:
     restart: always
     logging: *default-logging
   delta-producer-pub-graph-maintainer-public:
-    image: lblod/delta-producer-publication-graph-maintainer:0.15.1
+    image: lblod/delta-producer-publication-graph-maintainer:0.15.4
     environment:
       MAX_BODY_SIZE: "50mb"
       JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
@@ -418,25 +420,28 @@ services:
   # DELTA CONSUMERS
   ################################################################################
   ################################################################################
-  # LOKET MANDATEN CONSUMERS
+  # LOKET MANDATARISSEN CONSUMERS
   ################################################################################
-  mandaten-consumer:
-    image: lblod/delta-consumer:0.0.14
+  mandatarissen-consumer:
+    image: lblod/delta-consumer:0.0.17-rc.7
     environment:
-      DCR_SERVICE_NAME: "mandatendatabank-consumer"
+      DCR_SERVICE_NAME: "mandatarissen-consumer"
       DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
       DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/MandatarissenCacheGraphDump"
       DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen"
       DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen"
-      DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatendatabank-consumer"
+      DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatarissen-consumer"
       DCR_DISABLE_INITIAL_SYNC: "true"
       BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
+      BATCH_SIZE: 1000
       DCR_SYNC_FILES_PATH: "/sync/mandatarissen/files"
-      FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
-      KEEP_DELTA_FILES: "true"
+      DCR_KEEP_DELTA_FILES: "true"
+      DCR_ENABLE_DELTA_CONTEXT: "true"
+      DCR_LANDING_ZONE_DATABASE: "triplestore"
+      DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/mandatarissen"
     volumes:
-      - ./config/loket-consumer/mandate-dispatching:/config/triples-dispatching/custom-dispatching
-      - ./data/files/consumer-files/mandaten:/consumer-files/
+      - ./config/loket-consumer/mandatarissen-dispatching:/config/triples-dispatching/custom-dispatching
+      - ./data/files/consumer-files/mandatarissen:/consumer-files/
     restart: always
     labels:
       - "logging=true"
@@ -449,7 +454,7 @@ services:
   # LOKET LEIDINGGEVENDEN CONSUMERS
   ################################################################################
   leidinggevenden-consumer:
-    image: lblod/delta-consumer:0.0.14
+    image: lblod/delta-consumer:0.0.17-rc.7
     environment:
       DCR_SERVICE_NAME: "leidinggevenden-consumer"
       DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -460,8 +465,11 @@ services:
       DCR_DISABLE_INITIAL_SYNC: "true"
       BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
       DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files"
-      FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
-      KEEP_DELTA_FILES: "true"
+      DCR_KEEP_DELTA_FILES: "true"
+      BATCH_SIZE: 1000
+      DCR_ENABLE_DELTA_CONTEXT: "true"
+      DCR_LANDING_ZONE_DATABASE: "triplestore"
+      DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/leidinggevenden"
     volumes:
       - ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching
       - ./data/files/consumer-files/leidinggevenden:/consumer-files/
@@ -477,22 +485,24 @@ services:
   # LOKET WORSHIP SENSITIVE CONSUMERS
   ################################################################################
   worship-services-sensitive-consumer:
-    image: lblod/delta-consumer:0.0.14
+    image: lblod/delta-consumer:0.0.17-rc.7
     environment:
       DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
       DCR_SERVICE_NAME: "worship-services-sensitive-consumer"
-      DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files"
       DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
       DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
       DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
       DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer"
       DCR_DISABLE_INITIAL_SYNC: "true"
-      FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
+      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
+      DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files"
+      BATCH_SIZE: 1000
       DCR_KEEP_DELTA_FILES: "true"
-      DCR_DELTA_FILE_FOLDER: "/consumer-files"
-      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false"
+      DCR_ENABLE_DELTA_CONTEXT: "true"
+      DCR_LANDING_ZONE_DATABASE: "triplestore"
+      DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/worship-services-sensitive"
     volumes:
-      - ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching
+      - ./config/loket-consumer/worship-services-sensitive-dispatching:/config/triples-dispatching/custom-dispatching
       - ./data/files/consumer-files/worship-services-sensitive:/consumer-files/
     restart: always
     labels:
@@ -530,7 +540,7 @@ services:
       - db:database
 
   roles:
-    image: lblod/select-role-service
+    image: lblod/select-role-service:0.1.0
     labels:
       - "logging=true"
     links:

scripts/reset-elastic.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+echo "warning this will run queries on the triplestore and delete containers, you have 3 seconds to press ctrl+c"
+sleep 3
+docker-compose rm -fs elasticsearch search
+sudo rm -rf data/elasticsearch/
+docker-compose exec -T triplestore isql-v <<EOF
+SPARQL DELETE WHERE {   GRAPH <http://mu.semte.ch/authorization> {     ?s ?p ?o.   } };
+exec('checkpoint');
+exit;
+EOF
+docker-compose up -d --remove-orphan