From 2576f86068a7f57c4cc3368180a5f5df7b640a5d Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Tue, 10 Jan 2023 13:04:46 +0000 Subject: [PATCH 1/6] apply worship-app changes --- .../delta-sync-dispatching.js | 90 +-- .../initial-sync-dispatching.js | 35 +- .../leidinggevenden-dispatching/util.js | 54 +- .../delta-sync-dispatching.js | 90 +-- .../initial-sync-dispatching.js | 35 +- .../mandate-dispatching/util.js | 54 +- .../positions-dispatching/config.js | 29 + .../delta-sync-dispatching.js | 76 +++ .../initial-sync-dispatching.js | 59 ++ .../positions-dispatching/util.js | 153 +++++ .../reasoner/dl2op/main/inverse_relations.n3 | 33 ++ config/reasoner/dl2op/main/mapping.n3 | 49 +- .../dl2op/main/persoon_http_to_https.n3 | 534 ++++++++++++++++++ .../dl2op/main/retract_empty_strings.n3 | 27 + config/reasoner/dl2op/main/rrn.n3 | 62 ++ config/reasoner/dl2op/preprocess/mapping.n3 | 31 - docker-compose.yml | 37 +- 17 files changed, 1174 insertions(+), 274 deletions(-) create mode 100644 config/loket-consumer/positions-dispatching/config.js create mode 100644 config/loket-consumer/positions-dispatching/delta-sync-dispatching.js create mode 100644 config/loket-consumer/positions-dispatching/initial-sync-dispatching.js create mode 100644 config/loket-consumer/positions-dispatching/util.js create mode 100644 config/reasoner/dl2op/main/inverse_relations.n3 create mode 100644 config/reasoner/dl2op/main/persoon_http_to_https.n3 create mode 100644 config/reasoner/dl2op/main/retract_empty_strings.n3 create mode 100644 config/reasoner/dl2op/main/rrn.n3 delete mode 100644 config/reasoner/dl2op/preprocess/mapping.n3 diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js index 9f936df..c8d8ba1 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js +++ b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js @@ -1,4 +1,4 @@ -const { transformStatements, batchedDbUpdate, partition } = require('./util'); +const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, DIRECT_DATABASE_ENDPOINT, MU_CALL_SCOPE_ID_INITIAL_SYNC, @@ -30,14 +30,28 @@ async function dispatch(lib, data) { const { termObjectChangeSets } = data; for (let { deletes, inserts } of termObjectChangeSets) { - const deletePartitions = partition(deletes, o => o.subject.startsWith(' `${o.subject} ${o.predicate} ${o.object}.`); - const deleteStatements = regularDeletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); if (deleteStatements.length) { await transformStatements(fetch, deleteStatements).then( + transformedStatements => { + deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } + ) + } + + + if (insertStatements.length) { + await transformStatements(fetch, insertStatements).then( transformedStatements => { batchedDbUpdate( muAuthSudo.updateSudo, @@ -49,70 +63,6 @@ async function dispatch(lib, data) { MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "DELETE" - ); - } - ) - } - - if (fileDeleteStatements.length) { - await transformStatements(fetch, fileDeleteStatements).then( - transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - FILE_SYNC_GRAPH, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "DELETE" - ); - } - ) - } - - const insertPartitions = partition(inserts, o => o.subject.startsWith(' `${o.subject} ${o.predicate} ${o.object}.`); - const fileInsertStatements = fileInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - - - if (regularInsertStatements.length) { - await transformStatements(fetch, regularInsertStatements).then( - transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatements, - {}, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "INSERT" - ); - } - ) - } - - if (fileInsertStatements.length) { - await transformStatements(fetch, fileInsertStatements).then( - transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - FILE_SYNC_GRAPH, - transformedStatements, - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, "INSERT" ); } diff --git a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js index 69bcdb4..88017c1 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js +++ b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js @@ -28,45 +28,20 @@ async function dispatch(lib, data) { const { mu, muAuthSudo, fetch } = lib; const { termObjects } = data; - const partitions = partition(termObjects, o => o.subject.startsWith(' `${o.subject} ${o.predicate} ${o.object}.`) + if (termObjects.length) { + originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - await transformStatements(fetch, originalRegularInsertTriples).then( - transformedRegularInsertTriples => { - // console.log(transformedRegularInsertTriples); + await transformStatements(fetch, originalInsertTriples).then( + transformedInsertTriples => { batchedDbUpdate( muAuthSudo.updateSudo, INGEST_GRAPH, - transformedRegularInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ) - } - ) - } - - if (fileInserts.length) { - originalFileInsertTriples = fileInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - await transformStatements(fetch, originalFileInsertTriples).then( - transformedFileInsertTriples => { - // console.log(transformedFileInsertTriples); - batchedDbUpdate( - muAuthSudo.updateSudo, - FILE_SYNC_GRAPH, - transformedFileInsertTriples, + transformedInsertTriples, { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, endpoint, BATCH_SIZE, diff --git a/config/loket-consumer/leidinggevenden-dispatching/util.js b/config/loket-consumer/leidinggevenden-dispatching/util.js index 54bee47..ff3932f 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/util.js +++ b/config/loket-consumer/leidinggevenden-dispatching/util.js @@ -37,6 +37,39 @@ ${batch} } } +async function deleteFromAllGraphs(muUpdate, + triples, + extraHeaders, + endpoint, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, +) { + + for (const triple of triples) { + + console.log(`Deleting a triple from all graphs in triplestore`); + + const deleteCall = async () => { + await muUpdate(` + DELETE { + GRAPH ?g { + ${triple} + } + } WHERE { + GRAPH ?g { + ${triple} + } + } + `, extraHeaders, endpoint); + }; + + await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + async function operationWithRetry(callback, attempt, maxAttempts, @@ -83,27 +116,11 @@ function partition(arr, fn) { * */ function transformTriples(fetch, triples) { - return operationWithRetry(preProcess(fetch, triples), 0, - MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION) - .then(preprocessed => operationWithRetry(mainConversion(fetch, preprocessed), 0, - MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION)); + return operationWithRetry(mainConversion(fetch, triples), 0, + MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); } -function preProcess(fetch, triples) { - let formdata = new URLSearchParams(); - formdata.append("data", triples); - - let requestOptions = { - method: 'POST', - body: formdata, - redirect: 'follow' - }; - - return fetch("http://reasoner/reason/dl2op/preprocess", requestOptions) - .then(response => response.text()); -} - function mainConversion(fetch, triples) { let formdata = new URLSearchParams(); formdata.append("data", triples); @@ -130,6 +147,7 @@ function transformStatements(fetch, triples) { module.exports = { batchedDbUpdate, + deleteFromAllGraphs, partition, transformStatements }; \ No newline at end of file diff --git a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js index 9f936df..c8d8ba1 100644 --- a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js +++ b/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js @@ -1,4 +1,4 @@ -const { transformStatements, batchedDbUpdate, partition } = require('./util'); +const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, DIRECT_DATABASE_ENDPOINT, MU_CALL_SCOPE_ID_INITIAL_SYNC, @@ -30,14 +30,28 @@ async function dispatch(lib, data) { const { termObjectChangeSets } = data; for (let { deletes, inserts } of termObjectChangeSets) { - const deletePartitions = partition(deletes, o => o.subject.startsWith(' `${o.subject} ${o.predicate} ${o.object}.`); - const deleteStatements = regularDeletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); if (deleteStatements.length) { await transformStatements(fetch, deleteStatements).then( + transformedStatements => { + deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } + ) + } + + + if (insertStatements.length) { + await transformStatements(fetch, insertStatements).then( transformedStatements => { batchedDbUpdate( muAuthSudo.updateSudo, @@ -49,70 +63,6 @@ async function dispatch(lib, data) { MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "DELETE" - ); - } - ) - } - - if (fileDeleteStatements.length) { - await transformStatements(fetch, fileDeleteStatements).then( - transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - FILE_SYNC_GRAPH, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "DELETE" - ); - } - ) - } - - const insertPartitions = partition(inserts, o => o.subject.startsWith(' `${o.subject} ${o.predicate} ${o.object}.`); - const fileInsertStatements = fileInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - - - if (regularInsertStatements.length) { - await transformStatements(fetch, regularInsertStatements).then( - transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatements, - {}, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "INSERT" - ); - } - ) - } - - if (fileInsertStatements.length) { - await transformStatements(fetch, fileInsertStatements).then( - transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - FILE_SYNC_GRAPH, - transformedStatements, - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, "INSERT" ); } diff --git a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js index 69bcdb4..88017c1 100644 --- a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js +++ b/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js @@ -28,45 +28,20 @@ async function dispatch(lib, data) { const { mu, muAuthSudo, fetch } = lib; const { termObjects } = data; - const partitions = partition(termObjects, o => o.subject.startsWith(' `${o.subject} ${o.predicate} ${o.object}.`) + if (termObjects.length) { + originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - await transformStatements(fetch, originalRegularInsertTriples).then( - transformedRegularInsertTriples => { - // console.log(transformedRegularInsertTriples); + await transformStatements(fetch, originalInsertTriples).then( + transformedInsertTriples => { batchedDbUpdate( muAuthSudo.updateSudo, INGEST_GRAPH, - transformedRegularInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ) - } - ) - } - - if (fileInserts.length) { - originalFileInsertTriples = fileInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - await transformStatements(fetch, originalFileInsertTriples).then( - transformedFileInsertTriples => { - // console.log(transformedFileInsertTriples); - batchedDbUpdate( - muAuthSudo.updateSudo, - FILE_SYNC_GRAPH, - transformedFileInsertTriples, + transformedInsertTriples, { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, endpoint, BATCH_SIZE, diff --git a/config/loket-consumer/mandate-dispatching/util.js b/config/loket-consumer/mandate-dispatching/util.js index 54bee47..ff3932f 100644 --- a/config/loket-consumer/mandate-dispatching/util.js +++ b/config/loket-consumer/mandate-dispatching/util.js @@ -37,6 +37,39 @@ ${batch} } } +async function deleteFromAllGraphs(muUpdate, + triples, + extraHeaders, + endpoint, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, +) { + + for (const triple of triples) { + + console.log(`Deleting a triple from all graphs in triplestore`); + + const deleteCall = async () => { + await muUpdate(` + DELETE { + GRAPH ?g { + ${triple} + } + } WHERE { + GRAPH ?g { + ${triple} + } + } + `, extraHeaders, endpoint); + }; + + await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + async function operationWithRetry(callback, attempt, maxAttempts, @@ -83,27 +116,11 @@ function partition(arr, fn) { * */ function transformTriples(fetch, triples) { - return operationWithRetry(preProcess(fetch, triples), 0, - MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION) - .then(preprocessed => operationWithRetry(mainConversion(fetch, preprocessed), 0, - MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION)); + return operationWithRetry(mainConversion(fetch, triples), 0, + MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); } -function preProcess(fetch, triples) { - let formdata = new URLSearchParams(); - formdata.append("data", triples); - - let requestOptions = { - method: 'POST', - body: formdata, - redirect: 'follow' - }; - - return fetch("http://reasoner/reason/dl2op/preprocess", requestOptions) - .then(response => response.text()); -} - function mainConversion(fetch, triples) { let formdata = new URLSearchParams(); formdata.append("data", triples); @@ -130,6 +147,7 @@ function transformStatements(fetch, triples) { module.exports = { batchedDbUpdate, + deleteFromAllGraphs, partition, transformStatements }; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/config.js b/config/loket-consumer/positions-dispatching/config.js new file mode 100644 index 0000000..e280733 --- /dev/null +++ b/config/loket-consumer/positions-dispatching/config.js @@ -0,0 +1,29 @@ +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); +const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); +const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); +const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`; + + +if (!process.env.FILE_SYNC_GRAPH) + throw `Expected 'FILE_SYNC_GRAPH' to be provided.`; +const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH; + +module.exports = { + BATCH_SIZE, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MAX_DB_RETRY_ATTEMPTS, + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +}; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js new file mode 100644 index 0000000..c8d8ba1 --- /dev/null +++ b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js @@ -0,0 +1,76 @@ +const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); +const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +} = require('./config'); + +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo, fetch } = lib; + const { termObjectChangeSets } = data; + + for (let { deletes, inserts } of termObjectChangeSets) { + const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + if (deleteStatements.length) { + await transformStatements(fetch, deleteStatements).then( + transformedStatements => { + deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } + ) + } + + + if (insertStatements.length) { + await transformStatements(fetch, insertStatements).then( + transformedStatements => { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + "INSERT" + ); + } + ) + } + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js new file mode 100644 index 0000000..88017c1 --- /dev/null +++ b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js @@ -0,0 +1,59 @@ +const { transformStatements, batchedDbUpdate, partition } = require('./util'); +const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +} = require('./config'); +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo, fetch } = lib; + const { termObjects } = data; + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) + + await transformStatements(fetch, originalInsertTriples).then( + transformedInsertTriples => { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ) + } + ) + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/util.js b/config/loket-consumer/positions-dispatching/util.js new file mode 100644 index 0000000..ff3932f --- /dev/null +++ b/config/loket-consumer/positions-dispatching/util.js @@ -0,0 +1,153 @@ +const { + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION +} = require('./config'); + +async function batchedDbUpdate(muUpdate, + graph, + triples, + extraHeaders, + endpoint, + batchSize, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, + operation = 'INSERT' +) { + + for (let i = 0; i < triples.length; i += batchSize) { + console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); + + const batch = triples.slice(i, i + batchSize).join('\n'); + + const insertCall = async () => { + await muUpdate(` +${operation} DATA { +GRAPH <${graph}> { +${batch} +} +} +`, extraHeaders, endpoint); + }; + + await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); + + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function deleteFromAllGraphs(muUpdate, + triples, + extraHeaders, + endpoint, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, +) { + + for (const triple of triples) { + + console.log(`Deleting a triple from all graphs in triplestore`); + + const deleteCall = async () => { + await muUpdate(` + DELETE { + GRAPH ?g { + ${triple} + } + } WHERE { + GRAPH ?g { + ${triple} + } + } + `, extraHeaders, endpoint); + }; + + await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function operationWithRetry(callback, + attempt, + maxAttempts, + sleepTimeOnFail) { + try { + if (typeof callback === "function") + return await callback(); + else // Catch error from promise - not how I would do it normally, but allows re use of existing code. + return await callback; + } + catch (e) { + console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); + console.log(`Error: ${e}`); + console.log(`Sleeping ${sleepTimeOnFail} ms`); + + if (attempt >= maxAttempts) { + console.log(`Max attempts reached for ${callback.toString()}, giving up`); + throw e; + } + + await new Promise(r => setTimeout(r, sleepTimeOnFail)); + return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); + } +} + +/** + * Splits an array into two parts, a part that passes and a part that fails a predicate function. + * Credits: https://github.com/benjay10 + * @public + * @function partition + * @param {Array} arr - Array to be partitioned + * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. + * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively + */ +function partition(arr, fn) { + let passes = [], fails = []; + arr.forEach((item) => (fn(item) ? passes : fails).push(item)); + return { passes, fails }; +} + + +/** + * Send triples to reasoning service for conversion + * + */ +function transformTriples(fetch, triples) { + return operationWithRetry(mainConversion(fetch, triples), 0, + MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); +} + + +function mainConversion(fetch, triples) { + let formdata = new URLSearchParams(); + formdata.append("data", triples); + + let requestOptions = { + method: 'POST', + body: formdata, + redirect: 'follow' + }; + + return fetch("http://reasoner/reason/dl2op/main", requestOptions) + .then(response => response.text()); +} + +function transformStatements(fetch, triples) { + return transformTriples(fetch, triples.join('\n')).then( + graph => { + statements = graph.replace(/\n{2,}/g, '').split('\n') + console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`) + return statements + } + ) +} + +module.exports = { + batchedDbUpdate, + deleteFromAllGraphs, + partition, + transformStatements +}; \ No newline at end of file diff --git a/config/reasoner/dl2op/main/inverse_relations.n3 b/config/reasoner/dl2op/main/inverse_relations.n3 new file mode 100644 index 0000000..bfeda31 --- /dev/null +++ b/config/reasoner/dl2op/main/inverse_relations.n3 @@ -0,0 +1,33 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Inverse properties +{ + ?s org:holds ?o. +} => { + ?o org:heldBy ?s. +}. + +{ + ?s org:heldBy ?o. +} => { + ?o org:holds ?s. +}. diff --git a/config/reasoner/dl2op/main/mapping.n3 b/config/reasoner/dl2op/main/mapping.n3 index 0497727..82bebd9 100644 --- a/config/reasoner/dl2op/main/mapping.n3 +++ b/config/reasoner/dl2op/main/mapping.n3 @@ -1,18 +1,51 @@ PREFIX : <#> +PREFIX adms: PREFIX adres: PREFIX besluit: +PREFIX crypto: PREFIX e: PREFIX ex: PREFIX foaf: PREFIX func: +PREFIX generiek: PREFIX log: PREFIX mandaat: +PREFIX mu: PREFIX org: -PREFIX persoon: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: PREFIX string: PREFIX xsd: +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) +# Hence the mapping is in two steps. +# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart +# and drops the DL persoon:gebruikteVoornaam statements with e:becomes +# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam +# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames +# in OP + +# First name +# persoon:gebruikteVoornaam => foaf:givenName +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock DL +# persoon_loket:gebruikteVoornaam "Didier". + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s foaf:givenName ?o + } +} => { + ?s foaf:givenName ?o +}. + + # First name # persoon:gebruikteVoornaam => foaf:givenName (done in pre-processing) # Alternatieve voornaam @@ -102,4 +135,16 @@ PREFIX xsd: } } => { ?s adres:Adresvoorstelling.huisnummer ?o -}. \ No newline at end of file +}. + + +# Tijdspecialisatie +{ + { + ?s mandaat:isTijdspecialisatieVan ?o. + } e:becomes { + ?s generiek:isTijdspecialisatieVan ?o. + } +} => { + ?s generiek:isTijdspecialisatieVan ?o. +}. diff --git a/config/reasoner/dl2op/main/persoon_http_to_https.n3 b/config/reasoner/dl2op/main/persoon_http_to_https.n3 new file mode 100644 index 0000000..6ced405 --- /dev/null +++ b/config/reasoner/dl2op/main/persoon_http_to_https.n3 @@ -0,0 +1,534 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Map from http persoon to https persoon namespace. + +## classes: +{ + { + ?s a persoon_loket:Afstamming + } e:becomes { + ?s a persoon:Afstamming + } +} => { + ?s a persoon:Afstamming +}. + +{ + { + ?s a persoon_loket:BurgerlijkeStaat + } e:becomes { + ?s a persoon:BurgerlijkeStaat + } +} => { + ?s a persoon:BurgerlijkeStaat +}. + +{ + { + ?s a persoon_loket:Domicilie + } e:becomes { + ?s a persoon:Domicilie + } +} => { + ?s a persoon:Domicilie +}. + +{ + { + ?s a persoon_loket:Geboorte + } e:becomes { + ?s a persoon:Geboorte + } +} => { + ?s a persoon:Geboorte +}. + +{ + { + ?s a persoon_loket:GeenInwoner + } e:becomes { + ?s a persoon:GeenInwoner + } +} => { + ?s a persoon:GeenInwoner +}. + +{ + { + ?s a persoon_loket:GeregistreerdPersoon + } e:becomes { + ?s a persoon:GeregistreerdPersoon + } +} => { + ?s a persoon:GeregistreerdPersoon +}. + +{ + { + ?s a persoon_loket:Gezin + } e:becomes { + ?s a persoon:Gezin + } +} => { + ?s a persoon:Gezin +}. + +{ + { + ?s a persoon_loket:Gezinsrelatie + } e:becomes { + ?s a persoon:Gezinsrelatie + } +} => { + ?s a persoon:Gezinsrelatie +}. + +{ + { + ?s a persoon_loket:Huwelijk + } e:becomes { + ?s a persoon:Huwelijk + } +} => { + ?s a persoon:Huwelijk +}. + +{ + { + ?s a persoon_loket:Inwoner + } e:becomes { + ?s a persoon:Inwoner + } +} => { + ?s a persoon:Inwoner +}. + +{ + { + ?s a persoon_loket:Inwonerschap + } e:becomes { + ?s a persoon:Inwonerschap + } +} => { + ?s a persoon:Inwonerschap +}. + +{ + { + ?s a persoon_loket:Nationaliteit + } e:becomes { + ?s a persoon:Nationaliteit + } +} => { + ?s a persoon:Nationaliteit +}. + +{ + { + ?s a persoon_loket:Overlijden + } e:becomes { + ?s a persoon:Overlijden + } +} => { + ?s a persoon:Overlijden +}. + +{ + { + ?s a persoon_loket:PermanentInwoner + } e:becomes { + ?s a persoon:PermanentInwoner + } +} => { + ?s a persoon:PermanentInwoner +}. + +{ + { + ?s a persoon_loket:Persoonsgebeurtenis + } e:becomes { + ?s a persoon:Persoonsgebeurtenis + } +} => { + ?s a persoon:Persoonsgebeurtenis +}. + +{ + { + ?s a persoon_loket:Persoonsrelatie + } e:becomes { + ?s a persoon:Persoonsrelatie + } +} => { + ?s a persoon:Persoonsrelatie +}. + +{ + { + ?s a persoon_loket:Samenwonen + } e:becomes { + ?s a persoon:Samenwonen + } +} => { + ?s a persoon:Samenwonen +}. + +{ + { + ?s a persoon_loket:Staatsburger + } e:becomes { + ?s a persoon:Staatsburger + } +} => { + ?s a persoon:Staatsburger +}. + +{ + { + ?s a persoon_loket:Staatsburgerschap + } e:becomes { + ?s a persoon:Staatsburgerschap + } +} => { + ?s a persoon:Staatsburgerschap +}. + +{ + { + ?s a persoon_loket:TijdelijkInwoner + } e:becomes { + ?s a persoon:TijdelijkInwoner + } +} => { + ?s a persoon:TijdelijkInwoner +}. + +{ + { + ?s a persoon_loket:Verblijfplaats + } e:becomes { + ?s a persoon:Verblijfplaats + } +} => { + ?s a persoon:Verblijfplaats +}. + +{ + { + ?s a persoon_loket:Voogdij + } e:becomes { + ?s a persoon:Voogdij + } +} => { + ?s a persoon:Voogdij +}. + +{ + { + ?s a persoon_loket:Vreemdeling + } e:becomes { + ?s a persoon:Vreemdeling + } +} => { + ?s a persoon:Vreemdeling +}. + +## properties +{ + { + ?s persoon_loket:afstammingsType ?o + } e:becomes { + ?s persoon:afstammingsType ?o + } +} => { + ?s persoon:afstammingsType ?o +}. + +{ + { + ?s persoon_loket:alternatieveNaam ?o + } e:becomes { + ?s persoon:alternatieveNaam ?o + } +} => { + ?s persoon:alternatieveNaam ?o +}. + +{ + { + ?s persoon_loket:datum ?o + } e:becomes { + ?s persoon:datum ?o + } +} => { + ?s persoon:datum ?o +}. + +{ + { + ?s persoon_loket:datumVanAfstamming ?o + } e:becomes { + ?s persoon:datumVanAfstamming ?o + } +} => { + ?s persoon:datumVanAfstamming ?o +}. + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +{ + { + ?s persoon_loket:geslacht ?o + } e:becomes { + ?s persoon:geslacht ?o + } +} => { + ?s persoon:geslacht ?o +}. + +{ + { + ?s persoon_loket:gezinsadres ?o + } e:becomes { + ?s persoon:gezinsadres ?o + } +} => { + ?s persoon:gezinsadres ?o +}. + +{ + { + ?s persoon_loket:gezinsrelatietype ?o + } e:becomes { + ?s persoon:gezinsrelatietype ?o + } +} => { + ?s persoon:gezinsrelatietype ?o +}. + +{ + { + ?s persoon_loket:heeftBurgerlijkeStaat ?o + } e:becomes { + ?s persoon:heeftBurgerlijkeStaat ?o + } +} => { + ?s persoon:heeftBurgerlijkeStaat ?o +}. + +{ + { + ?s persoon_loket:heeftGeboorte ?o + } e:becomes { + ?s persoon:heeftGeboorte ?o + } +} => { + ?s persoon:heeftGeboorte ?o +}. + +{ + { + ?s persoon_loket:heeftInwonerschap ?o + } e:becomes { + ?s persoon:heeftInwonerschap ?o + } +} => { + ?s persoon:heeftInwonerschap ?o +}. + +{ + { + ?s persoon_loket:heeftNationaliteit ?o + } e:becomes { + ?s persoon:heeftNationaliteit ?o + } +} => { + ?s persoon:heeftNationaliteit ?o +}. + +{ + { + ?s persoon_loket:heeftOverlijden ?o + } e:becomes { + ?s persoon:heeftOverlijden ?o + } +} => { + ?s persoon:heeftOverlijden ?o +}. + +{ + { + ?s persoon_loket:heeftPersoonsrelatie ?o + } e:becomes { + ?s persoon:heeftPersoonsrelatie ?o + } +} => { + ?s persoon:heeftPersoonsrelatie ?o +}. + +{ + { + ?s persoon_loket:heeftStaatsburgerschap ?o + } e:becomes { + ?s persoon:heeftStaatsburgerschap ?o + } +} => { + ?s persoon:heeftStaatsburgerschap ?o +}. + +{ + { + ?s persoon_loket:heeftVerblijfplaats ?o + } e:becomes { + ?s persoon:heeftVerblijfplaats ?o + } +} => { + ?s persoon:heeftVerblijfplaats ?o +}. + +{ + { + ?s persoon_loket:Inwonerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Inwonerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Inwonerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:isHoofdVan ?o + } e:becomes { + ?s persoon:isHoofdVan ?o + } +} => { + ?s persoon:isHoofdVan ?o +}. + +{ + { + ?s persoon_loket:isLidVan ?o + } e:becomes { + ?s persoon:isLidVan ?o + } +} => { + ?s persoon:isLidVan ?o +}. + +{ + { + ?s persoon_loket:isRelatieMet ?o + } e:becomes { + ?s persoon:isRelatieMet ?o + } +} => { + ?s persoon:isRelatieMet ?o +}. + +{ + { + ?s persoon_loket:land ?o + } e:becomes { + ?s persoon:land ?o + } +} => { + ?s persoon:land ?o +}. + +{ + { + ?s persoon_loket:nationaliteit ?o + } e:becomes { + ?s persoon:nationaliteit ?o + } +} => { + ?s persoon:nationaliteit ?o +}. + +{ + { + ?s persoon_loket:plaats ?o + } e:becomes { + ?s persoon:plaats ?o + } +} => { + ?s persoon:plaats ?o +}. + +{ + { + ?s persoon_loket:registratie ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. + +{ + { + ?s persoon_loket:Staatsburgerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:type ?o + } e:becomes { + ?s persoon:type ?o + } +} => { + ?s persoon:type ?o +}. + +{ + { + ?s persoon_loket:verblijfsadres ?o + } e:becomes { + ?s persoon:verblijfsadres ?o + } +} => { + ?s persoon:verblijfsadres ?o +}. + +{ + { + ?s persoon_loket:volledigeNaam ?o + } e:becomes { + ?s persoon:volledigeNaam ?o + } +} => { + ?s persoon:volledigeNaam ?o +}. \ No newline at end of file diff --git a/config/reasoner/dl2op/main/retract_empty_strings.n3 b/config/reasoner/dl2op/main/retract_empty_strings.n3 new file mode 100644 index 0000000..f8204a7 --- /dev/null +++ b/config/reasoner/dl2op/main/retract_empty_strings.n3 @@ -0,0 +1,27 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Retract empty strings +{ + { + ?s ?p "" + } e:becomes {}. +} => {}. diff --git a/config/reasoner/dl2op/main/rrn.n3 b/config/reasoner/dl2op/main/rrn.n3 new file mode 100644 index 0000000..b45c688 --- /dev/null +++ b/config/reasoner/dl2op/main/rrn.n3 @@ -0,0 +1,62 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + + +# Rijksregisternummer +# Extend identifiers to match OP's model +# skos:notation "00010100105". +{ + ?identifier skos:notation ?notation. + ?notation string:notEqualIgnoringCase "Rijksregisternummer". + ?identifier log:uri ?identifierString. + (?identifierString "STRUCTURED_IDENTIFIER") string:concatenation ?hashInput. + ?hashInput crypto:md5 ?uuid. + ("http://data.lblod.info/id/gestructureerdeIdentificatoren/" ?uuid) string:concatenation ?structuredIdString. + ?structuredIdentifier log:uri ?structuredIdString. + + { + ?identifier skos:notation ?notation. + } e:becomes { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + }. + +} => { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +}. + +# identifier property in the export is only used for RRN. +{ + { + ?s adms:identifier ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. diff --git a/config/reasoner/dl2op/preprocess/mapping.n3 b/config/reasoner/dl2op/preprocess/mapping.n3 deleted file mode 100644 index 0c6ae69..0000000 --- a/config/reasoner/dl2op/preprocess/mapping.n3 +++ /dev/null @@ -1,31 +0,0 @@ -PREFIX e: -PREFIX foaf: -PREFIX persoon: - - -# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) -# Hence the mapping is in two steps. -# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart -# and drops the DL persoon:gebruikteVoornaam statements with e:becomes -# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam -# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames -# in OP - -# First name -# persoon:gebruikteVoornaam => foaf:givenName -# Alternatieve voornaam -# foaf:name => persoon:gebruikteVoornaam - -# Test Mock DL -# persoon:gebruikteVoornaam "Didier". - -{ - { - ?s persoon:gebruikteVoornaam ?o - } e:becomes { - ?s foaf:givenName ?o - } -} => { - ?s foaf:givenName ?o -}. - diff --git a/docker-compose.yml b/docker-compose.yml index e016296..afd9ba2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -201,7 +201,7 @@ services: restart: always logging: *default-logging reasoner: - image: bdevloed/mu-reasoner + image: eyereasoner/reasoning-service:1.0.0 volumes: - ./config/reasoner:/config ################################################################################ @@ -460,7 +460,7 @@ services: # LOKET MANDATEN CONSUMERS ################################################################################ mandaten-consumer: - image: bdevloed/delta-consumer:fetch + image: lblod/delta-consumer:0.0.14 environment: DCR_SERVICE_NAME: "mandatendatabank-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API @@ -487,7 +487,7 @@ services: # LOKET LEIDINGGEVENDEN CONSUMERS ################################################################################ leidinggevenden-consumer: - image: bdevloed/delta-consumer:fetch + image: lblod/delta-consumer:0.0.14 environment: DCR_SERVICE_NAME: "leidinggevenden-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API @@ -510,15 +510,42 @@ services: - triplestore:virtuoso logging: *default-logging + ################################################################################ + # LOKET WORSHIP SENSITIVE CONSUMERS + ################################################################################ + worship-services-sensitive-consumer: + image: lblod/delta-consumer:0.0.14 + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info/" + DCR_SERVICE_NAME: "worship-services-sensitive-consumer" + DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files" + DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer" + DCR_DISABLE_INITIAL_SYNC: "true" + DCR_KEEP_DELTA_FILES: "true" + DCR_DELTA_FILE_FOLDER: "/consumer-files" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false" + volumes: + - ./config/worship-services-sensitive-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching + restart: always + labels: + - "logging=true" + links: + - db:database + - triplestore:virtuoso + logging: *default-logging + ################################################################################ # DELTA CONSUMERS: END ################################################################################ ################################################################################ - # MU SEARCH + # MU SEARCH ################################################################################ search: - image: semtech/mu-search:0.9.0-beta.1 + image: semtech/mu-search:0.9.0-beta.1 links: - db:database restart: always From 94fc9ac78e76326ec72e691db8beb9c6be3dacdf Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 12 Jan 2023 10:26:26 +0000 Subject: [PATCH 2/6] dispatch main and privacy sensitive position info --- .../positions-dispatching/config.js | 9 +- .../delta-sync-dispatching.js | 57 +- .../initial-sync-dispatching.js | 51 +- .../positions-dispatching/util.js | 18 +- .../dl2op/main-info/inverse_relations.n3 | 33 ++ config/reasoner/dl2op/main-info/mapping.n3 | 150 +++++ .../dl2op/main-info/persoon_http_to_https.n3 | 534 +++++++++++++++++ config/reasoner/dl2op/main-info/query.n3q | 538 ++++++++++++++++++ .../dl2op/main-info/retract_empty_strings.n3 | 27 + config/reasoner/dl2op/main-info/rrn.n3 | 62 ++ .../dl2op/private-info/inverse_relations.n3 | 33 ++ config/reasoner/dl2op/private-info/mapping.n3 | 150 +++++ .../private-info/persoon_http_to_https.n3 | 534 +++++++++++++++++ config/reasoner/dl2op/private-info/query.n3q | 63 ++ .../private-info/retract_empty_strings.n3 | 27 + config/reasoner/dl2op/private-info/rrn.n3 | 62 ++ docker-compose.yml | 5 +- 17 files changed, 2308 insertions(+), 45 deletions(-) create mode 100644 config/reasoner/dl2op/main-info/inverse_relations.n3 create mode 100644 config/reasoner/dl2op/main-info/mapping.n3 create mode 100644 config/reasoner/dl2op/main-info/persoon_http_to_https.n3 create mode 100644 config/reasoner/dl2op/main-info/query.n3q create mode 100644 config/reasoner/dl2op/main-info/retract_empty_strings.n3 create mode 100644 config/reasoner/dl2op/main-info/rrn.n3 create mode 100644 config/reasoner/dl2op/private-info/inverse_relations.n3 create mode 100644 config/reasoner/dl2op/private-info/mapping.n3 create mode 100644 config/reasoner/dl2op/private-info/persoon_http_to_https.n3 create mode 100644 config/reasoner/dl2op/private-info/query.n3q create mode 100644 config/reasoner/dl2op/private-info/retract_empty_strings.n3 create mode 100644 config/reasoner/dl2op/private-info/rrn.n3 diff --git a/config/loket-consumer/positions-dispatching/config.js b/config/loket-consumer/positions-dispatching/config.js index e280733..911277b 100644 --- a/config/loket-consumer/positions-dispatching/config.js +++ b/config/loket-consumer/positions-dispatching/config.js @@ -8,6 +8,10 @@ const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000 const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`; +const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/privacy-centric-graph`; +const UNFILTERED_MAPPING = 'main'; +const MAIN_INFO_MAPPING = 'main-info'; +const PRIVATE_INFO_MAPPING = 'private-info'; if (!process.env.FILE_SYNC_GRAPH) @@ -25,5 +29,8 @@ module.exports = { SLEEP_TIME_AFTER_FAILED_DB_OPERATION, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, INGEST_GRAPH, - FILE_SYNC_GRAPH + PRIVACY_SENSITIVE_GRAPH, + UNFILTERED_MAPPING, + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING, }; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js index c8d8ba1..42c997d 100644 --- a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js +++ b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js @@ -1,13 +1,14 @@ const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); -const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, +const { BATCH_SIZE, MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, INGEST_GRAPH, - FILE_SYNC_GRAPH + PRIVACY_SENSITIVE_GRAPH, + UNFILTERED_MAPPING, + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING, } = require('./config'); /** @@ -34,7 +35,7 @@ async function dispatch(lib, data) { const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); if (deleteStatements.length) { - await transformStatements(fetch, deleteStatements).then( + await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING).then( transformedStatements => { deleteFromAllGraphs( muAuthSudo.updateSudo, @@ -51,22 +52,40 @@ async function dispatch(lib, data) { if (insertStatements.length) { - await transformStatements(fetch, insertStatements).then( + await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING).then( transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "INSERT" - ); + if (transformedStatements.length) { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } } - ) + ); + await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING).then( + transformedStatements => { + if (transformedStatements.length) { + batchedDbUpdate( + muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } + } + ); } } } diff --git a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js index 88017c1..f5fedc8 100644 --- a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js +++ b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js @@ -1,4 +1,4 @@ -const { transformStatements, batchedDbUpdate, partition } = require('./util'); +const { transformStatements, batchedDbUpdate, } = require('./util'); const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, DIRECT_DATABASE_ENDPOINT, MU_CALL_SCOPE_ID_INITIAL_SYNC, @@ -7,7 +7,9 @@ const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, INGEST_GRAPH, - FILE_SYNC_GRAPH + PRIVACY_SENSITIVE_GRAPH, + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING } = require('./config'); const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; @@ -36,21 +38,40 @@ async function dispatch(lib, data) { if (termObjects.length) { originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - await transformStatements(fetch, originalInsertTriples).then( + await transformStatements(fetch, originalInsertTriples, MAIN_INFO_MAPPING).then( transformedInsertTriples => { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ) + if (transformedInsertTriples.length) { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ) + } } - ) + ); + await transformStatements(fetch, originalInsertTriples, PRIVATE_INFO_MAPPING).then( + transformedInsertTriples => { + if (transformedInsertTriples.length) { + batchedDbUpdate( + muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ) + } + } + ); } } diff --git a/config/loket-consumer/positions-dispatching/util.js b/config/loket-consumer/positions-dispatching/util.js index ff3932f..cad25a3 100644 --- a/config/loket-consumer/positions-dispatching/util.js +++ b/config/loket-consumer/positions-dispatching/util.js @@ -115,13 +115,13 @@ function partition(arr, fn) { * Send triples to reasoning service for conversion * */ -function transformTriples(fetch, triples) { - return operationWithRetry(mainConversion(fetch, triples), 0, +function transformTriples(fetch, triples, mapping) { + return operationWithRetry(mainConversion(fetch, triples, mapping), 0, MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); } -function mainConversion(fetch, triples) { +function mainConversion(fetch, triples, mapping) { let formdata = new URLSearchParams(); formdata.append("data", triples); @@ -131,15 +131,17 @@ function mainConversion(fetch, triples) { redirect: 'follow' }; - return fetch("http://reasoner/reason/dl2op/main", requestOptions) + return fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) .then(response => response.text()); } -function transformStatements(fetch, triples) { - return transformTriples(fetch, triples.join('\n')).then( + +function transformStatements(fetch, triples, mapping) { + return transformTriples(fetch, triples.join('\n'), mapping).then( graph => { - statements = graph.replace(/\n{2,}/g, '').split('\n') - console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`) + statements = graph ? graph.split('\n') : []; + console.log(`CONVERSION ${mapping}: FROM ${triples.length} triples to ${statements.length}`) + console.log(statements) return statements } ) diff --git a/config/reasoner/dl2op/main-info/inverse_relations.n3 b/config/reasoner/dl2op/main-info/inverse_relations.n3 new file mode 100644 index 0000000..bfeda31 --- /dev/null +++ b/config/reasoner/dl2op/main-info/inverse_relations.n3 @@ -0,0 +1,33 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Inverse properties +{ + ?s org:holds ?o. +} => { + ?o org:heldBy ?s. +}. + +{ + ?s org:heldBy ?o. +} => { + ?o org:holds ?s. +}. diff --git a/config/reasoner/dl2op/main-info/mapping.n3 b/config/reasoner/dl2op/main-info/mapping.n3 new file mode 100644 index 0000000..82bebd9 --- /dev/null +++ b/config/reasoner/dl2op/main-info/mapping.n3 @@ -0,0 +1,150 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) +# Hence the mapping is in two steps. +# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart +# and drops the DL persoon:gebruikteVoornaam statements with e:becomes +# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam +# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames +# in OP + +# First name +# persoon:gebruikteVoornaam => foaf:givenName +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock DL +# persoon_loket:gebruikteVoornaam "Didier". + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s foaf:givenName ?o + } +} => { + ?s foaf:givenName ?o +}. + + +# First name +# persoon:gebruikteVoornaam => foaf:givenName (done in pre-processing) +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock LLB +# foaf:name "Didier". +{ + { + ?s foaf:name ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +# Test +# ex:s "2019-01-01T19:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:start ?startDateTime. + # use backward arrow rule defined below + ?startDateTime :dateFromDateTime ?startDate. + { + ?s mandaat:start ?startDateTime + } e:becomes { + ?s mandaat:start ?startDate + }. +} => { + ?s mandaat:start ?startDate +}. + +# ex:s "2019-02-01T10:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:einde ?eindeDateTime. + # use backward arrow rule defined below + ?eindeDateTime :dateFromDateTime ?eindeDate. + { + ?s mandaat:einde ?eindeDateTime + } e:becomes { + ?s mandaat:einde ?eindeDate + }. +} => { + ?s mandaat:einde ?eindeDate +}. + + +# Backward arrow rule is like a function call. +# Removes time info in the returned date +# The user in DL selects a date via the UI. +# These dates are persisted as midnight dateTimes in UTC. +# Meaning that 1st of July is stored as +# "2022-06-30T22:00:00Z"^^xsd:dateTime +# and 1 December +# "2022-11-30T23:00:00Z"^^xsd:dateTime +# So, the following rule adds 2 hours to the datetime, and then takes the date part of the UTC date time +# to come to the date that the DL user selected (in CE(S)T) +{ + ?dateTime :dateFromDateTime ?date +} <= { + (?dateTimeLiteral xsd:dateTime) log:dtlit ?dateTime. + (?dateTime "PT2H"^^xsd:dayTimeDuration) func:add-dayTimeDuration-to-dateTime ?localDateTime. + (?localDateTimeLiteral xsd:dateTime) log:dtlit ?localDateTime. + (?localDateTimeLiteral 0 11) func:substring ?dateLiteral. + (?dateLiteral xsd:date) log:dtlit ?date. +}. + + +# ex:orgA besluit:classificatie . +{ + { + ?s besluit:classificatie ?o + } e:becomes { + ?s org:classification ?o + } +} => { + ?s org:classification ?o +}. + +# fix typo V -> v +# "1" . +{ + { + ?s adres:AdresVoorstelling.huisnummer ?o + } e:becomes { + ?s adres:Adresvoorstelling.huisnummer ?o + } +} => { + ?s adres:Adresvoorstelling.huisnummer ?o +}. + + +# Tijdspecialisatie +{ + { + ?s mandaat:isTijdspecialisatieVan ?o. + } e:becomes { + ?s generiek:isTijdspecialisatieVan ?o. + } +} => { + ?s generiek:isTijdspecialisatieVan ?o. +}. diff --git a/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 b/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 new file mode 100644 index 0000000..6ced405 --- /dev/null +++ b/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 @@ -0,0 +1,534 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Map from http persoon to https persoon namespace. + +## classes: +{ + { + ?s a persoon_loket:Afstamming + } e:becomes { + ?s a persoon:Afstamming + } +} => { + ?s a persoon:Afstamming +}. + +{ + { + ?s a persoon_loket:BurgerlijkeStaat + } e:becomes { + ?s a persoon:BurgerlijkeStaat + } +} => { + ?s a persoon:BurgerlijkeStaat +}. + +{ + { + ?s a persoon_loket:Domicilie + } e:becomes { + ?s a persoon:Domicilie + } +} => { + ?s a persoon:Domicilie +}. + +{ + { + ?s a persoon_loket:Geboorte + } e:becomes { + ?s a persoon:Geboorte + } +} => { + ?s a persoon:Geboorte +}. + +{ + { + ?s a persoon_loket:GeenInwoner + } e:becomes { + ?s a persoon:GeenInwoner + } +} => { + ?s a persoon:GeenInwoner +}. + +{ + { + ?s a persoon_loket:GeregistreerdPersoon + } e:becomes { + ?s a persoon:GeregistreerdPersoon + } +} => { + ?s a persoon:GeregistreerdPersoon +}. + +{ + { + ?s a persoon_loket:Gezin + } e:becomes { + ?s a persoon:Gezin + } +} => { + ?s a persoon:Gezin +}. + +{ + { + ?s a persoon_loket:Gezinsrelatie + } e:becomes { + ?s a persoon:Gezinsrelatie + } +} => { + ?s a persoon:Gezinsrelatie +}. + +{ + { + ?s a persoon_loket:Huwelijk + } e:becomes { + ?s a persoon:Huwelijk + } +} => { + ?s a persoon:Huwelijk +}. + +{ + { + ?s a persoon_loket:Inwoner + } e:becomes { + ?s a persoon:Inwoner + } +} => { + ?s a persoon:Inwoner +}. + +{ + { + ?s a persoon_loket:Inwonerschap + } e:becomes { + ?s a persoon:Inwonerschap + } +} => { + ?s a persoon:Inwonerschap +}. + +{ + { + ?s a persoon_loket:Nationaliteit + } e:becomes { + ?s a persoon:Nationaliteit + } +} => { + ?s a persoon:Nationaliteit +}. + +{ + { + ?s a persoon_loket:Overlijden + } e:becomes { + ?s a persoon:Overlijden + } +} => { + ?s a persoon:Overlijden +}. + +{ + { + ?s a persoon_loket:PermanentInwoner + } e:becomes { + ?s a persoon:PermanentInwoner + } +} => { + ?s a persoon:PermanentInwoner +}. + +{ + { + ?s a persoon_loket:Persoonsgebeurtenis + } e:becomes { + ?s a persoon:Persoonsgebeurtenis + } +} => { + ?s a persoon:Persoonsgebeurtenis +}. + +{ + { + ?s a persoon_loket:Persoonsrelatie + } e:becomes { + ?s a persoon:Persoonsrelatie + } +} => { + ?s a persoon:Persoonsrelatie +}. + +{ + { + ?s a persoon_loket:Samenwonen + } e:becomes { + ?s a persoon:Samenwonen + } +} => { + ?s a persoon:Samenwonen +}. + +{ + { + ?s a persoon_loket:Staatsburger + } e:becomes { + ?s a persoon:Staatsburger + } +} => { + ?s a persoon:Staatsburger +}. + +{ + { + ?s a persoon_loket:Staatsburgerschap + } e:becomes { + ?s a persoon:Staatsburgerschap + } +} => { + ?s a persoon:Staatsburgerschap +}. + +{ + { + ?s a persoon_loket:TijdelijkInwoner + } e:becomes { + ?s a persoon:TijdelijkInwoner + } +} => { + ?s a persoon:TijdelijkInwoner +}. + +{ + { + ?s a persoon_loket:Verblijfplaats + } e:becomes { + ?s a persoon:Verblijfplaats + } +} => { + ?s a persoon:Verblijfplaats +}. + +{ + { + ?s a persoon_loket:Voogdij + } e:becomes { + ?s a persoon:Voogdij + } +} => { + ?s a persoon:Voogdij +}. + +{ + { + ?s a persoon_loket:Vreemdeling + } e:becomes { + ?s a persoon:Vreemdeling + } +} => { + ?s a persoon:Vreemdeling +}. + +## properties +{ + { + ?s persoon_loket:afstammingsType ?o + } e:becomes { + ?s persoon:afstammingsType ?o + } +} => { + ?s persoon:afstammingsType ?o +}. + +{ + { + ?s persoon_loket:alternatieveNaam ?o + } e:becomes { + ?s persoon:alternatieveNaam ?o + } +} => { + ?s persoon:alternatieveNaam ?o +}. + +{ + { + ?s persoon_loket:datum ?o + } e:becomes { + ?s persoon:datum ?o + } +} => { + ?s persoon:datum ?o +}. + +{ + { + ?s persoon_loket:datumVanAfstamming ?o + } e:becomes { + ?s persoon:datumVanAfstamming ?o + } +} => { + ?s persoon:datumVanAfstamming ?o +}. + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +{ + { + ?s persoon_loket:geslacht ?o + } e:becomes { + ?s persoon:geslacht ?o + } +} => { + ?s persoon:geslacht ?o +}. + +{ + { + ?s persoon_loket:gezinsadres ?o + } e:becomes { + ?s persoon:gezinsadres ?o + } +} => { + ?s persoon:gezinsadres ?o +}. + +{ + { + ?s persoon_loket:gezinsrelatietype ?o + } e:becomes { + ?s persoon:gezinsrelatietype ?o + } +} => { + ?s persoon:gezinsrelatietype ?o +}. + +{ + { + ?s persoon_loket:heeftBurgerlijkeStaat ?o + } e:becomes { + ?s persoon:heeftBurgerlijkeStaat ?o + } +} => { + ?s persoon:heeftBurgerlijkeStaat ?o +}. + +{ + { + ?s persoon_loket:heeftGeboorte ?o + } e:becomes { + ?s persoon:heeftGeboorte ?o + } +} => { + ?s persoon:heeftGeboorte ?o +}. + +{ + { + ?s persoon_loket:heeftInwonerschap ?o + } e:becomes { + ?s persoon:heeftInwonerschap ?o + } +} => { + ?s persoon:heeftInwonerschap ?o +}. + +{ + { + ?s persoon_loket:heeftNationaliteit ?o + } e:becomes { + ?s persoon:heeftNationaliteit ?o + } +} => { + ?s persoon:heeftNationaliteit ?o +}. + +{ + { + ?s persoon_loket:heeftOverlijden ?o + } e:becomes { + ?s persoon:heeftOverlijden ?o + } +} => { + ?s persoon:heeftOverlijden ?o +}. + +{ + { + ?s persoon_loket:heeftPersoonsrelatie ?o + } e:becomes { + ?s persoon:heeftPersoonsrelatie ?o + } +} => { + ?s persoon:heeftPersoonsrelatie ?o +}. + +{ + { + ?s persoon_loket:heeftStaatsburgerschap ?o + } e:becomes { + ?s persoon:heeftStaatsburgerschap ?o + } +} => { + ?s persoon:heeftStaatsburgerschap ?o +}. + +{ + { + ?s persoon_loket:heeftVerblijfplaats ?o + } e:becomes { + ?s persoon:heeftVerblijfplaats ?o + } +} => { + ?s persoon:heeftVerblijfplaats ?o +}. + +{ + { + ?s persoon_loket:Inwonerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Inwonerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Inwonerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:isHoofdVan ?o + } e:becomes { + ?s persoon:isHoofdVan ?o + } +} => { + ?s persoon:isHoofdVan ?o +}. + +{ + { + ?s persoon_loket:isLidVan ?o + } e:becomes { + ?s persoon:isLidVan ?o + } +} => { + ?s persoon:isLidVan ?o +}. + +{ + { + ?s persoon_loket:isRelatieMet ?o + } e:becomes { + ?s persoon:isRelatieMet ?o + } +} => { + ?s persoon:isRelatieMet ?o +}. + +{ + { + ?s persoon_loket:land ?o + } e:becomes { + ?s persoon:land ?o + } +} => { + ?s persoon:land ?o +}. + +{ + { + ?s persoon_loket:nationaliteit ?o + } e:becomes { + ?s persoon:nationaliteit ?o + } +} => { + ?s persoon:nationaliteit ?o +}. + +{ + { + ?s persoon_loket:plaats ?o + } e:becomes { + ?s persoon:plaats ?o + } +} => { + ?s persoon:plaats ?o +}. + +{ + { + ?s persoon_loket:registratie ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. + +{ + { + ?s persoon_loket:Staatsburgerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:type ?o + } e:becomes { + ?s persoon:type ?o + } +} => { + ?s persoon:type ?o +}. + +{ + { + ?s persoon_loket:verblijfsadres ?o + } e:becomes { + ?s persoon:verblijfsadres ?o + } +} => { + ?s persoon:verblijfsadres ?o +}. + +{ + { + ?s persoon_loket:volledigeNaam ?o + } e:becomes { + ?s persoon:volledigeNaam ?o + } +} => { + ?s persoon:volledigeNaam ?o +}. \ No newline at end of file diff --git a/config/reasoner/dl2op/main-info/query.n3q b/config/reasoner/dl2op/main-info/query.n3q new file mode 100644 index 0000000..4ecdf11 --- /dev/null +++ b/config/reasoner/dl2op/main-info/query.n3q @@ -0,0 +1,538 @@ +# Generated by producer-config-to-query.ipynb + +## CLASSES +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +# { +# ?s a +# } => { +# ?s a +# }. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +# { +# ?s a +# } => { +# ?s a +# }. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +## PROPERTIES +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + diff --git a/config/reasoner/dl2op/main-info/retract_empty_strings.n3 b/config/reasoner/dl2op/main-info/retract_empty_strings.n3 new file mode 100644 index 0000000..f8204a7 --- /dev/null +++ b/config/reasoner/dl2op/main-info/retract_empty_strings.n3 @@ -0,0 +1,27 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Retract empty strings +{ + { + ?s ?p "" + } e:becomes {}. +} => {}. diff --git a/config/reasoner/dl2op/main-info/rrn.n3 b/config/reasoner/dl2op/main-info/rrn.n3 new file mode 100644 index 0000000..b45c688 --- /dev/null +++ b/config/reasoner/dl2op/main-info/rrn.n3 @@ -0,0 +1,62 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + + +# Rijksregisternummer +# Extend identifiers to match OP's model +# skos:notation "00010100105". +{ + ?identifier skos:notation ?notation. + ?notation string:notEqualIgnoringCase "Rijksregisternummer". + ?identifier log:uri ?identifierString. + (?identifierString "STRUCTURED_IDENTIFIER") string:concatenation ?hashInput. + ?hashInput crypto:md5 ?uuid. + ("http://data.lblod.info/id/gestructureerdeIdentificatoren/" ?uuid) string:concatenation ?structuredIdString. + ?structuredIdentifier log:uri ?structuredIdString. + + { + ?identifier skos:notation ?notation. + } e:becomes { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + }. + +} => { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +}. + +# identifier property in the export is only used for RRN. +{ + { + ?s adms:identifier ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. diff --git a/config/reasoner/dl2op/private-info/inverse_relations.n3 b/config/reasoner/dl2op/private-info/inverse_relations.n3 new file mode 100644 index 0000000..bfeda31 --- /dev/null +++ b/config/reasoner/dl2op/private-info/inverse_relations.n3 @@ -0,0 +1,33 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Inverse properties +{ + ?s org:holds ?o. +} => { + ?o org:heldBy ?s. +}. + +{ + ?s org:heldBy ?o. +} => { + ?o org:holds ?s. +}. diff --git a/config/reasoner/dl2op/private-info/mapping.n3 b/config/reasoner/dl2op/private-info/mapping.n3 new file mode 100644 index 0000000..82bebd9 --- /dev/null +++ b/config/reasoner/dl2op/private-info/mapping.n3 @@ -0,0 +1,150 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) +# Hence the mapping is in two steps. +# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart +# and drops the DL persoon:gebruikteVoornaam statements with e:becomes +# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam +# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames +# in OP + +# First name +# persoon:gebruikteVoornaam => foaf:givenName +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock DL +# persoon_loket:gebruikteVoornaam "Didier". + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s foaf:givenName ?o + } +} => { + ?s foaf:givenName ?o +}. + + +# First name +# persoon:gebruikteVoornaam => foaf:givenName (done in pre-processing) +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock LLB +# foaf:name "Didier". +{ + { + ?s foaf:name ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +# Test +# ex:s "2019-01-01T19:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:start ?startDateTime. + # use backward arrow rule defined below + ?startDateTime :dateFromDateTime ?startDate. + { + ?s mandaat:start ?startDateTime + } e:becomes { + ?s mandaat:start ?startDate + }. +} => { + ?s mandaat:start ?startDate +}. + +# ex:s "2019-02-01T10:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:einde ?eindeDateTime. + # use backward arrow rule defined below + ?eindeDateTime :dateFromDateTime ?eindeDate. + { + ?s mandaat:einde ?eindeDateTime + } e:becomes { + ?s mandaat:einde ?eindeDate + }. +} => { + ?s mandaat:einde ?eindeDate +}. + + +# Backward arrow rule is like a function call. +# Removes time info in the returned date +# The user in DL selects a date via the UI. +# These dates are persisted as midnight dateTimes in UTC. +# Meaning that 1st of July is stored as +# "2022-06-30T22:00:00Z"^^xsd:dateTime +# and 1 December +# "2022-11-30T23:00:00Z"^^xsd:dateTime +# So, the following rule adds 2 hours to the datetime, and then takes the date part of the UTC date time +# to come to the date that the DL user selected (in CE(S)T) +{ + ?dateTime :dateFromDateTime ?date +} <= { + (?dateTimeLiteral xsd:dateTime) log:dtlit ?dateTime. + (?dateTime "PT2H"^^xsd:dayTimeDuration) func:add-dayTimeDuration-to-dateTime ?localDateTime. + (?localDateTimeLiteral xsd:dateTime) log:dtlit ?localDateTime. + (?localDateTimeLiteral 0 11) func:substring ?dateLiteral. + (?dateLiteral xsd:date) log:dtlit ?date. +}. + + +# ex:orgA besluit:classificatie . +{ + { + ?s besluit:classificatie ?o + } e:becomes { + ?s org:classification ?o + } +} => { + ?s org:classification ?o +}. + +# fix typo V -> v +# "1" . +{ + { + ?s adres:AdresVoorstelling.huisnummer ?o + } e:becomes { + ?s adres:Adresvoorstelling.huisnummer ?o + } +} => { + ?s adres:Adresvoorstelling.huisnummer ?o +}. + + +# Tijdspecialisatie +{ + { + ?s mandaat:isTijdspecialisatieVan ?o. + } e:becomes { + ?s generiek:isTijdspecialisatieVan ?o. + } +} => { + ?s generiek:isTijdspecialisatieVan ?o. +}. diff --git a/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 b/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 new file mode 100644 index 0000000..6ced405 --- /dev/null +++ b/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 @@ -0,0 +1,534 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Map from http persoon to https persoon namespace. + +## classes: +{ + { + ?s a persoon_loket:Afstamming + } e:becomes { + ?s a persoon:Afstamming + } +} => { + ?s a persoon:Afstamming +}. + +{ + { + ?s a persoon_loket:BurgerlijkeStaat + } e:becomes { + ?s a persoon:BurgerlijkeStaat + } +} => { + ?s a persoon:BurgerlijkeStaat +}. + +{ + { + ?s a persoon_loket:Domicilie + } e:becomes { + ?s a persoon:Domicilie + } +} => { + ?s a persoon:Domicilie +}. + +{ + { + ?s a persoon_loket:Geboorte + } e:becomes { + ?s a persoon:Geboorte + } +} => { + ?s a persoon:Geboorte +}. + +{ + { + ?s a persoon_loket:GeenInwoner + } e:becomes { + ?s a persoon:GeenInwoner + } +} => { + ?s a persoon:GeenInwoner +}. + +{ + { + ?s a persoon_loket:GeregistreerdPersoon + } e:becomes { + ?s a persoon:GeregistreerdPersoon + } +} => { + ?s a persoon:GeregistreerdPersoon +}. + +{ + { + ?s a persoon_loket:Gezin + } e:becomes { + ?s a persoon:Gezin + } +} => { + ?s a persoon:Gezin +}. + +{ + { + ?s a persoon_loket:Gezinsrelatie + } e:becomes { + ?s a persoon:Gezinsrelatie + } +} => { + ?s a persoon:Gezinsrelatie +}. + +{ + { + ?s a persoon_loket:Huwelijk + } e:becomes { + ?s a persoon:Huwelijk + } +} => { + ?s a persoon:Huwelijk +}. + +{ + { + ?s a persoon_loket:Inwoner + } e:becomes { + ?s a persoon:Inwoner + } +} => { + ?s a persoon:Inwoner +}. + +{ + { + ?s a persoon_loket:Inwonerschap + } e:becomes { + ?s a persoon:Inwonerschap + } +} => { + ?s a persoon:Inwonerschap +}. + +{ + { + ?s a persoon_loket:Nationaliteit + } e:becomes { + ?s a persoon:Nationaliteit + } +} => { + ?s a persoon:Nationaliteit +}. + +{ + { + ?s a persoon_loket:Overlijden + } e:becomes { + ?s a persoon:Overlijden + } +} => { + ?s a persoon:Overlijden +}. + +{ + { + ?s a persoon_loket:PermanentInwoner + } e:becomes { + ?s a persoon:PermanentInwoner + } +} => { + ?s a persoon:PermanentInwoner +}. + +{ + { + ?s a persoon_loket:Persoonsgebeurtenis + } e:becomes { + ?s a persoon:Persoonsgebeurtenis + } +} => { + ?s a persoon:Persoonsgebeurtenis +}. + +{ + { + ?s a persoon_loket:Persoonsrelatie + } e:becomes { + ?s a persoon:Persoonsrelatie + } +} => { + ?s a persoon:Persoonsrelatie +}. + +{ + { + ?s a persoon_loket:Samenwonen + } e:becomes { + ?s a persoon:Samenwonen + } +} => { + ?s a persoon:Samenwonen +}. + +{ + { + ?s a persoon_loket:Staatsburger + } e:becomes { + ?s a persoon:Staatsburger + } +} => { + ?s a persoon:Staatsburger +}. + +{ + { + ?s a persoon_loket:Staatsburgerschap + } e:becomes { + ?s a persoon:Staatsburgerschap + } +} => { + ?s a persoon:Staatsburgerschap +}. + +{ + { + ?s a persoon_loket:TijdelijkInwoner + } e:becomes { + ?s a persoon:TijdelijkInwoner + } +} => { + ?s a persoon:TijdelijkInwoner +}. + +{ + { + ?s a persoon_loket:Verblijfplaats + } e:becomes { + ?s a persoon:Verblijfplaats + } +} => { + ?s a persoon:Verblijfplaats +}. + +{ + { + ?s a persoon_loket:Voogdij + } e:becomes { + ?s a persoon:Voogdij + } +} => { + ?s a persoon:Voogdij +}. + +{ + { + ?s a persoon_loket:Vreemdeling + } e:becomes { + ?s a persoon:Vreemdeling + } +} => { + ?s a persoon:Vreemdeling +}. + +## properties +{ + { + ?s persoon_loket:afstammingsType ?o + } e:becomes { + ?s persoon:afstammingsType ?o + } +} => { + ?s persoon:afstammingsType ?o +}. + +{ + { + ?s persoon_loket:alternatieveNaam ?o + } e:becomes { + ?s persoon:alternatieveNaam ?o + } +} => { + ?s persoon:alternatieveNaam ?o +}. + +{ + { + ?s persoon_loket:datum ?o + } e:becomes { + ?s persoon:datum ?o + } +} => { + ?s persoon:datum ?o +}. + +{ + { + ?s persoon_loket:datumVanAfstamming ?o + } e:becomes { + ?s persoon:datumVanAfstamming ?o + } +} => { + ?s persoon:datumVanAfstamming ?o +}. + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +{ + { + ?s persoon_loket:geslacht ?o + } e:becomes { + ?s persoon:geslacht ?o + } +} => { + ?s persoon:geslacht ?o +}. + +{ + { + ?s persoon_loket:gezinsadres ?o + } e:becomes { + ?s persoon:gezinsadres ?o + } +} => { + ?s persoon:gezinsadres ?o +}. + +{ + { + ?s persoon_loket:gezinsrelatietype ?o + } e:becomes { + ?s persoon:gezinsrelatietype ?o + } +} => { + ?s persoon:gezinsrelatietype ?o +}. + +{ + { + ?s persoon_loket:heeftBurgerlijkeStaat ?o + } e:becomes { + ?s persoon:heeftBurgerlijkeStaat ?o + } +} => { + ?s persoon:heeftBurgerlijkeStaat ?o +}. + +{ + { + ?s persoon_loket:heeftGeboorte ?o + } e:becomes { + ?s persoon:heeftGeboorte ?o + } +} => { + ?s persoon:heeftGeboorte ?o +}. + +{ + { + ?s persoon_loket:heeftInwonerschap ?o + } e:becomes { + ?s persoon:heeftInwonerschap ?o + } +} => { + ?s persoon:heeftInwonerschap ?o +}. + +{ + { + ?s persoon_loket:heeftNationaliteit ?o + } e:becomes { + ?s persoon:heeftNationaliteit ?o + } +} => { + ?s persoon:heeftNationaliteit ?o +}. + +{ + { + ?s persoon_loket:heeftOverlijden ?o + } e:becomes { + ?s persoon:heeftOverlijden ?o + } +} => { + ?s persoon:heeftOverlijden ?o +}. + +{ + { + ?s persoon_loket:heeftPersoonsrelatie ?o + } e:becomes { + ?s persoon:heeftPersoonsrelatie ?o + } +} => { + ?s persoon:heeftPersoonsrelatie ?o +}. + +{ + { + ?s persoon_loket:heeftStaatsburgerschap ?o + } e:becomes { + ?s persoon:heeftStaatsburgerschap ?o + } +} => { + ?s persoon:heeftStaatsburgerschap ?o +}. + +{ + { + ?s persoon_loket:heeftVerblijfplaats ?o + } e:becomes { + ?s persoon:heeftVerblijfplaats ?o + } +} => { + ?s persoon:heeftVerblijfplaats ?o +}. + +{ + { + ?s persoon_loket:Inwonerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Inwonerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Inwonerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:isHoofdVan ?o + } e:becomes { + ?s persoon:isHoofdVan ?o + } +} => { + ?s persoon:isHoofdVan ?o +}. + +{ + { + ?s persoon_loket:isLidVan ?o + } e:becomes { + ?s persoon:isLidVan ?o + } +} => { + ?s persoon:isLidVan ?o +}. + +{ + { + ?s persoon_loket:isRelatieMet ?o + } e:becomes { + ?s persoon:isRelatieMet ?o + } +} => { + ?s persoon:isRelatieMet ?o +}. + +{ + { + ?s persoon_loket:land ?o + } e:becomes { + ?s persoon:land ?o + } +} => { + ?s persoon:land ?o +}. + +{ + { + ?s persoon_loket:nationaliteit ?o + } e:becomes { + ?s persoon:nationaliteit ?o + } +} => { + ?s persoon:nationaliteit ?o +}. + +{ + { + ?s persoon_loket:plaats ?o + } e:becomes { + ?s persoon:plaats ?o + } +} => { + ?s persoon:plaats ?o +}. + +{ + { + ?s persoon_loket:registratie ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. + +{ + { + ?s persoon_loket:Staatsburgerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:type ?o + } e:becomes { + ?s persoon:type ?o + } +} => { + ?s persoon:type ?o +}. + +{ + { + ?s persoon_loket:verblijfsadres ?o + } e:becomes { + ?s persoon:verblijfsadres ?o + } +} => { + ?s persoon:verblijfsadres ?o +}. + +{ + { + ?s persoon_loket:volledigeNaam ?o + } e:becomes { + ?s persoon:volledigeNaam ?o + } +} => { + ?s persoon:volledigeNaam ?o +}. \ No newline at end of file diff --git a/config/reasoner/dl2op/private-info/query.n3q b/config/reasoner/dl2op/private-info/query.n3q new file mode 100644 index 0000000..f4e3bd6 --- /dev/null +++ b/config/reasoner/dl2op/private-info/query.n3q @@ -0,0 +1,63 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + + +# Rijksregisternummer +# Extend identifiers to match OP's model +# skos:notation "00010100105". +{ + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +} => { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +}. + +# identifier property in the export is only used for RRN. +{ + ?s persoon:registratie ?o +} => { + ?s persoon:registratie ?o +}. + +# Birth and related attributes +{ + ?s a persoon:Geboorte; + ?p ?o. +} => { + ?s a persoon:Geboorte; + ?p ?o. +}. + +{ + ?s persoon:heeftGeboorte ?o +} => { + ?s persoon:heeftGeboorte ?o +}. \ No newline at end of file diff --git a/config/reasoner/dl2op/private-info/retract_empty_strings.n3 b/config/reasoner/dl2op/private-info/retract_empty_strings.n3 new file mode 100644 index 0000000..f8204a7 --- /dev/null +++ b/config/reasoner/dl2op/private-info/retract_empty_strings.n3 @@ -0,0 +1,27 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Retract empty strings +{ + { + ?s ?p "" + } e:becomes {}. +} => {}. diff --git a/config/reasoner/dl2op/private-info/rrn.n3 b/config/reasoner/dl2op/private-info/rrn.n3 new file mode 100644 index 0000000..b45c688 --- /dev/null +++ b/config/reasoner/dl2op/private-info/rrn.n3 @@ -0,0 +1,62 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + + +# Rijksregisternummer +# Extend identifiers to match OP's model +# skos:notation "00010100105". +{ + ?identifier skos:notation ?notation. + ?notation string:notEqualIgnoringCase "Rijksregisternummer". + ?identifier log:uri ?identifierString. + (?identifierString "STRUCTURED_IDENTIFIER") string:concatenation ?hashInput. + ?hashInput crypto:md5 ?uuid. + ("http://data.lblod.info/id/gestructureerdeIdentificatoren/" ?uuid) string:concatenation ?structuredIdString. + ?structuredIdentifier log:uri ?structuredIdString. + + { + ?identifier skos:notation ?notation. + } e:becomes { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + }. + +} => { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +}. + +# identifier property in the export is only used for RRN. +{ + { + ?s adms:identifier ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. diff --git a/docker-compose.yml b/docker-compose.yml index afd9ba2..bc9266d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -201,7 +201,7 @@ services: restart: always logging: *default-logging reasoner: - image: eyereasoner/reasoning-service:1.0.0 + image: eyereasoner/reasoning-service:1.0.1 volumes: - ./config/reasoner:/config ################################################################################ @@ -524,11 +524,12 @@ services: DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing" DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer" DCR_DISABLE_INITIAL_SYNC: "true" + FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data" DCR_KEEP_DELTA_FILES: "true" DCR_DELTA_FILE_FOLDER: "/consumer-files" BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false" volumes: - - ./config/worship-services-sensitive-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching + - ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching restart: always labels: - "logging=true" From d89064d1670da7e7f09f614862dacf245cd94313 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 12 Jan 2023 15:56:13 +0000 Subject: [PATCH 3/6] remove unnecessary log statement --- config/loket-consumer/positions-dispatching/util.js | 1 - 1 file changed, 1 deletion(-) diff --git a/config/loket-consumer/positions-dispatching/util.js b/config/loket-consumer/positions-dispatching/util.js index cad25a3..ce38369 100644 --- a/config/loket-consumer/positions-dispatching/util.js +++ b/config/loket-consumer/positions-dispatching/util.js @@ -141,7 +141,6 @@ function transformStatements(fetch, triples, mapping) { graph => { statements = graph ? graph.split('\n') : []; console.log(`CONVERSION ${mapping}: FROM ${triples.length} triples to ${statements.length}`) - console.log(statements) return statements } ) From c4c8f2332b25d96ead08c068b104351abf1cbdb4 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 12 Jan 2023 18:24:46 +0100 Subject: [PATCH 4/6] Ensure deletes and inserts happen in the correct order --- .../delta-sync-dispatching.js | 89 +++++++++---------- docker-compose.yml | 3 + 2 files changed, 44 insertions(+), 48 deletions(-) diff --git a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js index 42c997d..d826b2e 100644 --- a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js +++ b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js @@ -35,57 +35,50 @@ async function dispatch(lib, data) { const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); if (deleteStatements.length) { - await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING).then( - transformedStatements => { - deleteFromAllGraphs( - muAuthSudo.updateSudo, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - ); - } - ) + const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING); + if (transformedStatementsToDelete.length) { + await deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatementsToDelete, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } } - if (insertStatements.length) { - await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING).then( - transformedStatements => { - if (transformedStatements.length) { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } - } - ); - await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING).then( - transformedStatements => { - if (transformedStatements.length) { - batchedDbUpdate( - muAuthSudo.updateSudo, - PRIVACY_SENSITIVE_GRAPH, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } - } - ); + const transformedStatementsToInsert = await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING); + if (transformedStatementsToInsert.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatementsToInsert, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } + + const transformedStatementsToInsertPrivate = await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING); + if (transformedStatementsToInsertPrivate.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + transformedStatementsToInsertPrivate, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } } } } diff --git a/docker-compose.yml b/docker-compose.yml index bc9266d..2446211 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -475,6 +475,7 @@ services: KEEP_DELTA_FILES: "true" volumes: - ./config/loket-consumer/mandate-dispatching:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/mandaten:/consumer-files/ restart: always labels: - "logging=true" @@ -502,6 +503,7 @@ services: KEEP_DELTA_FILES: "true" volumes: - ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/leidinggevenden:/consumer-files/ restart: always labels: - "logging=true" @@ -530,6 +532,7 @@ services: BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false" volumes: - ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/worship-services-sensitive:/consumer-files/ restart: always labels: - "logging=true" From 8e6f6067dbd8a4a88c2c5e949374fb1f2e7a15df Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 12 Jan 2023 18:33:42 +0000 Subject: [PATCH 5/6] increased scope of privacy sensitive --- config/reasoner/dl2op/main-info/query.n3q | 20 +++++++++---------- config/reasoner/dl2op/private-info/query.n3q | 21 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/config/reasoner/dl2op/main-info/query.n3q b/config/reasoner/dl2op/main-info/query.n3q index 4ecdf11..09dd4fb 100644 --- a/config/reasoner/dl2op/main-info/query.n3q +++ b/config/reasoner/dl2op/main-info/query.n3q @@ -260,11 +260,11 @@ ?s ?o }. -{ - ?s ?o -} => { - ?s ?o -}. +# { +# ?s ?o +# } => { +# ?s ?o +# }. # { # ?s ?o @@ -272,11 +272,11 @@ # ?s ?o # }. -{ - ?s ?o -} => { - ?s ?o -}. +# { +# ?s ?o +# } => { +# ?s ?o +# }. { ?s ?o diff --git a/config/reasoner/dl2op/private-info/query.n3q b/config/reasoner/dl2op/private-info/query.n3q index f4e3bd6..12e94f2 100644 --- a/config/reasoner/dl2op/private-info/query.n3q +++ b/config/reasoner/dl2op/private-info/query.n3q @@ -47,6 +47,14 @@ PREFIX xsd: ?s persoon:registratie ?o }. +# Nationality +{ + ?s persoon:heeftNationaliteit ?o +} => { + ?s persoon:heeftNationaliteit ?o +}. + + # Birth and related attributes { ?s a persoon:Geboorte; @@ -60,4 +68,17 @@ PREFIX xsd: ?s persoon:heeftGeboorte ?o } => { ?s persoon:heeftGeboorte ?o +}. + +{ + ?s persoon:datum ?o +} => { + ?s persoon:datum ?o +}. + +# Gender +{ + ?s persoon:geslacht ?o +} => { + ?s persoon:geslacht ?o }. \ No newline at end of file From 075e56b1f2764ffb535a7c47768dca264aa77255 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Fri, 13 Jan 2023 08:45:50 +0000 Subject: [PATCH 6/6] Add identifier type to privacy graph --- config/reasoner/dl2op/private-info/query.n3q | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/reasoner/dl2op/private-info/query.n3q b/config/reasoner/dl2op/private-info/query.n3q index 12e94f2..bddbb11 100644 --- a/config/reasoner/dl2op/private-info/query.n3q +++ b/config/reasoner/dl2op/private-info/query.n3q @@ -47,6 +47,12 @@ PREFIX xsd: ?s persoon:registratie ?o }. +{ + ?s a + } => { + ?s a +}. + # Nationality { ?s persoon:heeftNationaliteit ?o