diff --git a/.gitignore b/.gitignore index cc42a73..d6b6a17 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ !/data/db/toLoad !/data/db/toLoad/* /data/files/* +/data/temp/* docker-compose.override.yml .idea *.iml diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 5f0fd27..23a6d85 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -1,38 +1,66 @@ { - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", "fileBaseName": "dump-administrative-units", - "compression": true + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", "fileBaseName": "dump-administrative-units", - "compression": true + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", - "compression": true + "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", + "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizationsdeltas", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", - "compression": true + "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", + "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizationsdeltas", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", - "compression": false + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", - "compression": false + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" : { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", + "fileBaseName": "dump-worship-posts", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" : { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", + "fileBaseName": "dump-worship-posts", + "targetDcatGraph": "http://mu.semte.ch/graphs/public", + "targetFilesGraph": "http://mu.semte.ch/graphs/public", + "cleanupOldDumps": true } } diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json new file mode 100644 index 0000000..9c4b9ad --- /dev/null +++ b/config/delta-producer/worship-posts/export.json @@ -0,0 +1,47 @@ +{ + "export": [ + { + "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role" + ], + "additionalFilter": "?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://www.w3.org/ns/org#hasPost" + ], + "additionalFilter": "?subject / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role", + "http://data.lblod.info/vocabularies/erediensten/functie" + ] + } + ] +} diff --git a/config/delta/rules.js b/config/delta/rules.js index 15a2309..2b2899f 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -156,6 +156,24 @@ export default [ optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] } }, + { + match: { + // anything + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-worship-posts/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, { match: { predicate: { diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index bdac90f..a86c47a 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -335,6 +335,14 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-organizations/files/" end + ################################################################# + # DELTA: posts + ################################################################# + + get "/sync/worship-posts/files/*path" do + Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-worship-posts/files/" + end + ############################################################### # frontend layer ############################################################### diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 928128b..5817d84 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -94,5 +94,37 @@ "nextIndex": "0" } ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextIndex": "0" + }, + { + "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "1" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "0" + } + ] } } diff --git a/config/loket-consumer/leidinggevenden-dispatching/config.js b/config/loket-consumer/leidinggevenden-dispatching/config.js new file mode 100644 index 0000000..e280733 --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/config.js @@ -0,0 +1,29 @@ +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); +const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); +const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); +const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`; + + +if (!process.env.FILE_SYNC_GRAPH) + throw `Expected 'FILE_SYNC_GRAPH' to be provided.`; +const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH; + +module.exports = { + BATCH_SIZE, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MAX_DB_RETRY_ATTEMPTS, + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +}; \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js new file mode 100644 index 0000000..c8d8ba1 --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js @@ -0,0 +1,76 @@ +const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); +const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +} = require('./config'); + +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo, fetch } = lib; + const { termObjectChangeSets } = data; + + for (let { deletes, inserts } of termObjectChangeSets) { + const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + if (deleteStatements.length) { + await transformStatements(fetch, deleteStatements).then( + transformedStatements => { + deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } + ) + } + + + if (insertStatements.length) { + await transformStatements(fetch, insertStatements).then( + transformedStatements => { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + "INSERT" + ); + } + ) + } + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js new file mode 100644 index 0000000..88017c1 --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js @@ -0,0 +1,59 @@ +const { transformStatements, batchedDbUpdate, partition } = require('./util'); +const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +} = require('./config'); +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo, fetch } = lib; + const { termObjects } = data; + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) + + await transformStatements(fetch, originalInsertTriples).then( + transformedInsertTriples => { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ) + } + ) + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden-dispatching/util.js b/config/loket-consumer/leidinggevenden-dispatching/util.js new file mode 100644 index 0000000..ff3932f --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/util.js @@ -0,0 +1,153 @@ +const { + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION +} = require('./config'); + +async function batchedDbUpdate(muUpdate, + graph, + triples, + extraHeaders, + endpoint, + batchSize, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, + operation = 'INSERT' +) { + + for (let i = 0; i < triples.length; i += batchSize) { + console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); + + const batch = triples.slice(i, i + batchSize).join('\n'); + + const insertCall = async () => { + await muUpdate(` +${operation} DATA { +GRAPH <${graph}> { +${batch} +} +} +`, extraHeaders, endpoint); + }; + + await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); + + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function deleteFromAllGraphs(muUpdate, + triples, + extraHeaders, + endpoint, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, +) { + + for (const triple of triples) { + + console.log(`Deleting a triple from all graphs in triplestore`); + + const deleteCall = async () => { + await muUpdate(` + DELETE { + GRAPH ?g { + ${triple} + } + } WHERE { + GRAPH ?g { + ${triple} + } + } + `, extraHeaders, endpoint); + }; + + await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function operationWithRetry(callback, + attempt, + maxAttempts, + sleepTimeOnFail) { + try { + if (typeof callback === "function") + return await callback(); + else // Catch error from promise - not how I would do it normally, but allows re use of existing code. + return await callback; + } + catch (e) { + console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); + console.log(`Error: ${e}`); + console.log(`Sleeping ${sleepTimeOnFail} ms`); + + if (attempt >= maxAttempts) { + console.log(`Max attempts reached for ${callback.toString()}, giving up`); + throw e; + } + + await new Promise(r => setTimeout(r, sleepTimeOnFail)); + return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); + } +} + +/** + * Splits an array into two parts, a part that passes and a part that fails a predicate function. + * Credits: https://github.com/benjay10 + * @public + * @function partition + * @param {Array} arr - Array to be partitioned + * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. + * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively + */ +function partition(arr, fn) { + let passes = [], fails = []; + arr.forEach((item) => (fn(item) ? passes : fails).push(item)); + return { passes, fails }; +} + + +/** + * Send triples to reasoning service for conversion + * + */ +function transformTriples(fetch, triples) { + return operationWithRetry(mainConversion(fetch, triples), 0, + MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); +} + + +function mainConversion(fetch, triples) { + let formdata = new URLSearchParams(); + formdata.append("data", triples); + + let requestOptions = { + method: 'POST', + body: formdata, + redirect: 'follow' + }; + + return fetch("http://reasoner/reason/dl2op/main", requestOptions) + .then(response => response.text()); +} + +function transformStatements(fetch, triples) { + return transformTriples(fetch, triples.join('\n')).then( + graph => { + statements = graph.replace(/\n{2,}/g, '').split('\n') + console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`) + return statements + } + ) +} + +module.exports = { + batchedDbUpdate, + deleteFromAllGraphs, + partition, + transformStatements +}; \ No newline at end of file diff --git a/config/loket-consumer/mandate-dispatching/config.js b/config/loket-consumer/mandate-dispatching/config.js new file mode 100644 index 0000000..e280733 --- /dev/null +++ b/config/loket-consumer/mandate-dispatching/config.js @@ -0,0 +1,29 @@ +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); +const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); +const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); +const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`; + + +if (!process.env.FILE_SYNC_GRAPH) + throw `Expected 'FILE_SYNC_GRAPH' to be provided.`; +const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH; + +module.exports = { + BATCH_SIZE, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MAX_DB_RETRY_ATTEMPTS, + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +}; \ No newline at end of file diff --git a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js new file mode 100644 index 0000000..c8d8ba1 --- /dev/null +++ b/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js @@ -0,0 +1,76 @@ +const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); +const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +} = require('./config'); + +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo, fetch } = lib; + const { termObjectChangeSets } = data; + + for (let { deletes, inserts } of termObjectChangeSets) { + const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + if (deleteStatements.length) { + await transformStatements(fetch, deleteStatements).then( + transformedStatements => { + deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } + ) + } + + + if (insertStatements.length) { + await transformStatements(fetch, insertStatements).then( + transformedStatements => { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatements, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + "INSERT" + ); + } + ) + } + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js new file mode 100644 index 0000000..88017c1 --- /dev/null +++ b/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js @@ -0,0 +1,59 @@ +const { transformStatements, batchedDbUpdate, partition } = require('./util'); +const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + INGEST_GRAPH, + FILE_SYNC_GRAPH +} = require('./config'); +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo, fetch } = lib; + const { termObjects } = data; + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) + + await transformStatements(fetch, originalInsertTriples).then( + transformedInsertTriples => { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ) + } + ) + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/loket-consumer/mandate-dispatching/util.js b/config/loket-consumer/mandate-dispatching/util.js new file mode 100644 index 0000000..ff3932f --- /dev/null +++ b/config/loket-consumer/mandate-dispatching/util.js @@ -0,0 +1,153 @@ +const { + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION +} = require('./config'); + +async function batchedDbUpdate(muUpdate, + graph, + triples, + extraHeaders, + endpoint, + batchSize, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, + operation = 'INSERT' +) { + + for (let i = 0; i < triples.length; i += batchSize) { + console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); + + const batch = triples.slice(i, i + batchSize).join('\n'); + + const insertCall = async () => { + await muUpdate(` +${operation} DATA { +GRAPH <${graph}> { +${batch} +} +} +`, extraHeaders, endpoint); + }; + + await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); + + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function deleteFromAllGraphs(muUpdate, + triples, + extraHeaders, + endpoint, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, +) { + + for (const triple of triples) { + + console.log(`Deleting a triple from all graphs in triplestore`); + + const deleteCall = async () => { + await muUpdate(` + DELETE { + GRAPH ?g { + ${triple} + } + } WHERE { + GRAPH ?g { + ${triple} + } + } + `, extraHeaders, endpoint); + }; + + await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function operationWithRetry(callback, + attempt, + maxAttempts, + sleepTimeOnFail) { + try { + if (typeof callback === "function") + return await callback(); + else // Catch error from promise - not how I would do it normally, but allows re use of existing code. + return await callback; + } + catch (e) { + console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); + console.log(`Error: ${e}`); + console.log(`Sleeping ${sleepTimeOnFail} ms`); + + if (attempt >= maxAttempts) { + console.log(`Max attempts reached for ${callback.toString()}, giving up`); + throw e; + } + + await new Promise(r => setTimeout(r, sleepTimeOnFail)); + return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); + } +} + +/** + * Splits an array into two parts, a part that passes and a part that fails a predicate function. + * Credits: https://github.com/benjay10 + * @public + * @function partition + * @param {Array} arr - Array to be partitioned + * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. + * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively + */ +function partition(arr, fn) { + let passes = [], fails = []; + arr.forEach((item) => (fn(item) ? passes : fails).push(item)); + return { passes, fails }; +} + + +/** + * Send triples to reasoning service for conversion + * + */ +function transformTriples(fetch, triples) { + return operationWithRetry(mainConversion(fetch, triples), 0, + MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); +} + + +function mainConversion(fetch, triples) { + let formdata = new URLSearchParams(); + formdata.append("data", triples); + + let requestOptions = { + method: 'POST', + body: formdata, + redirect: 'follow' + }; + + return fetch("http://reasoner/reason/dl2op/main", requestOptions) + .then(response => response.text()); +} + +function transformStatements(fetch, triples) { + return transformTriples(fetch, triples.join('\n')).then( + graph => { + statements = graph.replace(/\n{2,}/g, '').split('\n') + console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`) + return statements + } + ) +} + +module.exports = { + batchedDbUpdate, + deleteFromAllGraphs, + partition, + transformStatements +}; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/config.js b/config/loket-consumer/positions-dispatching/config.js new file mode 100644 index 0000000..911277b --- /dev/null +++ b/config/loket-consumer/positions-dispatching/config.js @@ -0,0 +1,36 @@ +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); +const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); +const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); +const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`; +const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/privacy-centric-graph`; +const UNFILTERED_MAPPING = 'main'; +const MAIN_INFO_MAPPING = 'main-info'; +const PRIVATE_INFO_MAPPING = 'private-info'; + + +if (!process.env.FILE_SYNC_GRAPH) + throw `Expected 'FILE_SYNC_GRAPH' to be provided.`; +const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH; + +module.exports = { + BATCH_SIZE, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MAX_DB_RETRY_ATTEMPTS, + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + INGEST_GRAPH, + PRIVACY_SENSITIVE_GRAPH, + UNFILTERED_MAPPING, + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING, +}; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js new file mode 100644 index 0000000..d826b2e --- /dev/null +++ b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js @@ -0,0 +1,88 @@ +const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); +const { + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + INGEST_GRAPH, + PRIVACY_SENSITIVE_GRAPH, + UNFILTERED_MAPPING, + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING, +} = require('./config'); + +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo, fetch } = lib; + const { termObjectChangeSets } = data; + + for (let { deletes, inserts } of termObjectChangeSets) { + const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + if (deleteStatements.length) { + const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING); + if (transformedStatementsToDelete.length) { + await deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatementsToDelete, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } + } + + if (insertStatements.length) { + const transformedStatementsToInsert = await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING); + if (transformedStatementsToInsert.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatementsToInsert, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } + + const transformedStatementsToInsertPrivate = await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING); + if (transformedStatementsToInsertPrivate.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + transformedStatementsToInsertPrivate, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } + } + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js new file mode 100644 index 0000000..f5fedc8 --- /dev/null +++ b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js @@ -0,0 +1,80 @@ +const { transformStatements, batchedDbUpdate, } = require('./util'); +const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + INGEST_GRAPH, + PRIVACY_SENSITIVE_GRAPH, + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING +} = require('./config'); +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo, fetch } = lib; + const { termObjects } = data; + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) + + await transformStatements(fetch, originalInsertTriples, MAIN_INFO_MAPPING).then( + transformedInsertTriples => { + if (transformedInsertTriples.length) { + batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ) + } + } + ); + await transformStatements(fetch, originalInsertTriples, PRIVATE_INFO_MAPPING).then( + transformedInsertTriples => { + if (transformedInsertTriples.length) { + batchedDbUpdate( + muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ) + } + } + ); + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/loket-consumer/positions-dispatching/util.js b/config/loket-consumer/positions-dispatching/util.js new file mode 100644 index 0000000..ce38369 --- /dev/null +++ b/config/loket-consumer/positions-dispatching/util.js @@ -0,0 +1,154 @@ +const { + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION +} = require('./config'); + +async function batchedDbUpdate(muUpdate, + graph, + triples, + extraHeaders, + endpoint, + batchSize, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, + operation = 'INSERT' +) { + + for (let i = 0; i < triples.length; i += batchSize) { + console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); + + const batch = triples.slice(i, i + batchSize).join('\n'); + + const insertCall = async () => { + await muUpdate(` +${operation} DATA { +GRAPH <${graph}> { +${batch} +} +} +`, extraHeaders, endpoint); + }; + + await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); + + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function deleteFromAllGraphs(muUpdate, + triples, + extraHeaders, + endpoint, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, +) { + + for (const triple of triples) { + + console.log(`Deleting a triple from all graphs in triplestore`); + + const deleteCall = async () => { + await muUpdate(` + DELETE { + GRAPH ?g { + ${triple} + } + } WHERE { + GRAPH ?g { + ${triple} + } + } + `, extraHeaders, endpoint); + }; + + await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function operationWithRetry(callback, + attempt, + maxAttempts, + sleepTimeOnFail) { + try { + if (typeof callback === "function") + return await callback(); + else // Catch error from promise - not how I would do it normally, but allows re use of existing code. + return await callback; + } + catch (e) { + console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); + console.log(`Error: ${e}`); + console.log(`Sleeping ${sleepTimeOnFail} ms`); + + if (attempt >= maxAttempts) { + console.log(`Max attempts reached for ${callback.toString()}, giving up`); + throw e; + } + + await new Promise(r => setTimeout(r, sleepTimeOnFail)); + return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); + } +} + +/** + * Splits an array into two parts, a part that passes and a part that fails a predicate function. + * Credits: https://github.com/benjay10 + * @public + * @function partition + * @param {Array} arr - Array to be partitioned + * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. + * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively + */ +function partition(arr, fn) { + let passes = [], fails = []; + arr.forEach((item) => (fn(item) ? passes : fails).push(item)); + return { passes, fails }; +} + + +/** + * Send triples to reasoning service for conversion + * + */ +function transformTriples(fetch, triples, mapping) { + return operationWithRetry(mainConversion(fetch, triples, mapping), 0, + MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); +} + + +function mainConversion(fetch, triples, mapping) { + let formdata = new URLSearchParams(); + formdata.append("data", triples); + + let requestOptions = { + method: 'POST', + body: formdata, + redirect: 'follow' + }; + + return fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) + .then(response => response.text()); +} + + +function transformStatements(fetch, triples, mapping) { + return transformTriples(fetch, triples.join('\n'), mapping).then( + graph => { + statements = graph ? graph.split('\n') : []; + console.log(`CONVERSION ${mapping}: FROM ${triples.length} triples to ${statements.length}`) + return statements + } + ) +} + +module.exports = { + batchedDbUpdate, + deleteFromAllGraphs, + partition, + transformStatements +}; \ No newline at end of file diff --git a/config/migrations/2022/20221206100001-delete-duplicate-persons-st-franciscus-glabbeek.sparql b/config/migrations/2022/20221206100001-delete-duplicate-persons-st-franciscus-glabbeek.sparql new file mode 100644 index 0000000..c72862a --- /dev/null +++ b/config/migrations/2022/20221206100001-delete-duplicate-persons-st-franciscus-glabbeek.sparql @@ -0,0 +1,83 @@ +# OP-1928 : Remove worship members in "Kerkfabriek St.-Franciscus van Glabbeek" + +# Delete duplicate persons +DELETE { + GRAPH { + ?pm ?om . + ?pj ?oj . + ?pd ?od . + ?pt ?ot . + } +} +WHERE { + GRAPH { + ?pm ?om . + ?pj ?oj . + ?pd ?od . + ?pt ?ot . + } +} +; +DELETE { + GRAPH { + ?pm ?om . + ?pj ?oj . + } +} +WHERE { + GRAPH { + ?pm ?om . + ?pj ?oj . + } +} +; +# Change Start Date +DELETE { + GRAPH ?g { + "2022-05-04"^^xsd:date . + } +} INSERT { + GRAPH ?g { + "2022-01-28"^^xsd:date . + } +} WHERE { + GRAPH ?g { + "2022-05-04"^^xsd:date . + } +} +; +DELETE { + GRAPH ?g { + "2022-05-04"^^xsd:date . + } +} INSERT { + GRAPH ?g { + "2022-01-28"^^xsd:date . + } +} WHERE { + GRAPH ?g { + "2022-05-04"^^xsd:date . + } +} + +# The associated call to trigger a reindex is : + +# curl --location --request POST '172.19.0.23/update' \ +# --header 'Content-Type: application/vnd.api+json' \ +# --data-raw ' [ +# { +# "deletes": [ +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/personen/635BD6EC8DE5818A7C9F4310" }, +# "predicate": { "type": "uri", "value": "http://mu.semte.ch/vocabularies/core/uuid" }, +# "object": { "type": "string", "value": "635BD6EC8DE5818A7C9F4310" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/personen/635BD7518DE5818A7C9F4313" }, +# "predicate": { "type": "uri", "value": "http://mu.semte.ch/vocabularies/core/uuid" }, +# "object": { "type": "string", "value": "635BD7518DE5818A7C9F4313" } +# } +# ], +# "inserts": [] +# } +# ]' \ No newline at end of file diff --git a/config/migrations/2022/20221208100000-delete-duplicate-person.sparql b/config/migrations/2022/20221208100000-delete-duplicate-person.sparql new file mode 100644 index 0000000..9e5fa96 --- /dev/null +++ b/config/migrations/2022/20221208100000-delete-duplicate-person.sparql @@ -0,0 +1,79 @@ +# OP-1929 : Delete duplicated members from worship service + +# Delete wrong peson and mandatarissen +DELETE { + GRAPH { ?s ?p ?o . } +} +WHERE { + GRAPH { + VALUES ?s { + + + + + + + + } + ?s ?p ?o . + } +} +; +# Delete sensitive data +DELETE { + GRAPH { + ?pp ?op . + } +} +WHERE { + GRAPH { + ?pp ?op . + } +} + +# The associated call to trigger a reindex is : + +# curl --location --request POST '172.20.0.26/update' \ +# --header 'Content-Type: application/vnd.api+json' \ +# --data-raw ' [ +# { +# "deletes": [ +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/personen/74c5800e8489c825f318acc9dcb031d0" }, +# "predicate": { "type": "uri", "value": "http://mu.semte.ch/vocabularies/core/uuid" }, +# "object": { "type": "string", "value": "74c5800e8489c825f318acc9dcb031d0" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/mandatarissen/6380A02F4B5FEAF28DEDE975" }, +# "predicate": { "type": "uri", "value": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" }, +# "object": { "type": "uri", "value": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/mandatarissen/6380BBB04B5FEAF28DEDE97E" }, +# "predicate": { "type": "uri", "value": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" }, +# "object": { "type": "uri", "value": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/mandatarissen/63809FC24B5FEAF28DEDE974" }, +# "predicate": { "type": "uri", "value": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" }, +# "object": { "type": "uri", "value": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/mandatarissen/63809F414B5FEAF28DEDE972" }, +# "predicate": { "type": "uri", "value": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" }, +# "object": { "type": "uri", "value": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/mandatarissen/63809F804B5FEAF28DEDE973" }, +# "predicate": { "type": "uri", "value": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" }, +# "object": { "type": "uri", "value": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/mandatarissen/6380D0854B5FEAF28DEDE97F" }, +# "predicate": { "type": "uri", "value": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" }, +# "object": { "type": "uri", "value": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" } +# } +# ], +# "inserts": [] +# } +# ]' \ No newline at end of file diff --git a/config/migrations/2022/20221209093743-add-missing-betrokken-besturen.graph b/config/migrations/2022/20221209093743-add-missing-betrokken-besturen.graph new file mode 100644 index 0000000..7489cd5 --- /dev/null +++ b/config/migrations/2022/20221209093743-add-missing-betrokken-besturen.graph @@ -0,0 +1 @@ +http://mu.semte.ch/graphs/organisatieportaal \ No newline at end of file diff --git a/config/migrations/2022/20221209093743-add-missing-betrokken-besturen.ttl b/config/migrations/2022/20221209093743-add-missing-betrokken-besturen.ttl new file mode 100644 index 0000000..a886300 --- /dev/null +++ b/config/migrations/2022/20221209093743-add-missing-betrokken-besturen.ttl @@ -0,0 +1,3070 @@ + + + a ; + 100; + ; + "39db6dfe-579c-4c1d-9851-f18547643a33"; + . + + + a ; + 100; + ; + "14d5d68e-ac34-4793-8fde-e43569f0c37d"; + . + + + a ; + 100; + ; + "54281766-9fce-42be-9c83-b8d1ee171e74"; + . + + + a ; + 100; + ; + "c0bcd3a6-517a-4669-a376-c31e3ae6cc17"; + . + + + a ; + 100; + ; + "cd8d283b-7e5d-427a-be95-6ff9b87db757"; + . + + + a ; + 100; + ; + "86b1eac5-aa95-475c-93df-b9586eb077de"; + . + + + a ; + 100; + ; + "15acbee6-a360-43ff-b5cc-fe5dc3bc03ce"; + . + + + a ; + 100; + ; + "296eff94-b3cc-473b-abef-5abf68590916"; + . + + + a ; + 100; + ; + "bba8e9b3-547d-4f52-88cf-36ba90b0cfd8"; + . + + + a ; + 100; + ; + "e2dfa734-ed22-4109-bae1-979575030fbc"; + . + + + a ; + 100; + ; + "291d611f-e5e8-4e65-b0d9-4ddd39026ce0"; + . + + + a ; + 100; + ; + "34c15ea2-7347-4ad7-a3fe-4c0540e6a00b"; + . + + + a ; + 100; + ; + "fcb1b005-226d-4b9a-94ee-25bab1950a52"; + . + + + a ; + 100; + ; + "134dec04-7788-49af-b539-1b2a3b7b2f9b"; + . + + + a ; + 100; + ; + "8f8d28c9-5776-4bb7-9da5-ca8498d36130"; + . + + + a ; + 100; + ; + "91d5cddd-2127-42c4-a14b-9ac09410d35f"; + . + + + a ; + 100; + ; + "8609c2af-ab32-4b1d-8166-5a8aa9549145"; + . + + + a ; + 100; + ; + "60535a06-07a0-4071-8a1b-f776f7d4d68d"; + . + + + a ; + 100; + ; + "a44ea2b7-65df-41ee-a747-7065a7f8df1a"; + . + + + a ; + 100; + ; + "339eb0e3-fbb1-4a6d-9284-e4b47c7afb82"; + . + + + a ; + 100; + ; + "206dfcd4-a6b2-4b04-b83a-3ef2b7a0d630"; + . + + + a ; + 100; + ; + "744e85a0-5217-4f1e-a863-5845f7b19b5a"; + . + + + a ; + 100; + ; + "306b56d0-5294-4ee6-bf22-b62cfdda8d32"; + . + + + a ; + 100; + ; + "7a4c7098-a524-4c1c-abdd-d674a31df7b1"; + . + + + a ; + 100; + ; + "127859a0-41f8-4593-9e65-a0ecdc354bdc"; + . + + + a ; + 100; + ; + "ce1026f8-7474-47f2-86bc-80851d14b15b"; + . + + + a ; + 100; + ; + "77e4d005-5acc-45f7-ab52-c37cdbd27840"; + . + + + a ; + 100; + ; + "0e305e28-e4c8-485a-9d78-4c8a4932108d"; + . + + + a ; + 100; + ; + "c86dfd33-5286-4981-911a-b14182685c0e"; + . + + + a ; + 100; + ; + "f857ddda-4349-4918-9eb0-7fa95c82f903"; + . + + + a ; + 100; + ; + "d8d2a773-c569-4ac5-b0f4-f67560705124"; + . + + + a ; + 100; + ; + "90a72485-877d-428e-9b7a-41a5e282acf3"; + . + + + a ; + 100; + ; + "a3dba296-e6a9-481c-b708-6c57b04c30e0"; + . + + + a ; + 100; + ; + "2fd9d299-0826-47ab-8a4d-e56247fe703a"; + . + + + a ; + 100; + ; + "04a73cb0-8e64-4609-804d-8d189f3e8ceb"; + . + + + a ; + 100; + ; + "8fe7a0f7-767f-4fd2-94f0-8e4c0c3b529e"; + . + + + a ; + 100; + ; + "7c989a6c-ef15-490c-a972-13b380892fb4"; + . + + + a ; + 100; + ; + "18ad5664-1873-4ef1-8be0-7235f903d4c2"; + . + + + a ; + 100; + ; + "8ebc757c-c4d8-486f-a01c-93e25c72708e"; + . + + + a ; + 100; + ; + "7316fb2f-388b-40f8-b6fa-49a91255d6c4"; + . + + + a ; + 100; + ; + "6018fdd0-5588-4652-935f-72fb9f48a03b"; + . + + + a ; + 100; + ; + "5bbe9ad0-2066-4ae4-b893-301ae1a3c8b8"; + . + + + a ; + 100; + ; + "d2a63b79-a907-4e67-a283-dde537b38eca"; + . + + + a ; + 100; + ; + "8c7ad881-4f96-4beb-b69a-0e4a9ac3048a"; + . + + + a ; + 100; + ; + "ad165bd6-0fb9-43df-8e74-28370f306aed"; + . + + + a ; + 100; + ; + "2b5d3333-d0c0-4d56-9435-fa56b32bba7a"; + . + + + a ; + 100; + ; + "c8a4618d-e6f8-4ed4-8dc0-0a1ebc19edf3"; + . + + + a ; + 100; + ; + "992017f5-247b-4e02-acfd-e9882a336cef"; + . + + + a ; + 100; + ; + "94ee9cd9-350e-4cf4-8037-d84e989b13cd"; + . + + + a ; + 100; + ; + "239249a4-1121-48fa-a4d2-99f57a1a6ffa"; + . + + + a ; + 100; + ; + "565404af-88c0-4005-a976-1fcd2f786e48"; + . + + + a ; + 100; + ; + "787879fb-d398-4cdf-9d3e-31bd3fcde711"; + . + + + a ; + 100; + ; + "13737cc0-59e5-439c-95b0-fddc4352a4dc"; + . + + + a ; + 100; + ; + "ca1146a6-786d-4e67-897a-6f9be3a63f8d"; + . + + + a ; + 100; + ; + "9dbc6857-0647-4e8c-85e9-83042bdc0e5c"; + . + + + a ; + 100; + ; + "f83ee6da-c5c6-48cf-b47a-bd7cde8ef7d1"; + . + + + a ; + 100; + ; + "4219f672-aa0c-4d1e-97b0-ed779d14e390"; + . + + + a ; + 100; + ; + "6798d9cd-f314-4bca-8488-2f376439abdd"; + . + + + a ; + 100; + ; + "428f7bb2-ec40-4ec8-be93-cc1f9cf118c2"; + . + + + a ; + 100; + ; + "64d7773a-810b-4eee-9172-bffaa2946ee1"; + . + + + a ; + 100; + ; + "e088c7f4-b5b4-4f55-9815-43c1be94bd2f"; + . + + + a ; + 100; + ; + "83cff3a2-7ab0-4cb7-b9b1-01f88265817e"; + . + + + a ; + 100; + ; + "d1917ea6-f1c5-41cf-b2af-447f1d370d01"; + . + + + a ; + 100; + ; + "64ae6d76-aa1c-4df7-99d3-4efd9a4320df"; + . + + + a ; + 100; + ; + "570ac453-b6b7-47ea-869e-2047b0f7906e"; + . + + + a ; + 100; + ; + "741ddc80-fdff-432f-895d-484ee3fb035c"; + . + + + a ; + 100; + ; + "f0c82c4c-e50d-4885-a769-3ecd1edd8ee7"; + . + + + a ; + 100; + ; + "dc963735-8bb7-4faa-9c57-c2bcdbc9782a"; + . + + + a ; + 100; + ; + "7a25c32f-e161-43da-af61-133179ff6fdf"; + . + + + a ; + 100; + ; + "25c11629-4084-42f8-94ac-918f3ef6cfd4"; + . + + + a ; + 100; + ; + "61e30f3a-54b2-41fb-b2ff-f96207a98d8c"; + . + + + a ; + 100; + ; + "278ccdf8-5170-4501-9daa-d4a853de40cd"; + . + + + a ; + 100; + ; + "cc97c7ae-8b3f-464d-89c9-30e74abaea79"; + . + + + a ; + 100; + ; + "9a3a5452-e626-4886-9499-d4826e70206c"; + . + + + a ; + 100; + ; + "b7695bf9-8f3e-4a90-b029-230e069fc38e"; + . + + + a ; + 100; + ; + "4ebb7193-400b-4415-9508-4a3af27a029a"; + . + + + a ; + 100; + ; + "31658bc3-3296-4f2f-9e91-0badefc5c790"; + . + + + a ; + 100; + ; + "28246920-b91c-4321-8e10-65c1edc3c7ab"; + . + + + a ; + 100; + ; + "250a4a81-2394-4828-8aa8-3ec6e2625908"; + . + + + a ; + 100; + ; + "868c83a4-bd00-436f-93a8-3fae5f32ccea"; + . + + + a ; + 100; + ; + "baa67736-47f5-4838-a5f0-3b2e50af4ae9"; + . + + + a ; + 100; + ; + "8e0e36f0-99fe-4a4f-af12-b0408163ce21"; + . + + + a ; + 100; + ; + "f5413fe1-5117-473a-8f7f-d9da25e2f5a6"; + . + + + a ; + 100; + ; + "e4455570-25ed-478e-8fa4-be7ecac66c11"; + . + + + a ; + 100; + ; + "50a9a3b9-a9ed-4d94-891d-4d70b8cc1a2a"; + . + + + a ; + 100; + ; + "62bd8d74-c266-4a42-8a43-bf8fb8887bf5"; + . + + + a ; + 100; + ; + "214bc598-39ea-496e-906b-60ec04543b08"; + . + + + a ; + 100; + ; + "49751f48-7076-40ac-8002-b0793aa4aa5d"; + . + + + a ; + 100; + ; + "68c31d29-4373-4215-a9dd-e26b90be567c"; + . + + + a ; + 100; + ; + "d243b200-81a3-43d7-8dbf-df8f89fae3a3"; + . + + + a ; + 100; + ; + "7bcd745d-6fa6-4a57-94ec-5aae6afd05a1"; + . + + + a ; + 100; + ; + "62786f76-4e5d-46df-958f-b45c95e6aa03"; + . + + + a ; + 100; + ; + "3c6efedb-efb6-4aaf-877d-7cbbe3bc9aa3"; + . + + + a ; + 100; + ; + "ed3bebca-9177-4e8c-8e65-27616d334a18"; + . + + + a ; + 100; + ; + "e9c7a18b-197c-401e-8239-14f990c950fa"; + . + + + a ; + 100; + ; + "568e868f-7005-40b9-957c-9e02054b1a80"; + . + + + a ; + 100; + ; + "162a3bc7-ec69-4dd2-b352-6848290f4f7a"; + . + + + a ; + 100; + ; + "12a63c62-dc35-40fa-a9c4-33890fb75b9e"; + . + + + a ; + 100; + ; + "fe93089a-d4c1-4dc7-a774-b3d01decb8c3"; + . + + + a ; + 100; + ; + "35c5df16-5ef4-43bf-97f0-681654a1549e"; + . + + + a ; + 100; + ; + "4ca67124-51c6-4c39-b2c2-76f332d74a90"; + . + + + a ; + 100; + ; + "ba45706a-e24e-4adc-96a1-f6475d9e72a0"; + . + + + a ; + 100; + ; + "a1ffa407-3e09-4c59-8666-41e4d06c93c1"; + . + + + a ; + 100; + ; + "e894ea54-e3f5-497b-b2db-99244d7b8492"; + . + + + a ; + 100; + ; + "b6e810a9-565a-4157-b3bb-dcc88e8e5338"; + . + + + a ; + 100; + ; + "e6b76587-cb8c-4689-9f21-5cfb9f108f9d"; + . + + + a ; + 100; + ; + "9f0c28c3-6859-46b9-ac36-b53efaa333d7"; + . + + + a ; + 100; + ; + "4fb4a21b-74f0-4e98-afb3-62e32c03f793"; + . + + + a ; + 100; + ; + "a11ea4bb-6f42-40b5-948f-2d4645e96b10"; + . + + + a ; + 100; + ; + "aa9f88e5-132b-4bfa-9c5f-b226c6ef44f1"; + . + + + a ; + 100; + ; + "d22dc214-5d35-46a8-85b6-8f0b2793b7eb"; + . + + + a ; + 100; + ; + "79c1a286-69a7-4c6d-bfd9-eb82749fd759"; + . + + + a ; + 100; + ; + "697412fc-de35-41ee-9786-81e62ef25f56"; + . + + + a ; + 100; + ; + "207be3f8-e1fc-46f0-acfb-3e9f9660b344"; + . + + + a ; + 100; + ; + "2ac22503-9450-42c3-82a5-96cab6bd72d4"; + . + + + a ; + 100; + ; + "6572b77d-be77-4c5f-b1fb-473f34ace8ac"; + . + + + a ; + 100; + ; + "dd510104-5352-4a44-962f-0424c960c251"; + . + + + a ; + 100; + ; + "d02b415a-c66c-42b6-a15b-0e5101edf9cc"; + . + + + a ; + 100; + ; + "0632f407-2829-4a17-afb6-5725fbcf80db"; + . + + + a ; + 100; + ; + "f1dcd064-35aa-4d47-a8a9-5f3532e4787f"; + . + + + a ; + 100; + ; + "e7c618b7-f47b-46cd-b081-59839d20398f"; + . + + + a ; + 100; + ; + "ff579e0c-fce9-4855-8a06-ede8de9e9754"; + . + + + a ; + 100; + ; + "eb38cf59-77d6-4fb1-8912-183d90a6affb"; + . + + + a ; + 100; + ; + "87d179f6-7413-4137-bdf6-63ac9d4c16d7"; + . + + + a ; + 100; + ; + "466e8074-bd15-46d7-95e1-02a03527948e"; + . + + + a ; + 100; + ; + "8579babe-cfa9-4dc5-92ed-15044ef5a098"; + . + + + a ; + 100; + ; + "71680486-630e-46a4-8325-37b3026b1385"; + . + + + a ; + 100; + ; + "6417268c-ec01-4873-89d1-7bb777ad1583"; + . + + + a ; + 100; + ; + "64128756-b6c8-4e0b-8fce-df7bbf327c0c"; + . + + + a ; + 100; + ; + "9905d0ad-9ca5-4077-b5b5-452f207b45f7"; + . + + + a ; + 100; + ; + "77e1f4cb-1055-46d1-b052-852bdd79fe68"; + . + + + a ; + 100; + ; + "d19a92da-1ddc-4410-9188-9c3941f41173"; + . + + + a ; + 100; + ; + "e0c8b859-c442-4e0f-8735-160a95ba8ac3"; + . + + + a ; + 100; + ; + "09dede7e-1e6e-4c0a-b0f6-2b3c52291d5b"; + . + + + a ; + 100; + ; + "937533ac-0cb8-4e78-91f7-c23dfee89bb2"; + . + + + a ; + 100; + ; + "f80a6a4c-9b90-42ae-ab5c-8ab9bbfe2724"; + . + + + a ; + 100; + ; + "089cb5a2-8b1c-4c5b-a49e-5a176a0d145a"; + . + + + a ; + 100; + ; + "706b2012-3aba-477e-9c9c-ce47f29e3840"; + . + + + a ; + 100; + ; + "b88e6c3d-21ac-4252-8526-a8fad885002e"; + . + + + a ; + 100; + ; + "3611f01a-6439-4631-9ae1-da38a5ba83f0"; + . + + + a ; + 100; + ; + "f74bebb6-1322-4d3d-b96c-acce0320ae36"; + . + + + a ; + 100; + ; + "d915a89f-703d-49c0-9dc2-7eaf5c419a99"; + . + + + a ; + 100; + ; + "ce3cce03-8a26-4eb2-8b22-887c99dd27b9"; + . + + + a ; + 100; + ; + "213e15a7-e334-4aa3-ad36-8cded8a27157"; + . + + + a ; + 100; + ; + "589e11e6-367b-4649-a15a-ac14ba1c4d8e"; + . + + + a ; + 100; + ; + "9246c603-62b1-4984-a90b-4f1608b600e4"; + . + + + a ; + 100; + ; + "35274e45-b5b6-46b2-afe1-98e0aaff5a58"; + . + + + a ; + 100; + ; + "5fb9cd00-c6b5-49b5-8eb5-381bb8167446"; + . + + + a ; + 100; + ; + "c34307ad-7555-4140-827d-34f49191622a"; + . + + + a ; + 100; + ; + "27a6cb6a-e17b-4f57-96e4-b3ff6f783e5a"; + . + + + a ; + 100; + ; + "128b0d77-a84f-404b-87c7-8a979cbfac44"; + . + + + a ; + 100; + ; + "70e8bbcc-3fec-45fb-855a-f10029468a19"; + . + + + a ; + 100; + ; + "9cba7852-98c8-4e71-b702-4d4057b8f2c5"; + . + + + a ; + 100; + ; + "b0a6cc03-f5d0-4232-89ed-9c9d4a82f211"; + . + + + a ; + 100; + ; + "7ad2d2ad-c003-472d-a9d3-cc0217e8244c"; + . + + + a ; + 100; + ; + "4a062eeb-8a0c-4fee-9d65-95ee2201f2a6"; + . + + + a ; + 100; + ; + "01a80a07-f933-4eb3-ad55-abbce96232f1"; + . + + + a ; + 100; + ; + "b7bbabee-aaea-4380-a6ea-ef80450a5419"; + . + + + a ; + 100; + ; + "50ae7530-3c5c-47c5-a4c3-e867f7b8d913"; + . + + + a ; + 100; + ; + "b91371e9-96fd-40e7-94ac-2dff4b25bf94"; + . + + + a ; + 100; + ; + "8a01e75c-a469-484c-b16a-925066ce84f6"; + . + + + a ; + 100; + ; + "fa7e9864-e350-4183-adb1-c448689fdc10"; + . + + + a ; + 100; + ; + "77bf3b5f-3f68-429a-885e-d88dbfefb3d2"; + . + + + a ; + 100; + ; + "c53d5dba-3877-43fd-9d41-d73b92549174"; + . + + + a ; + 100; + ; + "58808ab5-24a8-413b-80fb-68972c666bdd"; + . + + + a ; + 100; + ; + "3a91c63a-e100-49f8-9385-89c08c87675a"; + . + + + a ; + 100; + ; + "cabaeee7-baa5-453c-85c5-64b797f74734"; + . + + + a ; + 100; + ; + "2a51b7c0-2708-4c9b-9ce8-8eb2c1c16b11"; + . + + + a ; + 100; + ; + "0b24ff63-9e02-4149-b0da-5795de03ae6f"; + . + + + a ; + 100; + ; + "5464eefb-2fb0-4737-9304-466669a5ddc3"; + . + + + a ; + 100; + ; + "047418fa-8aa8-4524-a4a3-65fdde2bd492"; + . + + + a ; + 100; + ; + "b09b6515-5781-48cc-ba0b-2d22e765b432"; + . + + + a ; + 100; + ; + "fadbd6f2-0118-43a8-9eb7-e8d4a91bc8ed"; + . + + + a ; + 100; + ; + "80f91b42-8387-40e3-a57d-682f35b5954d"; + . + + + a ; + 100; + ; + "df27db01-f88c-4c60-988b-74fec112c8b4"; + . + + + a ; + 100; + ; + "81da9158-07fa-44bb-b598-31ad4ac20dc6"; + . + + + a ; + 100; + ; + "bfabc47a-37f0-4ed5-b4ba-70052e83d135"; + . + + + a ; + 100; + ; + "a259314c-5931-4122-bac1-e7f9813cc10c"; + . + + + a ; + 100; + ; + "16243d63-2cfd-42a4-834a-3ccb9a01a2f6"; + . + + + a ; + 100; + ; + "6a929c1b-82e4-4d93-a3fe-d91a75217f46"; + . + + + a ; + 100; + ; + "7ff5d6d2-d991-486a-ad03-2beba2a88f68"; + . + + + a ; + 100; + ; + "28b914fa-0e7d-47e2-99fa-9fa3a409be54"; + . + + + a ; + 100; + ; + "8b4a085c-99a1-4789-b566-b6ad837bcd44"; + . + + + a ; + 100; + ; + "ad535a13-cff6-422b-a083-c827cd77f232"; + . + + + a ; + 100; + ; + "709b701c-573c-42b3-ae08-4db84215308b"; + . + + + a ; + 100; + ; + "cb8b230f-f1e4-4992-981d-e4d0ecf889a3"; + . + + + a ; + 100; + ; + "c345fbfc-ab9b-4211-a032-e41753da5288"; + . + + + a ; + 100; + ; + "9a7efbb9-d5ff-418d-b02e-f9c09d8489f5"; + . + + + a ; + 100; + ; + "d7565813-18f0-4dad-b6b8-41b91947f105"; + . + + + a ; + 100; + ; + "f7054ad2-b4e6-4117-9f8a-d40e03da4ce5"; + . + + + a ; + 100; + ; + "5ec8ca42-4386-4888-a875-f2175106a813"; + . + + + a ; + 100; + ; + "4baf611b-0850-4c92-9dd2-8872c5c4f629"; + . + + + a ; + 100; + ; + "93db383d-b5f2-4bde-bf0e-2d36c5d2efa0"; + . + + + a ; + 100; + ; + "d2398702-63bf-498d-870d-6060cb716bc4"; + . + + + a ; + 100; + ; + "1047d898-f0a3-4f45-ae81-ea8e44e1599d"; + . + + + a ; + 100; + ; + "88a03a37-2ff7-47e5-a79b-9dc6e5b99a50"; + . + + + a ; + 100; + ; + "0206d727-0cf5-4cb7-bc80-9af44cb205e2"; + . + + + a ; + 100; + ; + "9985cafe-8eeb-4ca0-bdea-6f644f1466a1"; + . + + + a ; + 100; + ; + "54958763-5035-4429-a978-2f390c5e2b6f"; + . + + + a ; + 100; + ; + "d09c807b-a88a-452e-8522-ab1192e4c14f"; + . + + + a ; + 100; + ; + "2feff2ed-ea4d-451e-b128-de14e7c2327e"; + . + + + a ; + 100; + ; + "2ed9ac9d-5131-40aa-9881-c23860283973"; + . + + + a ; + 100; + ; + "dfa69fc9-3739-4e65-9d18-1c8fd7721e34"; + . + + + a ; + 100; + ; + "fe97d33f-d6f9-45c4-9526-caac68f2e880"; + . + + + a ; + 100; + ; + "7d22ca2f-1203-4333-bc93-f8e7826dfbf9"; + . + + + a ; + 100; + ; + "935cc3be-c4e1-4b3f-9aaf-c12c2b294b3d"; + . + + + a ; + 100; + ; + "b37752e5-6010-4bde-9528-d75267568b46"; + . + + + a ; + 100; + ; + "fd55d258-10c1-41fc-8158-ebe18ef9a3cf"; + . + + + a ; + 100; + ; + "b02539bd-7de9-4b53-914d-b77a0b70b346"; + . + + + a ; + 100; + ; + "c4ea099a-26b3-4a6c-8d81-6b7985579499"; + . + + + a ; + 100; + ; + "3440173c-27e5-4f0d-8f02-be3b91f605be"; + . + + + a ; + 100; + ; + "c4c82ddf-4cd4-4e82-8976-45987e24c0fe"; + . + + + a ; + 100; + ; + "7f467c15-7ac5-4fc5-92af-83aa20bc920b"; + . + + + a ; + 100; + ; + "ba677230-e9bd-4902-b763-7672e5a3a979"; + . + + + a ; + 100; + ; + "d8dfe222-6a06-4b8c-a925-84bbc1d1cace"; + . + + + a ; + 100; + ; + "efea1be4-8f83-41d3-a3b1-44aa09d1e258"; + . + + + a ; + 100; + ; + "509a0e42-bc30-442f-a872-d39fe8388f33"; + . + + + a ; + 100; + ; + "74abb738-8f89-413b-81b6-bc7f461b278a"; + . + + + a ; + 100; + ; + "bf6ecbe4-7ce3-4711-918f-04f498367d47"; + . + + + a ; + 100; + ; + "78aa54ff-a9ef-47d9-81d0-9266cb8ef940"; + . + + + a ; + 100; + ; + "e2620be3-8f6c-4dee-a0ef-ed02009aa7dc"; + . + + + a ; + 100; + ; + "5ed9eb05-627f-4b28-8ed5-813e4b3dbf38"; + . + + + a ; + 100; + ; + "cb597309-1b3e-41e2-8798-1670b0b3c2ad"; + . + + + a ; + 100; + ; + "a215fa25-a829-48b2-b40c-6edea1494795"; + . + + + a ; + 100; + ; + "a6309fc7-47e5-48a8-882b-2995f54463ad"; + . + + + a ; + 100; + ; + "961861e5-a0a8-4591-b50a-815de0c98136"; + . + + + a ; + 100; + ; + "bf59dce8-dc6d-4cd6-a47a-aebc02e50e5b"; + . + + + a ; + 100; + ; + "0ccd39a1-6007-49d8-b4a3-57b5ec6cfc3b"; + . + + + a ; + 100; + ; + "65771337-68d9-4e12-821f-21e8bb2f3cdf"; + . + + + a ; + 100; + ; + "5890a48a-bff5-4dce-869a-64f386827dc1"; + . + + + a ; + 100; + ; + "4c2744fb-08d7-4be9-8392-d1fdf4664095"; + . + + + a ; + 100; + ; + "5572181b-d91c-4258-828f-372ae64f10e9"; + . + + + a ; + 100; + ; + "1a01900a-a9ce-4d10-87aa-df79e0d529ad"; + . + + + a ; + 100; + ; + "cb39f5f3-ca4e-4d17-b9f2-9cd0b9a5cfdc"; + . + + + a ; + 100; + ; + "e356d500-f70b-4f96-b2aa-030331b3368c"; + . + + + a ; + 100; + ; + "f62426cf-e089-49f3-8222-7e5af777ad0a"; + . + + + a ; + 100; + ; + "af47b89b-a457-43c8-aad3-d0c2f03b097b"; + . + + + a ; + 100; + ; + "e19033b0-8f1e-4f67-b2b4-a293a59f4aab"; + . + + + a ; + 100; + ; + "af5ff968-30f0-48c6-80bd-17c56501ed60"; + . + + + a ; + 100; + ; + "84b30cfb-94c9-419a-8bf7-041a1ab94ac4"; + . + + + a ; + 100; + ; + "b96042b6-bfbb-4187-803d-6661675958c3"; + . + + + a ; + 100; + ; + "0ac22758-7380-46be-b881-cb962acbcc4f"; + . + + + a ; + 100; + ; + "d119dc08-213d-4030-ab98-fdbe796d4667"; + . + + + a ; + 100; + ; + "9fffe76f-9cfb-4e91-8408-d70c4feb75bc"; + . + + + a ; + 100; + ; + "64c9f77a-5078-449e-966d-10de580ae689"; + . + + + a ; + 100; + ; + "be57a498-43d8-4c3a-8a5e-a1dc5f9d1fc3"; + . + + + a ; + 100; + ; + "06d5bcbe-7917-48f8-9e55-3a24a632b627"; + . + + + a ; + 100; + ; + "a242d7d2-ab33-4cb6-8490-22584778764c"; + . + + + a ; + 100; + ; + "714a2f24-e9ba-4219-89ab-49129fba7986"; + . + + + a ; + 100; + ; + "8daf621b-766d-45d0-b071-9ca71bcb4ff5"; + . + + + a ; + 100; + ; + "a8f2b132-4257-422e-9bd0-0f8081311fef"; + . + + + a ; + 100; + ; + "65163706-f40e-4944-9f18-ac0bdc51a368"; + . + + + a ; + 100; + ; + "273f3fe0-fd88-4c20-9674-de1ac7a598f8"; + . + + + a ; + 100; + ; + "f1819db5-b9b5-45a2-9897-3fd3e551e0c3"; + . + + + a ; + 100; + ; + "281edce4-7008-4464-8ad5-18e74b878124"; + . + + + a ; + 100; + ; + "b663f2e9-22c5-4308-82da-668b33efbb2d"; + . + + + a ; + 100; + ; + "fba041d9-1d8b-4c86-8454-6ba72da0db82"; + . + + + a ; + 100; + ; + "c8dc25bf-8622-4f53-a61e-f2a852b8f441"; + . + + + a ; + 100; + ; + "7ec382e1-fb93-4fb8-8a33-35491e963ad1"; + . + + + a ; + 100; + ; + "14c3966b-ab20-4c0d-b61d-04c386072052"; + . + + + a ; + 100; + ; + "3a2fff57-3aca-4278-a7cc-c44ce6504d99"; + . + + + a ; + 100; + ; + "ae1fb5b6-b210-4b1a-81e7-429cfa4c6321"; + . + + + a ; + 100; + ; + "a3b5ba49-7826-4221-afbe-58ea09f7e9ae"; + . + + + a ; + 100; + ; + "a96b574f-354e-45b5-b5c7-323c3686b20f"; + . + + + a ; + 100; + ; + "d7cd7da2-d97c-4496-9ed3-c423a0db4292"; + . + + + a ; + 100; + ; + "c9fd8612-c459-40e7-bff9-de204a9e4733"; + . + + + a ; + 100; + ; + "6462d6a9-604d-4860-a480-06de8ab88488"; + . + + + a ; + 100; + ; + "5d2b4250-0927-4c24-aab7-08ba65a4f56a"; + . + + + a ; + 100; + ; + "cd84600f-5d7a-43c8-b971-8285b67ae173"; + . + + + a ; + 100; + ; + "d7672d1f-893b-496d-ae39-55fa40ab99b6"; + . + + + a ; + 100; + ; + "7a1bc677-2030-48fb-85c3-7633a5c6a0a6"; + . + + + a ; + 100; + ; + "4dd4ef9f-36cd-4a40-a16d-1cd65bbaf3ec"; + . + + + a ; + 100; + ; + "4bdbb2b6-3ce4-41c7-b05a-10867a142668"; + . + + + a ; + 100; + ; + "ec55bfbc-67cd-41d9-8823-3aca1d4c959b"; + . + + + a ; + 100; + ; + "8d13312e-69c3-4284-b371-737b58be2361"; + . + + + a ; + 100; + ; + "3a1038f8-6e42-49b2-9f25-3bab894414f1"; + . + + + a ; + 100; + ; + "4c392c17-bff7-4f24-b288-99e3df00b589"; + . + + + a ; + 100; + ; + "213df356-1b68-45c0-a1db-8feab78aec81"; + . + + + a ; + 100; + ; + "692921a6-2465-47eb-8e04-2c29b49f50de"; + . + + + a ; + 100; + ; + "dcb29e8e-263d-490a-97ef-b312252822d8"; + . + + + a ; + 100; + ; + "4dc96030-88e8-4748-adbf-1aebdc2ea2b3"; + . + + + a ; + 100; + ; + "ad539bb3-f41a-44a9-947f-0dc6ecb29fea"; + . + + + a ; + 100; + ; + "03ddc34f-7990-4bb2-888b-37bce5ce044e"; + . + + + a ; + 100; + ; + "9773c47e-4206-44cd-81ac-a437e080b9d0"; + . + + + a ; + 100; + ; + "637812c3-34c4-4339-8618-bcc2675d83d4"; + . + + + a ; + 100; + ; + "94f23563-6df6-4808-8324-5924c86b4f25"; + . + + + a ; + 100; + ; + "dd898f3b-87e5-432e-9e00-5d0d3fa54b05"; + . + + + a ; + 100; + ; + "849f49cf-bd09-4f88-96ea-e1bd702a94ca"; + . + + + a ; + 100; + ; + "5e40cc59-194b-4b42-b64a-6a53a8cb1152"; + . + + + a ; + 100; + ; + "37b2f71b-c14f-4fa5-ba7d-8a765c87859e"; + . + + + a ; + 100; + ; + "4ba2e1d8-fa5d-4002-9d1b-c168a2a84045"; + . + + + a ; + 100; + ; + "34837fc6-c5f9-424e-8d9e-b9fef8751a8a"; + . + + + a ; + 100; + ; + "aeeed194-f748-4ec1-a2c8-c54e8fd60545"; + . + + + a ; + 100; + ; + "b8d72a74-90c1-4241-8b3f-9df3abb6299b"; + . + + + a ; + 100; + ; + "d5f854b4-0751-4a3a-8f4b-227e3dd8252e"; + . + + + a ; + 100; + ; + "e395419c-b06e-484e-8302-cd2d1011601f"; + . + + + a ; + 100; + ; + "2924cc14-f51c-4692-ae4b-84f886d4daf9"; + . + + + a ; + 100; + ; + "c6e14db2-95da-4189-9d60-051164b5e0b5"; + . + + + a ; + 100; + ; + "9ae6d05c-a9db-429b-8c7c-65367ff00fcb"; + . + + + a ; + 100; + ; + "49d534b3-50ca-41ed-9fea-80cd49f67454"; + . + + + a ; + 100; + ; + "ef6637fa-f831-4c6a-92c7-2f8a1c29e383"; + . + + + a ; + 100; + ; + "7d08085a-3e2a-46ba-9347-8681b8173137"; + . + + + a ; + 100; + ; + "ade9acde-d2e6-4ae3-b733-733d8465e83b"; + . + + + a ; + 100; + ; + "44d35426-24a8-4516-b9bb-6c8b56417e85"; + . + + + a ; + 100; + ; + "2e60a1c4-358c-409f-94ab-cbf817d55569"; + . + + + a ; + 100; + ; + "626974f2-ad28-4a67-a59c-a60cd0c0ab77"; + . + + + a ; + 100; + ; + "a03b1945-3232-4561-b719-78b3e77b366e"; + . + + + a ; + 100; + ; + "40e1c80d-17ab-47ce-ba7c-bfcb39e7cd57"; + . + + + a ; + 100; + ; + "f09f7a20-5c11-4277-902c-851c1f98e031"; + . + + + a ; + 100; + ; + "f28864b3-d734-4c52-bf0c-e6335be2f714"; + . + + + a ; + 100; + ; + "12451bc0-ed31-4c6b-9d1e-0c68ec2d23b5"; + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + , + , + , + , + , + , + , + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . \ No newline at end of file diff --git a/config/migrations/2022/20221209164001-modify-bedienaar-role.sparql b/config/migrations/2022/20221209164001-modify-bedienaar-role.sparql new file mode 100644 index 0000000..be586f9 --- /dev/null +++ b/config/migrations/2022/20221209164001-modify-bedienaar-role.sparql @@ -0,0 +1,17 @@ +PREFIX org: +PREFIX ere: + +DELETE { + GRAPH ?g { + ?role ere:functie . # Priester + } +} INSERT { + GRAPH ?g { + ?role ere:functie . # Rector + } +} WHERE { + GRAPH ?g { + org:holds ?role . + ?role ere:functie . + } +} diff --git a/config/migrations/2022/20221220152700-modify-bedienaar-role.sparql b/config/migrations/2022/20221220152700-modify-bedienaar-role.sparql new file mode 100644 index 0000000..8a7e398 --- /dev/null +++ b/config/migrations/2022/20221220152700-modify-bedienaar-role.sparql @@ -0,0 +1,17 @@ +PREFIX org: +PREFIX ere: + +DELETE { + GRAPH ?g { + ?role ere:functie . # Priester + } +} INSERT { + GRAPH ?g { + ?role ere:functie . # Rector + } +} WHERE { + GRAPH ?g { + org:holds ?role . + ?role ere:functie . + } +} \ No newline at end of file diff --git a/config/migrations/2022/20221222160000-fix-kbo-uris.sparql b/config/migrations/2022/20221222160000-fix-kbo-uris.sparql new file mode 100644 index 0000000..54cc7f6 --- /dev/null +++ b/config/migrations/2022/20221222160000-fix-kbo-uris.sparql @@ -0,0 +1,80 @@ +# http://data.lblod.info/id/centraleBesturenVanDeEredienst/c5908a253d1aa2fccb75e6fff439c93a CKB Kruisem --> Stays +# http://data.lblod.info/id/centraleBesturenVanDeEredienst/64dcf0eaafd13dea75717d9014982106 CKB Kruishoutem --> DELETE KBO + +# http://data.lblod.info/id/centraleBesturenVanDeEredienst/6d4226a920cc47e4d4bbfddb451f4883 CKB Aalter NIET actief --> Change to "0535.589.755" + +# http://data.lblod.info/id/centraleBesturenVanDeEredienst/e08a8bc5828bd44e735724a79b588b13 CKB Aalter actief --> Stays +# http://data.lblod.info/id/centraleBesturenVanDeEredienst/9e0e8f060ca954da6263243642433f44 CKB De Pinte --> Stays +# http://data.lblod.info/id/centraleBesturenVanDeEredienst/fa51c0cd4530f0db4caea031f75305d3 CKB Sint-Martens-Latem --> DELETE KBO + +# http://data.lblod.info/id/besturenVanDeEredienst/619a5721094b28807dca8d04de69e3e0 Islamitische geloofsgemeenschap Yunus Emre van Genk --> Stays +# http://data.lblod.info/id/centraleBesturenVanDeEredienst/e17dabfaf8f562ad181f422006c42e97 CB van de Islamitische gemeenschappen in Limburg --> Delete KBO + + + +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX ch: +PREFIX code: +PREFIX dbpedia: +PREFIX dc_terms: +PREFIX ere: +PREFIX euro: +PREFIX euvoc: +PREFIX ext: +PREFIX foaf: +PREFIX generiek: +PREFIX lblodlg: +PREFIX locn: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX organisatie: +PREFIX person: +PREFIX persoon: +PREFIX prov: +PREFIX rdfs: +PREFIX regorg: +PREFIX schema: +PREFIX skos: +PREFIX vcard: +PREFIX xsd: + +DELETE { + GRAPH ?g { + ?bestuur + adms:identifier + ?identifier_kbo. + } +} INSERT { + GRAPH ?g { + + adms:identifier + . + + a adms:Identifier; + mu:uuid "d029ebdf899249f2b3d1ae88be0ebf65"; + skos:notation + "KBO nummer"; + generiek:gestructureerdeIdentificator + . + + a generiek:GestructureerdeIdentificator; + mu:uuid "6a197d71a59d41ca900009ef84c72002"; + generiek:lokaleIdentificator + "0535589755". + } +} WHERE { + GRAPH ?g { + VALUES ?bestuur { + + + + + } + + ?bestuur + adms:identifier + ?identifier_kbo. + ?identifier_kbo + skos:notation + "KBO nummer". + } +} \ No newline at end of file diff --git a/config/migrations/2022/20221227100001-modify-date-of-change-event-OP-1981.sparql b/config/migrations/2022/20221227100001-modify-date-of-change-event-OP-1981.sparql new file mode 100644 index 0000000..6e75cf9 --- /dev/null +++ b/config/migrations/2022/20221227100001-modify-date-of-change-event-OP-1981.sparql @@ -0,0 +1,17 @@ +PREFIX dct: + +DELETE { + GRAPH ?g { + dct:date "2022-12-21"^^xsd:date . + } +} INSERT { + GRAPH ?g { + dct:date "2022-12-20"^^xsd:date . + } +} WHERE { + GRAPH ?g { + dct:date "2022-12-21"^^xsd:date . + } +} + +# OP-1981 \ No newline at end of file diff --git a/config/migrations/2023/20230104152916-delete-orphan-person.sparql b/config/migrations/2023/20230104152916-delete-orphan-person.sparql new file mode 100644 index 0000000..8e886f1 --- /dev/null +++ b/config/migrations/2023/20230104152916-delete-orphan-person.sparql @@ -0,0 +1,65 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: + +DELETE { + GRAPH { + ?person ?ppPerson ?ooPerson. + ?heeftGeboorte ?p ?o. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + } +} + +WHERE { + GRAPH { + ?person a . + FILTER((NOT EXISTS { ?mandatories ?person. }) + && (NOT EXISTS { ?ministers ?person. })) + } + + GRAPH { + optional {?person ?ppPerson ?ooPerson.} + optional { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + } + + optional { + ?person persoon:registratie ?registration. + ?registration ?t ?j. + } + optional { + + ?registration ?identificator. + ?identificator ?x ?y. + } + } +} + +; + +DELETE { + GRAPH { + ?person ?pPerson ?oPerson. + } + +} + + + +WHERE { + GRAPH { + ?person a ; + ?pPerson ?oPerson. + FILTER((NOT EXISTS { ?mandatories ?person. }) && (NOT EXISTS { ?ministers ?person. })) + } + + +} diff --git a/config/migrations/2023/20230106110000-delete-duplicate-worship-services.sparql b/config/migrations/2023/20230106110000-delete-duplicate-worship-services.sparql new file mode 100644 index 0000000..e4d3e47 --- /dev/null +++ b/config/migrations/2023/20230106110000-delete-duplicate-worship-services.sparql @@ -0,0 +1,148 @@ +PREFIX owl: +PREFIX rdfs: +PREFIX skos: +PREFIX org: +PREFIX regorg: +PREFIX person: +PREFIX besluit: +PREFIX mandaat: +PREFIX organisatie: +PREFIX mu: +PREFIX ext: +PREFIX ere: + +DELETE { + GRAPH { + ?representatieveOrgaan ?bestuur . + ?bestuur ?pbestuur ?obestuur . + } +} WHERE { + GRAPH { + VALUES ?bestuur { + # OP-1980 + # OP-1978 + # OP-1979 + # OP-1977 + } + + OPTIONAL { ?representatieveOrgaan ?bestuur } + + ?bestuur ?pbestuur ?obestuur . + + } +} + +; + +DELETE { + GRAPH { + ?primarySite ?pprimarySite ?oprimarySite . + ?address ?paddress ?oaddress . + ?contactPoint ?pcontactPoint ?ocontactPoint . + } +} WHERE { + GRAPH { + VALUES ?bestuur { + # OP-1980 + # OP-1978 + # OP-1979 + # OP-1977 + } + + OPTIONAL { + ?bestuur ?primarySite. + ?primarySite ?pprimarySite ?oprimarySite . + } + + OPTIONAL { + ?primarySite ?address. + ?address ?paddress ?oaddress . + } + + OPTIONAL { + ?primarySite ?contactPoint . + ?contactPoint ?pcontactPoint ?ocontactPoint . + } + } +} + +; + +DELETE { + GRAPH { + ?identifier ?pidentifier ?oidentifier . + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} WHERE { + GRAPH { + VALUES ?bestuur { + # OP-1980 + # OP-1978 + # OP-1979 + # OP-1977 + } + + OPTIONAL { + ?bestuur ?identifier . + ?identifier ?pidentifier ?oidentifier . + } + + OPTIONAL { + ?identifier ?structuredIdentifier. + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } + } +} + +; + +# Clean reminiscences + +DELETE { + GRAPH { + ?s ?p ?o . + } +} WHERE { + GRAPH { + VALUES ?o { + # OP-1980 + # OP-1978 + # OP-1979 + # OP-1977 + } + + OPTIONAL{ ?s ?p ?o } + } +} + +# The associated call to trigger a reindex is : + +# curl --location --request POST '172.20.0.15/update' \ +# --header 'Content-Type: application/vnd.api+json' \ +# --data-raw ' [ +# { +# "deletes": [ +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/besturenVanDeEredienst/999b011fe6b90b3b546612c25db4b607" }, +# "predicate": { "type": "uri", "value": "http://mu.semte.ch/vocabularies/core/uuid" }, +# "object": { "type": "string", "value": "999b011fe6b90b3b546612c25db4b607" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/besturenVanDeEredienst/da2184c3dca9eb7c4828a50c0e9f5af4" }, +# "predicate": { "type": "uri", "value": "http://mu.semte.ch/vocabularies/core/uuid" }, +# "object": { "type": "string", "value": "da2184c3dca9eb7c4828a50c0e9f5af4" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/besturenVanDeEredienst/a6b5c5db3a261ca024a42fe9489496a9" }, +# "predicate": { "type": "uri", "value": "http://mu.semte.ch/vocabularies/core/uuid" }, +# "object": { "type": "string", "value": "a6b5c5db3a261ca024a42fe9489496a9" } +# }, +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/besturenVanDeEredienst/260312c07d8e00358064a362dee3506f" }, +# "predicate": { "type": "uri", "value": "http://mu.semte.ch/vocabularies/core/uuid" }, +# "object": { "type": "string", "value": "260312c07d8e00358064a362dee3506f" } +# } +# ], +# "inserts": [] +# } +# ]' \ No newline at end of file diff --git a/config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql b/config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql new file mode 100644 index 0000000..5be441c --- /dev/null +++ b/config/migrations/2023/20230106140000-publisher-config/20230106143000-delete-old-dump-info.sparql @@ -0,0 +1,34 @@ +PREFIX nie: +PREFIX dct: +PREFIX dcat: +PREFIX prov: + + +DELETE { + GRAPH { + ?sub + dct:subject ; + dcat:distribution ?distribution. + ?distribution + dct:subject ?file ; + dct:created ?created . + ?phyiscalFile + nie:dataSource ?file + } +} +WHERE { + GRAPH { + ?sub + dct:subject ; + dcat:distribution ?distribution. + FILTER NOT EXISTS { + ?revision + prov:wasRevisionOf ?sub + } + ?distribution + dct:subject ?file ; + dct:created ?created . + ?phyiscalFile + nie:dataSource ?file + } +} \ No newline at end of file diff --git a/config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql b/config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql new file mode 100644 index 0000000..6e74965 --- /dev/null +++ b/config/migrations/2023/20230106140000-publisher-config/20230106143000-organization-producer-account.sparql @@ -0,0 +1,16 @@ +PREFIX foaf: +INSERT DATA { + GRAPH { + + a foaf:OnlineAccount. + + + a foaf:Agent; + foaf:account . + + + a foaf:Group; + foaf:member ; + foaf:name "organizations". + } +} \ No newline at end of file diff --git a/config/reasoner/dl2op/main-info/inverse_relations.n3 b/config/reasoner/dl2op/main-info/inverse_relations.n3 new file mode 100644 index 0000000..bfeda31 --- /dev/null +++ b/config/reasoner/dl2op/main-info/inverse_relations.n3 @@ -0,0 +1,33 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Inverse properties +{ + ?s org:holds ?o. +} => { + ?o org:heldBy ?s. +}. + +{ + ?s org:heldBy ?o. +} => { + ?o org:holds ?s. +}. diff --git a/config/reasoner/dl2op/main-info/mapping.n3 b/config/reasoner/dl2op/main-info/mapping.n3 new file mode 100644 index 0000000..82bebd9 --- /dev/null +++ b/config/reasoner/dl2op/main-info/mapping.n3 @@ -0,0 +1,150 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) +# Hence the mapping is in two steps. +# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart +# and drops the DL persoon:gebruikteVoornaam statements with e:becomes +# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam +# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames +# in OP + +# First name +# persoon:gebruikteVoornaam => foaf:givenName +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock DL +# persoon_loket:gebruikteVoornaam "Didier". + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s foaf:givenName ?o + } +} => { + ?s foaf:givenName ?o +}. + + +# First name +# persoon:gebruikteVoornaam => foaf:givenName (done in pre-processing) +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock LLB +# foaf:name "Didier". +{ + { + ?s foaf:name ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +# Test +# ex:s "2019-01-01T19:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:start ?startDateTime. + # use backward arrow rule defined below + ?startDateTime :dateFromDateTime ?startDate. + { + ?s mandaat:start ?startDateTime + } e:becomes { + ?s mandaat:start ?startDate + }. +} => { + ?s mandaat:start ?startDate +}. + +# ex:s "2019-02-01T10:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:einde ?eindeDateTime. + # use backward arrow rule defined below + ?eindeDateTime :dateFromDateTime ?eindeDate. + { + ?s mandaat:einde ?eindeDateTime + } e:becomes { + ?s mandaat:einde ?eindeDate + }. +} => { + ?s mandaat:einde ?eindeDate +}. + + +# Backward arrow rule is like a function call. +# Removes time info in the returned date +# The user in DL selects a date via the UI. +# These dates are persisted as midnight dateTimes in UTC. +# Meaning that 1st of July is stored as +# "2022-06-30T22:00:00Z"^^xsd:dateTime +# and 1 December +# "2022-11-30T23:00:00Z"^^xsd:dateTime +# So, the following rule adds 2 hours to the datetime, and then takes the date part of the UTC date time +# to come to the date that the DL user selected (in CE(S)T) +{ + ?dateTime :dateFromDateTime ?date +} <= { + (?dateTimeLiteral xsd:dateTime) log:dtlit ?dateTime. + (?dateTime "PT2H"^^xsd:dayTimeDuration) func:add-dayTimeDuration-to-dateTime ?localDateTime. + (?localDateTimeLiteral xsd:dateTime) log:dtlit ?localDateTime. + (?localDateTimeLiteral 0 11) func:substring ?dateLiteral. + (?dateLiteral xsd:date) log:dtlit ?date. +}. + + +# ex:orgA besluit:classificatie . +{ + { + ?s besluit:classificatie ?o + } e:becomes { + ?s org:classification ?o + } +} => { + ?s org:classification ?o +}. + +# fix typo V -> v +# "1" . +{ + { + ?s adres:AdresVoorstelling.huisnummer ?o + } e:becomes { + ?s adres:Adresvoorstelling.huisnummer ?o + } +} => { + ?s adres:Adresvoorstelling.huisnummer ?o +}. + + +# Tijdspecialisatie +{ + { + ?s mandaat:isTijdspecialisatieVan ?o. + } e:becomes { + ?s generiek:isTijdspecialisatieVan ?o. + } +} => { + ?s generiek:isTijdspecialisatieVan ?o. +}. diff --git a/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 b/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 new file mode 100644 index 0000000..6ced405 --- /dev/null +++ b/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 @@ -0,0 +1,534 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Map from http persoon to https persoon namespace. + +## classes: +{ + { + ?s a persoon_loket:Afstamming + } e:becomes { + ?s a persoon:Afstamming + } +} => { + ?s a persoon:Afstamming +}. + +{ + { + ?s a persoon_loket:BurgerlijkeStaat + } e:becomes { + ?s a persoon:BurgerlijkeStaat + } +} => { + ?s a persoon:BurgerlijkeStaat +}. + +{ + { + ?s a persoon_loket:Domicilie + } e:becomes { + ?s a persoon:Domicilie + } +} => { + ?s a persoon:Domicilie +}. + +{ + { + ?s a persoon_loket:Geboorte + } e:becomes { + ?s a persoon:Geboorte + } +} => { + ?s a persoon:Geboorte +}. + +{ + { + ?s a persoon_loket:GeenInwoner + } e:becomes { + ?s a persoon:GeenInwoner + } +} => { + ?s a persoon:GeenInwoner +}. + +{ + { + ?s a persoon_loket:GeregistreerdPersoon + } e:becomes { + ?s a persoon:GeregistreerdPersoon + } +} => { + ?s a persoon:GeregistreerdPersoon +}. + +{ + { + ?s a persoon_loket:Gezin + } e:becomes { + ?s a persoon:Gezin + } +} => { + ?s a persoon:Gezin +}. + +{ + { + ?s a persoon_loket:Gezinsrelatie + } e:becomes { + ?s a persoon:Gezinsrelatie + } +} => { + ?s a persoon:Gezinsrelatie +}. + +{ + { + ?s a persoon_loket:Huwelijk + } e:becomes { + ?s a persoon:Huwelijk + } +} => { + ?s a persoon:Huwelijk +}. + +{ + { + ?s a persoon_loket:Inwoner + } e:becomes { + ?s a persoon:Inwoner + } +} => { + ?s a persoon:Inwoner +}. + +{ + { + ?s a persoon_loket:Inwonerschap + } e:becomes { + ?s a persoon:Inwonerschap + } +} => { + ?s a persoon:Inwonerschap +}. + +{ + { + ?s a persoon_loket:Nationaliteit + } e:becomes { + ?s a persoon:Nationaliteit + } +} => { + ?s a persoon:Nationaliteit +}. + +{ + { + ?s a persoon_loket:Overlijden + } e:becomes { + ?s a persoon:Overlijden + } +} => { + ?s a persoon:Overlijden +}. + +{ + { + ?s a persoon_loket:PermanentInwoner + } e:becomes { + ?s a persoon:PermanentInwoner + } +} => { + ?s a persoon:PermanentInwoner +}. + +{ + { + ?s a persoon_loket:Persoonsgebeurtenis + } e:becomes { + ?s a persoon:Persoonsgebeurtenis + } +} => { + ?s a persoon:Persoonsgebeurtenis +}. + +{ + { + ?s a persoon_loket:Persoonsrelatie + } e:becomes { + ?s a persoon:Persoonsrelatie + } +} => { + ?s a persoon:Persoonsrelatie +}. + +{ + { + ?s a persoon_loket:Samenwonen + } e:becomes { + ?s a persoon:Samenwonen + } +} => { + ?s a persoon:Samenwonen +}. + +{ + { + ?s a persoon_loket:Staatsburger + } e:becomes { + ?s a persoon:Staatsburger + } +} => { + ?s a persoon:Staatsburger +}. + +{ + { + ?s a persoon_loket:Staatsburgerschap + } e:becomes { + ?s a persoon:Staatsburgerschap + } +} => { + ?s a persoon:Staatsburgerschap +}. + +{ + { + ?s a persoon_loket:TijdelijkInwoner + } e:becomes { + ?s a persoon:TijdelijkInwoner + } +} => { + ?s a persoon:TijdelijkInwoner +}. + +{ + { + ?s a persoon_loket:Verblijfplaats + } e:becomes { + ?s a persoon:Verblijfplaats + } +} => { + ?s a persoon:Verblijfplaats +}. + +{ + { + ?s a persoon_loket:Voogdij + } e:becomes { + ?s a persoon:Voogdij + } +} => { + ?s a persoon:Voogdij +}. + +{ + { + ?s a persoon_loket:Vreemdeling + } e:becomes { + ?s a persoon:Vreemdeling + } +} => { + ?s a persoon:Vreemdeling +}. + +## properties +{ + { + ?s persoon_loket:afstammingsType ?o + } e:becomes { + ?s persoon:afstammingsType ?o + } +} => { + ?s persoon:afstammingsType ?o +}. + +{ + { + ?s persoon_loket:alternatieveNaam ?o + } e:becomes { + ?s persoon:alternatieveNaam ?o + } +} => { + ?s persoon:alternatieveNaam ?o +}. + +{ + { + ?s persoon_loket:datum ?o + } e:becomes { + ?s persoon:datum ?o + } +} => { + ?s persoon:datum ?o +}. + +{ + { + ?s persoon_loket:datumVanAfstamming ?o + } e:becomes { + ?s persoon:datumVanAfstamming ?o + } +} => { + ?s persoon:datumVanAfstamming ?o +}. + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +{ + { + ?s persoon_loket:geslacht ?o + } e:becomes { + ?s persoon:geslacht ?o + } +} => { + ?s persoon:geslacht ?o +}. + +{ + { + ?s persoon_loket:gezinsadres ?o + } e:becomes { + ?s persoon:gezinsadres ?o + } +} => { + ?s persoon:gezinsadres ?o +}. + +{ + { + ?s persoon_loket:gezinsrelatietype ?o + } e:becomes { + ?s persoon:gezinsrelatietype ?o + } +} => { + ?s persoon:gezinsrelatietype ?o +}. + +{ + { + ?s persoon_loket:heeftBurgerlijkeStaat ?o + } e:becomes { + ?s persoon:heeftBurgerlijkeStaat ?o + } +} => { + ?s persoon:heeftBurgerlijkeStaat ?o +}. + +{ + { + ?s persoon_loket:heeftGeboorte ?o + } e:becomes { + ?s persoon:heeftGeboorte ?o + } +} => { + ?s persoon:heeftGeboorte ?o +}. + +{ + { + ?s persoon_loket:heeftInwonerschap ?o + } e:becomes { + ?s persoon:heeftInwonerschap ?o + } +} => { + ?s persoon:heeftInwonerschap ?o +}. + +{ + { + ?s persoon_loket:heeftNationaliteit ?o + } e:becomes { + ?s persoon:heeftNationaliteit ?o + } +} => { + ?s persoon:heeftNationaliteit ?o +}. + +{ + { + ?s persoon_loket:heeftOverlijden ?o + } e:becomes { + ?s persoon:heeftOverlijden ?o + } +} => { + ?s persoon:heeftOverlijden ?o +}. + +{ + { + ?s persoon_loket:heeftPersoonsrelatie ?o + } e:becomes { + ?s persoon:heeftPersoonsrelatie ?o + } +} => { + ?s persoon:heeftPersoonsrelatie ?o +}. + +{ + { + ?s persoon_loket:heeftStaatsburgerschap ?o + } e:becomes { + ?s persoon:heeftStaatsburgerschap ?o + } +} => { + ?s persoon:heeftStaatsburgerschap ?o +}. + +{ + { + ?s persoon_loket:heeftVerblijfplaats ?o + } e:becomes { + ?s persoon:heeftVerblijfplaats ?o + } +} => { + ?s persoon:heeftVerblijfplaats ?o +}. + +{ + { + ?s persoon_loket:Inwonerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Inwonerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Inwonerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:isHoofdVan ?o + } e:becomes { + ?s persoon:isHoofdVan ?o + } +} => { + ?s persoon:isHoofdVan ?o +}. + +{ + { + ?s persoon_loket:isLidVan ?o + } e:becomes { + ?s persoon:isLidVan ?o + } +} => { + ?s persoon:isLidVan ?o +}. + +{ + { + ?s persoon_loket:isRelatieMet ?o + } e:becomes { + ?s persoon:isRelatieMet ?o + } +} => { + ?s persoon:isRelatieMet ?o +}. + +{ + { + ?s persoon_loket:land ?o + } e:becomes { + ?s persoon:land ?o + } +} => { + ?s persoon:land ?o +}. + +{ + { + ?s persoon_loket:nationaliteit ?o + } e:becomes { + ?s persoon:nationaliteit ?o + } +} => { + ?s persoon:nationaliteit ?o +}. + +{ + { + ?s persoon_loket:plaats ?o + } e:becomes { + ?s persoon:plaats ?o + } +} => { + ?s persoon:plaats ?o +}. + +{ + { + ?s persoon_loket:registratie ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. + +{ + { + ?s persoon_loket:Staatsburgerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:type ?o + } e:becomes { + ?s persoon:type ?o + } +} => { + ?s persoon:type ?o +}. + +{ + { + ?s persoon_loket:verblijfsadres ?o + } e:becomes { + ?s persoon:verblijfsadres ?o + } +} => { + ?s persoon:verblijfsadres ?o +}. + +{ + { + ?s persoon_loket:volledigeNaam ?o + } e:becomes { + ?s persoon:volledigeNaam ?o + } +} => { + ?s persoon:volledigeNaam ?o +}. \ No newline at end of file diff --git a/config/reasoner/dl2op/main-info/query.n3q b/config/reasoner/dl2op/main-info/query.n3q new file mode 100644 index 0000000..09dd4fb --- /dev/null +++ b/config/reasoner/dl2op/main-info/query.n3q @@ -0,0 +1,538 @@ +# Generated by producer-config-to-query.ipynb + +## CLASSES +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +# { +# ?s a +# } => { +# ?s a +# }. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +# { +# ?s a +# } => { +# ?s a +# }. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +{ + ?s a + } => { + ?s a +}. + +## PROPERTIES +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +# { +# ?s ?o +# } => { +# ?s ?o +# }. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + +{ + ?s ?o +} => { + ?s ?o +}. + diff --git a/config/reasoner/dl2op/main-info/retract_empty_strings.n3 b/config/reasoner/dl2op/main-info/retract_empty_strings.n3 new file mode 100644 index 0000000..f8204a7 --- /dev/null +++ b/config/reasoner/dl2op/main-info/retract_empty_strings.n3 @@ -0,0 +1,27 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Retract empty strings +{ + { + ?s ?p "" + } e:becomes {}. +} => {}. diff --git a/config/reasoner/dl2op/main-info/rrn.n3 b/config/reasoner/dl2op/main-info/rrn.n3 new file mode 100644 index 0000000..b45c688 --- /dev/null +++ b/config/reasoner/dl2op/main-info/rrn.n3 @@ -0,0 +1,62 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + + +# Rijksregisternummer +# Extend identifiers to match OP's model +# skos:notation "00010100105". +{ + ?identifier skos:notation ?notation. + ?notation string:notEqualIgnoringCase "Rijksregisternummer". + ?identifier log:uri ?identifierString. + (?identifierString "STRUCTURED_IDENTIFIER") string:concatenation ?hashInput. + ?hashInput crypto:md5 ?uuid. + ("http://data.lblod.info/id/gestructureerdeIdentificatoren/" ?uuid) string:concatenation ?structuredIdString. + ?structuredIdentifier log:uri ?structuredIdString. + + { + ?identifier skos:notation ?notation. + } e:becomes { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + }. + +} => { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +}. + +# identifier property in the export is only used for RRN. +{ + { + ?s adms:identifier ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. diff --git a/config/reasoner/dl2op/main/inverse_relations.n3 b/config/reasoner/dl2op/main/inverse_relations.n3 new file mode 100644 index 0000000..bfeda31 --- /dev/null +++ b/config/reasoner/dl2op/main/inverse_relations.n3 @@ -0,0 +1,33 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Inverse properties +{ + ?s org:holds ?o. +} => { + ?o org:heldBy ?s. +}. + +{ + ?s org:heldBy ?o. +} => { + ?o org:holds ?s. +}. diff --git a/config/reasoner/dl2op/main/mapping.n3 b/config/reasoner/dl2op/main/mapping.n3 new file mode 100644 index 0000000..82bebd9 --- /dev/null +++ b/config/reasoner/dl2op/main/mapping.n3 @@ -0,0 +1,150 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) +# Hence the mapping is in two steps. +# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart +# and drops the DL persoon:gebruikteVoornaam statements with e:becomes +# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam +# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames +# in OP + +# First name +# persoon:gebruikteVoornaam => foaf:givenName +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock DL +# persoon_loket:gebruikteVoornaam "Didier". + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s foaf:givenName ?o + } +} => { + ?s foaf:givenName ?o +}. + + +# First name +# persoon:gebruikteVoornaam => foaf:givenName (done in pre-processing) +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock LLB +# foaf:name "Didier". +{ + { + ?s foaf:name ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +# Test +# ex:s "2019-01-01T19:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:start ?startDateTime. + # use backward arrow rule defined below + ?startDateTime :dateFromDateTime ?startDate. + { + ?s mandaat:start ?startDateTime + } e:becomes { + ?s mandaat:start ?startDate + }. +} => { + ?s mandaat:start ?startDate +}. + +# ex:s "2019-02-01T10:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:einde ?eindeDateTime. + # use backward arrow rule defined below + ?eindeDateTime :dateFromDateTime ?eindeDate. + { + ?s mandaat:einde ?eindeDateTime + } e:becomes { + ?s mandaat:einde ?eindeDate + }. +} => { + ?s mandaat:einde ?eindeDate +}. + + +# Backward arrow rule is like a function call. +# Removes time info in the returned date +# The user in DL selects a date via the UI. +# These dates are persisted as midnight dateTimes in UTC. +# Meaning that 1st of July is stored as +# "2022-06-30T22:00:00Z"^^xsd:dateTime +# and 1 December +# "2022-11-30T23:00:00Z"^^xsd:dateTime +# So, the following rule adds 2 hours to the datetime, and then takes the date part of the UTC date time +# to come to the date that the DL user selected (in CE(S)T) +{ + ?dateTime :dateFromDateTime ?date +} <= { + (?dateTimeLiteral xsd:dateTime) log:dtlit ?dateTime. + (?dateTime "PT2H"^^xsd:dayTimeDuration) func:add-dayTimeDuration-to-dateTime ?localDateTime. + (?localDateTimeLiteral xsd:dateTime) log:dtlit ?localDateTime. + (?localDateTimeLiteral 0 11) func:substring ?dateLiteral. + (?dateLiteral xsd:date) log:dtlit ?date. +}. + + +# ex:orgA besluit:classificatie . +{ + { + ?s besluit:classificatie ?o + } e:becomes { + ?s org:classification ?o + } +} => { + ?s org:classification ?o +}. + +# fix typo V -> v +# "1" . +{ + { + ?s adres:AdresVoorstelling.huisnummer ?o + } e:becomes { + ?s adres:Adresvoorstelling.huisnummer ?o + } +} => { + ?s adres:Adresvoorstelling.huisnummer ?o +}. + + +# Tijdspecialisatie +{ + { + ?s mandaat:isTijdspecialisatieVan ?o. + } e:becomes { + ?s generiek:isTijdspecialisatieVan ?o. + } +} => { + ?s generiek:isTijdspecialisatieVan ?o. +}. diff --git a/config/reasoner/dl2op/main/persoon_http_to_https.n3 b/config/reasoner/dl2op/main/persoon_http_to_https.n3 new file mode 100644 index 0000000..6ced405 --- /dev/null +++ b/config/reasoner/dl2op/main/persoon_http_to_https.n3 @@ -0,0 +1,534 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Map from http persoon to https persoon namespace. + +## classes: +{ + { + ?s a persoon_loket:Afstamming + } e:becomes { + ?s a persoon:Afstamming + } +} => { + ?s a persoon:Afstamming +}. + +{ + { + ?s a persoon_loket:BurgerlijkeStaat + } e:becomes { + ?s a persoon:BurgerlijkeStaat + } +} => { + ?s a persoon:BurgerlijkeStaat +}. + +{ + { + ?s a persoon_loket:Domicilie + } e:becomes { + ?s a persoon:Domicilie + } +} => { + ?s a persoon:Domicilie +}. + +{ + { + ?s a persoon_loket:Geboorte + } e:becomes { + ?s a persoon:Geboorte + } +} => { + ?s a persoon:Geboorte +}. + +{ + { + ?s a persoon_loket:GeenInwoner + } e:becomes { + ?s a persoon:GeenInwoner + } +} => { + ?s a persoon:GeenInwoner +}. + +{ + { + ?s a persoon_loket:GeregistreerdPersoon + } e:becomes { + ?s a persoon:GeregistreerdPersoon + } +} => { + ?s a persoon:GeregistreerdPersoon +}. + +{ + { + ?s a persoon_loket:Gezin + } e:becomes { + ?s a persoon:Gezin + } +} => { + ?s a persoon:Gezin +}. + +{ + { + ?s a persoon_loket:Gezinsrelatie + } e:becomes { + ?s a persoon:Gezinsrelatie + } +} => { + ?s a persoon:Gezinsrelatie +}. + +{ + { + ?s a persoon_loket:Huwelijk + } e:becomes { + ?s a persoon:Huwelijk + } +} => { + ?s a persoon:Huwelijk +}. + +{ + { + ?s a persoon_loket:Inwoner + } e:becomes { + ?s a persoon:Inwoner + } +} => { + ?s a persoon:Inwoner +}. + +{ + { + ?s a persoon_loket:Inwonerschap + } e:becomes { + ?s a persoon:Inwonerschap + } +} => { + ?s a persoon:Inwonerschap +}. + +{ + { + ?s a persoon_loket:Nationaliteit + } e:becomes { + ?s a persoon:Nationaliteit + } +} => { + ?s a persoon:Nationaliteit +}. + +{ + { + ?s a persoon_loket:Overlijden + } e:becomes { + ?s a persoon:Overlijden + } +} => { + ?s a persoon:Overlijden +}. + +{ + { + ?s a persoon_loket:PermanentInwoner + } e:becomes { + ?s a persoon:PermanentInwoner + } +} => { + ?s a persoon:PermanentInwoner +}. + +{ + { + ?s a persoon_loket:Persoonsgebeurtenis + } e:becomes { + ?s a persoon:Persoonsgebeurtenis + } +} => { + ?s a persoon:Persoonsgebeurtenis +}. + +{ + { + ?s a persoon_loket:Persoonsrelatie + } e:becomes { + ?s a persoon:Persoonsrelatie + } +} => { + ?s a persoon:Persoonsrelatie +}. + +{ + { + ?s a persoon_loket:Samenwonen + } e:becomes { + ?s a persoon:Samenwonen + } +} => { + ?s a persoon:Samenwonen +}. + +{ + { + ?s a persoon_loket:Staatsburger + } e:becomes { + ?s a persoon:Staatsburger + } +} => { + ?s a persoon:Staatsburger +}. + +{ + { + ?s a persoon_loket:Staatsburgerschap + } e:becomes { + ?s a persoon:Staatsburgerschap + } +} => { + ?s a persoon:Staatsburgerschap +}. + +{ + { + ?s a persoon_loket:TijdelijkInwoner + } e:becomes { + ?s a persoon:TijdelijkInwoner + } +} => { + ?s a persoon:TijdelijkInwoner +}. + +{ + { + ?s a persoon_loket:Verblijfplaats + } e:becomes { + ?s a persoon:Verblijfplaats + } +} => { + ?s a persoon:Verblijfplaats +}. + +{ + { + ?s a persoon_loket:Voogdij + } e:becomes { + ?s a persoon:Voogdij + } +} => { + ?s a persoon:Voogdij +}. + +{ + { + ?s a persoon_loket:Vreemdeling + } e:becomes { + ?s a persoon:Vreemdeling + } +} => { + ?s a persoon:Vreemdeling +}. + +## properties +{ + { + ?s persoon_loket:afstammingsType ?o + } e:becomes { + ?s persoon:afstammingsType ?o + } +} => { + ?s persoon:afstammingsType ?o +}. + +{ + { + ?s persoon_loket:alternatieveNaam ?o + } e:becomes { + ?s persoon:alternatieveNaam ?o + } +} => { + ?s persoon:alternatieveNaam ?o +}. + +{ + { + ?s persoon_loket:datum ?o + } e:becomes { + ?s persoon:datum ?o + } +} => { + ?s persoon:datum ?o +}. + +{ + { + ?s persoon_loket:datumVanAfstamming ?o + } e:becomes { + ?s persoon:datumVanAfstamming ?o + } +} => { + ?s persoon:datumVanAfstamming ?o +}. + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +{ + { + ?s persoon_loket:geslacht ?o + } e:becomes { + ?s persoon:geslacht ?o + } +} => { + ?s persoon:geslacht ?o +}. + +{ + { + ?s persoon_loket:gezinsadres ?o + } e:becomes { + ?s persoon:gezinsadres ?o + } +} => { + ?s persoon:gezinsadres ?o +}. + +{ + { + ?s persoon_loket:gezinsrelatietype ?o + } e:becomes { + ?s persoon:gezinsrelatietype ?o + } +} => { + ?s persoon:gezinsrelatietype ?o +}. + +{ + { + ?s persoon_loket:heeftBurgerlijkeStaat ?o + } e:becomes { + ?s persoon:heeftBurgerlijkeStaat ?o + } +} => { + ?s persoon:heeftBurgerlijkeStaat ?o +}. + +{ + { + ?s persoon_loket:heeftGeboorte ?o + } e:becomes { + ?s persoon:heeftGeboorte ?o + } +} => { + ?s persoon:heeftGeboorte ?o +}. + +{ + { + ?s persoon_loket:heeftInwonerschap ?o + } e:becomes { + ?s persoon:heeftInwonerschap ?o + } +} => { + ?s persoon:heeftInwonerschap ?o +}. + +{ + { + ?s persoon_loket:heeftNationaliteit ?o + } e:becomes { + ?s persoon:heeftNationaliteit ?o + } +} => { + ?s persoon:heeftNationaliteit ?o +}. + +{ + { + ?s persoon_loket:heeftOverlijden ?o + } e:becomes { + ?s persoon:heeftOverlijden ?o + } +} => { + ?s persoon:heeftOverlijden ?o +}. + +{ + { + ?s persoon_loket:heeftPersoonsrelatie ?o + } e:becomes { + ?s persoon:heeftPersoonsrelatie ?o + } +} => { + ?s persoon:heeftPersoonsrelatie ?o +}. + +{ + { + ?s persoon_loket:heeftStaatsburgerschap ?o + } e:becomes { + ?s persoon:heeftStaatsburgerschap ?o + } +} => { + ?s persoon:heeftStaatsburgerschap ?o +}. + +{ + { + ?s persoon_loket:heeftVerblijfplaats ?o + } e:becomes { + ?s persoon:heeftVerblijfplaats ?o + } +} => { + ?s persoon:heeftVerblijfplaats ?o +}. + +{ + { + ?s persoon_loket:Inwonerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Inwonerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Inwonerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:isHoofdVan ?o + } e:becomes { + ?s persoon:isHoofdVan ?o + } +} => { + ?s persoon:isHoofdVan ?o +}. + +{ + { + ?s persoon_loket:isLidVan ?o + } e:becomes { + ?s persoon:isLidVan ?o + } +} => { + ?s persoon:isLidVan ?o +}. + +{ + { + ?s persoon_loket:isRelatieMet ?o + } e:becomes { + ?s persoon:isRelatieMet ?o + } +} => { + ?s persoon:isRelatieMet ?o +}. + +{ + { + ?s persoon_loket:land ?o + } e:becomes { + ?s persoon:land ?o + } +} => { + ?s persoon:land ?o +}. + +{ + { + ?s persoon_loket:nationaliteit ?o + } e:becomes { + ?s persoon:nationaliteit ?o + } +} => { + ?s persoon:nationaliteit ?o +}. + +{ + { + ?s persoon_loket:plaats ?o + } e:becomes { + ?s persoon:plaats ?o + } +} => { + ?s persoon:plaats ?o +}. + +{ + { + ?s persoon_loket:registratie ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. + +{ + { + ?s persoon_loket:Staatsburgerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:type ?o + } e:becomes { + ?s persoon:type ?o + } +} => { + ?s persoon:type ?o +}. + +{ + { + ?s persoon_loket:verblijfsadres ?o + } e:becomes { + ?s persoon:verblijfsadres ?o + } +} => { + ?s persoon:verblijfsadres ?o +}. + +{ + { + ?s persoon_loket:volledigeNaam ?o + } e:becomes { + ?s persoon:volledigeNaam ?o + } +} => { + ?s persoon:volledigeNaam ?o +}. \ No newline at end of file diff --git a/config/reasoner/dl2op/main/retract_empty_strings.n3 b/config/reasoner/dl2op/main/retract_empty_strings.n3 new file mode 100644 index 0000000..f8204a7 --- /dev/null +++ b/config/reasoner/dl2op/main/retract_empty_strings.n3 @@ -0,0 +1,27 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Retract empty strings +{ + { + ?s ?p "" + } e:becomes {}. +} => {}. diff --git a/config/reasoner/dl2op/main/rrn.n3 b/config/reasoner/dl2op/main/rrn.n3 new file mode 100644 index 0000000..b45c688 --- /dev/null +++ b/config/reasoner/dl2op/main/rrn.n3 @@ -0,0 +1,62 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + + +# Rijksregisternummer +# Extend identifiers to match OP's model +# skos:notation "00010100105". +{ + ?identifier skos:notation ?notation. + ?notation string:notEqualIgnoringCase "Rijksregisternummer". + ?identifier log:uri ?identifierString. + (?identifierString "STRUCTURED_IDENTIFIER") string:concatenation ?hashInput. + ?hashInput crypto:md5 ?uuid. + ("http://data.lblod.info/id/gestructureerdeIdentificatoren/" ?uuid) string:concatenation ?structuredIdString. + ?structuredIdentifier log:uri ?structuredIdString. + + { + ?identifier skos:notation ?notation. + } e:becomes { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + }. + +} => { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +}. + +# identifier property in the export is only used for RRN. +{ + { + ?s adms:identifier ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. diff --git a/config/reasoner/dl2op/private-info/inverse_relations.n3 b/config/reasoner/dl2op/private-info/inverse_relations.n3 new file mode 100644 index 0000000..bfeda31 --- /dev/null +++ b/config/reasoner/dl2op/private-info/inverse_relations.n3 @@ -0,0 +1,33 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Inverse properties +{ + ?s org:holds ?o. +} => { + ?o org:heldBy ?s. +}. + +{ + ?s org:heldBy ?o. +} => { + ?o org:holds ?s. +}. diff --git a/config/reasoner/dl2op/private-info/mapping.n3 b/config/reasoner/dl2op/private-info/mapping.n3 new file mode 100644 index 0000000..82bebd9 --- /dev/null +++ b/config/reasoner/dl2op/private-info/mapping.n3 @@ -0,0 +1,150 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) +# Hence the mapping is in two steps. +# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart +# and drops the DL persoon:gebruikteVoornaam statements with e:becomes +# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam +# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames +# in OP + +# First name +# persoon:gebruikteVoornaam => foaf:givenName +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock DL +# persoon_loket:gebruikteVoornaam "Didier". + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s foaf:givenName ?o + } +} => { + ?s foaf:givenName ?o +}. + + +# First name +# persoon:gebruikteVoornaam => foaf:givenName (done in pre-processing) +# Alternatieve voornaam +# foaf:name => persoon:gebruikteVoornaam + +# Test Mock LLB +# foaf:name "Didier". +{ + { + ?s foaf:name ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +# Test +# ex:s "2019-01-01T19:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:start ?startDateTime. + # use backward arrow rule defined below + ?startDateTime :dateFromDateTime ?startDate. + { + ?s mandaat:start ?startDateTime + } e:becomes { + ?s mandaat:start ?startDate + }. +} => { + ?s mandaat:start ?startDate +}. + +# ex:s "2019-02-01T10:32:52+00:00"^^xsd:dateTime. +{ + ?s mandaat:einde ?eindeDateTime. + # use backward arrow rule defined below + ?eindeDateTime :dateFromDateTime ?eindeDate. + { + ?s mandaat:einde ?eindeDateTime + } e:becomes { + ?s mandaat:einde ?eindeDate + }. +} => { + ?s mandaat:einde ?eindeDate +}. + + +# Backward arrow rule is like a function call. +# Removes time info in the returned date +# The user in DL selects a date via the UI. +# These dates are persisted as midnight dateTimes in UTC. +# Meaning that 1st of July is stored as +# "2022-06-30T22:00:00Z"^^xsd:dateTime +# and 1 December +# "2022-11-30T23:00:00Z"^^xsd:dateTime +# So, the following rule adds 2 hours to the datetime, and then takes the date part of the UTC date time +# to come to the date that the DL user selected (in CE(S)T) +{ + ?dateTime :dateFromDateTime ?date +} <= { + (?dateTimeLiteral xsd:dateTime) log:dtlit ?dateTime. + (?dateTime "PT2H"^^xsd:dayTimeDuration) func:add-dayTimeDuration-to-dateTime ?localDateTime. + (?localDateTimeLiteral xsd:dateTime) log:dtlit ?localDateTime. + (?localDateTimeLiteral 0 11) func:substring ?dateLiteral. + (?dateLiteral xsd:date) log:dtlit ?date. +}. + + +# ex:orgA besluit:classificatie . +{ + { + ?s besluit:classificatie ?o + } e:becomes { + ?s org:classification ?o + } +} => { + ?s org:classification ?o +}. + +# fix typo V -> v +# "1" . +{ + { + ?s adres:AdresVoorstelling.huisnummer ?o + } e:becomes { + ?s adres:Adresvoorstelling.huisnummer ?o + } +} => { + ?s adres:Adresvoorstelling.huisnummer ?o +}. + + +# Tijdspecialisatie +{ + { + ?s mandaat:isTijdspecialisatieVan ?o. + } e:becomes { + ?s generiek:isTijdspecialisatieVan ?o. + } +} => { + ?s generiek:isTijdspecialisatieVan ?o. +}. diff --git a/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 b/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 new file mode 100644 index 0000000..6ced405 --- /dev/null +++ b/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 @@ -0,0 +1,534 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Map from http persoon to https persoon namespace. + +## classes: +{ + { + ?s a persoon_loket:Afstamming + } e:becomes { + ?s a persoon:Afstamming + } +} => { + ?s a persoon:Afstamming +}. + +{ + { + ?s a persoon_loket:BurgerlijkeStaat + } e:becomes { + ?s a persoon:BurgerlijkeStaat + } +} => { + ?s a persoon:BurgerlijkeStaat +}. + +{ + { + ?s a persoon_loket:Domicilie + } e:becomes { + ?s a persoon:Domicilie + } +} => { + ?s a persoon:Domicilie +}. + +{ + { + ?s a persoon_loket:Geboorte + } e:becomes { + ?s a persoon:Geboorte + } +} => { + ?s a persoon:Geboorte +}. + +{ + { + ?s a persoon_loket:GeenInwoner + } e:becomes { + ?s a persoon:GeenInwoner + } +} => { + ?s a persoon:GeenInwoner +}. + +{ + { + ?s a persoon_loket:GeregistreerdPersoon + } e:becomes { + ?s a persoon:GeregistreerdPersoon + } +} => { + ?s a persoon:GeregistreerdPersoon +}. + +{ + { + ?s a persoon_loket:Gezin + } e:becomes { + ?s a persoon:Gezin + } +} => { + ?s a persoon:Gezin +}. + +{ + { + ?s a persoon_loket:Gezinsrelatie + } e:becomes { + ?s a persoon:Gezinsrelatie + } +} => { + ?s a persoon:Gezinsrelatie +}. + +{ + { + ?s a persoon_loket:Huwelijk + } e:becomes { + ?s a persoon:Huwelijk + } +} => { + ?s a persoon:Huwelijk +}. + +{ + { + ?s a persoon_loket:Inwoner + } e:becomes { + ?s a persoon:Inwoner + } +} => { + ?s a persoon:Inwoner +}. + +{ + { + ?s a persoon_loket:Inwonerschap + } e:becomes { + ?s a persoon:Inwonerschap + } +} => { + ?s a persoon:Inwonerschap +}. + +{ + { + ?s a persoon_loket:Nationaliteit + } e:becomes { + ?s a persoon:Nationaliteit + } +} => { + ?s a persoon:Nationaliteit +}. + +{ + { + ?s a persoon_loket:Overlijden + } e:becomes { + ?s a persoon:Overlijden + } +} => { + ?s a persoon:Overlijden +}. + +{ + { + ?s a persoon_loket:PermanentInwoner + } e:becomes { + ?s a persoon:PermanentInwoner + } +} => { + ?s a persoon:PermanentInwoner +}. + +{ + { + ?s a persoon_loket:Persoonsgebeurtenis + } e:becomes { + ?s a persoon:Persoonsgebeurtenis + } +} => { + ?s a persoon:Persoonsgebeurtenis +}. + +{ + { + ?s a persoon_loket:Persoonsrelatie + } e:becomes { + ?s a persoon:Persoonsrelatie + } +} => { + ?s a persoon:Persoonsrelatie +}. + +{ + { + ?s a persoon_loket:Samenwonen + } e:becomes { + ?s a persoon:Samenwonen + } +} => { + ?s a persoon:Samenwonen +}. + +{ + { + ?s a persoon_loket:Staatsburger + } e:becomes { + ?s a persoon:Staatsburger + } +} => { + ?s a persoon:Staatsburger +}. + +{ + { + ?s a persoon_loket:Staatsburgerschap + } e:becomes { + ?s a persoon:Staatsburgerschap + } +} => { + ?s a persoon:Staatsburgerschap +}. + +{ + { + ?s a persoon_loket:TijdelijkInwoner + } e:becomes { + ?s a persoon:TijdelijkInwoner + } +} => { + ?s a persoon:TijdelijkInwoner +}. + +{ + { + ?s a persoon_loket:Verblijfplaats + } e:becomes { + ?s a persoon:Verblijfplaats + } +} => { + ?s a persoon:Verblijfplaats +}. + +{ + { + ?s a persoon_loket:Voogdij + } e:becomes { + ?s a persoon:Voogdij + } +} => { + ?s a persoon:Voogdij +}. + +{ + { + ?s a persoon_loket:Vreemdeling + } e:becomes { + ?s a persoon:Vreemdeling + } +} => { + ?s a persoon:Vreemdeling +}. + +## properties +{ + { + ?s persoon_loket:afstammingsType ?o + } e:becomes { + ?s persoon:afstammingsType ?o + } +} => { + ?s persoon:afstammingsType ?o +}. + +{ + { + ?s persoon_loket:alternatieveNaam ?o + } e:becomes { + ?s persoon:alternatieveNaam ?o + } +} => { + ?s persoon:alternatieveNaam ?o +}. + +{ + { + ?s persoon_loket:datum ?o + } e:becomes { + ?s persoon:datum ?o + } +} => { + ?s persoon:datum ?o +}. + +{ + { + ?s persoon_loket:datumVanAfstamming ?o + } e:becomes { + ?s persoon:datumVanAfstamming ?o + } +} => { + ?s persoon:datumVanAfstamming ?o +}. + +{ + { + ?s persoon_loket:gebruikteVoornaam ?o + } e:becomes { + ?s persoon:gebruikteVoornaam ?o + } +} => { + ?s persoon:gebruikteVoornaam ?o +}. + +{ + { + ?s persoon_loket:geslacht ?o + } e:becomes { + ?s persoon:geslacht ?o + } +} => { + ?s persoon:geslacht ?o +}. + +{ + { + ?s persoon_loket:gezinsadres ?o + } e:becomes { + ?s persoon:gezinsadres ?o + } +} => { + ?s persoon:gezinsadres ?o +}. + +{ + { + ?s persoon_loket:gezinsrelatietype ?o + } e:becomes { + ?s persoon:gezinsrelatietype ?o + } +} => { + ?s persoon:gezinsrelatietype ?o +}. + +{ + { + ?s persoon_loket:heeftBurgerlijkeStaat ?o + } e:becomes { + ?s persoon:heeftBurgerlijkeStaat ?o + } +} => { + ?s persoon:heeftBurgerlijkeStaat ?o +}. + +{ + { + ?s persoon_loket:heeftGeboorte ?o + } e:becomes { + ?s persoon:heeftGeboorte ?o + } +} => { + ?s persoon:heeftGeboorte ?o +}. + +{ + { + ?s persoon_loket:heeftInwonerschap ?o + } e:becomes { + ?s persoon:heeftInwonerschap ?o + } +} => { + ?s persoon:heeftInwonerschap ?o +}. + +{ + { + ?s persoon_loket:heeftNationaliteit ?o + } e:becomes { + ?s persoon:heeftNationaliteit ?o + } +} => { + ?s persoon:heeftNationaliteit ?o +}. + +{ + { + ?s persoon_loket:heeftOverlijden ?o + } e:becomes { + ?s persoon:heeftOverlijden ?o + } +} => { + ?s persoon:heeftOverlijden ?o +}. + +{ + { + ?s persoon_loket:heeftPersoonsrelatie ?o + } e:becomes { + ?s persoon:heeftPersoonsrelatie ?o + } +} => { + ?s persoon:heeftPersoonsrelatie ?o +}. + +{ + { + ?s persoon_loket:heeftStaatsburgerschap ?o + } e:becomes { + ?s persoon:heeftStaatsburgerschap ?o + } +} => { + ?s persoon:heeftStaatsburgerschap ?o +}. + +{ + { + ?s persoon_loket:heeftVerblijfplaats ?o + } e:becomes { + ?s persoon:heeftVerblijfplaats ?o + } +} => { + ?s persoon:heeftVerblijfplaats ?o +}. + +{ + { + ?s persoon_loket:Inwonerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Inwonerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Inwonerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:isHoofdVan ?o + } e:becomes { + ?s persoon:isHoofdVan ?o + } +} => { + ?s persoon:isHoofdVan ?o +}. + +{ + { + ?s persoon_loket:isLidVan ?o + } e:becomes { + ?s persoon:isLidVan ?o + } +} => { + ?s persoon:isLidVan ?o +}. + +{ + { + ?s persoon_loket:isRelatieMet ?o + } e:becomes { + ?s persoon:isRelatieMet ?o + } +} => { + ?s persoon:isRelatieMet ?o +}. + +{ + { + ?s persoon_loket:land ?o + } e:becomes { + ?s persoon:land ?o + } +} => { + ?s persoon:land ?o +}. + +{ + { + ?s persoon_loket:nationaliteit ?o + } e:becomes { + ?s persoon:nationaliteit ?o + } +} => { + ?s persoon:nationaliteit ?o +}. + +{ + { + ?s persoon_loket:plaats ?o + } e:becomes { + ?s persoon:plaats ?o + } +} => { + ?s persoon:plaats ?o +}. + +{ + { + ?s persoon_loket:registratie ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. + +{ + { + ?s persoon_loket:Staatsburgerschap.binnenJurisdictie ?o + } e:becomes { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o + } +} => { + ?s persoon:Staatsburgerschap.binnenJurisdictie ?o +}. + +{ + { + ?s persoon_loket:type ?o + } e:becomes { + ?s persoon:type ?o + } +} => { + ?s persoon:type ?o +}. + +{ + { + ?s persoon_loket:verblijfsadres ?o + } e:becomes { + ?s persoon:verblijfsadres ?o + } +} => { + ?s persoon:verblijfsadres ?o +}. + +{ + { + ?s persoon_loket:volledigeNaam ?o + } e:becomes { + ?s persoon:volledigeNaam ?o + } +} => { + ?s persoon:volledigeNaam ?o +}. \ No newline at end of file diff --git a/config/reasoner/dl2op/private-info/query.n3q b/config/reasoner/dl2op/private-info/query.n3q new file mode 100644 index 0000000..bddbb11 --- /dev/null +++ b/config/reasoner/dl2op/private-info/query.n3q @@ -0,0 +1,90 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + + +# Rijksregisternummer +# Extend identifiers to match OP's model +# skos:notation "00010100105". +{ + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +} => { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +}. + +# identifier property in the export is only used for RRN. +{ + ?s persoon:registratie ?o +} => { + ?s persoon:registratie ?o +}. + +{ + ?s a + } => { + ?s a +}. + +# Nationality +{ + ?s persoon:heeftNationaliteit ?o +} => { + ?s persoon:heeftNationaliteit ?o +}. + + +# Birth and related attributes +{ + ?s a persoon:Geboorte; + ?p ?o. +} => { + ?s a persoon:Geboorte; + ?p ?o. +}. + +{ + ?s persoon:heeftGeboorte ?o +} => { + ?s persoon:heeftGeboorte ?o +}. + +{ + ?s persoon:datum ?o +} => { + ?s persoon:datum ?o +}. + +# Gender +{ + ?s persoon:geslacht ?o +} => { + ?s persoon:geslacht ?o +}. \ No newline at end of file diff --git a/config/reasoner/dl2op/private-info/retract_empty_strings.n3 b/config/reasoner/dl2op/private-info/retract_empty_strings.n3 new file mode 100644 index 0000000..f8204a7 --- /dev/null +++ b/config/reasoner/dl2op/private-info/retract_empty_strings.n3 @@ -0,0 +1,27 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + +# Retract empty strings +{ + { + ?s ?p "" + } e:becomes {}. +} => {}. diff --git a/config/reasoner/dl2op/private-info/rrn.n3 b/config/reasoner/dl2op/private-info/rrn.n3 new file mode 100644 index 0000000..b45c688 --- /dev/null +++ b/config/reasoner/dl2op/private-info/rrn.n3 @@ -0,0 +1,62 @@ +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + + + +# Rijksregisternummer +# Extend identifiers to match OP's model +# skos:notation "00010100105". +{ + ?identifier skos:notation ?notation. + ?notation string:notEqualIgnoringCase "Rijksregisternummer". + ?identifier log:uri ?identifierString. + (?identifierString "STRUCTURED_IDENTIFIER") string:concatenation ?hashInput. + ?hashInput crypto:md5 ?uuid. + ("http://data.lblod.info/id/gestructureerdeIdentificatoren/" ?uuid) string:concatenation ?structuredIdString. + ?structuredIdentifier log:uri ?structuredIdString. + + { + ?identifier skos:notation ?notation. + } e:becomes { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + }. + +} => { + ?identifier + generiek:gestructureerdeIdentificator ?structuredIdentifier ; + skos:notation "Rijksregisternummer" . + + ?structuredIdentifier a generiek:GestructureerdeIdentificator ; + mu:uuid ?uuid ; + generiek:lokaleIdentificator ?notation +}. + +# identifier property in the export is only used for RRN. +{ + { + ?s adms:identifier ?o + } e:becomes { + ?s persoon:registratie ?o + } +} => { + ?s persoon:registratie ?o +}. diff --git a/config/search/config.json b/config/search/config.json index bfb99e0..6649ae8 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -141,8 +141,8 @@ { "type": "mandatory", "on_path": "mandatories", - "rdf_type": "http://data.vlaanderen.be/ns/mandaat#Mandataris", - "sub_types": [ + "rdf_type": [ + "http://data.vlaanderen.be/ns/mandaat#Mandataris", "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" ], "properties": { @@ -150,11 +150,11 @@ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://xmlns.com/foaf/0.1/givenName" ], - "id": [ + "person_id": [ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://mu.semte.ch/vocabularies/core/uuid" ], - "uuid": [ + "person_uuid": [ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://mu.semte.ch/vocabularies/core/uuid" ], @@ -312,11 +312,11 @@ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://xmlns.com/foaf/0.1/givenName" ], - "id": [ + "person_id": [ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://mu.semte.ch/vocabularies/core/uuid" ], - "uuid": [ + "person_uuid": [ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://mu.semte.ch/vocabularies/core/uuid" ], @@ -468,8 +468,8 @@ { "type": "minister", "on_path": "ministers", - "rdf_type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", - "sub_types": [ + "rdf_type": [ + "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", "http://data.lblod.info/vocabularies/contacthub/AgentInPositie" ], "properties": { @@ -477,11 +477,11 @@ "http://www.w3.org/ns/org#heldBy", "http://xmlns.com/foaf/0.1/givenName" ], - "id": [ + "person_id": [ "http://www.w3.org/ns/org#heldBy", "http://mu.semte.ch/vocabularies/core/uuid" ], - "uuid": [ + "person_uuid": [ "http://www.w3.org/ns/org#heldBy", "http://mu.semte.ch/vocabularies/core/uuid" ], @@ -546,7 +546,7 @@ } } }, - "id": { + "person_id": { "type": "text", "fields": { "field": { @@ -638,11 +638,11 @@ } }, { - "name": "id", + "name": "person_id", "mappings": { - "minister": "id", - "agent": "id", - "mandatory": "id" + "minister": "person_id", + "agent": "person_id", + "mandatory": "person_id" } }, { @@ -814,8 +814,8 @@ { "type": "unit", "on_path": "units", - "rdf_type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", - "sub_types": [ + "rdf_type": [ + "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid", "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst" diff --git a/config/search/dev/config.json b/config/search/dev/config.json index 97a230f..153ec14 100644 --- a/config/search/dev/config.json +++ b/config/search/dev/config.json @@ -1,7 +1,7 @@ { "automatic_index_updates": true, "persist_indexes": true, - "number_of_threads": 11, + "number_of_threads": 8, "eager_indexing_groups": [ [ { @@ -46,27 +46,27 @@ } ] ], - "default_settings" : { + "default_settings": { "analysis": { "normalizer": { "custom_sort_normalizer": { - "type": "custom", - "char_filter": [], - "filter": [ - "lowercase", - "trim", - "asciifolding" - ] + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase", + "trim", + "asciifolding" + ] } - } + } } }, "types": [ { "type": "mandatory", "on_path": "mandatories", - "rdf_type": "http://data.vlaanderen.be/ns/mandaat#Mandataris", - "sub_types": [ + "rdf_type": [ + "http://data.vlaanderen.be/ns/mandaat#Mandataris", "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" ], "properties": { @@ -74,11 +74,11 @@ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://xmlns.com/foaf/0.1/givenName" ], - "id": [ + "person_id": [ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://mu.semte.ch/vocabularies/core/uuid" ], - "uuid": [ + "person_uuid": [ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://mu.semte.ch/vocabularies/core/uuid" ], @@ -153,7 +153,7 @@ } } }, - "id": { + "person_id": { "type": "text", "fields": { "field": { @@ -236,11 +236,11 @@ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://xmlns.com/foaf/0.1/givenName" ], - "id": [ + "person_id": [ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://mu.semte.ch/vocabularies/core/uuid" ], - "uuid": [ + "person_uuid": [ "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", "http://mu.semte.ch/vocabularies/core/uuid" ], @@ -315,7 +315,7 @@ } } }, - "id": { + "person_id": { "type": "text", "fields": { "field": { @@ -392,8 +392,8 @@ { "type": "minister", "on_path": "ministers", - "rdf_type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", - "sub_types": [ + "rdf_type": [ + "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", "http://data.lblod.info/vocabularies/contacthub/AgentInPositie" ], "properties": { @@ -401,11 +401,11 @@ "http://www.w3.org/ns/org#heldBy", "http://xmlns.com/foaf/0.1/givenName" ], - "id": [ + "person_id": [ "http://www.w3.org/ns/org#heldBy", "http://mu.semte.ch/vocabularies/core/uuid" ], - "uuid": [ + "person_uuid": [ "http://www.w3.org/ns/org#heldBy", "http://mu.semte.ch/vocabularies/core/uuid" ], @@ -470,7 +470,7 @@ } } }, - "id": { + "person_id": { "type": "text", "fields": { "field": { @@ -562,11 +562,11 @@ } }, { - "name": "id", + "name": "person_id", "mappings": { - "minister": "id", - "agent": "id", - "mandatory": "id" + "minister": "person_id", + "agent": "person_id", + "mandatory": "person_id" } }, { @@ -661,7 +661,7 @@ } } }, - "id": { + "person_id": { "type": "text", "fields": { "field": { @@ -739,8 +739,8 @@ { "type": "unit", "on_path": "units", - "rdf_type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", - "sub_types": [ + "rdf_type": [ + "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid", "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst" diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 33d98bd..5eecc28 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -71,3 +71,7 @@ services: restart: "no" construct-administrative-unit-relationships: restart: "no" + delta-producer-bg-jobs-initiator-worship-posts: + restart: "no" + delta-producer-pub-graph-maintainer-worship-posts: + restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index eba3edd..5f7d7b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,6 @@ -version: '3.4' +version: "3.4" -x-logging: - &default-logging +x-logging: &default-logging driver: "json-file" options: max-size: "10m" @@ -11,7 +10,7 @@ services: identifier: image: semtech/mu-identifier:1.9.1 environment: - DEFAULT_MU_AUTH_ALLOWED_GROUPS_HEADER: "[{\"variables\":[],\"name\":\"public\"}]" + DEFAULT_MU_AUTH_ALLOWED_GROUPS_HEADER: '[{"variables":[],"name":"public"}]' SESSION_COOKIE_SECURE: "on" labels: - "logging=true" @@ -92,7 +91,7 @@ services: LOGGING_LEVEL: "INFO" SPARQL_ENDPOINT: "http://db:8890/sparql" LIMIT_SIZE: "10000" - MU_AUTH_SUDO: "false" + MU_AUTH_SUDO: "true" labels: - "logging=true" restart: always @@ -158,7 +157,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.13.3 + image: lblod/frontend-organization-portal:1.14.0-0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: @@ -201,34 +200,35 @@ services: - db:database restart: always logging: *default-logging -################################################################################ -# DELTA GENERAL -################################################################################ + reasoner: + image: eyereasoner/reasoning-service:1.0.1 + volumes: + - ./config/reasoner:/config + ################################################################################ + # DELTA GENERAL + ################################################################################ delta-producer-dump-file-publisher: - image: bdevloed/delta-producer-dump-file-publisher:0.8.0-beta + image: lblod/delta-producer-dump-file-publisher:0.9.0 volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share - environment: - FILES_GRAPH: 'http://mu.semte.ch/graphs/public' # Note: this will be extracted in a config per job one day - DCAT_DATASET_GRAPH: 'http://mu.semte.ch/graphs/public' links: - db:database restart: always logging: *default-logging -################################################################################ -# DELTA ADMINISTRATIVE-UNITS: START -################################################################################ + ################################################################################ + # DELTA ADMINISTRATIVE-UNITS: START + ################################################################################ delta-producer-bg-jobs-initiator-administrative-units: image: lblod/delta-producer-background-jobs-initiator:0.3.1 environment: JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" - DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units' - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units' - HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units' - CRON_PATTERN_DUMP_JOB: '0 0 4 * * 6' # weekly on saturday - CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM - START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units" + CRON_PATTERN_DUMP_JOB: "0 0 4 * * 6" # weekly on saturday + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. labels: - "logging=true" links: @@ -241,12 +241,12 @@ services: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" RELATIVE_FILE_PATH: "deltas/administrative-units" - PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/administrative-units' - HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units' - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units' + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/administrative-units" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" QUEUE_POLL_INTERVAL: 3000 USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: 'true' + SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 volumes: - ./config/delta-producer/administrative-units:/config @@ -263,7 +263,7 @@ services: ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" RELATIVE_FILE_PATH: "deltas/administrative-units" PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/administrative-units" - PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/administrative-units' + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/administrative-units" DELTA_INTERVAL_MS: 10000 PRETTY_PRINT_DIFF_JSON: "true" volumes: @@ -274,22 +274,22 @@ services: - db:database restart: always logging: *default-logging -################################################################################ -# DELTA ADMINISTRATIVE-UNITS: END -################################################################################ -################################################################################ -# DELTA ORGANIZATIONS: START -################################################################################ + ################################################################################ + # DELTA ADMINISTRATIVE-UNITS: END + ################################################################################ + ################################################################################ + # DELTA ORGANIZATIONS: START + ################################################################################ delta-producer-bg-jobs-initiator-organizations: image: lblod/delta-producer-background-jobs-initiator:0.3.1 environment: JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" - DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations' - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations' - HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations' - CRON_PATTERN_DUMP_JOB: '42 * * * *' # At minute 42 every hour - CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM - START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" + CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. labels: - "logging=true" links: @@ -302,13 +302,15 @@ services: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" RELATIVE_FILE_PATH: "deltas/organizations" - PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/organizations' - HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations' - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations' + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" QUEUE_POLL_INTERVAL: 3000 USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: 'true' + SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + FILES_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-deltas" + ACCOUNT: "http://data.lblod.info/foaf/account/id/8df6f1e4-869f-4223-9834-961b01a7bd48" volumes: - ./config/delta-producer/organizations:/config - ./data/files/:/share @@ -324,7 +326,7 @@ services: ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" RELATIVE_FILE_PATH: "deltas/organizations" PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" - PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/organizations' + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" DELTA_INTERVAL_MS: 10000 PRETTY_PRINT_DIFF_JSON: "true" volumes: @@ -335,24 +337,23 @@ services: - db:database restart: always logging: *default-logging -################################################################################ -# DELTA ORGANIZATIONS: END -################################################################################ + ################################################################################ + # DELTA ORGANIZATIONS: END + ################################################################################ - -################################################################################ -# DELTA PUBLIC INFORMATION: START -################################################################################ + ################################################################################ + # DELTA PUBLIC INFORMATION: START + ################################################################################ delta-producer-bg-jobs-initiator-public: image: lblod/delta-producer-background-jobs-initiator:0.3.1 environment: JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" - DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public' - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public' - HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public' - CRON_PATTERN_DUMP_JOB: '11 */4 * * *' # At minute 11 every 4 hours - CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM - START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. labels: - "logging=true" links: @@ -365,12 +366,12 @@ services: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" RELATIVE_FILE_PATH: "deltas/public" - PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/public' - HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public' - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public' + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" QUEUE_POLL_INTERVAL: 3000 USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: 'true' + SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 volumes: - ./config/delta-producer/public:/config @@ -387,7 +388,7 @@ services: ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" RELATIVE_FILE_PATH: "deltas/public" PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" - PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/public' + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" DELTA_INTERVAL_MS: 10000 PRETTY_PRINT_DIFF_JSON: "true" volumes: @@ -398,16 +399,156 @@ services: - db:database restart: always logging: *default-logging -################################################################################ -# DELTA PUBLIC INFORMATION: END -################################################################################ + ################################################################################ + # DELTA PUBLIC INFORMATION: END + ################################################################################ + ################################################################################ + # DELTA WORSHIP POSTS: START + ################################################################################ + delta-producer-bg-jobs-initiator-worship-posts: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-worship-posts: + image: lblod/delta-producer-publication-graph-maintainer:0.15.1 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + RELATIVE_FILE_PATH: "deltas/worship-posts" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + volumes: + - ./config/delta-producer/worship-posts:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + logging: *default-logging + ################################################################################ + # DELTA WORSHIP POSTS: END + ################################################################################ -################################################################################ -# MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH -################################################################################ + ################################################################################ + # DELTA CONSUMERS + ################################################################################ + ################################################################################ + # LOKET MANDATEN CONSUMERS + ################################################################################ + mandaten-consumer: + image: lblod/delta-consumer:0.0.14 + environment: + DCR_SERVICE_NAME: "mandatendatabank-consumer" + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API + DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/MandatarissenCacheGraphDump" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatendatabank-consumer" + DCR_DISABLE_INITIAL_SYNC: "true" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" + DCR_SYNC_FILES_PATH: "/sync/mandatarissen/files" + FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data" + KEEP_DELTA_FILES: "true" + volumes: + - ./config/loket-consumer/mandate-dispatching:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/mandaten:/consumer-files/ + restart: always + labels: + - "logging=true" + links: + - db:database + - triplestore:virtuoso + logging: *default-logging + + ################################################################################ + # LOKET LEIDINGGEVENDEN CONSUMERS + ################################################################################ + leidinggevenden-consumer: + image: lblod/delta-consumer:0.0.14 + environment: + DCR_SERVICE_NAME: "leidinggevenden-consumer" + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API + DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/LeidinggevendenCacheGraphDump" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/leidinggevenden-consumer" + DCR_DISABLE_INITIAL_SYNC: "true" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" + DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files" + FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data" + KEEP_DELTA_FILES: "true" + volumes: + - ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/leidinggevenden:/consumer-files/ + restart: always + labels: + - "logging=true" + links: + - db:database + - triplestore:virtuoso + logging: *default-logging + + ################################################################################ + # LOKET WORSHIP SENSITIVE CONSUMERS + ################################################################################ + worship-services-sensitive-consumer: + image: lblod/delta-consumer:0.0.14 + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info/" + DCR_SERVICE_NAME: "worship-services-sensitive-consumer" + DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files" + DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer" + DCR_DISABLE_INITIAL_SYNC: "true" + FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data" + DCR_KEEP_DELTA_FILES: "true" + DCR_DELTA_FILE_FOLDER: "/consumer-files" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false" + volumes: + - ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/worship-services-sensitive:/consumer-files/ + restart: always + labels: + - "logging=true" + links: + - db:database + - triplestore:virtuoso + logging: *default-logging + + ################################################################################ + # DELTA CONSUMERS: END + ################################################################################ + + ################################################################################ + # MU SEARCH + ################################################################################ search: - image: nbittich/mu-search:wip-inheritance-alpha2 # Bump to fixed version when merged and release to the original repo + image: semtech/mu-search:0.9.0-beta.1 links: - db:database restart: always @@ -418,7 +559,7 @@ services: volumes: - ./data/elasticsearch/:/usr/share/elasticsearch/data environment: - - discovery.type=single-node + - discovery.type=single-node restart: always error-report-service: