felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

apply worship-app changes

Browse Source
This commit is contained in:
Boris De Vloed
2023-01-10 13:04:46 +00:00
parent 71e024c007
commit 2576f86068
17 changed files with 1174 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js
View File

@@ -1,4 +1,4 @@
const { transformStatements, batchedDbUpdate, partition } = require('./util');
const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util');
const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
@@ -30,14 +30,28 @@ async function dispatch(lib, data) {
const { termObjectChangeSets } = data;
for (let { deletes, inserts } of termObjectChangeSets) {
const deletePartitions = partition(deletes, o => o.subject.startsWith('<share://'));
const regularDeletes = deletePartitions.fails;
const fileDeletes = deletePartitions.passes;
const fileDeleteStatements = fileDeletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const deleteStatements = regularDeletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
if (deleteStatements.length) {
await transformStatements(fetch, deleteStatements).then(
transformedStatements => {
deleteFromAllGraphs(
muAuthSudo.updateSudo,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
);
}
)
}
if (insertStatements.length) {
await transformStatements(fetch, insertStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
@@ -49,70 +63,6 @@ async function dispatch(lib, data) {
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"DELETE"
);
}
)
}
if (fileDeleteStatements.length) {
await transformStatements(fetch, fileDeleteStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
FILE_SYNC_GRAPH,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"DELETE"
);
}
)
}
const insertPartitions = partition(inserts, o => o.subject.startsWith('<share://'));
const regularInserts = insertPartitions.fails;
const fileInserts = insertPartitions.passes;
const regularInsertStatements = regularInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const fileInsertStatements = fileInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
if (regularInsertStatements.length) {
await transformStatements(fetch, regularInsertStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
INGEST_GRAPH,
transformedStatements,
{},
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"INSERT"
);
}
)
}
if (fileInsertStatements.length) {
await transformStatements(fetch, fileInsertStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
FILE_SYNC_GRAPH,
transformedStatements,
{},
process.env.MU_SPARQL_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"INSERT"
);
}

35
config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js
View File

@@ -28,45 +28,20 @@ async function dispatch(lib, data) {
const { mu, muAuthSudo, fetch } = lib;
const { termObjects } = data;
const partitions = partition(termObjects, o => o.subject.startsWith('<share://'));
const regularInserts = partitions.fails;
const fileInserts = partitions.passes;
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
console.warn(`Service configured to skip MU_AUTH!`);
}
console.log(`Using ${endpoint} to insert triples`);
if (regularInserts.length) {
originalRegularInsertTriples = regularInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
if (termObjects.length) {
originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
await transformStatements(fetch, originalRegularInsertTriples).then(
transformedRegularInsertTriples => {
// console.log(transformedRegularInsertTriples);
await transformStatements(fetch, originalInsertTriples).then(
transformedInsertTriples => {
batchedDbUpdate(
muAuthSudo.updateSudo,
INGEST_GRAPH,
transformedRegularInsertTriples,
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
endpoint,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
)
}
)
}
if (fileInserts.length) {
originalFileInsertTriples = fileInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
await transformStatements(fetch, originalFileInsertTriples).then(
transformedFileInsertTriples => {
// console.log(transformedFileInsertTriples);
batchedDbUpdate(
muAuthSudo.updateSudo,
FILE_SYNC_GRAPH,
transformedFileInsertTriples,
transformedInsertTriples,
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
endpoint,
BATCH_SIZE,

54
config/loket-consumer/leidinggevenden-dispatching/util.js
View File

@@ -37,6 +37,39 @@ ${batch}
}
}
async function deleteFromAllGraphs(muUpdate,
triples,
extraHeaders,
endpoint,
maxAttempts,
sleepBetweenBatches = 1000,
sleepTimeOnFail = 1000,
) {
for (const triple of triples) {
console.log(`Deleting a triple from all graphs in triplestore`);
const deleteCall = async () => {
await muUpdate(`
DELETE {
GRAPH ?g {
${triple}
}
} WHERE {
GRAPH ?g {
${triple}
}
}
`, extraHeaders, endpoint);
};
await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
await new Promise(r => setTimeout(r, sleepBetweenBatches));
}
}
async function operationWithRetry(callback,
attempt,
maxAttempts,
@@ -83,27 +116,11 @@ function partition(arr, fn) {
*
*/
function transformTriples(fetch, triples) {
return operationWithRetry(preProcess(fetch, triples), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION)
.then(preprocessed => operationWithRetry(mainConversion(fetch, preprocessed), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION));
return operationWithRetry(mainConversion(fetch, triples), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
}
function preProcess(fetch, triples) {
let formdata = new URLSearchParams();
formdata.append("data", triples);
let requestOptions = {
method: 'POST',
body: formdata,
redirect: 'follow'
};
return fetch("http://reasoner/reason/dl2op/preprocess", requestOptions)
.then(response => response.text());
}
function mainConversion(fetch, triples) {
let formdata = new URLSearchParams();
formdata.append("data", triples);
@@ -130,6 +147,7 @@ function transformStatements(fetch, triples) {
module.exports = {
batchedDbUpdate,
deleteFromAllGraphs,
partition,
transformStatements
};

90
config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js
View File

@@ -1,4 +1,4 @@
const { transformStatements, batchedDbUpdate, partition } = require('./util');
const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util');
const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
@@ -30,14 +30,28 @@ async function dispatch(lib, data) {
const { termObjectChangeSets } = data;
for (let { deletes, inserts } of termObjectChangeSets) {
const deletePartitions = partition(deletes, o => o.subject.startsWith('<share://'));
const regularDeletes = deletePartitions.fails;
const fileDeletes = deletePartitions.passes;
const fileDeleteStatements = fileDeletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const deleteStatements = regularDeletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
if (deleteStatements.length) {
await transformStatements(fetch, deleteStatements).then(
transformedStatements => {
deleteFromAllGraphs(
muAuthSudo.updateSudo,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
);
}
)
}
if (insertStatements.length) {
await transformStatements(fetch, insertStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
@@ -49,70 +63,6 @@ async function dispatch(lib, data) {
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"DELETE"
);
}
)
}
if (fileDeleteStatements.length) {
await transformStatements(fetch, fileDeleteStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
FILE_SYNC_GRAPH,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"DELETE"
);
}
)
}
const insertPartitions = partition(inserts, o => o.subject.startsWith('<share://'));
const regularInserts = insertPartitions.fails;
const fileInserts = insertPartitions.passes;
const regularInsertStatements = regularInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const fileInsertStatements = fileInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
if (regularInsertStatements.length) {
await transformStatements(fetch, regularInsertStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
INGEST_GRAPH,
transformedStatements,
{},
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"INSERT"
);
}
)
}
if (fileInsertStatements.length) {
await transformStatements(fetch, fileInsertStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
FILE_SYNC_GRAPH,
transformedStatements,
{},
process.env.MU_SPARQL_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"INSERT"
);
}

35
config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js
View File

@@ -28,45 +28,20 @@ async function dispatch(lib, data) {
const { mu, muAuthSudo, fetch } = lib;
const { termObjects } = data;
const partitions = partition(termObjects, o => o.subject.startsWith('<share://'));
const regularInserts = partitions.fails;
const fileInserts = partitions.passes;
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
console.warn(`Service configured to skip MU_AUTH!`);
}
console.log(`Using ${endpoint} to insert triples`);
if (regularInserts.length) {
originalRegularInsertTriples = regularInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
if (termObjects.length) {
originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
await transformStatements(fetch, originalRegularInsertTriples).then(
transformedRegularInsertTriples => {
// console.log(transformedRegularInsertTriples);
await transformStatements(fetch, originalInsertTriples).then(
transformedInsertTriples => {
batchedDbUpdate(
muAuthSudo.updateSudo,
INGEST_GRAPH,
transformedRegularInsertTriples,
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
endpoint,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
)
}
)
}
if (fileInserts.length) {
originalFileInsertTriples = fileInserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
await transformStatements(fetch, originalFileInsertTriples).then(
transformedFileInsertTriples => {
// console.log(transformedFileInsertTriples);
batchedDbUpdate(
muAuthSudo.updateSudo,
FILE_SYNC_GRAPH,
transformedFileInsertTriples,
transformedInsertTriples,
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
endpoint,
BATCH_SIZE,

54
config/loket-consumer/mandate-dispatching/util.js
View File

@@ -37,6 +37,39 @@ ${batch}
}
}
async function deleteFromAllGraphs(muUpdate,
triples,
extraHeaders,
endpoint,
maxAttempts,
sleepBetweenBatches = 1000,
sleepTimeOnFail = 1000,
) {
for (const triple of triples) {
console.log(`Deleting a triple from all graphs in triplestore`);
const deleteCall = async () => {
await muUpdate(`
DELETE {
GRAPH ?g {
${triple}
}
} WHERE {
GRAPH ?g {
${triple}
}
}
`, extraHeaders, endpoint);
};
await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
await new Promise(r => setTimeout(r, sleepBetweenBatches));
}
}
async function operationWithRetry(callback,
attempt,
maxAttempts,
@@ -83,27 +116,11 @@ function partition(arr, fn) {
*
*/
function transformTriples(fetch, triples) {
return operationWithRetry(preProcess(fetch, triples), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION)
.then(preprocessed => operationWithRetry(mainConversion(fetch, preprocessed), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION));
return operationWithRetry(mainConversion(fetch, triples), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
}
function preProcess(fetch, triples) {
let formdata = new URLSearchParams();
formdata.append("data", triples);
let requestOptions = {
method: 'POST',
body: formdata,
redirect: 'follow'
};
return fetch("http://reasoner/reason/dl2op/preprocess", requestOptions)
.then(response => response.text());
}
function mainConversion(fetch, triples) {
let formdata = new URLSearchParams();
formdata.append("data", triples);
@@ -130,6 +147,7 @@ function transformStatements(fetch, triples) {
module.exports = {
batchedDbUpdate,
deleteFromAllGraphs,
partition,
transformStatements
};

29
config/loket-consumer/positions-dispatching/config.js Normal file
View File

@@ -0,0 +1,29 @@
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`;
if (!process.env.FILE_SYNC_GRAPH)
throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
module.exports = {
BATCH_SIZE,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MAX_DB_RETRY_ATTEMPTS,
MAX_REASONING_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
INGEST_GRAPH,
FILE_SYNC_GRAPH
};

76
config/loket-consumer/positions-dispatching/delta-sync-dispatching.js Normal file
View File

@@ -0,0 +1,76 @@
const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util');
const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
INGEST_GRAPH,
FILE_SYNC_GRAPH
} = require('./config');
/**
* Dispatch the fetched information to a target graph.
* Note: <share://file/data> will be ADDED to it's own graph.
* We take only care of adding them, not updating triples, this is a TODO
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
* [ {
* graph: "<http://foo>",
* subject: "<http://bar>",
* predicate: "<http://baz>",
* object: "<http://boom>^^<http://datatype>"
* }
* ]
* @return {void} Nothing
*/
async function dispatch(lib, data) {
const { mu, muAuthSudo, fetch } = lib;
const { termObjectChangeSets } = data;
for (let { deletes, inserts } of termObjectChangeSets) {
const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
if (deleteStatements.length) {
await transformStatements(fetch, deleteStatements).then(
transformedStatements => {
deleteFromAllGraphs(
muAuthSudo.updateSudo,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
);
}
)
}
if (insertStatements.length) {
await transformStatements(fetch, insertStatements).then(
transformedStatements => {
batchedDbUpdate(
muAuthSudo.updateSudo,
INGEST_GRAPH,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
"INSERT"
);
}
)
}
}
}
module.exports = {
dispatch
};

59
config/loket-consumer/positions-dispatching/initial-sync-dispatching.js Normal file
View File

@@ -0,0 +1,59 @@
const { transformStatements, batchedDbUpdate, partition } = require('./util');
const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
INGEST_GRAPH,
FILE_SYNC_GRAPH
} = require('./config');
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
/**
* Dispatch the fetched information to a target graph.
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
* [ {
* graph: "<http://foo>",
* subject: "<http://bar>",
* predicate: "<http://baz>",
* object: "<http://boom>^^<http://datatype>"
* }
* ]
* @return {void} Nothing
*/
async function dispatch(lib, data) {
const { mu, muAuthSudo, fetch } = lib;
const { termObjects } = data;
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
console.warn(`Service configured to skip MU_AUTH!`);
}
console.log(`Using ${endpoint} to insert triples`);
if (termObjects.length) {
originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
await transformStatements(fetch, originalInsertTriples).then(
transformedInsertTriples => {
batchedDbUpdate(
muAuthSudo.updateSudo,
INGEST_GRAPH,
transformedInsertTriples,
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
endpoint,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
)
}
)
}
}
module.exports = {
dispatch
};

153
config/loket-consumer/positions-dispatching/util.js Normal file
View File

@@ -0,0 +1,153 @@
const {
MAX_REASONING_RETRY_ATTEMPTS,
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
} = require('./config');
async function batchedDbUpdate(muUpdate,
graph,
triples,
extraHeaders,
endpoint,
batchSize,
maxAttempts,
sleepBetweenBatches = 1000,
sleepTimeOnFail = 1000,
operation = 'INSERT'
) {
for (let i = 0; i < triples.length; i += batchSize) {
console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
const batch = triples.slice(i, i + batchSize).join('\n');
const insertCall = async () => {
await muUpdate(`
${operation} DATA {
GRAPH <${graph}> {
${batch}
}
}
`, extraHeaders, endpoint);
};
await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
await new Promise(r => setTimeout(r, sleepBetweenBatches));
}
}
async function deleteFromAllGraphs(muUpdate,
triples,
extraHeaders,
endpoint,
maxAttempts,
sleepBetweenBatches = 1000,
sleepTimeOnFail = 1000,
) {
for (const triple of triples) {
console.log(`Deleting a triple from all graphs in triplestore`);
const deleteCall = async () => {
await muUpdate(`
DELETE {
GRAPH ?g {
${triple}
}
} WHERE {
GRAPH ?g {
${triple}
}
}
`, extraHeaders, endpoint);
};
await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
await new Promise(r => setTimeout(r, sleepBetweenBatches));
}
}
async function operationWithRetry(callback,
attempt,
maxAttempts,
sleepTimeOnFail) {
try {
if (typeof callback === "function")
return await callback();
else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
return await callback;
}
catch (e) {
console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
console.log(`Error: ${e}`);
console.log(`Sleeping ${sleepTimeOnFail} ms`);
if (attempt >= maxAttempts) {
console.log(`Max attempts reached for ${callback.toString()}, giving up`);
throw e;
}
await new Promise(r => setTimeout(r, sleepTimeOnFail));
return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
}
}
/**
* Splits an array into two parts, a part that passes and a part that fails a predicate function.
* Credits: https://github.com/benjay10
* @public
* @function partition
* @param {Array} arr - Array to be partitioned
* @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
* @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
*/
function partition(arr, fn) {
let passes = [], fails = [];
arr.forEach((item) => (fn(item) ? passes : fails).push(item));
return { passes, fails };
}
/**
* Send triples to reasoning service for conversion
*
*/
function transformTriples(fetch, triples) {
return operationWithRetry(mainConversion(fetch, triples), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
}
function mainConversion(fetch, triples) {
let formdata = new URLSearchParams();
formdata.append("data", triples);
let requestOptions = {
method: 'POST',
body: formdata,
redirect: 'follow'
};
return fetch("http://reasoner/reason/dl2op/main", requestOptions)
.then(response => response.text());
}
function transformStatements(fetch, triples) {
return transformTriples(fetch, triples.join('\n')).then(
graph => {
statements = graph.replace(/\n{2,}/g, '').split('\n')
console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`)
return statements
}
)
}
module.exports = {
batchedDbUpdate,
deleteFromAllGraphs,
partition,
transformStatements
};

33
config/reasoner/dl2op/main/inverse_relations.n3 Normal file
View File

@@ -0,0 +1,33 @@
PREFIX : <#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
PREFIX crypto: <http://www.w3.org/2000/10/swap/crypto#>
PREFIX e: <http://eulersharp.sourceforge.net/2003/03swap/log-rules#>
PREFIX ex: <http://example.org/test#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX func: <http://www.w3.org/2007/rif-builtin-function#>
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
PREFIX log: <http://www.w3.org/2000/10/swap/log#>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
# Inverse properties
{
?s org:holds ?o.
} => {
?o org:heldBy ?s.
}.
{
?s org:heldBy ?o.
} => {
?o org:holds ?s.
}.

49
config/reasoner/dl2op/main/mapping.n3
View File

@@ -1,18 +1,51 @@
PREFIX : <#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
PREFIX crypto: <http://www.w3.org/2000/10/swap/crypto#>
PREFIX e: <http://eulersharp.sourceforge.net/2003/03swap/log-rules#>
PREFIX ex: <http://example.org/test#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX func: <http://www.w3.org/2007/rif-builtin-function#>
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
PREFIX log: <http://www.w3.org/2000/10/swap/log#>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX persoon: <http://data.vlaanderen.be/ns/persoon#>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)
# Hence the mapping is in two steps.
# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart
# and drops the DL persoon:gebruikteVoornaam statements with e:becomes
# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam
# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames
# in OP
# First name
# persoon:gebruikteVoornaam => foaf:givenName
# Alternatieve voornaam
# foaf:name => persoon:gebruikteVoornaam
# Test Mock DL
# <http://example.org/person> persoon_loket:gebruikteVoornaam "Didier".
{
{
?s persoon_loket:gebruikteVoornaam ?o
} e:becomes {
?s foaf:givenName ?o
}
} => {
?s foaf:givenName ?o
}.
# First name
# persoon:gebruikteVoornaam => foaf:givenName (done in pre-processing)
# Alternatieve voornaam
@@ -102,4 +135,16 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
}
} => {
?s adres:Adresvoorstelling.huisnummer ?o
}.
}.
# Tijdspecialisatie
{
{
?s mandaat:isTijdspecialisatieVan ?o.
} e:becomes {
?s generiek:isTijdspecialisatieVan ?o.
}
} => {
?s generiek:isTijdspecialisatieVan ?o.
}.

534
config/reasoner/dl2op/main/persoon_http_to_https.n3 Normal file
View File

@@ -0,0 +1,534 @@
PREFIX : <#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
PREFIX crypto: <http://www.w3.org/2000/10/swap/crypto#>
PREFIX e: <http://eulersharp.sourceforge.net/2003/03swap/log-rules#>
PREFIX ex: <http://example.org/test#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX func: <http://www.w3.org/2007/rif-builtin-function#>
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
PREFIX log: <http://www.w3.org/2000/10/swap/log#>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
# Map from http persoon to https persoon namespace.
## classes:
{
{
?s a persoon_loket:Afstamming
} e:becomes {
?s a persoon:Afstamming
}
} => {
?s a persoon:Afstamming
}.
{
{
?s a persoon_loket:BurgerlijkeStaat
} e:becomes {
?s a persoon:BurgerlijkeStaat
}
} => {
?s a persoon:BurgerlijkeStaat
}.
{
{
?s a persoon_loket:Domicilie
} e:becomes {
?s a persoon:Domicilie
}
} => {
?s a persoon:Domicilie
}.
{
{
?s a persoon_loket:Geboorte
} e:becomes {
?s a persoon:Geboorte
}
} => {
?s a persoon:Geboorte
}.
{
{
?s a persoon_loket:GeenInwoner
} e:becomes {
?s a persoon:GeenInwoner
}
} => {
?s a persoon:GeenInwoner
}.
{
{
?s a persoon_loket:GeregistreerdPersoon
} e:becomes {
?s a persoon:GeregistreerdPersoon
}
} => {
?s a persoon:GeregistreerdPersoon
}.
{
{
?s a persoon_loket:Gezin
} e:becomes {
?s a persoon:Gezin
}
} => {
?s a persoon:Gezin
}.
{
{
?s a persoon_loket:Gezinsrelatie
} e:becomes {
?s a persoon:Gezinsrelatie
}
} => {
?s a persoon:Gezinsrelatie
}.
{
{
?s a persoon_loket:Huwelijk
} e:becomes {
?s a persoon:Huwelijk
}
} => {
?s a persoon:Huwelijk
}.
{
{
?s a persoon_loket:Inwoner
} e:becomes {
?s a persoon:Inwoner
}
} => {
?s a persoon:Inwoner
}.
{
{
?s a persoon_loket:Inwonerschap
} e:becomes {
?s a persoon:Inwonerschap
}
} => {
?s a persoon:Inwonerschap
}.
{
{
?s a persoon_loket:Nationaliteit
} e:becomes {
?s a persoon:Nationaliteit
}
} => {
?s a persoon:Nationaliteit
}.
{
{
?s a persoon_loket:Overlijden
} e:becomes {
?s a persoon:Overlijden
}
} => {
?s a persoon:Overlijden
}.
{
{
?s a persoon_loket:PermanentInwoner
} e:becomes {
?s a persoon:PermanentInwoner
}
} => {
?s a persoon:PermanentInwoner
}.
{
{
?s a persoon_loket:Persoonsgebeurtenis
} e:becomes {
?s a persoon:Persoonsgebeurtenis
}
} => {
?s a persoon:Persoonsgebeurtenis
}.
{
{
?s a persoon_loket:Persoonsrelatie
} e:becomes {
?s a persoon:Persoonsrelatie
}
} => {
?s a persoon:Persoonsrelatie
}.
{
{
?s a persoon_loket:Samenwonen
} e:becomes {
?s a persoon:Samenwonen
}
} => {
?s a persoon:Samenwonen
}.
{
{
?s a persoon_loket:Staatsburger
} e:becomes {
?s a persoon:Staatsburger
}
} => {
?s a persoon:Staatsburger
}.
{
{
?s a persoon_loket:Staatsburgerschap
} e:becomes {
?s a persoon:Staatsburgerschap
}
} => {
?s a persoon:Staatsburgerschap
}.
{
{
?s a persoon_loket:TijdelijkInwoner
} e:becomes {
?s a persoon:TijdelijkInwoner
}
} => {
?s a persoon:TijdelijkInwoner
}.
{
{
?s a persoon_loket:Verblijfplaats
} e:becomes {
?s a persoon:Verblijfplaats
}
} => {
?s a persoon:Verblijfplaats
}.
{
{
?s a persoon_loket:Voogdij
} e:becomes {
?s a persoon:Voogdij
}
} => {
?s a persoon:Voogdij
}.
{
{
?s a persoon_loket:Vreemdeling
} e:becomes {
?s a persoon:Vreemdeling
}
} => {
?s a persoon:Vreemdeling
}.
## properties
{
{
?s persoon_loket:afstammingsType ?o
} e:becomes {
?s persoon:afstammingsType ?o
}
} => {
?s persoon:afstammingsType ?o
}.
{
{
?s persoon_loket:alternatieveNaam ?o
} e:becomes {
?s persoon:alternatieveNaam ?o
}
} => {
?s persoon:alternatieveNaam ?o
}.
{
{
?s persoon_loket:datum ?o
} e:becomes {
?s persoon:datum ?o
}
} => {
?s persoon:datum ?o
}.
{
{
?s persoon_loket:datumVanAfstamming ?o
} e:becomes {
?s persoon:datumVanAfstamming ?o
}
} => {
?s persoon:datumVanAfstamming ?o
}.
{
{
?s persoon_loket:gebruikteVoornaam ?o
} e:becomes {
?s persoon:gebruikteVoornaam ?o
}
} => {
?s persoon:gebruikteVoornaam ?o
}.
{
{
?s persoon_loket:geslacht ?o
} e:becomes {
?s persoon:geslacht ?o
}
} => {
?s persoon:geslacht ?o
}.
{
{
?s persoon_loket:gezinsadres ?o
} e:becomes {
?s persoon:gezinsadres ?o
}
} => {
?s persoon:gezinsadres ?o
}.
{
{
?s persoon_loket:gezinsrelatietype ?o
} e:becomes {
?s persoon:gezinsrelatietype ?o
}
} => {
?s persoon:gezinsrelatietype ?o
}.
{
{
?s persoon_loket:heeftBurgerlijkeStaat ?o
} e:becomes {
?s persoon:heeftBurgerlijkeStaat ?o
}
} => {
?s persoon:heeftBurgerlijkeStaat ?o
}.
{
{
?s persoon_loket:heeftGeboorte ?o
} e:becomes {
?s persoon:heeftGeboorte ?o
}
} => {
?s persoon:heeftGeboorte ?o
}.
{
{
?s persoon_loket:heeftInwonerschap ?o
} e:becomes {
?s persoon:heeftInwonerschap ?o
}
} => {
?s persoon:heeftInwonerschap ?o
}.
{
{
?s persoon_loket:heeftNationaliteit ?o
} e:becomes {
?s persoon:heeftNationaliteit ?o
}
} => {
?s persoon:heeftNationaliteit ?o
}.
{
{
?s persoon_loket:heeftOverlijden ?o
} e:becomes {
?s persoon:heeftOverlijden ?o
}
} => {
?s persoon:heeftOverlijden ?o
}.
{
{
?s persoon_loket:heeftPersoonsrelatie ?o
} e:becomes {
?s persoon:heeftPersoonsrelatie ?o
}
} => {
?s persoon:heeftPersoonsrelatie ?o
}.
{
{
?s persoon_loket:heeftStaatsburgerschap ?o
} e:becomes {
?s persoon:heeftStaatsburgerschap ?o
}
} => {
?s persoon:heeftStaatsburgerschap ?o
}.
{
{
?s persoon_loket:heeftVerblijfplaats ?o
} e:becomes {
?s persoon:heeftVerblijfplaats ?o
}
} => {
?s persoon:heeftVerblijfplaats ?o
}.
{
{
?s persoon_loket:Inwonerschap.binnenJurisdictie ?o
} e:becomes {
?s persoon:Inwonerschap.binnenJurisdictie ?o
}
} => {
?s persoon:Inwonerschap.binnenJurisdictie ?o
}.
{
{
?s persoon_loket:isHoofdVan ?o
} e:becomes {
?s persoon:isHoofdVan ?o
}
} => {
?s persoon:isHoofdVan ?o
}.
{
{
?s persoon_loket:isLidVan ?o
} e:becomes {
?s persoon:isLidVan ?o
}
} => {
?s persoon:isLidVan ?o
}.
{
{
?s persoon_loket:isRelatieMet ?o
} e:becomes {
?s persoon:isRelatieMet ?o
}
} => {
?s persoon:isRelatieMet ?o
}.
{
{
?s persoon_loket:land ?o
} e:becomes {
?s persoon:land ?o
}
} => {
?s persoon:land ?o
}.
{
{
?s persoon_loket:nationaliteit ?o
} e:becomes {
?s persoon:nationaliteit ?o
}
} => {
?s persoon:nationaliteit ?o
}.
{
{
?s persoon_loket:plaats ?o
} e:becomes {
?s persoon:plaats ?o
}
} => {
?s persoon:plaats ?o
}.
{
{
?s persoon_loket:registratie ?o
} e:becomes {
?s persoon:registratie ?o
}
} => {
?s persoon:registratie ?o
}.
{
{
?s persoon_loket:Staatsburgerschap.binnenJurisdictie ?o
} e:becomes {
?s persoon:Staatsburgerschap.binnenJurisdictie ?o
}
} => {
?s persoon:Staatsburgerschap.binnenJurisdictie ?o
}.
{
{
?s persoon_loket:type ?o
} e:becomes {
?s persoon:type ?o
}
} => {
?s persoon:type ?o
}.
{
{
?s persoon_loket:verblijfsadres ?o
} e:becomes {
?s persoon:verblijfsadres ?o
}
} => {
?s persoon:verblijfsadres ?o
}.
{
{
?s persoon_loket:volledigeNaam ?o
} e:becomes {
?s persoon:volledigeNaam ?o
}
} => {
?s persoon:volledigeNaam ?o
}.

27
config/reasoner/dl2op/main/retract_empty_strings.n3 Normal file
View File

@@ -0,0 +1,27 @@
PREFIX : <#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
PREFIX crypto: <http://www.w3.org/2000/10/swap/crypto#>
PREFIX e: <http://eulersharp.sourceforge.net/2003/03swap/log-rules#>
PREFIX ex: <http://example.org/test#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX func: <http://www.w3.org/2007/rif-builtin-function#>
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
PREFIX log: <http://www.w3.org/2000/10/swap/log#>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
# Retract empty strings
{
{
?s ?p ""
} e:becomes {}.
} => {}.

62
config/reasoner/dl2op/main/rrn.n3 Normal file
View File

@@ -0,0 +1,62 @@
PREFIX : <#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
PREFIX crypto: <http://www.w3.org/2000/10/swap/crypto#>
PREFIX e: <http://eulersharp.sourceforge.net/2003/03swap/log-rules#>
PREFIX ex: <http://example.org/test#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX func: <http://www.w3.org/2007/rif-builtin-function#>
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
PREFIX log: <http://www.w3.org/2000/10/swap/log#>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
# Rijksregisternummer
# Extend identifiers to match OP's model
# <http://example.org/tst#subject> skos:notation "00010100105".
{
?identifier skos:notation ?notation.
?notation string:notEqualIgnoringCase "Rijksregisternummer".
?identifier log:uri ?identifierString.
(?identifierString "STRUCTURED_IDENTIFIER") string:concatenation ?hashInput.
?hashInput crypto:md5 ?uuid.
("http://data.lblod.info/id/gestructureerdeIdentificatoren/" ?uuid) string:concatenation ?structuredIdString.
?structuredIdentifier log:uri ?structuredIdString.
{
?identifier skos:notation ?notation.
} e:becomes {
?identifier
generiek:gestructureerdeIdentificator ?structuredIdentifier ;
skos:notation "Rijksregisternummer" .
}.
} => {
?identifier
generiek:gestructureerdeIdentificator ?structuredIdentifier ;
skos:notation "Rijksregisternummer" .
?structuredIdentifier a generiek:GestructureerdeIdentificator ;
mu:uuid ?uuid ;
generiek:lokaleIdentificator ?notation
}.
# identifier property in the export is only used for RRN.
{
{
?s adms:identifier ?o
} e:becomes {
?s persoon:registratie ?o
}
} => {
?s persoon:registratie ?o
}.

31
config/reasoner/dl2op/preprocess/mapping.n3
View File

@@ -1,31 +0,0 @@
PREFIX e: <http://eulersharp.sourceforge.net/2003/03swap/log-rules#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX persoon: <http://data.vlaanderen.be/ns/persoon#>
# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)
# Hence the mapping is in two steps.
# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart
# and drops the DL persoon:gebruikteVoornaam statements with e:becomes
# A second reasoning will map DL foaf:name to OP persoon:gebruikteVoornaam
# Doing both at once would lead all DL foaf:names to incorrectly become foaf:givennames
# in OP
# First name
# persoon:gebruikteVoornaam => foaf:givenName
# Alternatieve voornaam
# foaf:name => persoon:gebruikteVoornaam
# Test Mock DL
# <http://example.org/person> persoon:gebruikteVoornaam "Didier".
{
{
?s persoon:gebruikteVoornaam ?o
} e:becomes {
?s foaf:givenName ?o
}
} => {
?s foaf:givenName ?o
}.

37
docker-compose.yml
View File

@@ -201,7 +201,7 @@ services:
restart: always
logging: *default-logging
reasoner:
image: bdevloed/mu-reasoner
image: eyereasoner/reasoning-service:1.0.0
volumes:
- ./config/reasoner:/config
################################################################################
@@ -460,7 +460,7 @@ services:
# LOKET MANDATEN CONSUMERS
################################################################################
mandaten-consumer:
image: bdevloed/delta-consumer:fetch
image: lblod/delta-consumer:0.0.14
environment:
DCR_SERVICE_NAME: "mandatendatabank-consumer"
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -487,7 +487,7 @@ services:
# LOKET LEIDINGGEVENDEN CONSUMERS
################################################################################
leidinggevenden-consumer:
image: bdevloed/delta-consumer:fetch
image: lblod/delta-consumer:0.0.14
environment:
DCR_SERVICE_NAME: "leidinggevenden-consumer"
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -510,15 +510,42 @@ services:
- triplestore:virtuoso
logging: *default-logging
################################################################################
# LOKET WORSHIP SENSITIVE CONSUMERS
################################################################################
worship-services-sensitive-consumer:
image: lblod/delta-consumer:0.0.14
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
DCR_SERVICE_NAME: "worship-services-sensitive-consumer"
DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files"
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer"
DCR_DISABLE_INITIAL_SYNC: "true"
DCR_KEEP_DELTA_FILES: "true"
DCR_DELTA_FILE_FOLDER: "/consumer-files"
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false"
volumes:
- ./config/worship-services-sensitive-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching
restart: always
labels:
- "logging=true"
links:
- db:database
- triplestore:virtuoso
logging: *default-logging
################################################################################
# DELTA CONSUMERS: END
################################################################################
################################################################################
# MU SEARCH
# MU SEARCH
################################################################################
search:
image: semtech/mu-search:0.9.0-beta.1
image: semtech/mu-search:0.9.0-beta.1
links:
- db:database
restart: always