Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ae53746323 | ||
|
|
73ff073540 | ||
|
|
622dae7969 | ||
|
|
47a2ca5072 | ||
|
|
a4098a5b84 | ||
|
|
dfb7bbd7dc | ||
|
|
6293b77492 | ||
|
|
12bacd7391 | ||
|
|
627b4950dd | ||
|
|
1fbe0f2fda | ||
|
|
d1925167b3 | ||
|
|
f7df3a3cb0 | ||
|
|
b32930f4bb | ||
|
|
4454a95137 | ||
|
|
4ed997dabd | ||
|
|
02de06921e | ||
|
|
fefb103cb5 | ||
|
|
3aadf0960e | ||
|
|
666945f077 | ||
|
|
e3883db087 | ||
|
|
86e4304b7e | ||
|
|
64a2c22afc | ||
|
|
8d0686926e | ||
|
|
d08c88f312 | ||
|
|
9486643214 | ||
|
|
dc0aeec0dc | ||
|
|
53c449f08d | ||
|
|
f43ae9f642 | ||
|
|
4858aa51f3 | ||
|
|
6e4c5a45ed | ||
|
|
42dc3af86a | ||
|
|
99fe30d2ef | ||
|
|
b05f69a615 |
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,7 +1,7 @@
|
||||
# Virtuoso files /data/db
|
||||
/data/db.bck/**
|
||||
/data/db/*
|
||||
!/data/db/virtuoso.ini
|
||||
/data/db/virtuoso.ini
|
||||
!/data/db/toLoad
|
||||
!/data/db/toLoad/*
|
||||
/data/files/*
|
||||
|
||||
@@ -223,6 +223,18 @@
|
||||
"https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "http://www.w3.org/ns/adms#Identifier",
|
||||
"graphsFilter": [
|
||||
"http://mu.semte.ch/graphs/worship-service"
|
||||
],
|
||||
"properties": [
|
||||
"http://mu.semte.ch/vocabularies/core/uuid",
|
||||
"http://www.w3.org/2004/02/skos/core#notation",
|
||||
"https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator"
|
||||
],
|
||||
"additionalFilter": "?ro <http://www.w3.org/ns/adms#identifier> ?subject ; a <http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan> ."
|
||||
},
|
||||
{
|
||||
"type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
|
||||
"graphsFilter": [
|
||||
@@ -241,6 +253,17 @@
|
||||
"https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
|
||||
"graphsFilter": [
|
||||
"http://mu.semte.ch/graphs/worship-service"
|
||||
],
|
||||
"properties": [
|
||||
"http://mu.semte.ch/vocabularies/core/uuid",
|
||||
"https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
|
||||
],
|
||||
"additionalFilter": "?ro <http://www.w3.org/ns/adms#identifier>/<https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ?subject ; a <http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan> ."
|
||||
},
|
||||
{
|
||||
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode",
|
||||
"graphsFilter": [
|
||||
@@ -256,48 +279,21 @@
|
||||
},
|
||||
{
|
||||
"type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
|
||||
"strictTypeExport": "true",
|
||||
"graphsFilter": [
|
||||
"http://mu.semte.ch/graphs/shared",
|
||||
"http://mu.semte.ch/graphs/worship-service",
|
||||
"http://mu.semte.ch/graphs/administrative-unit"
|
||||
],
|
||||
"pathToConceptScheme": [
|
||||
"http://www.w3.org/ns/org#classification",
|
||||
"http://www.w3.org/2004/02/skos/core#inScheme"
|
||||
"http://mu.semte.ch/graphs/administrative-unit",
|
||||
"http://mu.semte.ch/graphs/worship-service"
|
||||
],
|
||||
"properties": [
|
||||
"http://mu.semte.ch/vocabularies/core/uuid",
|
||||
"http://data.vlaanderen.be/ns/mandaat#bindingEinde",
|
||||
"http://data.vlaanderen.be/ns/mandaat#bindingStart",
|
||||
"http://data.vlaanderen.be/ns/besluit#bestuurt",
|
||||
"http://www.w3.org/ns/org#classification",
|
||||
"https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
|
||||
"http://www.w3.org/ns/org#hasPost"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
|
||||
"strictTypeExport": "true",
|
||||
"graphsFilter": [
|
||||
"http://mu.semte.ch/graphs/shared",
|
||||
"http://mu.semte.ch/graphs/worship-service",
|
||||
"http://mu.semte.ch/graphs/administrative-unit"
|
||||
],
|
||||
"pathToConceptScheme": [
|
||||
"https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
|
||||
"http://www.w3.org/ns/org#classification",
|
||||
"http://www.w3.org/2004/02/skos/core#inScheme"
|
||||
],
|
||||
"properties": [
|
||||
"http://mu.semte.ch/vocabularies/core/uuid",
|
||||
"http://data.vlaanderen.be/ns/mandaat#bindingEinde",
|
||||
"http://data.vlaanderen.be/ns/mandaat#bindingStart",
|
||||
"http://data.vlaanderen.be/ns/besluit#bestuurt",
|
||||
"http://www.w3.org/ns/org#classification",
|
||||
"https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
|
||||
"http://www.w3.org/ns/org#hasPost"
|
||||
]
|
||||
],
|
||||
"additionalFilter": "?subject (<https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan>/<http://www.w3.org/ns/org#classification>/<http://www.w3.org/2004/02/skos/core#inScheme>)|(<http://www.w3.org/ns/org#classification>/<http://www.w3.org/2004/02/skos/core#inScheme>) <http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b> ."
|
||||
},
|
||||
{
|
||||
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",
|
||||
|
||||
@@ -1,18 +1,20 @@
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
|
||||
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
|
||||
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
|
||||
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
|
||||
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
|
||||
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
|
||||
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
|
||||
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
|
||||
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
|
||||
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
|
||||
const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
|
||||
|
||||
// Graphs
|
||||
const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
|
||||
const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
|
||||
|
||||
if (!process.env.FILE_SYNC_GRAPH)
|
||||
throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
|
||||
const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
|
||||
// ENDPOINTS
|
||||
const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
|
||||
const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
|
||||
|
||||
module.exports = {
|
||||
BATCH_SIZE,
|
||||
@@ -24,6 +26,7 @@ module.exports = {
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
INGEST_GRAPH,
|
||||
FILE_SYNC_GRAPH
|
||||
LANDING_ZONE_GRAPH,
|
||||
TARGET_GRAPH,
|
||||
LANDING_ZONE_DATABASE_ENDPOINT
|
||||
};
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
const contextConfig = {
|
||||
addTypes: {
|
||||
scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
|
||||
exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
|
||||
},
|
||||
contextQueries: []
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
contextConfig
|
||||
};
|
||||
@@ -1,11 +1,8 @@
|
||||
const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util');
|
||||
const {
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
INGEST_GRAPH
|
||||
} = require('./config');
|
||||
transformStatements,
|
||||
deleteFromTargetGraph,
|
||||
insertIntoTargetGraph,
|
||||
} = require('./util');
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
@@ -23,43 +20,61 @@ const {
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo, fetch } = lib;
|
||||
const { termObjectChangeSets } = data;
|
||||
const { mu, fetch } = lib;
|
||||
let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
|
||||
|
||||
for (let { deletes, inserts } of termObjectChangeSets) {
|
||||
const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
// Both arrays are the same length, so we can zip them together for easier processing
|
||||
zippedChangeSets = termObjectChangeSets.map((o, i) => ({
|
||||
original: o,
|
||||
withContext: termObjectChangeSetsWithContext[i]
|
||||
}));
|
||||
|
||||
if (deleteStatements.length) {
|
||||
const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements);
|
||||
if (transformedStatementsToDelete.length) {
|
||||
await deleteFromAllGraphs(
|
||||
muAuthSudo.updateSudo,
|
||||
transformedStatementsToDelete,
|
||||
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
|
||||
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
);
|
||||
console.log(`Received ${zippedChangeSets.length} change sets`)
|
||||
|
||||
for (let { original, withContext } of zippedChangeSets) {
|
||||
let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
|
||||
try {
|
||||
originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
} catch (e) {
|
||||
console.log(`Error while processing change set: ${e}`);
|
||||
console.log(`Original change set: ${JSON.stringify(original)}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Extra context needed for mapping from DL to OP model and filtering based on type.
|
||||
try {
|
||||
insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
} catch (e) {
|
||||
console.log(`Error while processing change set with context: ${e}`);
|
||||
console.log(`Change set with context: ${JSON.stringify(withContext)}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (originalDeletes.length) {
|
||||
// Map deletes from DL to OP model
|
||||
const transformedDeletes = await transformStatements(fetch, deletesWithContext);
|
||||
|
||||
if (!transformedDeletes.length) {
|
||||
console.log(`Warn: Delete statements mapped to empty result.`);
|
||||
console.log(`Input: ${deletesWithContext}`);
|
||||
console.log(`Output: ${transformedDeletes}`);
|
||||
} else {
|
||||
await deleteFromTargetGraph(lib, transformedDeletes);
|
||||
}
|
||||
}
|
||||
|
||||
if (insertStatements.length) {
|
||||
const transformedStatementsToInsert = await transformStatements(fetch, insertStatements);
|
||||
if (transformedStatementsToInsert.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
INGEST_GRAPH,
|
||||
transformedStatementsToInsert,
|
||||
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
|
||||
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
"INSERT"
|
||||
);
|
||||
if (originalInserts.length) {
|
||||
// Map inserts from DL to OP model
|
||||
const transformedInserts = await transformStatements(fetch, insertsWithContext);
|
||||
|
||||
if (!transformedInserts.length) {
|
||||
console.log(`Warn: Insert statements mapped to empty result.`);
|
||||
console.log(`Input: ${insertsWithContext}`);
|
||||
console.log(`Output: ${transformedInserts}`);
|
||||
} else {
|
||||
await insertIntoTargetGraph(lib, transformedInserts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,17 @@
|
||||
const { transformStatements, batchedDbUpdate } = require('./util');
|
||||
const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
|
||||
const {
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
INGEST_GRAPH
|
||||
TARGET_GRAPH,
|
||||
LANDING_ZONE_GRAPH,
|
||||
LANDING_ZONE_DATABASE_ENDPOINT
|
||||
} = require('./config');
|
||||
|
||||
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
|
||||
|
||||
/**
|
||||
@@ -24,34 +28,58 @@ const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo, fetch } = lib;
|
||||
const { mu, muAuthSudo } = lib;
|
||||
const { termObjects } = data;
|
||||
|
||||
// Steps:
|
||||
// 1. Write the triples to the landing zone graph - withouth any mapping or filtering
|
||||
// 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
|
||||
|
||||
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
|
||||
console.warn(`Service configured to skip MU_AUTH!`);
|
||||
}
|
||||
console.log(`Using ${endpoint} to insert triples`);
|
||||
|
||||
if (termObjects.length) {
|
||||
originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
|
||||
const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
|
||||
const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples);
|
||||
if (transformedStatementsToInsert.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
INGEST_GRAPH,
|
||||
transformedStatementsToInsert,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
LANDING_ZONE_GRAPH,
|
||||
originalInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
LANDING_ZONE_DATABASE_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function onFinishInitialIngest(lib) {
|
||||
const { muAuthSudo, fetch } = lib;
|
||||
|
||||
console.log(`!! On-finish triggered !!`);
|
||||
|
||||
const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint);
|
||||
|
||||
console.log(`Transformed ${transformedInsertTriples.length} triples`);
|
||||
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
transformedInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch
|
||||
dispatch,
|
||||
onFinishInitialIngest
|
||||
};
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
const {
|
||||
MAX_REASONING_RETRY_ATTEMPTS,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
TARGET_GRAPH,
|
||||
LANDING_ZONE_GRAPH,
|
||||
BATCH_SIZE,
|
||||
} = require('./config');
|
||||
|
||||
async function batchedDbUpdate(muUpdate,
|
||||
async function batchedDbUpdate(
|
||||
muUpdate,
|
||||
graph,
|
||||
triples,
|
||||
extraHeaders,
|
||||
@@ -37,39 +44,6 @@ ${batch}
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteFromAllGraphs(muUpdate,
|
||||
triples,
|
||||
extraHeaders,
|
||||
endpoint,
|
||||
maxAttempts,
|
||||
sleepBetweenBatches = 1000,
|
||||
sleepTimeOnFail = 1000,
|
||||
) {
|
||||
|
||||
for (const triple of triples) {
|
||||
|
||||
console.log(`Deleting a triple from all graphs in triplestore`);
|
||||
|
||||
const deleteCall = async () => {
|
||||
await muUpdate(`
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
${triple}
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH ?g {
|
||||
${triple}
|
||||
}
|
||||
}
|
||||
`, extraHeaders, endpoint);
|
||||
};
|
||||
|
||||
await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
|
||||
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
|
||||
await new Promise(r => setTimeout(r, sleepBetweenBatches));
|
||||
}
|
||||
}
|
||||
|
||||
async function operationWithRetry(callback,
|
||||
attempt,
|
||||
maxAttempts,
|
||||
@@ -110,18 +84,16 @@ function partition(arr, fn) {
|
||||
return { passes, fails };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Send triples to reasoning service for conversion
|
||||
*
|
||||
*/
|
||||
function transformTriples(fetch, triples) {
|
||||
return operationWithRetry(mainConversion(fetch, triples), 0,
|
||||
async function transformTriples(fetch, triples, mapping) {
|
||||
return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
|
||||
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
|
||||
}
|
||||
|
||||
|
||||
function mainConversion(fetch, triples) {
|
||||
async function mainConversion(fetch, triples, mapping) {
|
||||
let formdata = new URLSearchParams();
|
||||
formdata.append("data", triples);
|
||||
|
||||
@@ -131,23 +103,66 @@ function mainConversion(fetch, triples) {
|
||||
redirect: 'follow'
|
||||
};
|
||||
|
||||
return fetch("http://reasoner/reason/dl2op/main", requestOptions)
|
||||
.then(response => response.text());
|
||||
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
|
||||
return await response.text();
|
||||
}
|
||||
|
||||
function transformStatements(fetch, triples) {
|
||||
return transformTriples(fetch, triples.join('\n')).then(
|
||||
graph => {
|
||||
statements = graph.replace(/\n{2,}/g, '').split('\n')
|
||||
console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`)
|
||||
return statements
|
||||
}
|
||||
)
|
||||
async function transformStatements(fetch, triples, mapping = 'main') {
|
||||
console.log(`Received ${JSON.stringify(triples)} to transform`);
|
||||
const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
|
||||
statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
|
||||
console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
|
||||
return statements;
|
||||
}
|
||||
|
||||
async function deleteFromTargetGraph(lib, statements) {
|
||||
console.log(`Deleting ${statements.length} statements from target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'DELETE');
|
||||
}
|
||||
|
||||
async function insertIntoTargetGraph(lib, statements) {
|
||||
console.log(`Inserting ${statements.length} statements into target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'INSERT');
|
||||
}
|
||||
|
||||
async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
|
||||
console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
|
||||
|
||||
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
|
||||
const text = await response.text();
|
||||
const statements = text.replace(/\n{2,}/g, '').split('\n');
|
||||
|
||||
return statements;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
batchedDbUpdate,
|
||||
deleteFromAllGraphs,
|
||||
partition,
|
||||
transformStatements
|
||||
transformStatements,
|
||||
transformLandingZoneGraph,
|
||||
deleteFromTargetGraph,
|
||||
insertIntoTargetGraph
|
||||
};
|
||||
|
||||
@@ -1,18 +1,20 @@
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
|
||||
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
|
||||
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
|
||||
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
|
||||
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
|
||||
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
|
||||
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
|
||||
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
|
||||
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
|
||||
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
|
||||
const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
|
||||
|
||||
// Graphs
|
||||
const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
|
||||
const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
|
||||
|
||||
if (!process.env.FILE_SYNC_GRAPH)
|
||||
throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
|
||||
const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
|
||||
// ENDPOINTS
|
||||
const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
|
||||
const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
|
||||
|
||||
module.exports = {
|
||||
BATCH_SIZE,
|
||||
@@ -24,6 +26,8 @@ module.exports = {
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
INGEST_GRAPH,
|
||||
FILE_SYNC_GRAPH
|
||||
LANDING_ZONE_GRAPH,
|
||||
LANDING_ZONE_DATABASE_ENDPOINT,
|
||||
TARGET_GRAPH
|
||||
};
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
const contextConfig = {
|
||||
addTypes: {
|
||||
scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
|
||||
exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
|
||||
},
|
||||
contextQueries: []
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
contextConfig
|
||||
};
|
||||
@@ -0,0 +1,85 @@
|
||||
const {
|
||||
transformStatements,
|
||||
deleteFromTargetGraph,
|
||||
insertIntoTargetGraph,
|
||||
} = require('./util');
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* Note: <share://file/data> will be ADDED to it's own graph.
|
||||
* We take only care of adding them, not updating triples, this is a TODO
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, fetch } = lib;
|
||||
let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
|
||||
|
||||
// Both arrays are the same length, so we can zip them together for easier processing
|
||||
zippedChangeSets = termObjectChangeSets.map((o, i) => ({
|
||||
original: o,
|
||||
withContext: termObjectChangeSetsWithContext[i]
|
||||
}));
|
||||
|
||||
console.log(`Received ${zippedChangeSets.length} change sets`)
|
||||
|
||||
for (let { original, withContext } of zippedChangeSets) {
|
||||
let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
|
||||
try {
|
||||
originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
} catch (e) {
|
||||
console.log(`Error while processing change set: ${e}`);
|
||||
console.log(`Original change set: ${JSON.stringify(original)}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Extra context needed for mapping from DL to OP model and filtering based on type.
|
||||
try {
|
||||
insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
} catch (e) {
|
||||
console.log(`Error while processing change set with context: ${e}`);
|
||||
console.log(`Change set with context: ${JSON.stringify(withContext)}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (originalDeletes.length) {
|
||||
// Map deletes from DL to OP model
|
||||
const transformedDeletes = await transformStatements(fetch, deletesWithContext);
|
||||
|
||||
if (!transformedDeletes.length) {
|
||||
console.log(`Warn: Delete statements mapped to empty result.`);
|
||||
console.log(`Input: ${deletesWithContext}`);
|
||||
console.log(`Output: ${transformedDeletes}`);
|
||||
} else {
|
||||
await deleteFromTargetGraph(lib, transformedDeletes);
|
||||
}
|
||||
}
|
||||
|
||||
if (originalInserts.length) {
|
||||
// Map inserts from DL to OP model
|
||||
const transformedInserts = await transformStatements(fetch, insertsWithContext);
|
||||
|
||||
if (!transformedInserts.length) {
|
||||
console.log(`Warn: Insert statements mapped to empty result.`);
|
||||
console.log(`Input: ${insertsWithContext}`);
|
||||
console.log(`Output: ${transformedInserts}`);
|
||||
} else {
|
||||
await insertIntoTargetGraph(lib, transformedInserts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch
|
||||
};
|
||||
@@ -0,0 +1,85 @@
|
||||
const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
|
||||
const {
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
TARGET_GRAPH,
|
||||
LANDING_ZONE_GRAPH,
|
||||
LANDING_ZONE_DATABASE_ENDPOINT
|
||||
} = require('./config');
|
||||
|
||||
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo } = lib;
|
||||
const { termObjects } = data;
|
||||
|
||||
// Steps:
|
||||
// 1. Write the triples to the landing zone graph - withouth any mapping or filtering
|
||||
// 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
|
||||
|
||||
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
|
||||
console.warn(`Service configured to skip MU_AUTH!`);
|
||||
}
|
||||
console.log(`Using ${endpoint} to insert triples`);
|
||||
|
||||
if (termObjects.length) {
|
||||
const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
LANDING_ZONE_GRAPH,
|
||||
originalInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
LANDING_ZONE_DATABASE_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function onFinishInitialIngest(lib) {
|
||||
const { muAuthSudo, fetch } = lib;
|
||||
|
||||
console.log(`!! On-finish triggered !!`);
|
||||
|
||||
const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint);
|
||||
|
||||
console.log(`Transformed ${transformedInsertTriples.length} triples`);
|
||||
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
transformedInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch,
|
||||
onFinishInitialIngest
|
||||
};
|
||||
@@ -1,9 +1,16 @@
|
||||
const {
|
||||
MAX_REASONING_RETRY_ATTEMPTS,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
TARGET_GRAPH,
|
||||
LANDING_ZONE_GRAPH,
|
||||
BATCH_SIZE
|
||||
} = require('./config');
|
||||
|
||||
async function batchedDbUpdate(muUpdate,
|
||||
async function batchedDbUpdate(
|
||||
muUpdate,
|
||||
graph,
|
||||
triples,
|
||||
extraHeaders,
|
||||
@@ -37,39 +44,6 @@ ${batch}
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteFromAllGraphs(muUpdate,
|
||||
triples,
|
||||
extraHeaders,
|
||||
endpoint,
|
||||
maxAttempts,
|
||||
sleepBetweenBatches = 1000,
|
||||
sleepTimeOnFail = 1000,
|
||||
) {
|
||||
|
||||
for (const triple of triples) {
|
||||
|
||||
console.log(`Deleting a triple from all graphs in triplestore`);
|
||||
|
||||
const deleteCall = async () => {
|
||||
await muUpdate(`
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
${triple}
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH ?g {
|
||||
${triple}
|
||||
}
|
||||
}
|
||||
`, extraHeaders, endpoint);
|
||||
};
|
||||
|
||||
await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
|
||||
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
|
||||
await new Promise(r => setTimeout(r, sleepBetweenBatches));
|
||||
}
|
||||
}
|
||||
|
||||
async function operationWithRetry(callback,
|
||||
attempt,
|
||||
maxAttempts,
|
||||
@@ -110,18 +84,16 @@ function partition(arr, fn) {
|
||||
return { passes, fails };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Send triples to reasoning service for conversion
|
||||
*
|
||||
*/
|
||||
function transformTriples(fetch, triples, mapping) {
|
||||
return operationWithRetry(mainConversion(fetch, triples, mapping), 0,
|
||||
async function transformTriples(fetch, triples, mapping) {
|
||||
return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
|
||||
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
|
||||
}
|
||||
|
||||
|
||||
function mainConversion(fetch, triples, mapping) {
|
||||
async function mainConversion(fetch, triples, mapping) {
|
||||
let formdata = new URLSearchParams();
|
||||
formdata.append("data", triples);
|
||||
|
||||
@@ -131,24 +103,66 @@ function mainConversion(fetch, triples, mapping) {
|
||||
redirect: 'follow'
|
||||
};
|
||||
|
||||
return fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
|
||||
.then(response => response.text());
|
||||
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
|
||||
return await response.text();
|
||||
}
|
||||
|
||||
async function transformStatements(fetch, triples, mapping = 'main') {
|
||||
console.log(`Received ${JSON.stringify(triples)} to transform`);
|
||||
const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
|
||||
statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
|
||||
console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
|
||||
return statements;
|
||||
}
|
||||
|
||||
function transformStatements(fetch, triples, mapping) {
|
||||
return transformTriples(fetch, triples.join('\n'), mapping).then(
|
||||
graph => {
|
||||
statements = graph ? graph.split('\n') : [];
|
||||
console.log(`CONVERSION ${mapping}: FROM ${triples.length} triples to ${statements.length}`)
|
||||
return statements
|
||||
}
|
||||
)
|
||||
async function deleteFromTargetGraph(lib, statements) {
|
||||
console.log(`Deleting ${statements.length} statements from target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'DELETE');
|
||||
}
|
||||
|
||||
async function insertIntoTargetGraph(lib, statements) {
|
||||
console.log(`Inserting ${statements.length} statements into target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'INSERT');
|
||||
}
|
||||
|
||||
async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
|
||||
console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
|
||||
|
||||
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
|
||||
const text = await response.text();
|
||||
const statements = text.replace(/\n{2,}/g, '').split('\n');
|
||||
|
||||
return statements;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
batchedDbUpdate,
|
||||
deleteFromAllGraphs,
|
||||
partition,
|
||||
transformStatements
|
||||
transformStatements,
|
||||
transformLandingZoneGraph,
|
||||
deleteFromTargetGraph,
|
||||
insertIntoTargetGraph
|
||||
};
|
||||
@@ -1,70 +0,0 @@
|
||||
const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util');
|
||||
const {
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
INGEST_GRAPH
|
||||
} = require('./config');
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* Note: <share://file/data> will be ADDED to it's own graph.
|
||||
* We take only care of adding them, not updating triples, this is a TODO
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo, fetch } = lib;
|
||||
const { termObjectChangeSets } = data;
|
||||
|
||||
for (let { deletes, inserts } of termObjectChangeSets) {
|
||||
const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
|
||||
if (deleteStatements.length) {
|
||||
const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements);
|
||||
if (transformedStatementsToDelete.length) {
|
||||
await deleteFromAllGraphs(
|
||||
muAuthSudo.updateSudo,
|
||||
transformedStatementsToDelete,
|
||||
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
|
||||
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (insertStatements.length) {
|
||||
const transformedStatementsToInsert = await transformStatements(fetch, insertStatements);
|
||||
if (transformedStatementsToInsert.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
INGEST_GRAPH,
|
||||
transformedStatementsToInsert,
|
||||
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
|
||||
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
"INSERT"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch
|
||||
};
|
||||
@@ -1,57 +0,0 @@
|
||||
const { transformStatements, batchedDbUpdate } = require('./util');
|
||||
const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
INGEST_GRAPH
|
||||
} = require('./config');
|
||||
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo, fetch } = lib;
|
||||
const { termObjects } = data;
|
||||
|
||||
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
|
||||
console.warn(`Service configured to skip MU_AUTH!`);
|
||||
}
|
||||
console.log(`Using ${endpoint} to insert triples`);
|
||||
|
||||
if (termObjects.length) {
|
||||
originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
|
||||
|
||||
const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples);
|
||||
if (transformedStatementsToInsert.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
INGEST_GRAPH,
|
||||
transformedStatementsToInsert,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch
|
||||
};
|
||||
@@ -1,153 +0,0 @@
|
||||
const {
|
||||
MAX_REASONING_RETRY_ATTEMPTS,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
|
||||
} = require('./config');
|
||||
|
||||
async function batchedDbUpdate(muUpdate,
|
||||
graph,
|
||||
triples,
|
||||
extraHeaders,
|
||||
endpoint,
|
||||
batchSize,
|
||||
maxAttempts,
|
||||
sleepBetweenBatches = 1000,
|
||||
sleepTimeOnFail = 1000,
|
||||
operation = 'INSERT'
|
||||
) {
|
||||
|
||||
for (let i = 0; i < triples.length; i += batchSize) {
|
||||
console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
|
||||
|
||||
const batch = triples.slice(i, i + batchSize).join('\n');
|
||||
|
||||
const insertCall = async () => {
|
||||
await muUpdate(`
|
||||
${operation} DATA {
|
||||
GRAPH <${graph}> {
|
||||
${batch}
|
||||
}
|
||||
}
|
||||
`, extraHeaders, endpoint);
|
||||
};
|
||||
|
||||
await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
|
||||
|
||||
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
|
||||
await new Promise(r => setTimeout(r, sleepBetweenBatches));
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteFromAllGraphs(muUpdate,
|
||||
triples,
|
||||
extraHeaders,
|
||||
endpoint,
|
||||
maxAttempts,
|
||||
sleepBetweenBatches = 1000,
|
||||
sleepTimeOnFail = 1000,
|
||||
) {
|
||||
|
||||
for (const triple of triples) {
|
||||
|
||||
console.log(`Deleting a triple from all graphs in triplestore`);
|
||||
|
||||
const deleteCall = async () => {
|
||||
await muUpdate(`
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
${triple}
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH ?g {
|
||||
${triple}
|
||||
}
|
||||
}
|
||||
`, extraHeaders, endpoint);
|
||||
};
|
||||
|
||||
await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
|
||||
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
|
||||
await new Promise(r => setTimeout(r, sleepBetweenBatches));
|
||||
}
|
||||
}
|
||||
|
||||
async function operationWithRetry(callback,
|
||||
attempt,
|
||||
maxAttempts,
|
||||
sleepTimeOnFail) {
|
||||
try {
|
||||
if (typeof callback === "function")
|
||||
return await callback();
|
||||
else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
|
||||
return await callback;
|
||||
}
|
||||
catch (e) {
|
||||
console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
|
||||
console.log(`Error: ${e}`);
|
||||
console.log(`Sleeping ${sleepTimeOnFail} ms`);
|
||||
|
||||
if (attempt >= maxAttempts) {
|
||||
console.log(`Max attempts reached for ${callback.toString()}, giving up`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
await new Promise(r => setTimeout(r, sleepTimeOnFail));
|
||||
return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits an array into two parts, a part that passes and a part that fails a predicate function.
|
||||
* Credits: https://github.com/benjay10
|
||||
* @public
|
||||
* @function partition
|
||||
* @param {Array} arr - Array to be partitioned
|
||||
* @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
|
||||
* @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
|
||||
*/
|
||||
function partition(arr, fn) {
|
||||
let passes = [], fails = [];
|
||||
arr.forEach((item) => (fn(item) ? passes : fails).push(item));
|
||||
return { passes, fails };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Send triples to reasoning service for conversion
|
||||
*
|
||||
*/
|
||||
function transformTriples(fetch, triples) {
|
||||
return operationWithRetry(mainConversion(fetch, triples), 0,
|
||||
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
|
||||
}
|
||||
|
||||
|
||||
function mainConversion(fetch, triples) {
|
||||
let formdata = new URLSearchParams();
|
||||
formdata.append("data", triples);
|
||||
|
||||
let requestOptions = {
|
||||
method: 'POST',
|
||||
body: formdata,
|
||||
redirect: 'follow'
|
||||
};
|
||||
|
||||
return fetch("http://reasoner/reason/dl2op/main", requestOptions)
|
||||
.then(response => response.text());
|
||||
}
|
||||
|
||||
function transformStatements(fetch, triples) {
|
||||
return transformTriples(fetch, triples.join('\n')).then(
|
||||
graph => {
|
||||
statements = graph.replace(/\n{2,}/g, '').split('\n')
|
||||
console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`)
|
||||
return statements
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
batchedDbUpdate,
|
||||
deleteFromAllGraphs,
|
||||
partition,
|
||||
transformStatements
|
||||
};
|
||||
@@ -1,51 +0,0 @@
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
|
||||
const MU_CALL_SCOPE_ID_INITIAL_SYNC =
|
||||
process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC ||
|
||||
"http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync";
|
||||
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES =
|
||||
process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == "true" ? true : false;
|
||||
const DIRECT_DATABASE_ENDPOINT =
|
||||
process.env.DIRECT_DATABASE_ENDPOINT || "http://virtuoso:8890/sparql";
|
||||
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
|
||||
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(
|
||||
process.env.MAX_REASONING_RETRY_ATTEMPTS || 5
|
||||
);
|
||||
const SLEEP_BETWEEN_BATCHES = parseInt(
|
||||
process.env.SLEEP_BETWEEN_BATCHES || 1000
|
||||
);
|
||||
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(
|
||||
process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000
|
||||
);
|
||||
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(
|
||||
process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000
|
||||
);
|
||||
const INGEST_GRAPH =
|
||||
process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/worship-service`;
|
||||
const PRIVACY_SENSITIVE_GRAPH =
|
||||
process.env.PRIVACY_SENSITIVE_GRAPH ||
|
||||
`http://mu.semte.ch/graphs/worship-privacy-centric-graph`;
|
||||
const UNFILTERED_MAPPING = "main";
|
||||
const MAIN_INFO_MAPPING = "main-info";
|
||||
const PRIVATE_INFO_MAPPING = "private-info";
|
||||
|
||||
if (!process.env.FILE_SYNC_GRAPH)
|
||||
throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
|
||||
const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
|
||||
|
||||
module.exports = {
|
||||
BATCH_SIZE,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
MAX_REASONING_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
INGEST_GRAPH,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
UNFILTERED_MAPPING,
|
||||
MAIN_INFO_MAPPING,
|
||||
PRIVATE_INFO_MAPPING,
|
||||
FILE_SYNC_GRAPH,
|
||||
};
|
||||
@@ -1,88 +0,0 @@
|
||||
const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util');
|
||||
const {
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
INGEST_GRAPH,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
UNFILTERED_MAPPING,
|
||||
MAIN_INFO_MAPPING,
|
||||
PRIVATE_INFO_MAPPING,
|
||||
} = require('./config');
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* Note: <share://file/data> will be ADDED to it's own graph.
|
||||
* We take only care of adding them, not updating triples, this is a TODO
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo, fetch } = lib;
|
||||
const { termObjectChangeSets } = data;
|
||||
|
||||
for (let { deletes, inserts } of termObjectChangeSets) {
|
||||
const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
|
||||
if (deleteStatements.length) {
|
||||
const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING);
|
||||
if (transformedStatementsToDelete.length) {
|
||||
await deleteFromAllGraphs(
|
||||
muAuthSudo.updateSudo,
|
||||
transformedStatementsToDelete,
|
||||
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
|
||||
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (insertStatements.length) {
|
||||
const transformedStatementsToInsert = await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING);
|
||||
if (transformedStatementsToInsert.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
INGEST_GRAPH,
|
||||
transformedStatementsToInsert,
|
||||
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
|
||||
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
|
||||
const transformedStatementsToInsertPrivate = await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING);
|
||||
if (transformedStatementsToInsertPrivate.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
transformedStatementsToInsertPrivate,
|
||||
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
|
||||
process.env.MU_SPARQL_ENDPOINT, // Note: this is the default endpoint through auth
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch
|
||||
};
|
||||
@@ -1,75 +0,0 @@
|
||||
const { transformStatements, batchedDbUpdate, } = require('./util');
|
||||
const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
INGEST_GRAPH,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
MAIN_INFO_MAPPING,
|
||||
PRIVATE_INFO_MAPPING
|
||||
} = require('./config');
|
||||
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo, fetch } = lib;
|
||||
const { termObjects } = data;
|
||||
|
||||
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
|
||||
console.warn(`Service configured to skip MU_AUTH!`);
|
||||
}
|
||||
console.log(`Using ${endpoint} to insert triples`);
|
||||
|
||||
if (termObjects.length) {
|
||||
originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
|
||||
|
||||
const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples, MAIN_INFO_MAPPING);
|
||||
if (transformedStatementsToInsert.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
INGEST_GRAPH,
|
||||
transformedStatementsToInsert,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
|
||||
const transformedStatementsToInsertPrivate = await transformStatements(fetch, originalInsertTriples, PRIVATE_INFO_MAPPING);
|
||||
if (transformedStatementsToInsertPrivate.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
transformedStatementsToInsertPrivate,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch
|
||||
};
|
||||
@@ -0,0 +1,40 @@
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
|
||||
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
|
||||
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
|
||||
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
|
||||
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
|
||||
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
|
||||
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
|
||||
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
|
||||
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
|
||||
|
||||
// Graphs
|
||||
const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/worship-service`;
|
||||
const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/worship-privacy-centric-graph`;
|
||||
const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
|
||||
|
||||
// Mappings
|
||||
const MAIN_INFO_MAPPING = "main-info";
|
||||
const PRIVATE_INFO_MAPPING = "private-info";
|
||||
|
||||
// ENDPOINTS
|
||||
const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
|
||||
const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
|
||||
|
||||
module.exports = {
|
||||
BATCH_SIZE,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
MAX_REASONING_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
LANDING_ZONE_GRAPH,
|
||||
LANDING_ZONE_DATABASE_ENDPOINT,
|
||||
TARGET_GRAPH,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
MAIN_INFO_MAPPING,
|
||||
PRIVATE_INFO_MAPPING
|
||||
};
|
||||
@@ -0,0 +1,11 @@
|
||||
const contextConfig = {
|
||||
addTypes: {
|
||||
scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
|
||||
exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
|
||||
},
|
||||
contextQueries: []
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
contextConfig
|
||||
};
|
||||
@@ -0,0 +1,114 @@
|
||||
const {
|
||||
transformStatements,
|
||||
deleteFromTargetGraph,
|
||||
insertIntoTargetGraph,
|
||||
deleteFromPrivateGraph,
|
||||
insertIntoPrivateGraph
|
||||
} = require('./util');
|
||||
|
||||
const {
|
||||
MAIN_INFO_MAPPING,
|
||||
PRIVATE_INFO_MAPPING
|
||||
} = require('./config');
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* Note: <share://file/data> will be ADDED to it's own graph.
|
||||
* We take only care of adding them, not updating triples, this is a TODO
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, fetch } = lib;
|
||||
let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
|
||||
|
||||
// Both arrays are the same length, so we can zip them together for easier processing
|
||||
zippedChangeSets = termObjectChangeSets.map((o, i) => ({
|
||||
original: o,
|
||||
withContext: termObjectChangeSetsWithContext[i]
|
||||
}));
|
||||
|
||||
console.log(`Received ${zippedChangeSets.length} change sets`)
|
||||
|
||||
for (let { original, withContext } of zippedChangeSets) {
|
||||
let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
|
||||
try {
|
||||
originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
} catch (e) {
|
||||
console.log(`Error while processing change set: ${e}`);
|
||||
console.log(`Original change set: ${JSON.stringify(original)}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Extra context needed for mapping from DL to OP model and filtering based on type.
|
||||
try {
|
||||
insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
} catch (e) {
|
||||
console.log(`Error while processing change set with context: ${e}`);
|
||||
console.log(`Change set with context: ${JSON.stringify(withContext)}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (originalDeletes.length) {
|
||||
// Map main deletes from DL to OP model
|
||||
const transformedMainDeletes = await transformStatements(fetch, deletesWithContext, MAIN_INFO_MAPPING);
|
||||
|
||||
if (!transformedMainDeletes.length) {
|
||||
console.log(`Warn: Delete main statements mapped to empty result.`);
|
||||
console.log(`Input: ${deletesWithContext}`);
|
||||
console.log(`Output: ${transformedMainDeletes}`);
|
||||
} else {
|
||||
await deleteFromTargetGraph(lib, transformedMainDeletes);
|
||||
}
|
||||
|
||||
// Map private deletes from DL to OP model
|
||||
const transformedPrivateDeletes = await transformStatements(fetch, deletesWithContext, PRIVATE_INFO_MAPPING);
|
||||
|
||||
if (!transformedPrivateDeletes.length) {
|
||||
console.log(`Warn: Delete private statements mapped to empty result.`);
|
||||
console.log(`Input: ${deletesWithContext}`);
|
||||
console.log(`Output: ${transformedPrivateDeletes}`);
|
||||
} else {
|
||||
await deleteFromPrivateGraph(lib, transformedDeletes);
|
||||
}
|
||||
}
|
||||
|
||||
if (originalInserts.length) {
|
||||
// Map main inserts from DL to OP model
|
||||
const transformedMainInserts = await transformStatements(fetch, insertsWithContext, MAIN_INFO_MAPPING);
|
||||
|
||||
if (!transformedMainInserts.length) {
|
||||
console.log(`Warn: Insert statements mapped to empty result.`);
|
||||
console.log(`Input: ${insertsWithContext}`);
|
||||
console.log(`Output: ${transformedMainInserts}`);
|
||||
} else {
|
||||
await insertIntoTargetGraph(lib, transformedMainInserts);
|
||||
}
|
||||
|
||||
// Map private inserts from DL to OP model
|
||||
const transformedPrivateInserts = await transformStatements(fetch, insertsWithContext, PRIVATE_INFO_MAPPING);
|
||||
|
||||
if (!transformedPrivateInserts.length) {
|
||||
console.log(`Warn: Insert statements mapped to empty result.`);
|
||||
console.log(`Input: ${insertsWithContext}`);
|
||||
console.log(`Output: ${transformedPrivateInserts}`);
|
||||
} else {
|
||||
await insertIntoPrivateGraph(lib, transformedPrivateInserts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch
|
||||
};
|
||||
@@ -0,0 +1,106 @@
|
||||
const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
|
||||
const {
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
TARGET_GRAPH,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
LANDING_ZONE_GRAPH,
|
||||
LANDING_ZONE_DATABASE_ENDPOINT,
|
||||
MAIN_INFO_MAPPING,
|
||||
PRIVATE_INFO_MAPPING
|
||||
} = require('./config');
|
||||
|
||||
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo } = lib;
|
||||
const { termObjects } = data;
|
||||
|
||||
// Steps:
|
||||
// 1. Write the triples to the landing zone graph - withouth any mapping or filtering
|
||||
// 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
|
||||
|
||||
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
|
||||
console.warn(`Service configured to skip MU_AUTH!`);
|
||||
}
|
||||
console.log(`Using ${endpoint} to insert triples`);
|
||||
|
||||
if (termObjects.length) {
|
||||
const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
LANDING_ZONE_GRAPH,
|
||||
originalInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
LANDING_ZONE_DATABASE_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function onFinishInitialIngest(lib) {
|
||||
const { muAuthSudo, fetch } = lib;
|
||||
|
||||
console.log(`!! On-finish triggered !!`);
|
||||
|
||||
const transformedMainInsertTriples = await transformLandingZoneGraph(fetch, endpoint, MAIN_INFO_MAPPING);
|
||||
console.log(`Transformed ${transformedMainInsertTriples.length} main triples`);
|
||||
|
||||
if (transformedMainInsertTriples.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
transformedMainInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
|
||||
const transformedPrivateInsertTriples = await transformLandingZoneGraph(fetch, endpoint, PRIVATE_INFO_MAPPING);
|
||||
console.log(`Transformed ${transformedPrivateInsertTriples.length} private triples`);
|
||||
|
||||
if (transformedPrivateInsertTriples.length) {
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
transformedPrivateInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch,
|
||||
onFinishInitialIngest
|
||||
};
|
||||
@@ -0,0 +1,204 @@
|
||||
const {
|
||||
MAX_REASONING_RETRY_ATTEMPTS,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
TARGET_GRAPH,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
LANDING_ZONE_GRAPH,
|
||||
BATCH_SIZE
|
||||
} = require('./config');
|
||||
|
||||
async function batchedDbUpdate(
|
||||
muUpdate,
|
||||
graph,
|
||||
triples,
|
||||
extraHeaders,
|
||||
endpoint,
|
||||
batchSize,
|
||||
maxAttempts,
|
||||
sleepBetweenBatches = 1000,
|
||||
sleepTimeOnFail = 1000,
|
||||
operation = 'INSERT'
|
||||
) {
|
||||
|
||||
for (let i = 0; i < triples.length; i += batchSize) {
|
||||
console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
|
||||
|
||||
const batch = triples.slice(i, i + batchSize).join('\n');
|
||||
|
||||
const insertCall = async () => {
|
||||
await muUpdate(`
|
||||
${operation} DATA {
|
||||
GRAPH <${graph}> {
|
||||
${batch}
|
||||
}
|
||||
}
|
||||
`, extraHeaders, endpoint);
|
||||
};
|
||||
|
||||
await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
|
||||
|
||||
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
|
||||
await new Promise(r => setTimeout(r, sleepBetweenBatches));
|
||||
}
|
||||
}
|
||||
|
||||
async function operationWithRetry(callback,
|
||||
attempt,
|
||||
maxAttempts,
|
||||
sleepTimeOnFail) {
|
||||
try {
|
||||
if (typeof callback === "function")
|
||||
return await callback();
|
||||
else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
|
||||
return await callback;
|
||||
}
|
||||
catch (e) {
|
||||
console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
|
||||
console.log(`Error: ${e}`);
|
||||
console.log(`Sleeping ${sleepTimeOnFail} ms`);
|
||||
|
||||
if (attempt >= maxAttempts) {
|
||||
console.log(`Max attempts reached for ${callback.toString()}, giving up`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
await new Promise(r => setTimeout(r, sleepTimeOnFail));
|
||||
return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits an array into two parts, a part that passes and a part that fails a predicate function.
|
||||
* Credits: https://github.com/benjay10
|
||||
* @public
|
||||
* @function partition
|
||||
* @param {Array} arr - Array to be partitioned
|
||||
* @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
|
||||
* @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
|
||||
*/
|
||||
function partition(arr, fn) {
|
||||
let passes = [], fails = [];
|
||||
arr.forEach((item) => (fn(item) ? passes : fails).push(item));
|
||||
return { passes, fails };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send triples to reasoning service for conversion
|
||||
*
|
||||
*/
|
||||
async function transformTriples(fetch, triples, mapping) {
|
||||
return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
|
||||
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
|
||||
}
|
||||
|
||||
async function mainConversion(fetch, triples, mapping) {
|
||||
let formdata = new URLSearchParams();
|
||||
formdata.append("data", triples);
|
||||
|
||||
let requestOptions = {
|
||||
method: 'POST',
|
||||
body: formdata,
|
||||
redirect: 'follow'
|
||||
};
|
||||
|
||||
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
|
||||
return await response.text();
|
||||
}
|
||||
|
||||
async function transformStatements(fetch, triples, mapping = 'main') {
|
||||
console.log(`Received ${JSON.stringify(triples)} to transform`);
|
||||
const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
|
||||
statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
|
||||
console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
|
||||
return statements;
|
||||
}
|
||||
|
||||
async function deleteFromTargetGraph(lib, statements) {
|
||||
console.log(`Deleting ${statements.length} statements from target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'DELETE');
|
||||
}
|
||||
|
||||
async function deleteFromPrivateGraph(lib, statements) {
|
||||
console.log(`Deleting ${statements.length} statements from target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'DELETE');
|
||||
}
|
||||
|
||||
async function insertIntoTargetGraph(lib, statements) {
|
||||
console.log(`Inserting ${statements.length} statements into target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'INSERT');
|
||||
}
|
||||
|
||||
async function insertIntoPrivateGraph(lib, statements) {
|
||||
console.log(`Inserting ${statements.length} statements into target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
PRIVACY_SENSITIVE_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'INSERT');
|
||||
}
|
||||
|
||||
async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
|
||||
console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
|
||||
|
||||
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
|
||||
const text = await response.text();
|
||||
const statements = text.replace(/\n{2,}/g, '').split('\n');
|
||||
|
||||
return statements;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
batchedDbUpdate,
|
||||
partition,
|
||||
transformStatements,
|
||||
transformLandingZoneGraph,
|
||||
deleteFromTargetGraph,
|
||||
insertIntoTargetGraph,
|
||||
deleteFromPrivateGraph,
|
||||
insertIntoPrivateGraph
|
||||
};
|
||||
@@ -0,0 +1,46 @@
|
||||
DELETE DATA {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
<http://data.lblod.info/id/besturenVanDeEredienst/d640003a744edbe1a0597ab57243ab5e>
|
||||
<http://www.w3.org/ns/regorg#orgStatus> <http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311> .
|
||||
}
|
||||
}
|
||||
;
|
||||
INSERT DATA {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
<http://data.lblod.info/id/besturenVanDeEredienst/d640003a744edbe1a0597ab57243ab5e>
|
||||
<http://www.w3.org/ns/regorg#orgStatus> <http://lblod.data.gift/concepts/63cc561de9188d64ba5840a42ae8f0d6> .
|
||||
|
||||
<http://data.lblod.info/id/eredienstbestuursorganen/18d843d47e223409b9bb6e7af8a4d8a8>
|
||||
a <http://data.vlaanderen.be/ns/besluit#Bestuursorgaan> ;
|
||||
<http://mu.semte.ch/vocabularies/core/uuid> "18d843d47e223409b9bb6e7af8a4d8a8" ;
|
||||
<https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan> <http://data.lblod.info/id/eredienstbestuursorganen/520e20dd93d0eb546ee2402f124b499e> ;
|
||||
<http://data.vlaanderen.be/ns/mandaat#bindingStart> "2020-04-01T00:00:00"^^xsd:dateTime ;
|
||||
<http://data.vlaanderen.be/ns/mandaat#bindingEinde> "2023-03-31T00:00:00"^^xsd:dateTime ;
|
||||
<http://www.w3.org/ns/org#hasPost> <http://data.lblod.info/id/mandaten/c6d754b17be9634e629f7d751753bb27> ;
|
||||
<http://www.w3.org/ns/org#hasPost> <http://data.lblod.info/id/mandaten/37fc68e584d93a8c165a2b9b5b0f183a> ;
|
||||
<http://www.w3.org/ns/org#hasPost> <http://data.lblod.info/id/mandaten/daac7f53-b020-443d-92a7-9d47be6cb186> ;
|
||||
<http://www.w3.org/ns/org#hasPost> <http://data.lblod.info/id/mandaten/3618fc2a-83f7-45dd-969f-06afd0c3277b> ;
|
||||
<http://www.w3.org/ns/org#hasPost> <http://data.lblod.info/id/mandaten/e4b05858-c66d-4098-9c43-59af0359e603> ;
|
||||
<http://www.w3.org/ns/org#hasPost> <http://data.lblod.info/id/mandaten/5147f880-b0a5-4145-ac01-db8323a5410d> .
|
||||
|
||||
<http://data.lblod.info/id/mandaten/daac7f53-b020-443d-92a7-9d47be6cb186>
|
||||
a <http://data.vlaanderen.be/ns/mandaat#Mandaat> ;
|
||||
<http://mu.semte.ch/vocabularies/core/uuid> "daac7f53-b020-443d-92a7-9d47be6cb186" ;
|
||||
<http://www.w3.org/ns/org#role> <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/2962f0bd-2836-4758-9866-8ce8ea2c536f> .
|
||||
|
||||
<http://data.lblod.info/id/mandaten/3618fc2a-83f7-45dd-969f-06afd0c3277b>
|
||||
a <http://data.vlaanderen.be/ns/mandaat#Mandaat> ;
|
||||
<http://mu.semte.ch/vocabularies/core/uuid> "3618fc2a-83f7-45dd-969f-06afd0c3277b" ;
|
||||
<http://www.w3.org/ns/org#role> <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/67e6e585166cd97575b3e17ffc430a43> .
|
||||
|
||||
<http://data.lblod.info/id/mandaten/e4b05858-c66d-4098-9c43-59af0359e603>
|
||||
a <http://data.vlaanderen.be/ns/mandaat#Mandaat> ;
|
||||
<http://mu.semte.ch/vocabularies/core/uuid> "e4b05858-c66d-4098-9c43-59af0359e603" ;
|
||||
<http://www.w3.org/ns/org#role> <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/5ac134b9800b81da3c450d6b9605cef2> .
|
||||
|
||||
<http://data.lblod.info/id/mandaten/5147f880-b0a5-4145-ac01-db8323a5410d>
|
||||
a <http://data.vlaanderen.be/ns/mandaat#Mandaat> ;
|
||||
<http://mu.semte.ch/vocabularies/core/uuid> "5147f880-b0a5-4145-ac01-db8323a5410d" ;
|
||||
<http://www.w3.org/ns/org#role> <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/180d13930d6f1a3938e0aa7fa9990002> .
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
# We have two bestuursorganen of the same classification for 2 bestuurseenheden.
|
||||
# Then need to be merged, as we should only have one orgaan (not in time) per classification.
|
||||
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?otherBestuursorgaan ?potherBestuursorgaan ?ootherBestuursorgaan .
|
||||
?s ?p ?otherBestuursorgaan .
|
||||
}
|
||||
} INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?s ?p ?bestuursorgaan .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
VALUES ?type {
|
||||
ere:BestuurVanDeEredienst
|
||||
ere:CentraalBestuurVanDeEredienst
|
||||
}
|
||||
|
||||
?bestuurseenheid a ?type .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?bestuurseenheid <http://www.w3.org/ns/regorg#orgStatus> <http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311> .
|
||||
}
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid ;
|
||||
<http://www.w3.org/ns/org#classification> ?classif .
|
||||
|
||||
?otherBestuursorgaan besluit:bestuurt ?bestuurseenheid ;
|
||||
<http://www.w3.org/ns/org#classification> ?classif ;
|
||||
?potherBestuursorgaan ?ootherBestuursorgaan .
|
||||
|
||||
?s ?p ?otherBestuursorgaan .
|
||||
|
||||
FILTER (str(?bestuursorgaan) > str(?otherBestuursorgaan))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime a besluit:Bestuursorgaan ;
|
||||
mu:uuid ?uuidOrgaanInTime ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
}
|
||||
}
|
||||
WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
VALUES ?type {
|
||||
ere:BestuurVanDeEredienst
|
||||
ere:CentraalBestuurVanDeEredienst
|
||||
}
|
||||
|
||||
?bestuurseenheid a ?type .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?bestuurseenheid <http://www.w3.org/ns/regorg#orgStatus> <http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311> .
|
||||
}
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?missingOrgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
?missingOrgaanInTime mandaat:bindingStart ?start .
|
||||
FILTER CONTAINS(STR(?start), "2023")
|
||||
}
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?missingOrgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
?missingOrgaanInTime mandaat:bindingEinde ?end .
|
||||
FILTER CONTAINS(STR(?end), "2026")
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?bestuurseenheid, ?bestuursorgaan, "2023-2026")) as ?uuidOrgaanInTime)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/bestuursorganen/", ?uuidOrgaanInTime)) AS ?orgaanInTime)
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
# Add missing mandates to central worship services
|
||||
|
||||
## 2023 - 2026
|
||||
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?expert .
|
||||
|
||||
?expert a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidExpert ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/8c91c321ad477c4fc372ee36358d3ed4> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:CentraalBestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?expert .
|
||||
?expert org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/8c91c321ad477c4fc372ee36358d3ed4> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Expert 2023-2026")) as ?uuidExpert)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidExpert)) AS ?expert)
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?vertegenwoordiger .
|
||||
|
||||
?vertegenwoordiger a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidVertegenwoordiger ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/6e26e94ea4b127eeb850fb6debe07271> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:CentraalBestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?vertegenwoordiger .
|
||||
?vertegenwoordiger org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/6e26e94ea4b127eeb850fb6debe07271> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Vertegenwoordiger 2023-2026")) as ?uuidVertegenwoordiger)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidVertegenwoordiger)) AS ?vertegenwoordiger)
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?secretaris .
|
||||
|
||||
?secretaris a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidSecretaris ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/e2af0ea1a6af96cfb698ac39ad985eea> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:CentraalBestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?secretaris .
|
||||
?secretaris org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/e2af0ea1a6af96cfb698ac39ad985eea> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Secretaris 2023-2026")) as ?uuidSecretaris)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidSecretaris)) AS ?secretaris)
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?voorzitter .
|
||||
|
||||
?voorzitter a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidVoorzitter ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/5960262f753661cf84329f3afa9f7df7> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:CentraalBestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?voorzitter .
|
||||
?voorzitter org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/5960262f753661cf84329f3afa9f7df7> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Voorzitter 2023-2026")) as ?uuidVoorzitter)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidVoorzitter)) AS ?voorzitter)
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?bestuurslid .
|
||||
|
||||
?bestuurslid a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidBestuurslid ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/f848fa3cc2c5fb7c581a116866293925> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:CentraalBestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?worshipService <http://www.w3.org/ns/regorg#orgStatus> <http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311> .
|
||||
}
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?bestuurslid .
|
||||
?bestuurslid org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/f848fa3cc2c5fb7c581a116866293925> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Bestuurslid 2023-2026")) as ?uuidBestuurslid)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidBestuurslid)) AS ?bestuurslid)
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
# Add missing mandates to worship services for period 2023-2026
|
||||
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
|
||||
## Case LidVanRechtswege : shared mandate for all the organen in time
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?lidVanRechtswege .
|
||||
|
||||
?lidVanRechtswege a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidLidVanRechtswege ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/5972fccd87f864c4ec06bfbd20b5008b> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
VALUES (?start ?end) {
|
||||
("2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime>)
|
||||
}
|
||||
|
||||
?worshipService a ere:BestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart ?start ;
|
||||
mandaat:bindingEinde ?end .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?lidVanRechtswege .
|
||||
?lidVanRechtswege org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/5972fccd87f864c4ec06bfbd20b5008b> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Rechtswege")) as ?uuidLidVanRechtswege)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidLidVanRechtswege)) AS ?lidVanRechtswege)
|
||||
}
|
||||
|
||||
## Case Bestuursleden, small or big half : 2023 is the beggining of a KH
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?bestuurslidGH .
|
||||
|
||||
?bestuurslidGH a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidBestuurslidGH ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/a8b5509b-f86b-48f8-94d6-fe463a9b77e3> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:BestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?bestuurslidGH .
|
||||
?bestuurslidGH org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/a8b5509b-f86b-48f8-94d6-fe463a9b77e3> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "BestuurslidGH 2020-2026")) as ?uuidBestuurslidGH)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidBestuurslidGH)) AS ?bestuurslidGH)
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?bestuurslidKH .
|
||||
|
||||
?bestuurslidKH a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidBestuurslidKH ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/2962f0bd-2836-4758-9866-8ce8ea2c536f> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
VALUES (?start ?end) {
|
||||
("2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime>)
|
||||
}
|
||||
|
||||
?worshipService a ere:BestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart ?start ;
|
||||
mandaat:bindingEinde ?end .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?bestuurslidKH .
|
||||
?bestuurslidKH org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/2962f0bd-2836-4758-9866-8ce8ea2c536f> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "BestuurslidKH 2023-2029")) as ?uuidBestuurslidKH)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidBestuurslidKH)) AS ?bestuurslidKH)
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
## Other cases are simpler, one mandate is linked to only one orgaan in time. Creating them when missing.
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?voorzitter .
|
||||
|
||||
?voorzitter a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidVoorzitter ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/67e6e585166cd97575b3e17ffc430a43> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:BestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?voorzitter .
|
||||
?voorzitter org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/67e6e585166cd97575b3e17ffc430a43> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Voorzitter 2023-2026")) as ?uuidVoorzitter)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidVoorzitter)) AS ?voorzitter)
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?secretaris .
|
||||
|
||||
?secretaris a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidSecretaris ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/5ac134b9800b81da3c450d6b9605cef2> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:BestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?secretaris .
|
||||
?secretaris org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/5ac134b9800b81da3c450d6b9605cef2> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Secretaris 2023-2026")) as ?uuidSecretaris)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidSecretaris)) AS ?secretaris)
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?orgaanInTime org:hasPost ?penningmeester .
|
||||
|
||||
?penningmeester a mandaat:Mandaat ;
|
||||
mu:uuid ?uuidPenningmeester ;
|
||||
org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/180d13930d6f1a3938e0aa7fa9990002> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipService a ere:BestuurVanDeEredienst .
|
||||
?orgaan besluit:bestuurt ?worshipService .
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ;
|
||||
mandaat:bindingStart "2023-04-01T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> ;
|
||||
mandaat:bindingEinde "2026-03-31T00:00:00"^^<http://www.w3.org/2001/XMLSchema#dateTime> .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?orgaanInTime org:hasPost ?penningmeester .
|
||||
?penningmeester org:role <http://data.vlaanderen.be/id/concept/BestuursfunctieCode/180d13930d6f1a3938e0aa7fa9990002> .
|
||||
}
|
||||
}
|
||||
|
||||
BIND(MD5(CONCAT(?worshipService, ?orgaan, "Penningmeester 2023-2026")) as ?uuidPenningmeester)
|
||||
BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidPenningmeester)) AS ?penningmeester)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
|
||||
PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
|
||||
PREFIX dbpedia: <http://dbpedia.org/ontology/>
|
||||
PREFIX dc_terms: <http://purl.org/dc/terms/>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX euro: <http://data.europa.eu/m8g/>
|
||||
PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
|
||||
PREFIX locn: <http://www.w3.org/ns/locn#>
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
|
||||
PREFIX person: <http://www.w3.org/ns/person#>
|
||||
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
||||
PREFIX regorg: <http://www.w3.org/ns/regorg#>
|
||||
PREFIX schema: <http://schema.org/>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/shared> {
|
||||
?person
|
||||
?p ?o
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/shared> {
|
||||
?person a person:Person ;
|
||||
?p ?o.
|
||||
}
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
VALUES ?typePosition {
|
||||
<http://data.lblod.info/vocabularies/leidinggevenden/Functionaris>
|
||||
<http://data.vlaanderen.be/ns/mandaat#Mandataris>
|
||||
}
|
||||
|
||||
?position mandaat:isBestuurlijkeAliasVan ?person ;
|
||||
a ?typePosition .
|
||||
}
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?worshipPosition <http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan> ?person .
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
|
||||
PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
|
||||
PREFIX dbpedia: <http://dbpedia.org/ontology/>
|
||||
PREFIX dc_terms: <http://purl.org/dc/terms/>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX euro: <http://data.europa.eu/m8g/>
|
||||
PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
|
||||
PREFIX locn: <http://www.w3.org/ns/locn#>
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
|
||||
PREFIX person: <http://www.w3.org/ns/person#>
|
||||
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
||||
PREFIX regorg: <http://www.w3.org/ns/regorg#>
|
||||
PREFIX schema: <http://schema.org/>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?functionaris ?p ?o.
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?functionaris a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
|
||||
org:holds ?bestuursFunctie ;
|
||||
?p ?o .
|
||||
}
|
||||
|
||||
?bestuursorgaanInTime lblodlg:heeftBestuursfunctie ?bestuursFunctie ;
|
||||
generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
VALUES ?typeBestuur {
|
||||
<http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>
|
||||
<http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>
|
||||
}
|
||||
?bestuurseenheid a ?typeBestuur .
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
|
||||
PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
|
||||
PREFIX dbpedia: <http://dbpedia.org/ontology/>
|
||||
PREFIX dc_terms: <http://purl.org/dc/terms/>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX euro: <http://data.europa.eu/m8g/>
|
||||
PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
|
||||
PREFIX locn: <http://www.w3.org/ns/locn#>
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
|
||||
PREFIX person: <http://www.w3.org/ns/person#>
|
||||
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
||||
PREFIX regorg: <http://www.w3.org/ns/regorg#>
|
||||
PREFIX schema: <http://schema.org/>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?mandataris ?p ?o.
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?mandataris a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
|
||||
org:holds ?mandaat ;
|
||||
?p ?o .
|
||||
}
|
||||
|
||||
?bestuursorgaanInTime org:hasPost ?mandaat ;
|
||||
generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
VALUES ?typeBestuur {
|
||||
<http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>
|
||||
<http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>
|
||||
}
|
||||
?bestuurseenheid a ?typeBestuur .
|
||||
}
|
||||
}
|
||||
LIMIT 1000000
|
||||
@@ -0,0 +1,53 @@
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
|
||||
PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
|
||||
PREFIX dbpedia: <http://dbpedia.org/ontology/>
|
||||
PREFIX dc_terms: <http://purl.org/dc/terms/>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX euro: <http://data.europa.eu/m8g/>
|
||||
PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
|
||||
PREFIX locn: <http://www.w3.org/ns/locn#>
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
|
||||
PREFIX person: <http://www.w3.org/ns/person#>
|
||||
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
||||
PREFIX regorg: <http://www.w3.org/ns/regorg#>
|
||||
PREFIX schema: <http://schema.org/>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?mandataris ?p ?o.
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?mandataris a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
|
||||
org:holds ?mandaat ;
|
||||
?p ?o .
|
||||
}
|
||||
|
||||
?bestuursorgaanInTime org:hasPost ?mandaat ;
|
||||
generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
VALUES ?typeBestuur {
|
||||
<http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>
|
||||
<http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>
|
||||
}
|
||||
?bestuurseenheid a ?typeBestuur .
|
||||
}
|
||||
}
|
||||
LIMIT 1000000
|
||||
@@ -0,0 +1,52 @@
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX adres: <https://data.vlaanderen.be/ns/adres#>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
|
||||
PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
|
||||
PREFIX dbpedia: <http://dbpedia.org/ontology/>
|
||||
PREFIX dc_terms: <http://purl.org/dc/terms/>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX euro: <http://data.europa.eu/m8g/>
|
||||
PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
|
||||
PREFIX locn: <http://www.w3.org/ns/locn#>
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
|
||||
PREFIX person: <http://www.w3.org/ns/person#>
|
||||
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
||||
PREFIX regorg: <http://www.w3.org/ns/regorg#>
|
||||
PREFIX schema: <http://schema.org/>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX vcard: <http://www.w3.org/2006/vcard/ns#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?mandataris ?p ?o.
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?mandataris a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
|
||||
org:holds ?mandaat ;
|
||||
?p ?o .
|
||||
}
|
||||
|
||||
?bestuursorgaanInTime org:hasPost ?mandaat ;
|
||||
generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
VALUES ?typeBestuur {
|
||||
<http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>
|
||||
<http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>
|
||||
}
|
||||
?bestuurseenheid a ?typeBestuur .
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?minister ?p ?o .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?minister a ere:RolBedienaar ;
|
||||
org:holds ?ministerPosition ;
|
||||
?p ?o .
|
||||
|
||||
VALUES ?status {
|
||||
<http://lblod.data.gift/concepts/63cc561de9188d64ba5840a42ae8f0d6> # Actief
|
||||
<http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311> # Niet Actief
|
||||
}
|
||||
|
||||
?worshipService <http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor> ?ministerPosition ;
|
||||
<http://www.w3.org/ns/regorg#orgStatus> ?status .
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
delete {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
?position <http://data.lblod.info/vocabularies/erediensten/functie> ?function.
|
||||
}
|
||||
}
|
||||
|
||||
insert {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
?position <http://data.lblod.info/vocabularies/erediensten/functie> <http://lblod.data.gift/concepts/7d8e0c15-7726-4196-9416-6dc40ed858bb>.
|
||||
}
|
||||
}
|
||||
where {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
?position <http://data.lblod.info/vocabularies/erediensten/functie> ?function.
|
||||
?worshipService <http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor> ?position;
|
||||
<http://www.w3.org/ns/regorg#orgStatus> ?worshipStatus.
|
||||
|
||||
filter (?function in (<http://lblod.data.gift/concepts/5ebb798bd59c3b48c25116032caa02b7>,
|
||||
<http://lblod.data.gift/concepts/c4a3fd586211b17b06f574885e23f355>,
|
||||
<http://lblod.data.gift/concepts/e357bc8f4cc3a694fde2239748768a22>)
|
||||
&& ?worshipStatus = <http://lblod.data.gift/concepts/abf4fee82019f88cf122f986830621ab>)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://www.w3.org/ns/org#Post> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://data.vlaanderen.be/ns/mandaat#Mandaat> .
|
||||
}
|
||||
}
|
||||
|
||||
INSERT {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?s a <http://www.w3.org/ns/org#Post> .
|
||||
}
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/worship-service> {
|
||||
?s a <http://data.vlaanderen.be/ns/mandaat#Mandaat> .
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
delete {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/642E815DE025A7C706F41E5D> <http://purl.org/dc/terms/date> "2023-03-30"^^xsd:date .
|
||||
}
|
||||
}
|
||||
insert {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/642E815DE025A7C706F41E5D> <http://purl.org/dc/terms/date> "2023-04-06"^^xsd:date .
|
||||
}
|
||||
}
|
||||
where {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/642E815DE025A7C706F41E5D> <http://purl.org/dc/terms/date> "2023-03-30"^^xsd:date .
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
DELETE DATA {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
<http://data.lblod.info/id/bestuursorganen/ad933a3247ab634eb1a8c7233e77d6ea8cd2b6e1950d302832d065dcb516bcf5> <http://www.w3.org/2004/02/skos/core#prefLabel> "OCMWraad(Rechtstreeksverkiezing) Voeren" .
|
||||
<http://data.lblod.info/id/bestuursorganen/082976d57de0655ac4f123b57d5cb008f612956d0a6de1742c02913dcbfcf125> <http://www.w3.org/2004/02/skos/core#prefLabel> "OCMWraad(Rechtstreeksverkiezing) Drogenbos" .
|
||||
<http://data.lblod.info/id/bestuursorganen/cc803c53b2e0b17aac1069f5566fb8e1c8dad2f702845515a662229721803c00> <http://www.w3.org/2004/02/skos/core#prefLabel> "OCMWraad(Rechtstreeksverkiezing) Wezembeek-Oppem" .
|
||||
<http://data.lblod.info/id/bestuursorganen/f373ece36dc05b29737c0adc14de95682161bb78a55cb85f786fac3fbd5b1cfc> <http://www.w3.org/2004/02/skos/core#prefLabel> "OCMWraad(Rechtstreeksverkiezing) Sint-Genesius-Rode" .
|
||||
<http://data.lblod.info/id/bestuursorganen/317126204426dc7cabb60ecb09cbe4ca4830882652884e1e5800e60231195f5c> <http://www.w3.org/2004/02/skos/core#prefLabel> "OCMWraad(Rechtstreeksverkiezing) Linkebeek" .
|
||||
<http://data.lblod.info/id/bestuursorganen/db8c3e2da85d7621120a986adc19fd8d21f501b39c01952238073abfe7d4a7e7> <http://www.w3.org/2004/02/skos/core#prefLabel> "OCMWraad(Rechtstreeksverkiezing) Wemmel" .
|
||||
<http://data.lblod.info/id/bestuursorganen/d7fffe5c17d1936d46b92620d1e454f38393736988514b5f339910a17c7134f0> <http://www.w3.org/2004/02/skos/core#prefLabel> "OCMWraad(Rechtstreeksverkiezing) Kraainem" .
|
||||
}
|
||||
}
|
||||
;
|
||||
INSERT DATA {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
<http://data.lblod.info/id/bestuursorganen/f373ece36dc05b29737c0adc14de95682161bb78a55cb85f786fac3fbd5b1cfc> <http://www.w3.org/2004/02/skos/core#prefLabel> "Raad voor Maatschappelijk Welzijn Sint-Genesius-Rode" .
|
||||
<http://data.lblod.info/id/bestuursorganen/317126204426dc7cabb60ecb09cbe4ca4830882652884e1e5800e60231195f5c> <http://www.w3.org/2004/02/skos/core#prefLabel> "Raad voor Maatschappelijk Welzijn Linkebeek" .
|
||||
<http://data.lblod.info/id/bestuursorganen/cc803c53b2e0b17aac1069f5566fb8e1c8dad2f702845515a662229721803c00> <http://www.w3.org/2004/02/skos/core#prefLabel> "Raad voor Maatschappelijk Welzijn Wezembeek-Oppem" .
|
||||
<http://data.lblod.info/id/bestuursorganen/ad933a3247ab634eb1a8c7233e77d6ea8cd2b6e1950d302832d065dcb516bcf5> <http://www.w3.org/2004/02/skos/core#prefLabel> "Raad voor Maatschappelijk Welzijn Voeren" .
|
||||
<http://data.lblod.info/id/bestuursorganen/082976d57de0655ac4f123b57d5cb008f612956d0a6de1742c02913dcbfcf125> <http://www.w3.org/2004/02/skos/core#prefLabel> "Raad voor Maatschappelijk Welzijn Drogenbos" .
|
||||
<http://data.lblod.info/id/bestuursorganen/d7fffe5c17d1936d46b92620d1e454f38393736988514b5f339910a17c7134f0> <http://www.w3.org/2004/02/skos/core#prefLabel> "Raad voor Maatschappelijk Welzijn Kraainem" .
|
||||
<http://data.lblod.info/id/bestuursorganen/db8c3e2da85d7621120a986adc19fd8d21f501b39c01952238073abfe7d4a7e7> <http://www.w3.org/2004/02/skos/core#prefLabel> "Raad voor Maatschappelijk Welzijn Wemmel" .
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
delete {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/644616FDF08BCC5A4747B5A2> <http://purl.org/dc/terms/date> ?date.
|
||||
}
|
||||
}
|
||||
insert {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/644616FDF08BCC5A4747B5A2> <http://purl.org/dc/terms/date> "2022-04-22".
|
||||
}
|
||||
}
|
||||
where {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/644616FDF08BCC5A4747B5A2> <http://purl.org/dc/terms/date> ?date.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?orgaanInTime mandaat:bindingEinde ?end .
|
||||
}
|
||||
} INSERT {
|
||||
GRAPH ?g {
|
||||
?orgaanInTime mandaat:bindingEinde ?endDateTime .
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?g {
|
||||
<http://mu.semte.ch/graphs/worship-service>
|
||||
<http://mu.semte.ch/graphs/administrative-unit>
|
||||
}
|
||||
GRAPH ?g {
|
||||
?bestuurseenheid a ?type .
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
|
||||
?orgaanInTime mandaat:bindingEinde ?end .
|
||||
FILTER (datatype(?end) = xsd:date)
|
||||
BIND(strdt(concat(str(?end), 'T01:00:00'), xsd:dateTime) as ?endDateTime)
|
||||
}
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?orgaanInTime mandaat:bindingStart ?start .
|
||||
}
|
||||
} INSERT {
|
||||
GRAPH ?g {
|
||||
?orgaanInTime mandaat:bindingStart ?startDateTime .
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?g {
|
||||
<http://mu.semte.ch/graphs/worship-service>
|
||||
<http://mu.semte.ch/graphs/administrative-unit>
|
||||
}
|
||||
GRAPH ?g {
|
||||
?bestuurseenheid a ?type .
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
|
||||
?orgaanInTime mandaat:bindingStart ?start .
|
||||
FILTER (datatype(?start) = xsd:date)
|
||||
BIND(strdt(concat(str(?start), 'T01:00:00'), xsd:dateTime) as ?startDateTime)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?orgaanInTime mandaat:bindingEinde ?end .
|
||||
}
|
||||
} INSERT {
|
||||
GRAPH ?g {
|
||||
?orgaanInTime mandaat:bindingEinde ?endDateTime .
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?g {
|
||||
<http://mu.semte.ch/graphs/worship-service>
|
||||
<http://mu.semte.ch/graphs/administrative-unit>
|
||||
}
|
||||
GRAPH ?g {
|
||||
?bestuurseenheid a ?type .
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
|
||||
?orgaanInTime mandaat:bindingEinde ?end .
|
||||
|
||||
FILTER (HOURS(?end) = 1)
|
||||
BIND(strdt(concat(xsd:date(?end), 'T00:00:00'), xsd:dateTime) as ?endDateTime)
|
||||
}
|
||||
}
|
||||
|
||||
;
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?orgaanInTime mandaat:bindingStart ?start .
|
||||
}
|
||||
} INSERT {
|
||||
GRAPH ?g {
|
||||
?orgaanInTime mandaat:bindingStart ?startDateTime .
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?g {
|
||||
<http://mu.semte.ch/graphs/worship-service>
|
||||
<http://mu.semte.ch/graphs/administrative-unit>
|
||||
}
|
||||
GRAPH ?g {
|
||||
?bestuurseenheid a ?type .
|
||||
|
||||
?bestuursorgaan besluit:bestuurt ?bestuurseenheid .
|
||||
|
||||
?orgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan .
|
||||
|
||||
?orgaanInTime mandaat:bindingStart ?start .
|
||||
|
||||
FILTER (HOURS(?start) = 1)
|
||||
BIND(strdt(concat(xsd:date(?start), 'T00:00:00'), xsd:dateTime) as ?startDateTime)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s ?p ?o .
|
||||
}
|
||||
}
|
||||
WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://data.vlaanderen.be/ns/mandaat#Fractie> ;
|
||||
?p ?o .
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
|
||||
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
||||
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX concepts: <http://lblod.data.gift/concepts/>
|
||||
PREFIX bestuur: <http://data.lblod.info/id/besturenVanDeEredienst/>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
||||
PREFIX contacthub: <http://data.lblod.info/vocabularies/contacthub/>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
PREFIX ce: <http://data.lblod.info/id/veranderingsgebeurtenissen/>
|
||||
PREFIX cer: <http://data.lblod.info/id/veranderingsgebeurtenis-resultaten/>
|
||||
PREFIX m8g: <http://data.europa.eu/m8g/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX eli: <http://data.europa.eu/eli/ontology#>
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
|
||||
|
||||
DELETE {
|
||||
graph <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?decision ?decisionP ?decisionO.
|
||||
}
|
||||
}
|
||||
INSERT {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
?decision ?decisionP ?decisionO.
|
||||
}
|
||||
}
|
||||
WHERE {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
?changeEvent a <http://www.w3.org/ns/org#ChangeEvent>;
|
||||
<http://data.europa.eu/m8g/hasFormalFramework> ?decision.
|
||||
}
|
||||
graph <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?decision ?decisionP ?decisionO.
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
;
|
||||
|
||||
|
||||
DELETE {
|
||||
graph <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?decisionActivity ?p ?o.
|
||||
}
|
||||
}
|
||||
INSERT {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
?decisionActivity ?p ?o.
|
||||
}
|
||||
}
|
||||
WHERE {
|
||||
graph <http://mu.semte.ch/graphs/worship-service> {
|
||||
?decision a <http://data.vlaanderen.be/ns/besluit#Besluit>.
|
||||
}
|
||||
graph <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?decisionActivity a <https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit>;
|
||||
<http://www.w3.org/ns/prov#generated> ?decision;
|
||||
?p ?o.
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
|
||||
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
||||
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX concepts: <http://lblod.data.gift/concepts/>
|
||||
PREFIX bestuur: <http://data.lblod.info/id/besturenVanDeEredienst/>
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
|
||||
PREFIX contacthub: <http://data.lblod.info/vocabularies/contacthub/>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
PREFIX ce: <http://data.lblod.info/id/veranderingsgebeurtenissen/>
|
||||
PREFIX cer: <http://data.lblod.info/id/veranderingsgebeurtenis-resultaten/>
|
||||
PREFIX m8g: <http://data.europa.eu/m8g/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
|
||||
PREFIX eli: <http://data.europa.eu/eli/ontology#>
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
|
||||
|
||||
DELETE {
|
||||
graph ?g {
|
||||
?changeEvent ?p ?o.
|
||||
?changeEventResult ?cP ?cO.
|
||||
?decision ?decisionP ?decisionO.
|
||||
?decisionActivity ?decisionActivityP ?desitionActivityO.
|
||||
}
|
||||
}
|
||||
WHERE {
|
||||
graph ?g {
|
||||
?changeEvent a <http://www.w3.org/ns/org#ChangeEvent>;
|
||||
<http://lblod.data.gift/vocabularies/organisatie/veranderingsgebeurtenisResultaat> ?changeEventResult;
|
||||
mu:uuid ?uuidChangeEvent;
|
||||
?p ?o.
|
||||
?changeEventResult a <http://lblod.data.gift/vocabularies/organisatie/VeranderingsgebeurtenisResultaat>;
|
||||
mu:uuid ?uuidChangeEventResult;
|
||||
?cP ?cO.
|
||||
|
||||
optional {
|
||||
?changeEvent <http://data.europa.eu/m8g/hasFormalFramework> ?decision.
|
||||
?decision ?decisionP ?decisionO.
|
||||
optional {
|
||||
?decisionActivity a <https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit>;
|
||||
<http://www.w3.org/ns/prov#generated> ?decision;
|
||||
?decisionActivityP ?desitionActivityO.
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
filter(?uuidChangeEvent in ("63343865036B579F0A26FF5C", "63343E99036B579F0A26FF6F", "633446D6036B579F0A26FF87","63345772036B579F0A26FF93"))
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
delete {
|
||||
graph ?g {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/620E674C5747C883CA1D2EE4> <http://data.lblod.info/vocabularies/contacthub/typeWijziging> <http://lblod.data.gift/concepts/9f19f14910245f9b14737f1f1a2067f2>.
|
||||
}
|
||||
}
|
||||
insert {
|
||||
graph ?g {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/620E674C5747C883CA1D2EE4> <http://data.lblod.info/vocabularies/contacthub/typeWijziging> <http://lblod.data.gift/concepts/4297bc56c1a874240e4566e2c5b8816f>.
|
||||
|
||||
}
|
||||
}
|
||||
where {
|
||||
graph ?g {
|
||||
<http://data.lblod.info/id/veranderingsgebeurtenissen/620E674C5747C883CA1D2EE4> <http://data.lblod.info/vocabularies/contacthub/typeWijziging> <http://lblod.data.gift/concepts/9f19f14910245f9b14737f1f1a2067f2>.
|
||||
}
|
||||
}
|
||||
@@ -293,16 +293,6 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
?s persoon:datumVanAfstamming ?o
|
||||
}.
|
||||
|
||||
{
|
||||
{
|
||||
?s persoon_loket:gebruikteVoornaam ?o
|
||||
} e:becomes {
|
||||
?s persoon:gebruikteVoornaam ?o
|
||||
}
|
||||
} => {
|
||||
?s persoon:gebruikteVoornaam ?o
|
||||
}.
|
||||
|
||||
{
|
||||
{
|
||||
?s persoon_loket:geslacht ?o
|
||||
|
||||
@@ -49,12 +49,6 @@
|
||||
?s a <http://data.vlaanderen.be/ns/besluit#Bestuursorgaan>
|
||||
}.
|
||||
|
||||
{
|
||||
?s a <http://data.vlaanderen.be/ns/mandaat#Fractie>
|
||||
} => {
|
||||
?s a <http://data.vlaanderen.be/ns/mandaat#Fractie>
|
||||
}.
|
||||
|
||||
{
|
||||
?s a <http://data.vlaanderen.be/ns/mandaat#Mandaat>
|
||||
} => {
|
||||
|
||||
@@ -18,7 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
|
||||
# Inverse properties
|
||||
{
|
||||
?s org:holds ?o.
|
||||
|
||||
@@ -18,7 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
|
||||
# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)
|
||||
# Hence the mapping is in two steps.
|
||||
# The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart
|
||||
@@ -92,6 +91,31 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
?s mandaat:einde ?eindeDate
|
||||
}.
|
||||
|
||||
{
|
||||
?s mandaat:bindingStart ?bindingStartDateTime.
|
||||
# use backward arrow rule defined below
|
||||
?bindingStartDateTime :dateFromDateTime ?bindingStartDate.
|
||||
{
|
||||
?s mandaat:bindingStart ?bindingStartDateTime
|
||||
} e:becomes {
|
||||
?s mandaat:bindingStart ?bindingStartDate
|
||||
}.
|
||||
} => {
|
||||
?s mandaat:bindingStart ?bindingStartDate
|
||||
}.
|
||||
|
||||
{
|
||||
?s mandaat:bindingEinde ?bindingEindeDateTime.
|
||||
# use backward arrow rule defined below
|
||||
?bindingEindeDateTime :dateFromDateTime ?bindingEindeDate.
|
||||
{
|
||||
?s mandaat:bindingEinde ?bindingEindeDateTime
|
||||
} e:becomes {
|
||||
?s mandaat:bindingEinde ?bindingEindeDate
|
||||
}.
|
||||
} => {
|
||||
?s mandaat:bindingEinde ?bindingEindeDate
|
||||
}.
|
||||
|
||||
# Backward arrow rule is like a function call.
|
||||
# Removes time info in the returned date
|
||||
|
||||
@@ -18,7 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
|
||||
# Map from http persoon to https persoon namespace.
|
||||
|
||||
## classes:
|
||||
@@ -293,16 +292,6 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
?s persoon:datumVanAfstamming ?o
|
||||
}.
|
||||
|
||||
{
|
||||
{
|
||||
?s persoon_loket:gebruikteVoornaam ?o
|
||||
} e:becomes {
|
||||
?s persoon:gebruikteVoornaam ?o
|
||||
}
|
||||
} => {
|
||||
?s persoon:gebruikteVoornaam ?o
|
||||
}.
|
||||
|
||||
{
|
||||
{
|
||||
?s persoon_loket:geslacht ?o
|
||||
|
||||
56
config/reasoner/dl2op/main/query.n3q
Normal file
56
config/reasoner/dl2op/main/query.n3q
Normal file
@@ -0,0 +1,56 @@
|
||||
# Filtering classes to ingest
|
||||
|
||||
{
|
||||
?s a <http://www.w3.org/ns/person#Person> ;
|
||||
?p ?o .
|
||||
} => {
|
||||
?s a <http://www.w3.org/ns/person#Person> ;
|
||||
?p ?o .
|
||||
}.
|
||||
|
||||
{
|
||||
?s a <http://www.w3.org/ns/locn#Address> ;
|
||||
?p ?o .
|
||||
} => {
|
||||
?s a <http://www.w3.org/ns/locn#Address> ;
|
||||
?p ?o .
|
||||
}.
|
||||
|
||||
{
|
||||
?s a <http://schema.org/ContactPoint> ;
|
||||
?p ?o .
|
||||
} => {
|
||||
?s a <http://schema.org/ContactPoint> ;
|
||||
?p ?o .
|
||||
}.
|
||||
|
||||
{
|
||||
?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
|
||||
?p ?o .
|
||||
} => {
|
||||
?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris> ;
|
||||
?p ?o .
|
||||
}.
|
||||
|
||||
{
|
||||
?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
|
||||
?p ?o .
|
||||
} => {
|
||||
?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris> ;
|
||||
?p ?o .
|
||||
}.
|
||||
|
||||
{
|
||||
?s a <http://www.w3.org/ns/org#Membership> ;
|
||||
?p ?o .
|
||||
} => {
|
||||
?s a <http://www.w3.org/ns/org#Membership> ;
|
||||
?p ?o .
|
||||
}.
|
||||
{
|
||||
?s a <http://www.w3.org/ns/prov#Location> ;
|
||||
?p ?o .
|
||||
} => {
|
||||
?s a <http://www.w3.org/ns/prov#Location> ;
|
||||
?p ?o .
|
||||
}.
|
||||
@@ -18,7 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
|
||||
# Retract empty strings
|
||||
{
|
||||
{
|
||||
|
||||
@@ -18,8 +18,6 @@ PREFIX skos:<http://www.w3.org/2004/02/skos/core#>
|
||||
PREFIX string: <http://www.w3.org/2000/10/swap/string#>
|
||||
PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
|
||||
|
||||
|
||||
# Rijksregisternummer
|
||||
# Extend identifiers to match OP's model
|
||||
# <http://example.org/tst#subject> skos:notation "00010100105".
|
||||
|
||||
@@ -293,16 +293,6 @@ PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
|
||||
?s persoon:datumVanAfstamming ?o
|
||||
}.
|
||||
|
||||
{
|
||||
{
|
||||
?s persoon_loket:gebruikteVoornaam ?o
|
||||
} e:becomes {
|
||||
?s persoon:gebruikteVoornaam ?o
|
||||
}
|
||||
} => {
|
||||
?s persoon:gebruikteVoornaam ?o
|
||||
}.
|
||||
|
||||
{
|
||||
{
|
||||
?s persoon_loket:geslacht ?o
|
||||
|
||||
@@ -578,11 +578,11 @@
|
||||
"class": "besluit:Bestuursorgaan",
|
||||
"attributes": {
|
||||
"start-date": {
|
||||
"type": "date",
|
||||
"type": "datetime",
|
||||
"predicate": "mandaat:bindingStart"
|
||||
},
|
||||
"end-date": {
|
||||
"type": "date",
|
||||
"type": "datetime",
|
||||
"predicate": "mandaat:bindingEinde"
|
||||
}
|
||||
},
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
(defparameter *cache-model-properties* t)
|
||||
(defparameter *cache-count-queries* t)
|
||||
(defparameter *supply-cache-headers-p* t)
|
||||
(defparameter sparql:*experimental-no-application-graph-for-sudo-select-queries* t)
|
||||
|
||||
(read-domain-file "domain.json")
|
||||
(read-domain-file "auth.json")
|
||||
|
||||
@@ -35,7 +35,7 @@ services:
|
||||
restart: always
|
||||
logging: *default-logging
|
||||
db:
|
||||
image: semtech/mu-authorization:feature-service-roam-r1
|
||||
image: semtech/mu-authorization:feature-service-roam-r1.1
|
||||
environment:
|
||||
MU_SPARQL_ENDPOINT: "http://triplestore:8890/sparql"
|
||||
LOG_OUTGOING_SPARQL_QUERIES: "true"
|
||||
@@ -43,6 +43,8 @@ services:
|
||||
LOG_ACCESS_RIGHTS: "true"
|
||||
LOG_OUTGOING_SPARQL_QUERY_ROUNDTRIP: "true"
|
||||
ERROR_ON_UNWRITTEN_DATA: "true"
|
||||
QUERY_MAX_PROCESSING_TIME: "300000"
|
||||
QUERY_MAX_EXECUTION_TIME: "300000"
|
||||
volumes:
|
||||
- ./config/authorization:/config
|
||||
labels:
|
||||
@@ -70,7 +72,7 @@ services:
|
||||
restart: always
|
||||
logging: *default-logging
|
||||
resource:
|
||||
image: semtech/mu-cl-resources:1.21.1
|
||||
image: semtech/mu-cl-resources:1.22.1
|
||||
environment:
|
||||
CACHE_CLEAR_PATH: "http://cache/.mu/clear-keys"
|
||||
links:
|
||||
@@ -82,7 +84,7 @@ services:
|
||||
restart: always
|
||||
logging: *default-logging
|
||||
cache:
|
||||
image: semtech/mu-cache:2.0.1
|
||||
image: semtech/mu-cache:2.0.2
|
||||
links:
|
||||
- resource:backend
|
||||
labels:
|
||||
@@ -173,13 +175,13 @@ services:
|
||||
links:
|
||||
- db:database
|
||||
adressenregister:
|
||||
image: lblod/adressenregister-fuzzy-search-service:0.6.2
|
||||
image: lblod/adressenregister-fuzzy-search-service:0.6.3
|
||||
labels:
|
||||
- "logging=true"
|
||||
restart: always
|
||||
logging: *default-logging
|
||||
frontend:
|
||||
image: lblod/frontend-organization-portal:1.16.0-1
|
||||
image: lblod/frontend-organization-portal:1.17.0
|
||||
volumes:
|
||||
- ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf
|
||||
environment:
|
||||
@@ -218,19 +220,22 @@ services:
|
||||
- ./config/jobs-controller/:/config/
|
||||
construct-administrative-unit-relationships:
|
||||
image: lblod/construct-administrative-unit-relationships-service:0.1.1
|
||||
environment:
|
||||
START_DATE_WORSHIP_GOVERNING_BODY: "2023-04-01T00:00:00"
|
||||
END_DATE_WORSHIP_GOVERNING_BODY: "2026-03-31T00:00:00"
|
||||
links:
|
||||
- db:database
|
||||
restart: always
|
||||
logging: *default-logging
|
||||
reasoner:
|
||||
image: eyereasoner/reasoning-service:1.0.1
|
||||
image: eyereasoner/reasoning-service:1.0.2
|
||||
volumes:
|
||||
- ./config/reasoner:/config
|
||||
################################################################################
|
||||
# DELTA GENERAL
|
||||
################################################################################
|
||||
delta-producer-dump-file-publisher:
|
||||
image: lblod/delta-producer-dump-file-publisher:0.9.0
|
||||
image: lblod/delta-producer-dump-file-publisher:0.9.1
|
||||
volumes:
|
||||
- ./config/delta-producer/dump-file-publisher:/config
|
||||
- ./data/files:/share
|
||||
@@ -320,7 +325,7 @@ services:
|
||||
restart: always
|
||||
logging: *default-logging
|
||||
delta-producer-pub-graph-maintainer-public:
|
||||
image: lblod/delta-producer-publication-graph-maintainer:0.15.1
|
||||
image: lblod/delta-producer-publication-graph-maintainer:0.15.4
|
||||
environment:
|
||||
MAX_BODY_SIZE: "50mb"
|
||||
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
|
||||
@@ -415,25 +420,28 @@ services:
|
||||
# DELTA CONSUMERS
|
||||
################################################################################
|
||||
################################################################################
|
||||
# LOKET MANDATEN CONSUMERS
|
||||
# LOKET MANDATARISSEN CONSUMERS
|
||||
################################################################################
|
||||
mandaten-consumer:
|
||||
image: lblod/delta-consumer:0.0.14
|
||||
mandatarissen-consumer:
|
||||
image: lblod/delta-consumer:0.0.17-rc.7
|
||||
environment:
|
||||
DCR_SERVICE_NAME: "mandatendatabank-consumer"
|
||||
DCR_SERVICE_NAME: "mandatarissen-consumer"
|
||||
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
|
||||
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/MandatarissenCacheGraphDump"
|
||||
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen"
|
||||
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen"
|
||||
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatendatabank-consumer"
|
||||
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatarissen-consumer"
|
||||
DCR_DISABLE_INITIAL_SYNC: "true"
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
|
||||
BATCH_SIZE: 1000
|
||||
DCR_SYNC_FILES_PATH: "/sync/mandatarissen/files"
|
||||
FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
|
||||
KEEP_DELTA_FILES: "true"
|
||||
DCR_KEEP_DELTA_FILES: "true"
|
||||
DCR_ENABLE_DELTA_CONTEXT: "true"
|
||||
DCR_LANDING_ZONE_DATABASE: "triplestore"
|
||||
DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/mandatarissen"
|
||||
volumes:
|
||||
- ./config/loket-consumer/mandate-dispatching:/config/triples-dispatching/custom-dispatching
|
||||
- ./data/files/consumer-files/mandaten:/consumer-files/
|
||||
- ./config/loket-consumer/mandatarissen-dispatching:/config/triples-dispatching/custom-dispatching
|
||||
- ./data/files/consumer-files/mandatarissen:/consumer-files/
|
||||
restart: always
|
||||
labels:
|
||||
- "logging=true"
|
||||
@@ -446,7 +454,7 @@ services:
|
||||
# LOKET LEIDINGGEVENDEN CONSUMERS
|
||||
################################################################################
|
||||
leidinggevenden-consumer:
|
||||
image: lblod/delta-consumer:0.0.14
|
||||
image: lblod/delta-consumer:0.0.17-rc.7
|
||||
environment:
|
||||
DCR_SERVICE_NAME: "leidinggevenden-consumer"
|
||||
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
|
||||
@@ -457,8 +465,11 @@ services:
|
||||
DCR_DISABLE_INITIAL_SYNC: "true"
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
|
||||
DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files"
|
||||
FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
|
||||
KEEP_DELTA_FILES: "true"
|
||||
DCR_KEEP_DELTA_FILES: "true"
|
||||
BATCH_SIZE: 1000
|
||||
DCR_ENABLE_DELTA_CONTEXT: "true"
|
||||
DCR_LANDING_ZONE_DATABASE: "triplestore"
|
||||
DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/leidinggevenden"
|
||||
volumes:
|
||||
- ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching
|
||||
- ./data/files/consumer-files/leidinggevenden:/consumer-files/
|
||||
@@ -474,22 +485,24 @@ services:
|
||||
# LOKET WORSHIP SENSITIVE CONSUMERS
|
||||
################################################################################
|
||||
worship-services-sensitive-consumer:
|
||||
image: lblod/delta-consumer:0.0.14
|
||||
image: lblod/delta-consumer:0.0.17-rc.7
|
||||
environment:
|
||||
DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
|
||||
DCR_SERVICE_NAME: "worship-services-sensitive-consumer"
|
||||
DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files"
|
||||
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
|
||||
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
|
||||
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
|
||||
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer"
|
||||
DCR_DISABLE_INITIAL_SYNC: "true"
|
||||
FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
|
||||
DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files"
|
||||
BATCH_SIZE: 1000
|
||||
DCR_KEEP_DELTA_FILES: "true"
|
||||
DCR_DELTA_FILE_FOLDER: "/consumer-files"
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false"
|
||||
DCR_ENABLE_DELTA_CONTEXT: "true"
|
||||
DCR_LANDING_ZONE_DATABASE: "triplestore"
|
||||
DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/worship-services-sensitive"
|
||||
volumes:
|
||||
- ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching
|
||||
- ./config/loket-consumer/worship-services-sensitive-dispatching:/config/triples-dispatching/custom-dispatching
|
||||
- ./data/files/consumer-files/worship-services-sensitive:/consumer-files/
|
||||
restart: always
|
||||
labels:
|
||||
@@ -507,7 +520,7 @@ services:
|
||||
# MU SEARCH
|
||||
################################################################################
|
||||
search:
|
||||
image: semtech/mu-search:0.9.0-beta.2
|
||||
image: semtech/mu-search:0.9.0-beta.3
|
||||
links:
|
||||
- db:database
|
||||
restart: always
|
||||
@@ -527,7 +540,7 @@ services:
|
||||
- db:database
|
||||
|
||||
roles:
|
||||
image: lblod/select-role-service
|
||||
image: lblod/select-role-service:0.1.0
|
||||
labels:
|
||||
- "logging=true"
|
||||
links:
|
||||
|
||||
11
scripts/reset-elastic.sh
Normal file
11
scripts/reset-elastic.sh
Normal file
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
echo "warning this will run queries on the triplestore and delete containers, you have 3 seconds to press ctrl+c"
|
||||
sleep 3
|
||||
docker-compose rm -fs elasticsearch search
|
||||
sudo rm -rf data/elasticsearch/
|
||||
docker-compose exec -T triplestore isql-v <<EOF
|
||||
SPARQL DELETE WHERE { GRAPH <http://mu.semte.ch/authorization> { ?s ?p ?o. } };
|
||||
exec('checkpoint');
|
||||
exit;
|
||||
EOF
|
||||
docker-compose up -d --remove-orphan
|
||||
Reference in New Issue
Block a user