felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #507 from lblod/chore/bump-mandatarissen-consumer

Browse Source 
Bump mandatarissen consumer to v0.1.0
This commit is contained in:
Claire Lovisa
2025-02-03 16:11:15 +01:00
committed by GitHub
parent 0b8530db34 f79724132a
commit 404a282a7d
28 changed files with 560 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
CHANGELOG.md
View File

@@ -3,6 +3,7 @@
## 1.29.0 (TODO)
### Backend
#### Consumer
- Upgraded `mandatarissen-consumer` [OP-3510]
- Upgraded `leidinggevenden-consumer` [OP-3533], [OP-3511]
- Upgraded `worship-services-sensitive-consumer` [OP-3337]
@@ -10,12 +11,18 @@
```
drc down;
```
Update `docker-compose.override.yml` to remove the config of `worship-services-sensitive-consumer` and replace it by:
Update `docker-compose.override.yml` to remove the config of the consumers and replace it by:
```
mandatarissen-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
DCR_DISABLE_DELTA_INGEST: "false"
DCR_DISABLE_INITIAL_SYNC: "false"
leidinggevenden-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
DCR_DISABLE_DELTA_INGEST: "false"
@@ -42,15 +49,21 @@ Update `docker-compose.override.yml` to remove the config of `worship-services-s
Then:
```
drc up -d triplestore migrations
drc up -d db leidinggevenden-consumer worship-services-main-info-consumer worship-services-private-info-consumer
drc up -d db mandatarissen-consumer leidinggevenden-consumer worship-services-main-info-consumer worship-services-private-info-consumer
# Wait until success of the previous step
```
Then, update `docker-compose.override.yml` to:
```
mandatarissen-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint
DCR_LANDING_ZONE_DATABASE: "db"
DCR_REMAPPING_DATABASE: "db"
DCR_DISABLE_DELTA_INGEST: "false"
DCR_DISABLE_INITIAL_SYNC: "false"
leidinggevenden-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # choose the correct endpoint
DCR_LANDING_ZONE_DATABASE: "db"
DCR_REMAPPING_DATABASE: "db"
DCR_DISABLE_DELTA_INGEST: "false"

40
README.md
View File

@@ -12,10 +12,48 @@ Backend for the organization portal application, based on the mu.semte.ch micros
You can shut down using `docker-compose stop` and remove everything using `docker-compose rm`.
#### Setup consumers
The application depends on external data. You'll have to perform extra steps if you want this data to be available.
These are multiple data sets, we'll try to be exhaustive on how to ingest these. Even though the procedures are similar.
#### Setup http://data.lblod.info/vocabularies/leidinggevenden/Functionaris
#### Setup mandatarissen sync
Note: this is assuming you have never ran this before.
```
drc down;
```
Update `docker-compose.override.yml` to:
```
mandatarissen-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint
DCR_LANDING_ZONE_DATABASE: "triplestore"
DCR_REMAPPING_DATABASE: "triplestore"
DCR_DISABLE_DELTA_INGEST: "false"
DCR_DISABLE_INITIAL_SYNC: "false"
```
Then:
```
drc up -d migrations
drc up -d db mandatarissen-consumer
# Wait until success of the previous step
```
Then, update `docker-compose.override.yml` to:
```
mandatarissen-consumer:
environment:
DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint
DCR_LANDING_ZONE_DATABASE: "db"
DCR_REMAPPING_DATABASE: "db"
DCR_DISABLE_DELTA_INGEST: "false"
DCR_DISABLE_INITIAL_SYNC: "false"
```
Finally, put the stack back up and reindex elasticsearch
```
drc up -d
sh scripts/reset-elastic.sh
```
#### Setup leidinggevenden sync (http://data.lblod.info/vocabularies/leidinggevenden/Functionaris)
Note: this is assuming you have never ran this before.
```

33
config/loket-consumer/mandatarissen-dispatching/config.js
View File

@@ -1,33 +0,0 @@
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
// Graphs
const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
// ENDPOINTS
const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
module.exports = {
BATCH_SIZE,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MAX_DB_RETRY_ATTEMPTS,
MAX_REASONING_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
LANDING_ZONE_GRAPH,
LANDING_ZONE_DATABASE_ENDPOINT,
TARGET_GRAPH
};

72
config/loket-consumer/mandatarissen-dispatching/delta-context-config.js
View File

@@ -1,72 +0,0 @@
const PREFIXES = `
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>`
const contextConfig = {
addTypes: {
scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
},
contextQueries: [
// Case Mandatarissen
// They can be triggered when being inserted or when their link to a mandate is added
{
trigger: { // subjectType or predicateValue
subjectType: "mandaat:Mandataris"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
?mandataris org:holds ?mandate ;
?p ?o .
?orgaanInTime org:hasPost ?mandate ;
mandaat:isTijdspecialisatieVan ?orgaan .
?orgaan besluit:bestuurt ?eenheid .
?eenheid besluit:classificatie ?classification .
} WHERE {
GRAPH <http://mu.semte.ch/graphs/landing-zone/mandatarissen> {
VALUES ?mandataris { ${subject} }
?mandataris org:holds ?mandate ;
?p ?o .
?orgaanInTime org:hasPost ?mandate ;
mandaat:isTijdspecialisatieVan ?orgaan .
?orgaan besluit:bestuurt ?eenheid .
?eenheid besluit:classificatie ?classification .
}
}`
},
{
trigger: {
predicateValue: "org:holds"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
?mandataris org:holds ?mandate ;
?p ?o .
?orgaanInTime org:hasPost ?mandate ;
mandaat:isTijdspecialisatieVan ?orgaan .
?orgaan besluit:bestuurt ?eenheid .
?eenheid besluit:classificatie ?classification .
} WHERE {
GRAPH <http://mu.semte.ch/graphs/landing-zone/mandatarissen> {
VALUES ?mandataris { ${subject} }
?mandataris org:holds ?mandate ;
?p ?o .
?orgaanInTime org:hasPost ?mandate ;
mandaat:isTijdspecialisatieVan ?orgaan .
?orgaan besluit:bestuurt ?eenheid .
?eenheid besluit:classificatie ?classification .
}
}`
}
]
}
module.exports = {
contextConfig,
PREFIXES
};

85
config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js
View File

@@ -1,85 +0,0 @@
const {
transformStatements,
deleteFromTargetGraph,
insertIntoTargetGraph,
} = require('./util');
/**
* Dispatch the fetched information to a target graph.
* Note: <share://file/data> will be ADDED to it's own graph.
* We take only care of adding them, not updating triples, this is a TODO
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
* [ {
* graph: "<http://foo>",
* subject: "<http://bar>",
* predicate: "<http://baz>",
* object: "<http://boom>^^<http://datatype>"
* }
* ]
* @return {void} Nothing
*/
async function dispatch(lib, data) {
const { mu, fetch } = lib;
let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
// Both arrays are the same length, so we can zip them together for easier processing
zippedChangeSets = termObjectChangeSets.map((o, i) => ({
original: o,
withContext: termObjectChangeSetsWithContext[i]
}));
console.log(`Received ${zippedChangeSets.length} change sets`)
for (let { original, withContext } of zippedChangeSets) {
let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
try {
originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
} catch (e) {
console.log(`Error while processing change set: ${e}`);
console.log(`Original change set: ${JSON.stringify(original)}`);
throw e;
}
// Extra context needed for mapping from DL to OP model and filtering based on type.
try {
insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
} catch (e) {
console.log(`Error while processing change set with context: ${e}`);
console.log(`Change set with context: ${JSON.stringify(withContext)}`);
throw e;
}
if (originalDeletes.length) {
// Map deletes from DL to OP model
const transformedDeletes = await transformStatements(fetch, deletesWithContext);
if (!transformedDeletes.length) {
console.log(`Warn: Delete statements mapped to empty result.`);
console.log(`Input: ${deletesWithContext}`);
console.log(`Output: ${transformedDeletes}`);
} else {
await deleteFromTargetGraph(lib, transformedDeletes);
}
}
if (originalInserts.length) {
// Map inserts from DL to OP model
const transformedInserts = await transformStatements(fetch, insertsWithContext);
if (!transformedInserts.length) {
console.log(`Warn: Insert statements mapped to empty result.`);
console.log(`Input: ${insertsWithContext}`);
console.log(`Output: ${transformedInserts}`);
} else {
await insertIntoTargetGraph(lib, transformedInserts);
}
}
}
}
module.exports = {
dispatch
};

85
config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js
View File

@@ -1,85 +0,0 @@
const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
const {
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
TARGET_GRAPH,
LANDING_ZONE_GRAPH,
LANDING_ZONE_DATABASE_ENDPOINT
} = require('./config');
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
/**
* Dispatch the fetched information to a target graph.
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
* [ {
* graph: "<http://foo>",
* subject: "<http://bar>",
* predicate: "<http://baz>",
* object: "<http://boom>^^<http://datatype>"
* }
* ]
* @return {void} Nothing
*/
async function dispatch(lib, data) {
const { mu, muAuthSudo } = lib;
const { termObjects } = data;
// Steps:
// 1. Write the triples to the landing zone graph - withouth any mapping or filtering
// 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
console.warn(`Service configured to skip MU_AUTH!`);
}
console.log(`Using ${endpoint} to insert triples`);
if (termObjects.length) {
const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
await batchedDbUpdate(
muAuthSudo.updateSudo,
LANDING_ZONE_GRAPH,
originalInsertTriples,
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
LANDING_ZONE_DATABASE_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
);
}
}
async function onFinishInitialIngest(lib) {
const { muAuthSudo, fetch } = lib;
console.log(`!! On-finish triggered !!`);
const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint);
console.log(`Transformed ${transformedInsertTriples.length} triples`);
await batchedDbUpdate(
muAuthSudo.updateSudo,
TARGET_GRAPH,
transformedInsertTriples,
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
endpoint,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
);
}
module.exports = {
dispatch,
onFinishInitialIngest
};

168
config/loket-consumer/mandatarissen-dispatching/util.js
View File

@@ -1,168 +0,0 @@
const {
MAX_REASONING_RETRY_ATTEMPTS,
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
TARGET_GRAPH,
LANDING_ZONE_GRAPH,
BATCH_SIZE
} = require('./config');
async function batchedDbUpdate(
muUpdate,
graph,
triples,
extraHeaders,
endpoint,
batchSize,
maxAttempts,
sleepBetweenBatches = 1000,
sleepTimeOnFail = 1000,
operation = 'INSERT'
) {
for (let i = 0; i < triples.length; i += batchSize) {
console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
const batch = triples.slice(i, i + batchSize).join('\n');
const insertCall = async () => {
await muUpdate(`
${operation} DATA {
GRAPH <${graph}> {
${batch}
}
}
`, extraHeaders, endpoint);
};
await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
await new Promise(r => setTimeout(r, sleepBetweenBatches));
}
}
async function operationWithRetry(callback,
attempt,
maxAttempts,
sleepTimeOnFail) {
try {
if (typeof callback === "function")
return await callback();
else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
return await callback;
}
catch (e) {
console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
console.log(`Error: ${e}`);
console.log(`Sleeping ${sleepTimeOnFail} ms`);
if (attempt >= maxAttempts) {
console.log(`Max attempts reached for ${callback.toString()}, giving up`);
throw e;
}
await new Promise(r => setTimeout(r, sleepTimeOnFail));
return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
}
}
/**
* Splits an array into two parts, a part that passes and a part that fails a predicate function.
* Credits: https://github.com/benjay10
* @public
* @function partition
* @param {Array} arr - Array to be partitioned
* @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
* @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
*/
function partition(arr, fn) {
let passes = [], fails = [];
arr.forEach((item) => (fn(item) ? passes : fails).push(item));
return { passes, fails };
}
/**
* Send triples to reasoning service for conversion
*
*/
async function transformTriples(fetch, triples, mapping) {
return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
}
async function mainConversion(fetch, triples, mapping) {
let formdata = new URLSearchParams();
formdata.append("data", triples);
let requestOptions = {
method: 'POST',
body: formdata,
redirect: 'follow'
};
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
return await response.text();
}
async function transformStatements(fetch, triples, mapping = 'main') {
console.log(`Received ${JSON.stringify(triples)} to transform`);
const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
return statements;
}
async function deleteFromTargetGraph(lib, statements) {
console.log(`Deleting ${statements.length} statements from target graph`);
console.log(`Statements: ${JSON.stringify(statements)}`)
await batchedDbUpdate(
lib.muAuthSudo.updateSudo,
TARGET_GRAPH,
statements,
{},
process.env.MU_SPARQL_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
'DELETE');
}
async function insertIntoTargetGraph(lib, statements) {
console.log(`Inserting ${statements.length} statements into target graph`);
console.log(`Statements: ${JSON.stringify(statements)}`)
await batchedDbUpdate(
lib.muAuthSudo.updateSudo,
TARGET_GRAPH,
statements,
{},
process.env.MU_SPARQL_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
'INSERT');
}
async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
const text = await response.text();
const statements = text.replace(/\n{2,}/g, '').split('\n');
return statements;
}
module.exports = {
batchedDbUpdate,
partition,
transformStatements,
transformLandingZoneGraph,
deleteFromTargetGraph,
insertIntoTargetGraph
};

14
config/loket-consumer/mandatarissen/mapping/queries/dl2op/inheritance/Agent.rq Normal file
View File

@@ -0,0 +1,14 @@
PREFIX dc_terms: <http://purl.org/dc/terms/>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
PREFIX person: <http://www.w3.org/ns/person#>
CONSTRUCT {
?s a dc_terms:Agent .
} WHERE {
?s a ?type .
VALUES ?type {
mandaat:Mandataris
person:Person
}
}

11
config/loket-consumer/mandatarissen/mapping/queries/dl2op/inheritance/AgentInPositie.rq Normal file
View File

@@ -0,0 +1,11 @@
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
CONSTRUCT {
?s a ch:AgentInPositie .
} WHERE {
?s a ?type .
VALUES ?type {
mandaat:Mandataris
}
}

8
config/loket-consumer/mandatarissen/mapping/queries/dl2op/inverse/holds-to-held-by.rq Normal file
View File

@@ -0,0 +1,8 @@
PREFIX org: <http://www.w3.org/ns/org#>
CONSTRUCT {
?o org:heldBy ?s.
}
WHERE {
?s org:holds ?o.
}

25
config/loket-consumer/mandatarissen/mapping/queries/dl2op/pass/mandataris.sparql Normal file
View File

@@ -0,0 +1,25 @@
CONSTRUCT {
?s ?p ?o.
} WHERE {
?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris>;
<http://www.w3.org/ns/org#holds> ?mandate;
?p ?o.
?orgaanInTime <http://www.w3.org/ns/org#hasPost> ?mandate ;
<http://data.vlaanderen.be/ns/mandaat#isTijdspecialisatieVan> ?orgaan .
?orgaan <http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuur .
?bestuur <http://data.vlaanderen.be/ns/besluit#classificatie> ?classification .
FILTER(?classification IN (
<http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000000>,
<http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000001>,
<http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000002>,
<http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/5ab0e9b8a3b2ca7c5e000003>
))
FILTER NOT EXISTS {
?s ?p "" .
}
}

16
config/loket-consumer/mandatarissen/mapping/queries/dl2op/pass/person.sparql Normal file
View File

@@ -0,0 +1,16 @@
CONSTRUCT {
?s ?p ?o.
} WHERE {
?s a <http://www.w3.org/ns/person#Person>;
?p ?o.
FILTER (?p IN (
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>,
<http://xmlns.com/foaf/0.1/familyName>,
<http://mu.semte.ch/vocabularies/core/uuid>
))
FILTER NOT EXISTS {
?s ?p "" .
}
}

21
config/loket-consumer/mandatarissen/mapping/queries/dl2op/predicate/person-foaf-name.rq Normal file
View File

@@ -0,0 +1,21 @@
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
PREFIX person: <http://www.w3.org/ns/person#>
# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)
# First name: persoon:gebruikteVoornaam => foaf:givenName
# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam
# We do both in one query with an OPTIONAL because they need to happen simultaneously
# not to end up in a loop
# Part 1
CONSTRUCT {
?s persoon:gebruikteVoornaam ?name .
} WHERE {
?s a person:Person;
foaf:name ?name .
}

22
config/loket-consumer/mandatarissen/mapping/queries/dl2op/predicate/person-gebruikte-voornaam.rq Normal file
View File

@@ -0,0 +1,22 @@
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
PREFIX person: <http://www.w3.org/ns/person#>
# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)
# First name: persoon:gebruikteVoornaam => foaf:givenName
# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam
# We do both in one query with an OPTIONAL because they need to happen simultaneously
# not to end up in a loop
# Part 2
CONSTRUCT {
?s foaf:givenName ?gebruikteVoornaam .
} WHERE {
?s a person:Person ;
persoon_loket:gebruikteVoornaam ?gebruikteVoornaam .
}

37
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2023/20250122204102-flush-mandatarissen-consumer-containers.sparql Normal file
View File

@@ -0,0 +1,37 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?container ?containerP ?containerO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
task:operation ?operation;
dct:created ?created.
FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime)
?task dct:isPartOf ?job;
task:resultsContainer ?container.
?container a nfo:DataContainer;
?containerP ?containerO.
}
}

34
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2023/20250122204103-flush-mandatarissen-consumer-tasks.sparql Normal file
View File

@@ -0,0 +1,34 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?task ?taskP ?taskO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
dct:created ?created;
task:operation ?operation.
FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime)
?task dct:isPartOf ?job;
?taskP ?taskO.
}
}

32
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2023/20250122204104-flush-mandatarissen-consumer-jobs.sparql Normal file
View File

@@ -0,0 +1,32 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?job ?jobP ?jobO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
dct:created ?created;
task:operation ?operation;
?jobP ?jobO.
FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime)
}
}

37
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2024/20250122204105-flush-mandatarissen-consumer-containers.sparql Normal file
View File

@@ -0,0 +1,37 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?container ?containerP ?containerO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
task:operation ?operation;
dct:created ?created.
FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime)
?task dct:isPartOf ?job;
task:resultsContainer ?container.
?container a nfo:DataContainer;
?containerP ?containerO.
}
}

34
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2024/20250122204106-flush-mandatarissen-consumer-tasks.sparql Normal file
View File

@@ -0,0 +1,34 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?task ?taskP ?taskO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
dct:created ?created;
task:operation ?operation.
FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime)
?task dct:isPartOf ?job;
?taskP ?taskO.
}
}

32
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2024/20250122204107-flush-mandatarissen-consumer-jobs.sparql Normal file
View File

@@ -0,0 +1,32 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?job ?jobP ?jobO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
dct:created ?created;
task:operation ?operation;
?jobP ?jobO.
FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime)
}
}

34
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2025/20250122204108-flush-mandatarissen-consumer-containers.sparql Normal file
View File

@@ -0,0 +1,34 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?container ?containerP ?containerO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
task:operation ?operation.
?task dct:isPartOf ?job;
task:resultsContainer ?container.
?container a nfo:DataContainer;
?containerP ?containerO.
}
}

31
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2025/20250122204109-flush-mandatarissen-consumer-tasks.sparql Normal file
View File

@@ -0,0 +1,31 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?task ?taskP ?taskO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
task:operation ?operation.
?task dct:isPartOf ?job;
?taskP ?taskO.
}
}

29
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204101-flush-old-jobs/2025/20250122204110-flush-mandatarissen-consumer-jobs.sparql Normal file
View File

@@ -0,0 +1,29 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
PREFIX dct: <http://purl.org/dc/terms/>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
PREFIX oslc: <http://open-services.net/ns/core#>
PREFIX cogs: <http://vocab.deri.ie/cogs#>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
PREFIX dbpedia: <http://dbpedia.org/resource/>
DELETE {
GRAPH ?g {
?job ?jobP ?jobO.
}
} WHERE {
VALUES ?operation {
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen>
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen>
}
GRAPH ?g {
?job a <http://vocab.deri.ie/cogs#Job>;
task:operation ?operation;
?jobP ?jobO.
}
}

25
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204111-flush-existing-mandatarissen-data.sparql Normal file
View File

@@ -0,0 +1,25 @@
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
DELETE {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?s a <http://www.w3.org/ns/person#Person>;
?p ?o.
}
} WHERE {
VALUES ?p {
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>
<http://mu.semte.ch/vocabularies/core/uuid>
foaf:givenName
foaf:familyName
<http://data.vlaanderen.be/ns/persoon#gebruikteVoornaam>
}
?mandataris a <http://data.vlaanderen.be/ns/mandaat#Mandataris>;
<http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan> ?s.
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?s a <http://www.w3.org/ns/person#Person>;
?p ?o.
}
}

21
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204112-flush-existing-mandatarissen-data.sparql Normal file
View File

@@ -0,0 +1,21 @@
DELETE {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?s a <http://www.w3.org/ns/org#Membership>;
?p ?o.
}
} WHERE {
VALUES ?p {
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>
<http://mu.semte.ch/vocabularies/core/uuid>
<http://www.w3.org/ns/org#organisation>
}
?mandataris a <http://data.vlaanderen.be/ns/mandaat#Mandataris>;
<http://www.w3.org/ns/org#hasMembership> ?s.
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?s a <http://www.w3.org/ns/org#Membership>;
?p ?o.
}
}

29
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204113-flush-existing-mandatarissen-data.sparql Normal file
View File

@@ -0,0 +1,29 @@
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
DELETE {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris>;
?p ?o.
}
} WHERE {
VALUES ?p {
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>
<http://mu.semte.ch/vocabularies/core/uuid>
<http://www.w3.org/ns/org#holds>
<http://data.vlaanderen.be/ns/mandaat#status>
<http://data.vlaanderen.be/ns/mandaat#start>
<http://data.vlaanderen.be/ns/mandaat#einde>
<http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan>
<http://data.vlaanderen.be/ns/mandaat#isTijdelijkVervangenDoor>
<http://www.w3.org/ns/org#hasMembership>
<http://data.vlaanderen.be/ns/mandaat#beleidsdomein>
<http://data.vlaanderen.be/ns/mandaat#rangorde>
}
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?s a <http://data.vlaanderen.be/ns/mandaat#Mandataris>;
?p ?o.
}
}

1
config/migrations/2025/20250122204100-update-mandatarissen-consumer/20250122204114-flush-landing-graph.sparql Normal file
View File

@@ -0,0 +1 @@
CLEAR GRAPH <http://mu.semte.ch/graphs/landing-zone/mandatarissen>

16
docker-compose.yml
View File

@@ -370,24 +370,26 @@ services:
# LOKET MANDATARISSEN CONSUMERS
################################################################################
mandatarissen-consumer:
image: lblod/delta-consumer:0.0.24
image: lblod/delta-consumer:0.1.0
environment:
DCR_SERVICE_NAME: "mandatarissen-consumer"
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
DCR_SYNC_FILES_PATH: "/sync/mandatarissen/files"
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/MandatarissenCacheGraphDump"
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen"
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen"
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatarissen-consumer"
DCR_DISABLE_INITIAL_SYNC: "true"
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
BATCH_SIZE: 1000
DCR_SYNC_FILES_PATH: "/sync/mandatarissen/files"
DCR_KEEP_DELTA_FILES: "true"
DCR_ENABLE_DELTA_CONTEXT: "true"
DCR_LANDING_ZONE_DATABASE: "triplestore"
DCR_DELTA_JOBS_RETENTION_PERIOD: "30"
DCR_ENABLE_TRIPLE_REMAPPING: "true"
DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/mandatarissen"
DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/administrative-unit"
DCR_BATCH_SIZE: 1000
SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503"
SUDO_QUERY_RETRY: "true"
volumes:
- ./config/loket-consumer/mandatarissen-dispatching:/config/triples-dispatching/custom-dispatching
- ./config/loket-consumer/mandatarissen/mapping:/config/mapping
- ./data/files/consumer-files/mandatarissen:/consumer-files/
restart: always
labels: