Revert "Chore/update leidinggevenden consumer"
This commit is contained in:
Claire Lovisa
GitHub
39
CHANGELOG.md
39
CHANGELOG.md
@@ -1,47 +1,12 @@
|
||||
# Changelog
|
||||
|
||||
## 1.28.5 (TODO)
|
||||
### Frontend
|
||||
- Bump to [v1.28.2](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.2)
|
||||
### Backend
|
||||
- Datafix: correct merger change events for worship organisations (OP-3534)
|
||||
#### Consumer
|
||||
- Upgraded `leidinggevenden-consumer` [OP-3533], [OP-3511]
|
||||
### Deploy notes
|
||||
```
|
||||
drc down;
|
||||
```
|
||||
Update `docker-compose.override.yml` to:
|
||||
```
|
||||
leidinggevenden-consumer:
|
||||
environment:
|
||||
DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint
|
||||
DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
|
||||
DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
|
||||
DCR_DISABLE_DELTA_INGEST: "false"
|
||||
DCR_DISABLE_INITIAL_SYNC: "false"
|
||||
```
|
||||
Then:
|
||||
```
|
||||
drc up -d migrations migrations-triggering-indexing; drc logs -ft --tail=200 migrations migrations-triggering-indexing
|
||||
drc up -d db leidinggevenden-consumer
|
||||
# Wait until success of the previous step
|
||||
```
|
||||
Then, update `docker-compose.override.yml` to:
|
||||
```
|
||||
leidinggevenden-consumer:
|
||||
environment:
|
||||
DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint
|
||||
DCR_LANDING_ZONE_DATABASE: "db"
|
||||
DCR_REMAPPING_DATABASE: "db"
|
||||
DCR_DISABLE_DELTA_INGEST: "false"
|
||||
DCR_DISABLE_INITIAL_SYNC: "false"
|
||||
```
|
||||
```
|
||||
drc pull frontend
|
||||
drc up -d
|
||||
sh scripts/reset-elastic.sh
|
||||
```
|
||||
- `drc restart migrations-triggering-indexing; drc logs -ft --tail=200 migrations-triggering-indexing`
|
||||
- `drc pull frontend; drc up -d frontend`
|
||||
|
||||
## 1.28.4 (2025-01-16)
|
||||
### Backend
|
||||
|
||||
39
README.md
39
README.md
@@ -11,45 +11,6 @@ Backend for the organization portal application, based on the mu.semte.ch micros
|
||||
|
||||
You can shut down using `docker-compose stop` and remove everything using `docker-compose rm`.
|
||||
|
||||
#### Setup consumers
|
||||
The application depends on external data. You'll have to perform extra steps if you want this data to be available.
|
||||
These are multiple data sets, we'll try to be exhaustive on how to ingest these. Even though the procedures are similar.
|
||||
|
||||
#### Setup http://data.lblod.info/vocabularies/leidinggevenden/Functionaris
|
||||
Note: this is assuming you have never ran this before.
|
||||
```
|
||||
drc down;
|
||||
```
|
||||
Update `docker-compose.override.yml` to:
|
||||
```
|
||||
leidinggevenden-consumer:
|
||||
environment:
|
||||
DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint
|
||||
DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
|
||||
DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso
|
||||
DCR_DISABLE_DELTA_INGEST: "false"
|
||||
DCR_DISABLE_INITIAL_SYNC: "false"
|
||||
```
|
||||
Then:
|
||||
```
|
||||
drc up -d migrations
|
||||
drc up -d db leidinggevenden-consumer
|
||||
# Wait until success of the previous step
|
||||
```
|
||||
Then, update `docker-compose.override.yml` to:
|
||||
```
|
||||
leidinggevenden-consumer:
|
||||
environment:
|
||||
DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint
|
||||
DCR_LANDING_ZONE_DATABASE: "db"
|
||||
DCR_REMAPPING_DATABASE: "db"
|
||||
DCR_DISABLE_DELTA_INGEST: "false"
|
||||
DCR_DISABLE_INITIAL_SYNC: "false"
|
||||
```
|
||||
```
|
||||
drc up -d
|
||||
sh scripts/reset-elastic.sh
|
||||
```
|
||||
|
||||
### Setting up the delta-producers related services
|
||||
To make sure the app can share data, producers need to be set up. There is an intial sync, that is potentially very expensive, and must be started manually
|
||||
|
||||
32
config/loket-consumer/leidinggevenden-dispatching/config.js
Normal file
32
config/loket-consumer/leidinggevenden-dispatching/config.js
Normal file
@@ -0,0 +1,32 @@
|
||||
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
|
||||
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
|
||||
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
|
||||
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
|
||||
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
|
||||
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
|
||||
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
|
||||
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
|
||||
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
|
||||
|
||||
// Graphs
|
||||
const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
|
||||
const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
|
||||
|
||||
// ENDPOINTS
|
||||
const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
|
||||
const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
|
||||
|
||||
module.exports = {
|
||||
BATCH_SIZE,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
MAX_REASONING_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
LANDING_ZONE_GRAPH,
|
||||
TARGET_GRAPH,
|
||||
LANDING_ZONE_DATABASE_ENDPOINT
|
||||
};
|
||||
@@ -0,0 +1,72 @@
|
||||
|
||||
const PREFIXES = `
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
PREFIX mandaat: <http://data.vlaanderen.be/ns/mandaat#>
|
||||
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
|
||||
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>`
|
||||
|
||||
const contextConfig = {
|
||||
addTypes: {
|
||||
scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
|
||||
exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
|
||||
},
|
||||
contextQueries: [
|
||||
// Case Leidnggevenden
|
||||
// They can be triggered when being inserted or when their link to a bestuursfunctie is added
|
||||
{
|
||||
trigger: { // subjectType or predicateValue
|
||||
subjectType: "lblodlg:Functionaris"
|
||||
},
|
||||
queryTemplate: (subject) => `
|
||||
${PREFIXES}
|
||||
CONSTRUCT {
|
||||
?functionaris org:holds ?bestuursfunctie ;
|
||||
?p ?o .
|
||||
?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ;
|
||||
mandaat:isTijdspecialisatieVan ?orgaan .
|
||||
?orgaan besluit:bestuurt ?eenheid .
|
||||
?eenheid besluit:classificatie ?classification .
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/landing-zone/leidinggevenden> {
|
||||
VALUES ?functionaris { ${subject} }
|
||||
?functionaris org:holds ?bestuursfunctie ;
|
||||
?p ?o .
|
||||
?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ;
|
||||
mandaat:isTijdspecialisatieVan ?orgaan .
|
||||
?orgaan besluit:bestuurt ?eenheid .
|
||||
?eenheid besluit:classificatie ?classification .
|
||||
}
|
||||
}`
|
||||
},
|
||||
{
|
||||
trigger: {
|
||||
predicateValue: "org:holds"
|
||||
},
|
||||
queryTemplate: (subject) => `
|
||||
${PREFIXES}
|
||||
CONSTRUCT {
|
||||
?functionaris org:holds ?bestuursfunctie ;
|
||||
?p ?o .
|
||||
?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ;
|
||||
mandaat:isTijdspecialisatieVan ?orgaan .
|
||||
?orgaan besluit:bestuurt ?eenheid .
|
||||
?eenheid besluit:classificatie ?classification .
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/landing-zone/leidinggevenden> {
|
||||
VALUES ?functionaris { ${subject} }
|
||||
?functionaris org:holds ?bestuursfunctie ;
|
||||
?p ?o .
|
||||
?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ;
|
||||
mandaat:isTijdspecialisatieVan ?orgaan .
|
||||
?orgaan besluit:bestuurt ?eenheid .
|
||||
?eenheid besluit:classificatie ?classification .
|
||||
}
|
||||
}`
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
contextConfig,
|
||||
PREFIXES
|
||||
};
|
||||
@@ -0,0 +1,85 @@
|
||||
const {
|
||||
transformStatements,
|
||||
deleteFromTargetGraph,
|
||||
insertIntoTargetGraph,
|
||||
} = require('./util');
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* Note: <share://file/data> will be ADDED to it's own graph.
|
||||
* We take only care of adding them, not updating triples, this is a TODO
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, fetch } = lib;
|
||||
let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
|
||||
|
||||
// Both arrays are the same length, so we can zip them together for easier processing
|
||||
zippedChangeSets = termObjectChangeSets.map((o, i) => ({
|
||||
original: o,
|
||||
withContext: termObjectChangeSetsWithContext[i]
|
||||
}));
|
||||
|
||||
console.log(`Received ${zippedChangeSets.length} change sets`)
|
||||
|
||||
for (let { original, withContext } of zippedChangeSets) {
|
||||
let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
|
||||
try {
|
||||
originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
} catch (e) {
|
||||
console.log(`Error while processing change set: ${e}`);
|
||||
console.log(`Original change set: ${JSON.stringify(original)}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
// Extra context needed for mapping from DL to OP model and filtering based on type.
|
||||
try {
|
||||
insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
} catch (e) {
|
||||
console.log(`Error while processing change set with context: ${e}`);
|
||||
console.log(`Change set with context: ${JSON.stringify(withContext)}`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (originalDeletes.length) {
|
||||
// Map deletes from DL to OP model
|
||||
const transformedDeletes = await transformStatements(fetch, deletesWithContext);
|
||||
|
||||
if (!transformedDeletes.length) {
|
||||
console.log(`Warn: Delete statements mapped to empty result.`);
|
||||
console.log(`Input: ${deletesWithContext}`);
|
||||
console.log(`Output: ${transformedDeletes}`);
|
||||
} else {
|
||||
await deleteFromTargetGraph(lib, transformedDeletes);
|
||||
}
|
||||
}
|
||||
|
||||
if (originalInserts.length) {
|
||||
// Map inserts from DL to OP model
|
||||
const transformedInserts = await transformStatements(fetch, insertsWithContext);
|
||||
|
||||
if (!transformedInserts.length) {
|
||||
console.log(`Warn: Insert statements mapped to empty result.`);
|
||||
console.log(`Input: ${insertsWithContext}`);
|
||||
console.log(`Output: ${transformedInserts}`);
|
||||
} else {
|
||||
await insertIntoTargetGraph(lib, transformedInserts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch
|
||||
};
|
||||
@@ -0,0 +1,85 @@
|
||||
const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
|
||||
const {
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
|
||||
DIRECT_DATABASE_ENDPOINT,
|
||||
MU_CALL_SCOPE_ID_INITIAL_SYNC,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
TARGET_GRAPH,
|
||||
LANDING_ZONE_GRAPH,
|
||||
LANDING_ZONE_DATABASE_ENDPOINT
|
||||
} = require('./config');
|
||||
|
||||
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
|
||||
|
||||
/**
|
||||
* Dispatch the fetched information to a target graph.
|
||||
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
|
||||
* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
|
||||
* [ {
|
||||
* graph: "<http://foo>",
|
||||
* subject: "<http://bar>",
|
||||
* predicate: "<http://baz>",
|
||||
* object: "<http://boom>^^<http://datatype>"
|
||||
* }
|
||||
* ]
|
||||
* @return {void} Nothing
|
||||
*/
|
||||
async function dispatch(lib, data) {
|
||||
const { mu, muAuthSudo } = lib;
|
||||
const { termObjects } = data;
|
||||
|
||||
// Steps:
|
||||
// 1. Write the triples to the landing zone graph - withouth any mapping or filtering
|
||||
// 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
|
||||
|
||||
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
|
||||
console.warn(`Service configured to skip MU_AUTH!`);
|
||||
}
|
||||
console.log(`Using ${endpoint} to insert triples`);
|
||||
|
||||
if (termObjects.length) {
|
||||
const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
|
||||
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
LANDING_ZONE_GRAPH,
|
||||
originalInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
LANDING_ZONE_DATABASE_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function onFinishInitialIngest(lib) {
|
||||
const { muAuthSudo, fetch } = lib;
|
||||
|
||||
console.log(`!! On-finish triggered !!`);
|
||||
|
||||
const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint);
|
||||
|
||||
console.log(`Transformed ${transformedInsertTriples.length} triples`);
|
||||
|
||||
await batchedDbUpdate(
|
||||
muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
transformedInsertTriples,
|
||||
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
|
||||
endpoint,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dispatch,
|
||||
onFinishInitialIngest
|
||||
};
|
||||
168
config/loket-consumer/leidinggevenden-dispatching/util.js
Normal file
168
config/loket-consumer/leidinggevenden-dispatching/util.js
Normal file
@@ -0,0 +1,168 @@
|
||||
const {
|
||||
MAX_REASONING_RETRY_ATTEMPTS,
|
||||
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
TARGET_GRAPH,
|
||||
LANDING_ZONE_GRAPH,
|
||||
BATCH_SIZE,
|
||||
} = require('./config');
|
||||
|
||||
async function batchedDbUpdate(
|
||||
muUpdate,
|
||||
graph,
|
||||
triples,
|
||||
extraHeaders,
|
||||
endpoint,
|
||||
batchSize,
|
||||
maxAttempts,
|
||||
sleepBetweenBatches = 1000,
|
||||
sleepTimeOnFail = 1000,
|
||||
operation = 'INSERT'
|
||||
) {
|
||||
|
||||
for (let i = 0; i < triples.length; i += batchSize) {
|
||||
console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
|
||||
|
||||
const batch = triples.slice(i, i + batchSize).join('\n');
|
||||
|
||||
const insertCall = async () => {
|
||||
await muUpdate(`
|
||||
${operation} DATA {
|
||||
GRAPH <${graph}> {
|
||||
${batch}
|
||||
}
|
||||
}
|
||||
`, extraHeaders, endpoint);
|
||||
};
|
||||
|
||||
await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
|
||||
|
||||
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
|
||||
await new Promise(r => setTimeout(r, sleepBetweenBatches));
|
||||
}
|
||||
}
|
||||
|
||||
async function operationWithRetry(callback,
|
||||
attempt,
|
||||
maxAttempts,
|
||||
sleepTimeOnFail) {
|
||||
try {
|
||||
if (typeof callback === "function")
|
||||
return await callback();
|
||||
else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
|
||||
return await callback;
|
||||
}
|
||||
catch (e) {
|
||||
console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
|
||||
console.log(`Error: ${e}`);
|
||||
console.log(`Sleeping ${sleepTimeOnFail} ms`);
|
||||
|
||||
if (attempt >= maxAttempts) {
|
||||
console.log(`Max attempts reached for ${callback.toString()}, giving up`);
|
||||
throw e;
|
||||
}
|
||||
|
||||
await new Promise(r => setTimeout(r, sleepTimeOnFail));
|
||||
return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits an array into two parts, a part that passes and a part that fails a predicate function.
|
||||
* Credits: https://github.com/benjay10
|
||||
* @public
|
||||
* @function partition
|
||||
* @param {Array} arr - Array to be partitioned
|
||||
* @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
|
||||
* @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
|
||||
*/
|
||||
function partition(arr, fn) {
|
||||
let passes = [], fails = [];
|
||||
arr.forEach((item) => (fn(item) ? passes : fails).push(item));
|
||||
return { passes, fails };
|
||||
}
|
||||
|
||||
/**
|
||||
* Send triples to reasoning service for conversion
|
||||
*
|
||||
*/
|
||||
async function transformTriples(fetch, triples, mapping) {
|
||||
return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
|
||||
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
|
||||
}
|
||||
|
||||
async function mainConversion(fetch, triples, mapping) {
|
||||
let formdata = new URLSearchParams();
|
||||
formdata.append("data", triples);
|
||||
|
||||
let requestOptions = {
|
||||
method: 'POST',
|
||||
body: formdata,
|
||||
redirect: 'follow'
|
||||
};
|
||||
|
||||
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
|
||||
return await response.text();
|
||||
}
|
||||
|
||||
async function transformStatements(fetch, triples, mapping = 'main') {
|
||||
console.log(`Received ${JSON.stringify(triples)} to transform`);
|
||||
const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
|
||||
statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
|
||||
console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
|
||||
return statements;
|
||||
}
|
||||
|
||||
async function deleteFromTargetGraph(lib, statements) {
|
||||
console.log(`Deleting ${statements.length} statements from target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'DELETE');
|
||||
}
|
||||
|
||||
async function insertIntoTargetGraph(lib, statements) {
|
||||
console.log(`Inserting ${statements.length} statements into target graph`);
|
||||
console.log(`Statements: ${JSON.stringify(statements)}`)
|
||||
|
||||
await batchedDbUpdate(
|
||||
lib.muAuthSudo.updateSudo,
|
||||
TARGET_GRAPH,
|
||||
statements,
|
||||
{},
|
||||
process.env.MU_SPARQL_ENDPOINT,
|
||||
BATCH_SIZE,
|
||||
MAX_DB_RETRY_ATTEMPTS,
|
||||
SLEEP_BETWEEN_BATCHES,
|
||||
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
|
||||
'INSERT');
|
||||
}
|
||||
|
||||
async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
|
||||
console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
|
||||
|
||||
const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
|
||||
const text = await response.text();
|
||||
const statements = text.replace(/\n{2,}/g, '').split('\n');
|
||||
|
||||
return statements;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
batchedDbUpdate,
|
||||
partition,
|
||||
transformStatements,
|
||||
transformLandingZoneGraph,
|
||||
deleteFromTargetGraph,
|
||||
insertIntoTargetGraph
|
||||
};
|
||||
@@ -1,21 +0,0 @@
|
||||
PREFIX schema: <http://schema.org/>
|
||||
|
||||
CONSTRUCT {
|
||||
?s schema:telephone ?cleanedPhoneNumber .
|
||||
} WHERE {
|
||||
?s schema:telephone ?phoneNumber .
|
||||
|
||||
# Remove all non digits
|
||||
BIND(REPLACE(?phoneNumber, """[^+\\d]""", "") AS ?scrapeResult)
|
||||
|
||||
# Replace leading 00 with +
|
||||
BIND(REPLACE(?scrapeResult, """^00""", "+") AS ?replacedZeroZero)
|
||||
|
||||
BIND(REPLACE(?replacedZeroZero, """^0""", "+32") AS ?defaultBe)
|
||||
|
||||
BIND(REPLACE(?defaultBe, """\\+320""", "+32") AS ?extendedDefaultBe)
|
||||
|
||||
BIND(REPLACE(?extendedDefaultBe, """^320""", "+32") AS ?finalResult)
|
||||
|
||||
BIND(IF(REGEX(?finalResult, """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$"""), CONCAT("tel:", ?finalResult), ?phoneNumber) AS ?cleanedPhoneNumber)
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
PREFIX dc_terms: <http://purl.org/dc/terms/>
|
||||
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
|
||||
PREFIX person: <http://www.w3.org/ns/person#>
|
||||
|
||||
CONSTRUCT {
|
||||
?s a dc_terms:Agent .
|
||||
} WHERE {
|
||||
?s a ?type .
|
||||
|
||||
VALUES ?type {
|
||||
lblodlg:Functionaris
|
||||
person:Person
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
PREFIX ch: <http://data.lblod.info/vocabularies/contacthub/>
|
||||
PREFIX lblodlg: <http://data.lblod.info/vocabularies/leidinggevenden/>
|
||||
|
||||
CONSTRUCT {
|
||||
?s a ch:AgentInPositie .
|
||||
} WHERE {
|
||||
?s a ?type .
|
||||
VALUES ?type {
|
||||
lblodlg:Functionaris
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
PREFIX org: <http://www.w3.org/ns/org#>
|
||||
|
||||
CONSTRUCT {
|
||||
?o org:heldBy ?s.
|
||||
}
|
||||
WHERE {
|
||||
?s org:holds ?o.
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
CONSTRUCT {
|
||||
?s ?p ?o.
|
||||
} WHERE {
|
||||
?s a <http://www.w3.org/ns/locn#Address>;
|
||||
?p ?o.
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?s ?p "" .
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
CONSTRUCT {
|
||||
?s <http://schema.org/contactPoint> ?contactpoint.
|
||||
}
|
||||
WHERE {
|
||||
?s a <http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie>.
|
||||
?s <http://schema.org/contactPoint> ?contactpoint.
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
CONSTRUCT {
|
||||
?s ?p ?o.
|
||||
} WHERE {
|
||||
?s a <http://schema.org/ContactPoint>;
|
||||
?p ?o.
|
||||
FILTER (?p NOT IN (<http://schema.org/telephone>))
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?s ?p "" .
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
CONSTRUCT {
|
||||
?s ?p ?o.
|
||||
} WHERE {
|
||||
?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris>;
|
||||
?p ?o.
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?s ?p "" .
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
CONSTRUCT {
|
||||
?s ?p ?o.
|
||||
} WHERE {
|
||||
?s a <http://www.w3.org/ns/person#Person>;
|
||||
?p ?o.
|
||||
|
||||
FILTER (?p IN (
|
||||
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>,
|
||||
<http://xmlns.com/foaf/0.1/familyName>,
|
||||
<http://mu.semte.ch/vocabularies/core/uuid>
|
||||
))
|
||||
|
||||
FILTER NOT EXISTS {
|
||||
?s ?p "" .
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX person: <http://www.w3.org/ns/person#>
|
||||
|
||||
# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)
|
||||
|
||||
# First name: persoon:gebruikteVoornaam => foaf:givenName
|
||||
# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam
|
||||
|
||||
# We do both in one query with an OPTIONAL because they need to happen simultaneously
|
||||
# not to end up in a loop
|
||||
|
||||
# Part 1
|
||||
|
||||
CONSTRUCT {
|
||||
?s persoon:gebruikteVoornaam ?name .
|
||||
} WHERE {
|
||||
?s a person:Person;
|
||||
foaf:name ?name .
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
PREFIX persoon: <https://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX persoon_loket: <http://data.vlaanderen.be/ns/persoon#>
|
||||
PREFIX person: <http://www.w3.org/ns/person#>
|
||||
|
||||
# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!)
|
||||
|
||||
# First name: persoon:gebruikteVoornaam => foaf:givenName
|
||||
# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam
|
||||
|
||||
# We do both in one query with an OPTIONAL because they need to happen simultaneously
|
||||
# not to end up in a loop
|
||||
|
||||
# Part 2
|
||||
|
||||
CONSTRUCT {
|
||||
?s foaf:givenName ?gebruikteVoornaam .
|
||||
} WHERE {
|
||||
?s a person:Person ;
|
||||
persoon_loket:gebruikteVoornaam ?gebruikteVoornaam .
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://www.w3.org/ns/person#Person>;
|
||||
?p ?o.
|
||||
}
|
||||
} WHERE {
|
||||
|
||||
VALUES ?p {
|
||||
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>
|
||||
<http://mu.semte.ch/vocabularies/core/uuid>
|
||||
foaf:givenName
|
||||
foaf:familyName
|
||||
<http://data.vlaanderen.be/ns/persoon#gebruikteVoornaam>
|
||||
}
|
||||
|
||||
?functionaris a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris>;
|
||||
<http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan> ?s.
|
||||
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://www.w3.org/ns/person#Person>;
|
||||
?p ?o.
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris>;
|
||||
?p ?o.
|
||||
}
|
||||
|
||||
} WHERE {
|
||||
|
||||
VALUES ?p {
|
||||
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>
|
||||
<http://mu.semte.ch/vocabularies/core/uuid>
|
||||
<http://www.w3.org/ns/org#holds>
|
||||
<http://data.vlaanderen.be/ns/mandaat#status>
|
||||
<http://data.vlaanderen.be/ns/mandaat#start>
|
||||
<http://data.vlaanderen.be/ns/mandaat#einde>
|
||||
<http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan>
|
||||
}
|
||||
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://data.lblod.info/vocabularies/leidinggevenden/Functionaris>;
|
||||
?p ?o.
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://www.w3.org/ns/locn#Address>;
|
||||
?p ?o.
|
||||
}
|
||||
|
||||
} WHERE {
|
||||
|
||||
VALUES ?p {
|
||||
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>
|
||||
<http://mu.semte.ch/vocabularies/core/uuid>
|
||||
<https://data.vlaanderen.be/ns/adres#verwijstNaar>
|
||||
<https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer>
|
||||
<http://www.w3.org/ns/locn#postCode>
|
||||
<http://www.w3.org/ns/locn#thoroughfare>
|
||||
<http://www.w3.org/ns/locn#fullAddress>
|
||||
<https://data.vlaanderen.be/ns/adres#gemeentenaam>
|
||||
<https://data.vlaanderen.be/ns/adres#land>
|
||||
<http://www.w3.org/ns/locn#adminUnit1>
|
||||
<http://www.w3.org/ns/locn#adminUnit2>
|
||||
<http://www.w3.org/ns/locn#addressArea>
|
||||
<https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer>
|
||||
}
|
||||
|
||||
?bestuursfunctie a <http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie>;
|
||||
<http://schema.org/contactPoint> ?contactpoint.
|
||||
|
||||
?contactpoint a <http://schema.org/ContactPoint>;
|
||||
<http://www.w3.org/ns/locn#address> ?s.
|
||||
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://www.w3.org/ns/locn#Address>;
|
||||
?p ?o.
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://schema.org/ContactPoint>;
|
||||
?p ?o.
|
||||
}
|
||||
|
||||
} WHERE {
|
||||
|
||||
VALUES ?p {
|
||||
<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>
|
||||
<http://mu.semte.ch/vocabularies/core/uuid>
|
||||
<http://schema.org/telephone>
|
||||
<http://schema.org/email>
|
||||
<http://www.w3.org/ns/locn#address>
|
||||
<http://schema.org/faxNumber>
|
||||
<http://schema.org/url>
|
||||
}
|
||||
|
||||
?bestuursfunctie a <http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie>;
|
||||
<http://schema.org/contactPoint> ?s.
|
||||
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?s a <http://schema.org/ContactPoint>;
|
||||
?p ?o.
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
|
||||
|
||||
DELETE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?bestuursfunctie <http://schema.org/contactPoint> ?contactpoint.
|
||||
}
|
||||
|
||||
} WHERE {
|
||||
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
|
||||
?bestuursfunctie a <http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie>;
|
||||
<http://schema.org/contactPoint> ?contactpoint.
|
||||
}
|
||||
}
|
||||
@@ -1 +0,0 @@
|
||||
CLEAR GRAPH <http://mu.semte.ch/graphs/landing-zone/leidinggevenden>
|
||||
@@ -1,37 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?container ?containerP ?containerO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
task:operation ?operation;
|
||||
dct:created ?created.
|
||||
|
||||
FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime)
|
||||
|
||||
?task dct:isPartOf ?job;
|
||||
task:resultsContainer ?container.
|
||||
|
||||
?container a nfo:DataContainer;
|
||||
?containerP ?containerO.
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?task ?taskP ?taskO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
dct:created ?created;
|
||||
task:operation ?operation.
|
||||
|
||||
FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime)
|
||||
|
||||
?task dct:isPartOf ?job;
|
||||
?taskP ?taskO.
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?job ?jobP ?jobO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
dct:created ?created;
|
||||
task:operation ?operation;
|
||||
?jobP ?jobO.
|
||||
|
||||
FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime)
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?container ?containerP ?containerO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
task:operation ?operation;
|
||||
dct:created ?created.
|
||||
|
||||
FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime)
|
||||
|
||||
?task dct:isPartOf ?job;
|
||||
task:resultsContainer ?container.
|
||||
|
||||
?container a nfo:DataContainer;
|
||||
?containerP ?containerO.
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?task ?taskP ?taskO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
dct:created ?created;
|
||||
task:operation ?operation.
|
||||
|
||||
FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime)
|
||||
|
||||
?task dct:isPartOf ?job;
|
||||
?taskP ?taskO.
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?job ?jobP ?jobO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
dct:created ?created;
|
||||
task:operation ?operation;
|
||||
?jobP ?jobO.
|
||||
|
||||
FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime)
|
||||
}
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?container ?containerP ?containerO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
task:operation ?operation.
|
||||
|
||||
?task dct:isPartOf ?job;
|
||||
task:resultsContainer ?container.
|
||||
|
||||
?container a nfo:DataContainer;
|
||||
?containerP ?containerO.
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?task ?taskP ?taskO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
task:operation ?operation.
|
||||
|
||||
?task dct:isPartOf ?job;
|
||||
?taskP ?taskO.
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
|
||||
PREFIX task: <http://redpencil.data.gift/vocabularies/tasks/>
|
||||
PREFIX dct: <http://purl.org/dc/terms/>
|
||||
PREFIX prov: <http://www.w3.org/ns/prov#>
|
||||
PREFIX nie: <http://www.semanticdesktop.org/ontologies/2007/01/19/nie#>
|
||||
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>
|
||||
PREFIX oslc: <http://open-services.net/ns/core#>
|
||||
PREFIX cogs: <http://vocab.deri.ie/cogs#>
|
||||
PREFIX adms: <http://www.w3.org/ns/adms#>
|
||||
PREFIX nfo: <http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#>
|
||||
PREFIX dbpedia: <http://dbpedia.org/resource/>
|
||||
|
||||
|
||||
DELETE {
|
||||
GRAPH ?g {
|
||||
?job ?jobP ?jobO.
|
||||
}
|
||||
} WHERE {
|
||||
VALUES ?operation {
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden>
|
||||
<http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden>
|
||||
}
|
||||
|
||||
GRAPH ?g {
|
||||
?job a <http://vocab.deri.ie/cogs#Job>;
|
||||
task:operation ?operation;
|
||||
?jobP ?jobO.
|
||||
}
|
||||
}
|
||||
@@ -401,26 +401,24 @@ services:
|
||||
# LOKET LEIDINGGEVENDEN CONSUMERS
|
||||
################################################################################
|
||||
leidinggevenden-consumer:
|
||||
image: lblod/delta-consumer:0.1.0
|
||||
image: lblod/delta-consumer:0.0.24
|
||||
environment:
|
||||
DCR_SERVICE_NAME: "leidinggevenden-consumer"
|
||||
DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
|
||||
DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files"
|
||||
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/LeidinggevendenCacheGraphDump"
|
||||
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden"
|
||||
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden"
|
||||
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/leidinggevenden-consumer"
|
||||
DCR_DISABLE_INITIAL_SYNC: "true"
|
||||
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
|
||||
DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files"
|
||||
DCR_KEEP_DELTA_FILES: "true"
|
||||
DCR_DELTA_JOBS_RETENTION_PERIOD: "30"
|
||||
DCR_ENABLE_TRIPLE_REMAPPING: "true"
|
||||
BATCH_SIZE: 1000
|
||||
DCR_ENABLE_DELTA_CONTEXT: "true"
|
||||
DCR_LANDING_ZONE_DATABASE: "triplestore"
|
||||
DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/leidinggevenden"
|
||||
DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/administrative-unit"
|
||||
DCR_BATCH_SIZE: 1000
|
||||
SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503"
|
||||
SUDO_QUERY_RETRY: "true"
|
||||
volumes:
|
||||
- ./config/loket-consumer/leidinggevenden/mapping:/config/mapping
|
||||
- ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching
|
||||
- ./data/files/consumer-files/leidinggevenden:/consumer-files/
|
||||
restart: always
|
||||
labels:
|
||||
|
||||
Reference in New Issue
Block a user