felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added first version of the clb consumer

Browse Source
This commit is contained in:
Oscar Rodriguez Villalobos
2024-02-16 13:09:48 +01:00
parent a12293e460
commit 64044b2ce0
6 changed files with 847 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
config/contact-data/config.js Normal file
View File

@@ -0,0 +1,32 @@
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 1000;
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 100);
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
// Graphs
const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/organizations`;
const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/op`;
// ENDPOINTS
const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'database';
const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
module.exports = {
BATCH_SIZE,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MAX_DB_RETRY_ATTEMPTS,
MAX_REASONING_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
LANDING_ZONE_GRAPH,
TARGET_GRAPH,
LANDING_ZONE_DATABASE_ENDPOINT
};

256
config/contact-data/delta-context-config.js Normal file
View File

@@ -0,0 +1,256 @@
const PREFIXES = `
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX lblodgeneriek: <https://data.lblod.info/vocabularies/generiek/>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX schema: <http://schema.org/>
PREFIX locn: <http://www.w3.org/ns/locn#>
PREFIX ext: <http://mu.semte.ch/vocabularies/ext/>`
const contextConfig = {
addTypes: {
scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
},
contextQueries: [
{
trigger: { // subjectType or predicateValue
subjectType: "org:Site"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
${subject} ext:goesInGraph ?graphOne.
${subject} ext:goesInGraph ?graphTwo.
${subject} ext:goesInGraph ?graphThree.
${subject} ext:goesInGraph ?graphFour.
} WHERE {
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?adminUnitOne org:hasPrimarySite ${subject};
}
BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
}
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?adminUnitTwo org:hasSite ${subject};
}
BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphTwo)
}
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?adminUnitThree org:hasPrimarySite ${subject};
}
BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphThree)
}
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?adminUnitFour org:hasSite ${subject};
}
BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphFour)
}
}`
},
{
trigger: { // subjectType or predicateValue
subjectType: "schema:ContactPoint"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
${subject} ext:goesInGraph ?graphOne.
${subject} ext:goesInGraph ?graphTwo.
} WHERE {
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?siteOne org:siteAddress ${subject}.
}
BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
}
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?siteTwo org:siteAddress ${subject}.
}
BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphTwo)
}
}`
},
{
trigger: { // subjectType or predicateValue
subjectType: "locn:Address"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
${subject} ext:goesInGraph ?graphOne.
${subject} ext:goesInGraph ?graphTwo.
} WHERE {
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?siteOne organisatie:bestaatUit ${subject}.
}
BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
}
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?siteTwo organisatie:bestaatUit ${subject}.
}
BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphTwo)
}
}`
},
//Predicate triggers to solve out of order deltas
{
trigger: { // subjectType or predicateValue
predicateValue: "org:hasSite"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
${subject} ext:contextDataGoesInGraph ?graphOne.
${subject} ext:contextDataGoesInGraph ?graphTwo.
?site ?pSite ?oSite.
?contact ?pContact ?oContact.
?address ?pAddress ?oAddress.
?addressContact ?pAddressContact ?oAddressContact.
} WHERE {
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
${subject} a ?type.
}
BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
}
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
${subject} a ?type.
}
BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphTwo)
}
${subject} org:hasSite ?site.
?site ?pSite ?oSite.
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
?contact locn:address ?addressContact.
?addressContact ?pAddressContact ?oAddressContact.
}
UNION
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
}
UNION
{
?site organisatie:bestaatUit ?address.
?address ?pAddress ?oAddress.
}
BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph)
}`
},
{
trigger: { // subjectType or predicateValue
predicateValue: "org:hasPrimarySite"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
${subject} ext:contextDataGoesInGraph ?graphOne.
${subject} ext:contextDataGoesInGraph ?graphTwo.
?site ?pSite ?oSite.
?contact ?pContact ?oContact.
?address ?pAddress ?oAddress.
?addressContact ?pAddressContact ?oAddressContact.
} WHERE {
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
${subject} a ?type.
}
BIND(<http://mu.semte.ch/graphs/administrative-unit> AS ?graphOne)
}
OPTIONAL {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
${subject} a ?type.
}
BIND(<http://mu.semte.ch/graphs/worship-service> AS ?graphTwo)
}
${subject} org:hasPrimarySite ?site.
?site ?pSite ?oSite.
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
?contact locn:address ?addressContact.
?addressContact ?pAddressContact ?oAddressContact.
}
UNION
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
}
UNION
{
?site organisatie:bestaatUit ?address.
?address ?pAddress ?oAddress.
}
BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph)
}`
},
{
trigger: { // subjectType or predicateValue
predicateValue: "org:siteAddress"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
?contact ?pContact ?oContact.
?addressContact ?pAddressContact ?oAddressContact.
} WHERE {
${subject} org:siteAddress ?contact.
?contact ?pContact ?oContact.
OPTIONAL {
?contact locn:address ?addressContact.
?addressContact ?pAddressContact ?oAddressContact.
}
}`
},
{
trigger: { // subjectType or predicateValue
predicateValue: "locn:address"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
?addressContact ?pAddressContact ?oAddressContact.
} WHERE {
${subject} locn:address ?addressContact.
?addressContact ?pAddressContact ?oAddressContact.
}`
},
{
trigger: { // subjectType or predicateValue
predicateValue: "organisatie:bestaatUit"
},
queryTemplate: (subject) => `
${PREFIXES}
CONSTRUCT {
?address ?pAddress ?oAddress.
} WHERE {
${subject} organisatie:bestaatUit ?address.
?address ?pAddress ?oAddress.
}`
}
]
}
module.exports = {
contextConfig,
PREFIXES
};

122
config/contact-data/delta-sync-dispatching.js Normal file
View File

@@ -0,0 +1,122 @@
const {
deleteFromPublicGraph,
insertIntoPublicGraph,
deleteFromSpecificGraphs,
insertIntoSpecificGraphs
} = require('./util');
const {
LANDING_ZONE_GRAPH
} = require('./config')
/**
* Dispatch the fetched information to a target graph.
* Note: <share://file/data> will be ADDED to it's own graph.
* We take only care of adding them, not updating triples, this is a TODO
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
* [ {
* graph: "<http://foo>",
* subject: "<http://bar>",
* predicate: "<http://baz>",
* object: "<http://boom>^^<http://datatype>"
* }
* ]
* @return {void} Nothing
*/
const publicTypes = [
'<http://lblod.data.gift/vocabularies/organisatie/TypeVestiging>'
]
async function dispatch(lib, data) {
const { mu, fetch } = lib;
let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
// Both arrays are the same length, so we can zip them together for easier processing
zippedChangeSets = termObjectChangeSets.map((o, i) => ({
original: o,
withContext: termObjectChangeSetsWithContext && termObjectChangeSetsWithContext[i]
}));
console.log(`Received ${zippedChangeSets.length} change sets`)
for (let { original, withContext } of zippedChangeSets) {
const insertsOnPublic = [];
const insertsOnGraphs = {};
for(let insert of original.inserts) {
const subject = insert.subject;
const contextTriples = withContext.inserts.filter((context) => context.subject === subject);
const graphTriple = contextTriples.find((context) => context.predicate === '<http://mu.semte.ch/vocabularies/ext/goesInGraph>' && context.object !== `<${LANDING_ZONE_GRAPH}>`)
const otherContextTriples = withContext.inserts.filter((context) => context.predicate !== '<http://mu.semte.ch/vocabularies/ext/goesInGraph>' && context.predicate !== '<http://mu.semte.ch/vocabularies/ext/contextDataGoesInGraph>' && !(context.subject === subject && context.predicate === '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>'));
if(graphTriple) {
const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">"
if(!insertsOnGraphs[graph]) {
insertsOnGraphs[graph] = [`${insert.subject} ${insert.predicate} ${insert.object}.`]
} else {
insertsOnGraphs[graph].push(`${insert.subject} ${insert.predicate} ${insert.object}.`)
}
for(let triple of otherContextTriples) {
insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`)
}
}
const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>');
if(typeTriple) {
const type = typeTriple.object;
if(publicTypes.includes(type)) {
insertsOnPublic.push(`${insert.subject} ${insert.predicate} ${insert.object}.`)
//For the case where the subject is an admin unit but everything else must go in private graph
const contextGraphTriple = contextTriples.find((context) => context.predicate === '<http://mu.semte.ch/vocabularies/ext/contextDataGoesInGraph>' && context.object !== `<${LANDING_ZONE_GRAPH}>`)
for(let triple of otherContextTriples) {
if(contextGraphTriple) {
const graph = contextGraphTriple.object.slice(1,-1);
if(!insertsOnGraphs[graph]) {
insertsOnGraphs[graph] = [`${triple.subject} ${triple.predicate} ${triple.object}.`]
} else {
insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`)
}
} else {
insertsOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`)
}
}
}
}
}
const deletesOnPublic = [];
const deletesOnGraphs = {};
for(let deletion of original.deletes) {
const subject = deletion.subject;
const contextTriples = withContext.deletes.filter((context) => context.subject === subject);
const graphTriple = contextTriples.find((context) => context.predicate === '<http://mu.semte.ch/vocabularies/ext/goesInGraph>' && context.object !== `<${LANDING_ZONE_GRAPH}>`)
const otherContextTriples = withContext.deletes.filter((context) => context.predicate !== '<http://mu.semte.ch/vocabularies/ext/goesInGraph>' && context.predicate !== '<http://mu.semte.ch/vocabularies/ext/contextDataGoesInGraph>' && !(context.subject === subject && context.predicate === '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>'));
if(graphTriple) {
const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">"
if(!deletesOnGraphs[graph]) {
deletesOnGraphs[graph] = [`${deletion.subject} ${deletion.predicate} ${deletion.object}.`]
} else {
deletesOnGraphs[graph].push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`)
}
for(let triple of otherContextTriples) {
deletesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`)
}
}
const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === '<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>');
if(!typeTriple) continue;
const type = typeTriple.object;
if(publicTypes.includes(type)) {
deletesOnPublic.push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`)
for(let triple of otherContextTriples) {
deletesOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`)
}
}
}
await deleteFromPublicGraph(lib, deletesOnPublic);
await deleteFromSpecificGraphs(lib, deletesOnGraphs);
await insertIntoPublicGraph(lib, insertsOnPublic);
await insertIntoSpecificGraphs(lib, insertsOnGraphs);
}
}
module.exports = {
dispatch
};

69
config/contact-data/initial-sync-dispatching.js Normal file
View File

@@ -0,0 +1,69 @@
const { batchedDbUpdate, moveDataAfterInitialSync } = require('./util');
const {
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
LANDING_ZONE_GRAPH,
} = require('./config');
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
/**
* Dispatch the fetched information to a target graph.
* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
* [ {
* graph: "<http://foo>",
* subject: "<http://bar>",
* predicate: "<http://baz>",
* object: "<http://boom>^^<http://datatype>"
* }
* ]
* @return {void} Nothing
*/
async function dispatch(lib, data) {
const { mu, muAuthSudo } = lib;
const { termObjects } = data;
// Steps:
// 1. Write the triples to the landing zone graph - withouth any mapping or filtering
// 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
console.warn(`Service configured to skip MU_AUTH!`);
}
console.log(`Using ${endpoint} to insert triples`);
if (termObjects.length) {
const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
await batchedDbUpdate(
muAuthSudo.updateSudo,
LANDING_ZONE_GRAPH,
originalInsertTriples,
{ 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
endpoint,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
);
}
}
async function onFinishInitialIngest(lib,data) {
const { muAuthSudo } = lib;
console.log(`!! On-finish triggered !!`);
await moveDataAfterInitialSync(muAuthSudo.updateSudo, endpoint)
}
module.exports = {
dispatch,
onFinishInitialIngest
};

337
config/contact-data/util.js Normal file
View File

@@ -0,0 +1,337 @@
const {
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
LANDING_ZONE_GRAPH,
BATCH_SIZE,
} = require('./config');
const prefixes = `
PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
PREFIX lblodgeneriek: <https://data.lblod.info/vocabularies/generiek/>
PREFIX org: <http://www.w3.org/ns/org#>
PREFIX code: <http://lblod.data.gift/vocabularies/organisatie/>
PREFIX adms: <http://www.w3.org/ns/adms#>
PREFIX generiek: <https://data.vlaanderen.be/ns/generiek#>
PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
PREFIX organisatie: <https://data.vlaanderen.be/ns/organisatie#>
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX euvoc: <http://publications.europa.eu/ontology/euvoc#>
PREFIX prov: <http://www.w3.org/ns/prov#>
PREFIX schema: <http://schema.org/>
PREFIX locn: <http://www.w3.org/ns/locn#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX ext:<http://mu.semte.ch/vocabularies/ext/>
`
async function batchedDbUpdate(
muUpdate,
graph,
triples,
extraHeaders,
endpoint,
batchSize,
maxAttempts,
sleepBetweenBatches = 1000,
sleepTimeOnFail = 1000,
operation = 'INSERT'
) {
for (let i = 0; i < triples.length; i += batchSize) {
console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
const batch = triples.slice(i, i + batchSize).join('\n');
const insertCall = async () => {
await muUpdate(`
${operation} DATA {
GRAPH <${graph}> {
${batch}
}
}
`, extraHeaders, endpoint);
};
await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
await new Promise(r => setTimeout(r, sleepBetweenBatches));
}
}
async function operationWithRetry(callback,
attempt,
maxAttempts,
sleepTimeOnFail) {
try {
if (typeof callback === "function")
return await callback();
else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
return await callback;
}
catch (e) {
console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
console.log(`Error: ${e}`);
console.log(`Sleeping ${sleepTimeOnFail} ms`);
if (attempt >= maxAttempts) {
console.log(`Max attempts reached for ${callback.toString()}, giving up`);
throw e;
}
await new Promise(r => setTimeout(r, sleepTimeOnFail));
return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
}
}
async function insertIntoPublicGraph(lib, statements) {
console.log(`Inserting ${statements.length} statements into public graph`);
await batchedDbUpdate(
lib.muAuthSudo.updateSudo,
'http://mu.semte.ch/graphs/public',
statements,
{},
process.env.MU_SPARQL_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
'INSERT');
}
async function insertIntoSpecificGraphs(lib, statementsWithGraphs) {
for( let graph in statementsWithGraphs) {
console.log(`Inserting ${statementsWithGraphs[graph].length} statements into ${graph} graph`);
await batchedDbUpdate(
lib.muAuthSudo.updateSudo,
graph,
statementsWithGraphs[graph],
{},
process.env.MU_SPARQL_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
'INSERT');
}
}
async function deleteFromPublicGraph(lib, statements) {
console.log(`Deleting ${statements.length} statements from public graph`);
await batchedDbUpdate(
lib.muAuthSudo.updateSudo,
'http://mu.semte.ch/graphs/public',
statements,
{},
process.env.MU_SPARQL_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
'DELETE');
}
async function deleteFromSpecificGraphs(lib, statementsWithGraphs) {
for( let graph in statementsWithGraphs) {
console.log(`Deleting ${statementsWithGraphs[graph].length} statements from ${graph} graph`);
await batchedDbUpdate(
lib.muAuthSudo.updateSudo,
graph,
statementsWithGraphs[graph],
{},
process.env.MU_SPARQL_ENDPOINT,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
'DELETE');
}
}
async function moveDataAfterInitialSync(muUpdate, endpoint) {
// Move primary sites admin units
await muUpdate(`
${prefixes}
INSERT {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?adminUnit org:hasPrimarySite ?site.
?site ?pSite ?oSite.
?contact ?pContact ?oContact.
?address ?pAddress ?oAddress.
?addressContact ?pAddressContact ?oAddressContact.
}
} WHERE {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?adminUnit a ?type.
}
GRAPH <${LANDING_ZONE_GRAPH}> {
?adminUnit org:hasPrimarySite ?site.
?site ?pSite ?oSite.
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
?contact locn:address ?addressContact.
?addressContact ?pAddressContact ?oAddressContact.
}
UNION
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
}
UNION
{
?site organisatie:bestaatUit ?address.
?address ?pAddress ?oAddress.
}
}
}
`, undefined, endpoint)
// Move sites administrative units
await muUpdate(`
${prefixes}
INSERT {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?adminUnit org:hasSite ?site.
?site ?pSite ?oSite.
?contact ?pContact ?oContact.
?address ?pAddress ?oAddress.
?addressContact ?pAddressContact ?oAddressContact.
}
} WHERE {
GRAPH <http://mu.semte.ch/graphs/administrative-unit> {
?adminUnit a ?type.
}
GRAPH <${LANDING_ZONE_GRAPH}> {
?adminUnit org:hasSite ?site.
?site ?pSite ?oSite.
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
?contact locn:address ?addressContact.
?addressContact ?pAddressContact ?oAddressContact.
}
UNION
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
}
UNION
{
?site organisatie:bestaatUit ?address.
?address ?pAddress ?oAddress.
}
}
}
`, undefined, endpoint)
// Move primary sites worships
await muUpdate(`
${prefixes}
INSERT {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?adminUnit org:hasPrimarySite ?site.
?site ?pSite ?oSite.
?contact ?pContact ?oContact.
?address ?pAddress ?oAddress.
?addressContact ?pAddressContact ?oAddressContact.
}
} WHERE {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?adminUnit a ?type.
}
GRAPH <${LANDING_ZONE_GRAPH}> {
?adminUnit org:hasPrimarySite ?site.
?site ?pSite ?oSite.
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
?contact locn:address ?addressContact.
?addressContact ?pAddressContact ?oAddressContact.
}
UNION
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
}
UNION
{
?site organisatie:bestaatUit ?address.
?address ?pAddress ?oAddress.
}
}
}
`, undefined, endpoint)
// Move sites worships
await muUpdate(`
${prefixes}
INSERT {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?adminUnit org:hasSite ?site.
?site ?pSite ?oSite.
?contact ?pContact ?oContact.
?address ?pAddress ?oAddress.
?addressContact ?pAddressContact ?oAddressContact.
}
} WHERE {
GRAPH <http://mu.semte.ch/graphs/worship-service> {
?adminUnit a ?type.
}
GRAPH <${LANDING_ZONE_GRAPH}> {
?adminUnit org:hasSite ?site.
?site ?pSite ?oSite.
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
?contact locn:address ?addressContact.
?addressContact ?pAddressContact ?oAddressContact.
}
UNION
{
?site org:siteAddress ?contact.
?contact ?pContact ?oContact.
}
UNION
{
?site organisatie:bestaatUit ?address.
?address ?pAddress ?oAddress.
}
}
}
`, undefined, endpoint)
// Move sites worships
await muUpdate(`
${prefixes}
INSERT {
GRAPH <http://mu.semte.ch/graphs/public> {
?type a <http://lblod.data.gift/vocabularies/organisatie/TypeVestiging>.
?type ?a ?b.
}
} WHERE {
GRAPH <${LANDING_ZONE_GRAPH}> {
?type a <http://lblod.data.gift/vocabularies/organisatie/TypeVestiging>.
?type ?a ?b.
}
}
`, undefined, endpoint)
}
module.exports = {
batchedDbUpdate,
moveDataAfterInitialSync,
insertIntoPublicGraph,
deleteFromPublicGraph,
insertIntoSpecificGraphs,
deleteFromSpecificGraphs,
};

31
docker-compose.yml
View File

@@ -633,6 +633,37 @@ services:
- triplestore:virtuoso
logging: *default-logging
################################################################################
# CLB CONSUMERS
################################################################################
clb-contact-data-consumer:
image: lblod/delta-consumer:0.0.17-rc.7
environment:
DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/"
DCR_SERVICE_NAME: "clb-contact-data-consumer"
DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/ContactGegevensCacheGraphDump"
DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactData"
DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactDataDeltaFileSyncing"
DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/clb-contact-data-consumer"
DCR_DISABLE_INITIAL_SYNC: "false"
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
DCR_SYNC_FILES_PATH: "/sync/contact-data/files"
BATCH_SIZE: 1000
DCR_KEEP_DELTA_FILES: "true"
DCR_ENABLE_DELTA_CONTEXT: "true"
DCR_LANDING_ZONE_DATABASE: "triplestore"
DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/clb-contact-data"
volumes:
- ./config/clb-consumer/contact-data:/config/triples-dispatching/custom-dispatching
- ./data/files/consumer-files/contact-data:/consumer-files/
restart: always
labels:
- "logging=true"
links:
- db:database
- triplestore:virtuoso
logging: *default-logging
################################################################################
# DELTA CONSUMERS: END
################################################################################