From 64044b2ce09bf593fb1e310787089902f0f8027e Mon Sep 17 00:00:00 2001 From: Oscar Rodriguez Villalobos Date: Fri, 16 Feb 2024 13:09:48 +0100 Subject: [PATCH 01/13] Added first version of the clb consumer --- config/contact-data/config.js | 32 ++ config/contact-data/delta-context-config.js | 256 +++++++++++++ config/contact-data/delta-sync-dispatching.js | 122 +++++++ .../contact-data/initial-sync-dispatching.js | 69 ++++ config/contact-data/util.js | 337 ++++++++++++++++++ docker-compose.yml | 31 ++ 6 files changed, 847 insertions(+) create mode 100644 config/contact-data/config.js create mode 100644 config/contact-data/delta-context-config.js create mode 100644 config/contact-data/delta-sync-dispatching.js create mode 100644 config/contact-data/initial-sync-dispatching.js create mode 100644 config/contact-data/util.js diff --git a/config/contact-data/config.js b/config/contact-data/config.js new file mode 100644 index 0000000..7c4b8f0 --- /dev/null +++ b/config/contact-data/config.js @@ -0,0 +1,32 @@ +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 1000; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); +const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); +const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 100); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); + +// Graphs +const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/organizations`; +const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/op`; + +// ENDPOINTS +const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'database'; +const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; + +module.exports = { + BATCH_SIZE, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MAX_DB_RETRY_ATTEMPTS, + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + LANDING_ZONE_GRAPH, + TARGET_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT +}; \ No newline at end of file diff --git a/config/contact-data/delta-context-config.js b/config/contact-data/delta-context-config.js new file mode 100644 index 0000000..d2d40f5 --- /dev/null +++ b/config/contact-data/delta-context-config.js @@ -0,0 +1,256 @@ + +const PREFIXES = ` +PREFIX besluit: + PREFIX skos: + PREFIX lblodgeneriek: + PREFIX org: + PREFIX code: + PREFIX adms: + PREFIX generiek: + PREFIX ere: + PREFIX organisatie: + PREFIX mu: + PREFIX euvoc: + PREFIX prov: + PREFIX schema: + PREFIX locn: + PREFIX ext: ` + +const contextConfig = { + addTypes: { + scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both + exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta + }, + contextQueries: [ + { + trigger: { // subjectType or predicateValue + subjectType: "org:Site" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. + ${subject} ext:goesInGraph ?graphThree. + ${subject} ext:goesInGraph ?graphFour. + } WHERE { + OPTIONAL { + GRAPH { + ?adminUnitOne org:hasPrimarySite ${subject}; + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ?adminUnitTwo org:hasSite ${subject}; + } + BIND( AS ?graphTwo) + } + OPTIONAL { + GRAPH { + ?adminUnitThree org:hasPrimarySite ${subject}; + } + BIND( AS ?graphThree) + } + OPTIONAL { + GRAPH { + ?adminUnitFour org:hasSite ${subject}; + } + BIND( AS ?graphFour) + } + }` + }, + { + trigger: { // subjectType or predicateValue + subjectType: "schema:ContactPoint" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. + } WHERE { + OPTIONAL { + GRAPH { + ?siteOne org:siteAddress ${subject}. + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ?siteTwo org:siteAddress ${subject}. + } + BIND( AS ?graphTwo) + } + }` + }, + { + trigger: { // subjectType or predicateValue + subjectType: "locn:Address" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. + } WHERE { + OPTIONAL { + GRAPH { + ?siteOne organisatie:bestaatUit ${subject}. + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ?siteTwo organisatie:bestaatUit ${subject}. + } + BIND( AS ?graphTwo) + } + }` + }, + //Predicate triggers to solve out of order deltas + { + trigger: { // subjectType or predicateValue + predicateValue: "org:hasSite" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:contextDataGoesInGraph ?graphOne. + ${subject} ext:contextDataGoesInGraph ?graphTwo. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } WHERE { + OPTIONAL { + GRAPH { + ${subject} a ?type. + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ${subject} a ?type. + } + BIND( AS ?graphTwo) + } + ${subject} org:hasSite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph) + }` + }, + { + trigger: { // subjectType or predicateValue + predicateValue: "org:hasPrimarySite" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:contextDataGoesInGraph ?graphOne. + ${subject} ext:contextDataGoesInGraph ?graphTwo. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } WHERE { + OPTIONAL { + GRAPH { + ${subject} a ?type. + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ${subject} a ?type. + } + BIND( AS ?graphTwo) + } + ${subject} org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph) + }` + }, + { + trigger: { // subjectType or predicateValue + predicateValue: "org:siteAddress" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?contact ?pContact ?oContact. + ?addressContact ?pAddressContact ?oAddressContact. + } WHERE { + ${subject} org:siteAddress ?contact. + ?contact ?pContact ?oContact. + OPTIONAL { + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + }` + }, + { + trigger: { // subjectType or predicateValue + predicateValue: "locn:address" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?addressContact ?pAddressContact ?oAddressContact. + } WHERE { + ${subject} locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + }` + }, + { + trigger: { // subjectType or predicateValue + predicateValue: "organisatie:bestaatUit" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?address ?pAddress ?oAddress. + } WHERE { + ${subject} organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + }` + } + ] +} + +module.exports = { + contextConfig, + PREFIXES +}; \ No newline at end of file diff --git a/config/contact-data/delta-sync-dispatching.js b/config/contact-data/delta-sync-dispatching.js new file mode 100644 index 0000000..3bf9577 --- /dev/null +++ b/config/contact-data/delta-sync-dispatching.js @@ -0,0 +1,122 @@ +const { + deleteFromPublicGraph, + insertIntoPublicGraph, + deleteFromSpecificGraphs, + insertIntoSpecificGraphs +} = require('./util'); + +const { + LANDING_ZONE_GRAPH +} = require('./config') +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ + +const publicTypes = [ + '' +] +async function dispatch(lib, data) { + const { mu, fetch } = lib; + let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; + + // Both arrays are the same length, so we can zip them together for easier processing + zippedChangeSets = termObjectChangeSets.map((o, i) => ({ + original: o, + withContext: termObjectChangeSetsWithContext && termObjectChangeSetsWithContext[i] + })); + + console.log(`Received ${zippedChangeSets.length} change sets`) + + for (let { original, withContext } of zippedChangeSets) { + const insertsOnPublic = []; + const insertsOnGraphs = {}; + for(let insert of original.inserts) { + const subject = insert.subject; + const contextTriples = withContext.inserts.filter((context) => context.subject === subject); + const graphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) + const otherContextTriples = withContext.inserts.filter((context) => context.predicate !== '' && context.predicate !== '' && !(context.subject === subject && context.predicate === '')); + if(graphTriple) { + const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" + if(!insertsOnGraphs[graph]) { + insertsOnGraphs[graph] = [`${insert.subject} ${insert.predicate} ${insert.object}.`] + } else { + insertsOnGraphs[graph].push(`${insert.subject} ${insert.predicate} ${insert.object}.`) + } + for(let triple of otherContextTriples) { + insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === ''); + if(typeTriple) { + const type = typeTriple.object; + if(publicTypes.includes(type)) { + insertsOnPublic.push(`${insert.subject} ${insert.predicate} ${insert.object}.`) + //For the case where the subject is an admin unit but everything else must go in private graph + const contextGraphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) + for(let triple of otherContextTriples) { + if(contextGraphTriple) { + const graph = contextGraphTriple.object.slice(1,-1); + if(!insertsOnGraphs[graph]) { + insertsOnGraphs[graph] = [`${triple.subject} ${triple.predicate} ${triple.object}.`] + } else { + insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } else { + insertsOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + } + } + } + const deletesOnPublic = []; + const deletesOnGraphs = {}; + for(let deletion of original.deletes) { + const subject = deletion.subject; + const contextTriples = withContext.deletes.filter((context) => context.subject === subject); + const graphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) + const otherContextTriples = withContext.deletes.filter((context) => context.predicate !== '' && context.predicate !== '' && !(context.subject === subject && context.predicate === '')); + if(graphTriple) { + const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" + if(!deletesOnGraphs[graph]) { + deletesOnGraphs[graph] = [`${deletion.subject} ${deletion.predicate} ${deletion.object}.`] + } else { + deletesOnGraphs[graph].push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`) + } + for(let triple of otherContextTriples) { + deletesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === ''); + if(!typeTriple) continue; + const type = typeTriple.object; + if(publicTypes.includes(type)) { + deletesOnPublic.push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`) + for(let triple of otherContextTriples) { + deletesOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + } + + await deleteFromPublicGraph(lib, deletesOnPublic); + await deleteFromSpecificGraphs(lib, deletesOnGraphs); + await insertIntoPublicGraph(lib, insertsOnPublic); + await insertIntoSpecificGraphs(lib, insertsOnGraphs); + + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/contact-data/initial-sync-dispatching.js b/config/contact-data/initial-sync-dispatching.js new file mode 100644 index 0000000..d2adc14 --- /dev/null +++ b/config/contact-data/initial-sync-dispatching.js @@ -0,0 +1,69 @@ +const { batchedDbUpdate, moveDataAfterInitialSync } = require('./util'); +const { + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + LANDING_ZONE_GRAPH, +} = require('./config'); + +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo } = lib; + const { termObjects } = data; + + // Steps: + // 1. Write the triples to the landing zone graph - withouth any mapping or filtering + // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + await batchedDbUpdate( + muAuthSudo.updateSudo, + LANDING_ZONE_GRAPH, + originalInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } +} + +async function onFinishInitialIngest(lib,data) { + const { muAuthSudo } = lib; + + console.log(`!! On-finish triggered !!`); + + await moveDataAfterInitialSync(muAuthSudo.updateSudo, endpoint) +} + +module.exports = { + dispatch, + onFinishInitialIngest +}; \ No newline at end of file diff --git a/config/contact-data/util.js b/config/contact-data/util.js new file mode 100644 index 0000000..1913027 --- /dev/null +++ b/config/contact-data/util.js @@ -0,0 +1,337 @@ +const { + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + LANDING_ZONE_GRAPH, + BATCH_SIZE, +} = require('./config'); + +const prefixes = ` +PREFIX besluit: +PREFIX skos: +PREFIX lblodgeneriek: +PREFIX org: +PREFIX code: +PREFIX adms: +PREFIX generiek: +PREFIX ere: +PREFIX organisatie: +PREFIX mu: +PREFIX euvoc: +PREFIX prov: +PREFIX schema: +PREFIX locn: +PREFIX foaf: +PREFIX ext: +` + +async function batchedDbUpdate( + muUpdate, + graph, + triples, + extraHeaders, + endpoint, + batchSize, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, + operation = 'INSERT' +) { + for (let i = 0; i < triples.length; i += batchSize) { + console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); + + const batch = triples.slice(i, i + batchSize).join('\n'); + + const insertCall = async () => { + await muUpdate(` +${operation} DATA { +GRAPH <${graph}> { +${batch} +} +} +`, extraHeaders, endpoint); + }; + + await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); + + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function operationWithRetry(callback, + attempt, + maxAttempts, + sleepTimeOnFail) { + try { + if (typeof callback === "function") + return await callback(); + else // Catch error from promise - not how I would do it normally, but allows re use of existing code. + return await callback; + } + catch (e) { + console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); + console.log(`Error: ${e}`); + console.log(`Sleeping ${sleepTimeOnFail} ms`); + + if (attempt >= maxAttempts) { + console.log(`Max attempts reached for ${callback.toString()}, giving up`); + throw e; + } + + await new Promise(r => setTimeout(r, sleepTimeOnFail)); + return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); + } +} + + +async function insertIntoPublicGraph(lib, statements) { + console.log(`Inserting ${statements.length} statements into public graph`); + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + 'http://mu.semte.ch/graphs/public', + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); +} + +async function insertIntoSpecificGraphs(lib, statementsWithGraphs) { + + for( let graph in statementsWithGraphs) { + console.log(`Inserting ${statementsWithGraphs[graph].length} statements into ${graph} graph`); + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + graph, + statementsWithGraphs[graph], + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); + } + +} + +async function deleteFromPublicGraph(lib, statements) { + console.log(`Deleting ${statements.length} statements from public graph`); + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + 'http://mu.semte.ch/graphs/public', + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); +} + +async function deleteFromSpecificGraphs(lib, statementsWithGraphs) { + + + for( let graph in statementsWithGraphs) { + console.log(`Deleting ${statementsWithGraphs[graph].length} statements from ${graph} graph`); + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + graph, + statementsWithGraphs[graph], + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); + } + +} + + +async function moveDataAfterInitialSync(muUpdate, endpoint) { + // Move primary sites admin units + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?adminUnit org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } + } WHERE { + GRAPH { + ?adminUnit a ?type. + } + GRAPH <${LANDING_ZONE_GRAPH}> { + ?adminUnit org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + } + } + `, undefined, endpoint) + + // Move sites administrative units + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?adminUnit org:hasSite ?site. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } + } WHERE { + GRAPH { + ?adminUnit a ?type. + } + GRAPH <${LANDING_ZONE_GRAPH}> { + ?adminUnit org:hasSite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + } + } + `, undefined, endpoint) + + // Move primary sites worships + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?adminUnit org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } + } WHERE { + GRAPH { + ?adminUnit a ?type. + } + GRAPH <${LANDING_ZONE_GRAPH}> { + ?adminUnit org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + } + } + `, undefined, endpoint) + + // Move sites worships + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?adminUnit org:hasSite ?site. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } + } WHERE { + GRAPH { + ?adminUnit a ?type. + } + GRAPH <${LANDING_ZONE_GRAPH}> { + ?adminUnit org:hasSite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + } + } + `, undefined, endpoint) + + // Move sites worships + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?type a . + ?type ?a ?b. + } + } WHERE { + GRAPH <${LANDING_ZONE_GRAPH}> { + ?type a . + ?type ?a ?b. + } + } + `, undefined, endpoint) +} + +module.exports = { + batchedDbUpdate, + moveDataAfterInitialSync, + insertIntoPublicGraph, + deleteFromPublicGraph, + insertIntoSpecificGraphs, + deleteFromSpecificGraphs, +}; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0dbf73a..0ab9682 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -633,6 +633,37 @@ services: - triplestore:virtuoso logging: *default-logging +################################################################################ + # CLB CONSUMERS + ################################################################################ + clb-contact-data-consumer: + image: lblod/delta-consumer:0.0.17-rc.7 + environment: + DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" + DCR_SERVICE_NAME: "clb-contact-data-consumer" + DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/ContactGegevensCacheGraphDump" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactData" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactDataDeltaFileSyncing" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/clb-contact-data-consumer" + DCR_DISABLE_INITIAL_SYNC: "false" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" + DCR_SYNC_FILES_PATH: "/sync/contact-data/files" + BATCH_SIZE: 1000 + DCR_KEEP_DELTA_FILES: "true" + DCR_ENABLE_DELTA_CONTEXT: "true" + DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/clb-contact-data" + volumes: + - ./config/clb-consumer/contact-data:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/contact-data:/consumer-files/ + restart: always + labels: + - "logging=true" + links: + - db:database + - triplestore:virtuoso + logging: *default-logging + ################################################################################ # DELTA CONSUMERS: END ################################################################################ From d569d1536006d7d174f8dfcc9b7ed8cfac38d00c Mon Sep 17 00:00:00 2001 From: Oscar Rodriguez Villalobos Date: Wed, 21 Feb 2024 12:03:20 +0100 Subject: [PATCH 02/13] bug fixes --- config/contact-data/delta-context-config.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/contact-data/delta-context-config.js b/config/contact-data/delta-context-config.js index d2d40f5..c2097aa 100644 --- a/config/contact-data/delta-context-config.js +++ b/config/contact-data/delta-context-config.js @@ -36,25 +36,25 @@ const contextConfig = { } WHERE { OPTIONAL { GRAPH { - ?adminUnitOne org:hasPrimarySite ${subject}; + ?adminUnitOne org:hasPrimarySite ${subject}. } BIND( AS ?graphOne) } OPTIONAL { GRAPH { - ?adminUnitTwo org:hasSite ${subject}; + ?adminUnitTwo org:hasSite ${subject}. } BIND( AS ?graphTwo) } OPTIONAL { GRAPH { - ?adminUnitThree org:hasPrimarySite ${subject}; + ?adminUnitThree org:hasPrimarySite ${subject}. } BIND( AS ?graphThree) } OPTIONAL { GRAPH { - ?adminUnitFour org:hasSite ${subject}; + ?adminUnitFour org:hasSite ${subject}. } BIND( AS ?graphFour) } From 02b0e825f118ab71d9e406452846e4cb9f77b444 Mon Sep 17 00:00:00 2001 From: Oscar Date: Thu, 22 Feb 2024 16:01:27 +0100 Subject: [PATCH 03/13] bug fixes --- config/{ => clb-consumer}/contact-data/config.js | 0 config/{ => clb-consumer}/contact-data/delta-context-config.js | 0 .../{ => clb-consumer}/contact-data/delta-sync-dispatching.js | 1 + .../{ => clb-consumer}/contact-data/initial-sync-dispatching.js | 0 config/{ => clb-consumer}/contact-data/util.js | 0 docker-compose.yml | 2 +- 6 files changed, 2 insertions(+), 1 deletion(-) rename config/{ => clb-consumer}/contact-data/config.js (100%) rename config/{ => clb-consumer}/contact-data/delta-context-config.js (100%) rename config/{ => clb-consumer}/contact-data/delta-sync-dispatching.js (99%) rename config/{ => clb-consumer}/contact-data/initial-sync-dispatching.js (100%) rename config/{ => clb-consumer}/contact-data/util.js (100%) diff --git a/config/contact-data/config.js b/config/clb-consumer/contact-data/config.js similarity index 100% rename from config/contact-data/config.js rename to config/clb-consumer/contact-data/config.js diff --git a/config/contact-data/delta-context-config.js b/config/clb-consumer/contact-data/delta-context-config.js similarity index 100% rename from config/contact-data/delta-context-config.js rename to config/clb-consumer/contact-data/delta-context-config.js diff --git a/config/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js similarity index 99% rename from config/contact-data/delta-sync-dispatching.js rename to config/clb-consumer/contact-data/delta-sync-dispatching.js index 3bf9577..320bed6 100644 --- a/config/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -39,6 +39,7 @@ async function dispatch(lib, data) { console.log(`Received ${zippedChangeSets.length} change sets`) + for (let { original, withContext } of zippedChangeSets) { const insertsOnPublic = []; const insertsOnGraphs = {}; diff --git a/config/contact-data/initial-sync-dispatching.js b/config/clb-consumer/contact-data/initial-sync-dispatching.js similarity index 100% rename from config/contact-data/initial-sync-dispatching.js rename to config/clb-consumer/contact-data/initial-sync-dispatching.js diff --git a/config/contact-data/util.js b/config/clb-consumer/contact-data/util.js similarity index 100% rename from config/contact-data/util.js rename to config/clb-consumer/contact-data/util.js diff --git a/docker-compose.yml b/docker-compose.yml index 0ab9682..531d1cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -655,7 +655,7 @@ services: DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/clb-contact-data" volumes: - ./config/clb-consumer/contact-data:/config/triples-dispatching/custom-dispatching - - ./data/files/consumer-files/contact-data:/consumer-files/ + - ./data/files/consumer-files/clb-contact-data:/consumer-files/ restart: always labels: - "logging=true" From 1bf7b59905bf58171ab6ee1a7b3e62776fc2b679 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 10:27:21 +0100 Subject: [PATCH 04/13] Delete target graph as it's not used --- config/clb-consumer/contact-data/config.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/clb-consumer/contact-data/config.js b/config/clb-consumer/contact-data/config.js index 7c4b8f0..966cb4c 100644 --- a/config/clb-consumer/contact-data/config.js +++ b/config/clb-consumer/contact-data/config.js @@ -9,7 +9,6 @@ const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_T const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); // Graphs -const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/organizations`; const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/op`; // ENDPOINTS @@ -27,6 +26,5 @@ module.exports = { SLEEP_TIME_AFTER_FAILED_DB_OPERATION, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, LANDING_ZONE_GRAPH, - TARGET_GRAPH, LANDING_ZONE_DATABASE_ENDPOINT }; \ No newline at end of file From 0b1d8f3b12a02b1488496adb3fec50498afb46e2 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 10:31:01 +0100 Subject: [PATCH 05/13] avoid code between function and documentation --- .../clb-consumer/contact-data/delta-sync-dispatching.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index 320bed6..8263e47 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -8,6 +8,11 @@ const { const { LANDING_ZONE_GRAPH } = require('./config') + +const publicTypes = [ + '' +] + /** * Dispatch the fetched information to a target graph. * Note: will be ADDED to it's own graph. @@ -23,10 +28,6 @@ const { * ] * @return {void} Nothing */ - -const publicTypes = [ - '' -] async function dispatch(lib, data) { const { mu, fetch } = lib; let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; From c0e802d1f38495862729a8ff6cdb2da1cb8df9fb Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 10:32:33 +0100 Subject: [PATCH 06/13] Remove unused graph bindings --- config/clb-consumer/contact-data/delta-context-config.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-context-config.js b/config/clb-consumer/contact-data/delta-context-config.js index c2097aa..a5c078d 100644 --- a/config/clb-consumer/contact-data/delta-context-config.js +++ b/config/clb-consumer/contact-data/delta-context-config.js @@ -153,7 +153,6 @@ const contextConfig = { ?site organisatie:bestaatUit ?address. ?address ?pAddress ?oAddress. } - BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph) }` }, { @@ -200,7 +199,6 @@ const contextConfig = { ?site organisatie:bestaatUit ?address. ?address ?pAddress ?oAddress. } - BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph) }` }, { From ce93e787e6ee6f4669a3ae7d563fa3521fcbbe24 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 10:42:59 +0100 Subject: [PATCH 07/13] typo cleanup and update consumer --- config/clb-consumer/contact-data/delta-sync-dispatching.js | 1 - config/clb-consumer/contact-data/initial-sync-dispatching.js | 2 +- docker-compose.yml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index 8263e47..7171c21 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -29,7 +29,6 @@ const publicTypes = [ * @return {void} Nothing */ async function dispatch(lib, data) { - const { mu, fetch } = lib; let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; // Both arrays are the same length, so we can zip them together for easier processing diff --git a/config/clb-consumer/contact-data/initial-sync-dispatching.js b/config/clb-consumer/contact-data/initial-sync-dispatching.js index d2adc14..d3b037f 100644 --- a/config/clb-consumer/contact-data/initial-sync-dispatching.js +++ b/config/clb-consumer/contact-data/initial-sync-dispatching.js @@ -30,7 +30,7 @@ async function dispatch(lib, data) { const { termObjects } = data; // Steps: - // 1. Write the triples to the landing zone graph - withouth any mapping or filtering + // 1. Write the triples to the landing zone graph - without any mapping or filtering // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { diff --git a/docker-compose.yml b/docker-compose.yml index 531d1cb..64bb30d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -637,7 +637,7 @@ services: # CLB CONSUMERS ################################################################################ clb-contact-data-consumer: - image: lblod/delta-consumer:0.0.17-rc.7 + image: lblod/delta-consumer:0.0.18 environment: DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" DCR_SERVICE_NAME: "clb-contact-data-consumer" From c483546709dd9dbd615b3d1db42342838e888188 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 13:14:28 +0100 Subject: [PATCH 08/13] improve readability of the code --- .../contact-data/delta-sync-dispatching.js | 114 +++++++----------- 1 file changed, 45 insertions(+), 69 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index 7171c21..a85ea43 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -39,76 +39,10 @@ async function dispatch(lib, data) { console.log(`Received ${zippedChangeSets.length} change sets`) - for (let { original, withContext } of zippedChangeSets) { - const insertsOnPublic = []; - const insertsOnGraphs = {}; - for(let insert of original.inserts) { - const subject = insert.subject; - const contextTriples = withContext.inserts.filter((context) => context.subject === subject); - const graphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) - const otherContextTriples = withContext.inserts.filter((context) => context.predicate !== '' && context.predicate !== '' && !(context.subject === subject && context.predicate === '')); - if(graphTriple) { - const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(!insertsOnGraphs[graph]) { - insertsOnGraphs[graph] = [`${insert.subject} ${insert.predicate} ${insert.object}.`] - } else { - insertsOnGraphs[graph].push(`${insert.subject} ${insert.predicate} ${insert.object}.`) - } - for(let triple of otherContextTriples) { - insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === ''); - if(typeTriple) { - const type = typeTriple.object; - if(publicTypes.includes(type)) { - insertsOnPublic.push(`${insert.subject} ${insert.predicate} ${insert.object}.`) - //For the case where the subject is an admin unit but everything else must go in private graph - const contextGraphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) - for(let triple of otherContextTriples) { - if(contextGraphTriple) { - const graph = contextGraphTriple.object.slice(1,-1); - if(!insertsOnGraphs[graph]) { - insertsOnGraphs[graph] = [`${triple.subject} ${triple.predicate} ${triple.object}.`] - } else { - insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } else { - insertsOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - } - } - } - const deletesOnPublic = []; - const deletesOnGraphs = {}; - for(let deletion of original.deletes) { - const subject = deletion.subject; - const contextTriples = withContext.deletes.filter((context) => context.subject === subject); - const graphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) - const otherContextTriples = withContext.deletes.filter((context) => context.predicate !== '' && context.predicate !== '' && !(context.subject === subject && context.predicate === '')); - if(graphTriple) { - const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(!deletesOnGraphs[graph]) { - deletesOnGraphs[graph] = [`${deletion.subject} ${deletion.predicate} ${deletion.object}.`] - } else { - deletesOnGraphs[graph].push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`) - } - for(let triple of otherContextTriples) { - deletesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === ''); - if(!typeTriple) continue; - const type = typeTriple.object; - if(publicTypes.includes(type)) { - deletesOnPublic.push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`) - for(let triple of otherContextTriples) { - deletesOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - } + + const {public: insertsOnPublic, target: insertsOnGraphs} = separateBetweenTargetAndPublic(original.inserts, withContext.inserts) + const {public: deletesOnPublic, target: deletesOnGraphs} = separateBetweenTargetAndPublic(original.deletes, withContext.deletes) await deleteFromPublicGraph(lib, deletesOnPublic); await deleteFromSpecificGraphs(lib, deletesOnGraphs); @@ -118,6 +52,48 @@ async function dispatch(lib, data) { } } +function separateBetweenTargetAndPublic(changes, context) { + const changesOnPublic = []; + const changesOnGraphs = {}; + for(let change of changes) { + const subject = change.subject; + const contextTriples = context.filter((context) => context.subject === subject); + const graphTriple = contextTriples.find( + (context) => context.predicate === '' + && context.object !== `<${LANDING_ZONE_GRAPH}>` + ) + const otherContextTriples = context.filter( + (context) => context.predicate !== '' + && context.predicate !== '' + && !(context.subject === subject && context.predicate === '') + ); + if(graphTriple) { + const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" + if(!changesOnGraphs[graph]) { + changesOnGraphs[graph] = [`${change.subject} ${change.predicate} ${change.object}.`] + } else { + changesOnGraphs[graph].push(`${change.subject} ${change.predicate} ${change.object}.`) + } + for(let triple of otherContextTriples) { + changesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + const typeTriple = contextTriples.find( + (context) => context.subject === subject + && context.predicate === '' + ); + if(!typeTriple) continue; + const type = typeTriple.object; + if(publicTypes.includes(type)) { + changesOnPublic.push(`${change.subject} ${change.predicate} ${change.object}.`) + for(let triple of otherContextTriples) { + changesOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + } + return {public: changesOnPublic, target: changesOnGraphs} +} + module.exports = { dispatch }; \ No newline at end of file From c34ff821254fd19d70b1af90208846c7fe16bb6f Mon Sep 17 00:00:00 2001 From: Oscar Date: Mon, 11 Mar 2024 12:41:35 +0100 Subject: [PATCH 09/13] changed default landing zone graph --- config/clb-consumer/contact-data/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/clb-consumer/contact-data/config.js b/config/clb-consumer/contact-data/config.js index 966cb4c..052d8f7 100644 --- a/config/clb-consumer/contact-data/config.js +++ b/config/clb-consumer/contact-data/config.js @@ -9,7 +9,7 @@ const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_T const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); // Graphs -const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/op`; +const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/clb-contact-data`; // ENDPOINTS const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'database'; From 3052bf831512f7408aed13b681b513cea6709443 Mon Sep 17 00:00:00 2001 From: Oscar Date: Thu, 28 Mar 2024 11:25:32 +0100 Subject: [PATCH 10/13] fix bug site and primary site was not being added to the correct graph --- config/clb-consumer/contact-data/delta-context-config.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-context-config.js b/config/clb-consumer/contact-data/delta-context-config.js index a5c078d..b15b10e 100644 --- a/config/clb-consumer/contact-data/delta-context-config.js +++ b/config/clb-consumer/contact-data/delta-context-config.js @@ -116,8 +116,8 @@ const contextConfig = { queryTemplate: (subject) => ` ${PREFIXES} CONSTRUCT { - ${subject} ext:contextDataGoesInGraph ?graphOne. - ${subject} ext:contextDataGoesInGraph ?graphTwo. + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. ?site ?pSite ?oSite. ?contact ?pContact ?oContact. ?address ?pAddress ?oAddress. @@ -162,8 +162,8 @@ const contextConfig = { queryTemplate: (subject) => ` ${PREFIXES} CONSTRUCT { - ${subject} ext:contextDataGoesInGraph ?graphOne. - ${subject} ext:contextDataGoesInGraph ?graphTwo. + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. ?site ?pSite ?oSite. ?contact ?pContact ?oContact. ?address ?pAddress ?oAddress. From 188d7f0e18d739b627104ee610115cd163cac177 Mon Sep 17 00:00:00 2001 From: Oscar Date: Mon, 8 Apr 2024 10:35:02 +0200 Subject: [PATCH 11/13] Fix bugs (not tested) --- .../contact-data/delta-sync-dispatching.js | 13 ++++++++----- docker-compose.yml | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index a85ea43..95941e9 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -69,12 +69,15 @@ function separateBetweenTargetAndPublic(changes, context) { ); if(graphTriple) { const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(!changesOnGraphs[graph]) { - changesOnGraphs[graph] = [`${change.subject} ${change.predicate} ${change.object}.`] - } else { - changesOnGraphs[graph].push(`${change.subject} ${change.predicate} ${change.object}.`) - } + if(!change.object === '') { + if(!changesOnGraphs[graph]) { + changesOnGraphs[graph] = [`${change.subject} ${change.predicate} ${change.object}.`] + } else { + changesOnGraphs[graph].push(`${change.subject} ${change.predicate} ${change.object}.`) + } + } for(let triple of otherContextTriples) { + if(triple.object === '') continue; changesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) } } diff --git a/docker-compose.yml b/docker-compose.yml index 64bb30d..a9bdabf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -637,7 +637,7 @@ services: # CLB CONSUMERS ################################################################################ clb-contact-data-consumer: - image: lblod/delta-consumer:0.0.18 + image: lblod/delta-consumer:0.0.24 environment: DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" DCR_SERVICE_NAME: "clb-contact-data-consumer" From 41f5b74b8ff3601f2a46e1765845707344b63807 Mon Sep 17 00:00:00 2001 From: Oscar Date: Wed, 10 Apr 2024 10:00:59 +0200 Subject: [PATCH 12/13] revert to old version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a9bdabf..64bb30d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -637,7 +637,7 @@ services: # CLB CONSUMERS ################################################################################ clb-contact-data-consumer: - image: lblod/delta-consumer:0.0.24 + image: lblod/delta-consumer:0.0.18 environment: DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" DCR_SERVICE_NAME: "clb-contact-data-consumer" From 9e3f365ee381b626a0cee2bc694ed38eaf841f47 Mon Sep 17 00:00:00 2001 From: Oscar Date: Thu, 11 Apr 2024 14:56:28 +0200 Subject: [PATCH 13/13] fixed bug with object not present in triggering triple --- .../contact-data/delta-sync-dispatching.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index 95941e9..982980d 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -69,7 +69,7 @@ function separateBetweenTargetAndPublic(changes, context) { ); if(graphTriple) { const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(!change.object === '') { + if(change.object !== '') { if(!changesOnGraphs[graph]) { changesOnGraphs[graph] = [`${change.subject} ${change.predicate} ${change.object}.`] } else { @@ -77,8 +77,12 @@ function separateBetweenTargetAndPublic(changes, context) { } } for(let triple of otherContextTriples) { - if(triple.object === '') continue; - changesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + if(triple.object === '') continue; + if(!changesOnGraphs[graph]) { + changesOnGraphs[graph] = [`${triple.subject} ${triple.predicate} ${triple.object}.`] + } else { + changesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } } } const typeTriple = contextTriples.find(