From 40d7c72b59acda19c3f4814ea46455849a23023e Mon Sep 17 00:00:00 2001 From: mirdono Date: Wed, 9 Oct 2024 09:31:24 +0200 Subject: [PATCH] chore: remove the CLB consumer for contact data OP will take over the editing of contact data again. This removes the CLB consumer from OP altogether such that it cannot be accidentally be reactivated. --- config/clb-consumer/contact-data/config.js | 30 -- .../contact-data/delta-context-config.js | 268 -------------- .../contact-data/delta-sync-dispatching.js | 104 ------ .../contact-data/initial-sync-dispatching.js | 69 ---- config/clb-consumer/contact-data/util.js | 337 ------------------ docker-compose.yml | 31 -- 6 files changed, 839 deletions(-) delete mode 100644 config/clb-consumer/contact-data/config.js delete mode 100644 config/clb-consumer/contact-data/delta-context-config.js delete mode 100644 config/clb-consumer/contact-data/delta-sync-dispatching.js delete mode 100644 config/clb-consumer/contact-data/initial-sync-dispatching.js delete mode 100644 config/clb-consumer/contact-data/util.js diff --git a/config/clb-consumer/contact-data/config.js b/config/clb-consumer/contact-data/config.js deleted file mode 100644 index 052d8f7..0000000 --- a/config/clb-consumer/contact-data/config.js +++ /dev/null @@ -1,30 +0,0 @@ -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 1000; -const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; -const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; -const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; -const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); -const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); -const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 100); -const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); -const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); - -// Graphs -const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/clb-contact-data`; - -// ENDPOINTS -const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'database'; -const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; - -module.exports = { - BATCH_SIZE, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MAX_DB_RETRY_ATTEMPTS, - MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - LANDING_ZONE_GRAPH, - LANDING_ZONE_DATABASE_ENDPOINT -}; \ No newline at end of file diff --git a/config/clb-consumer/contact-data/delta-context-config.js b/config/clb-consumer/contact-data/delta-context-config.js deleted file mode 100644 index edbe30b..0000000 --- a/config/clb-consumer/contact-data/delta-context-config.js +++ /dev/null @@ -1,268 +0,0 @@ - -const PREFIXES = ` -PREFIX besluit: - PREFIX skos: - PREFIX lblodgeneriek: - PREFIX org: - PREFIX code: - PREFIX adms: - PREFIX generiek: - PREFIX ere: - PREFIX organisatie: - PREFIX mu: - PREFIX euvoc: - PREFIX prov: - PREFIX schema: - PREFIX locn: - PREFIX ext: ` - -const contextConfig = { - addTypes: { - scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both - exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta - }, - contextQueries: [ - { - trigger: { // subjectType or predicateValue - subjectType: "org:Site" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - ${subject} ext:goesInGraph ?graphThree. - ${subject} ext:goesInGraph ?graphFour. - } WHERE { - OPTIONAL { - GRAPH { - ?adminUnitOne org:hasPrimarySite ${subject}. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ?adminUnitTwo org:hasSite ${subject}. - } - BIND( AS ?graphTwo) - } - OPTIONAL { - GRAPH { - ?adminUnitThree org:hasPrimarySite ${subject}. - } - BIND( AS ?graphThree) - } - OPTIONAL { - GRAPH { - ?adminUnitFour org:hasSite ${subject}. - } - BIND( AS ?graphFour) - } - }` - }, - { - trigger: { // subjectType or predicateValue - subjectType: "schema:ContactPoint" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - } WHERE { - OPTIONAL { - GRAPH { - ?siteOne org:siteAddress ${subject}. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ?siteTwo org:siteAddress ${subject}. - } - BIND( AS ?graphTwo) - } - }` - }, - { - trigger: { // subjectType or predicateValue - subjectType: "locn:Address" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - } WHERE { - OPTIONAL { - GRAPH { - ?siteOne organisatie:bestaatUit ${subject}. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ?siteTwo organisatie:bestaatUit ${subject}. - } - BIND( AS ?graphTwo) - } - }` - }, - //Predicate triggers to solve out of order deltas - { - trigger: { // subjectType or predicateValue - predicateValue: "org:hasSite" - }, - //We assume that only administrative units are in the shared graph of OP due to the need of sharing the prefLabel with worships services - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } WHERE { - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphTwo) - } - ${subject} org:hasSite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - }` - }, - { - trigger: { // subjectType or predicateValue - predicateValue: "org:hasPrimarySite" - }, - //We assume that only administrative units are in the shared graph of OP due to the need of sharing the prefLabel with worships services - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } WHERE { - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphTwo) - } - ${subject} org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - }` - }, - { - trigger: { // subjectType or predicateValue - predicateValue: "org:siteAddress" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ?contact ?pContact ?oContact. - ?addressContact ?pAddressContact ?oAddressContact. - } WHERE { - ${subject} org:siteAddress ?contact. - ?contact ?pContact ?oContact. - OPTIONAL { - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - }` - }, - { - trigger: { // subjectType or predicateValue - predicateValue: "locn:address" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ?addressContact ?pAddressContact ?oAddressContact. - } WHERE { - ${subject} locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - }` - }, - { - trigger: { // subjectType or predicateValue - predicateValue: "organisatie:bestaatUit" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ?address ?pAddress ?oAddress. - } WHERE { - ${subject} organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - }` - } - ] -} - -module.exports = { - contextConfig, - PREFIXES -}; \ No newline at end of file diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js deleted file mode 100644 index 5f8c603..0000000 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ /dev/null @@ -1,104 +0,0 @@ -const { - deleteFromPublicGraph, - insertIntoPublicGraph, - deleteFromSpecificGraphs, - insertIntoSpecificGraphs -} = require('./util'); - -const { - LANDING_ZONE_GRAPH -} = require('./config') - -const publicTypes = [ - '' -] - -/** -* Dispatch the fetched information to a target graph. -* Note: will be ADDED to it's own graph. -* We take only care of adding them, not updating triples, this is a TODO -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; - - // Both arrays are the same length, so we can zip them together for easier processing - zippedChangeSets = termObjectChangeSets.map((o, i) => ({ - original: o, - withContext: termObjectChangeSetsWithContext && termObjectChangeSetsWithContext[i] - })); - - console.log(`Received ${zippedChangeSets.length} change sets`) - - for (let { original, withContext } of zippedChangeSets) { - - const {public: insertsOnPublic, target: insertsOnGraphs} = separateBetweenTargetAndPublic(original.inserts, withContext.inserts) - const {public: deletesOnPublic, target: deletesOnGraphs} = separateBetweenTargetAndPublic(original.deletes, withContext.deletes) - - await deleteFromPublicGraph(lib, deletesOnPublic); - await deleteFromSpecificGraphs(lib, deletesOnGraphs); - await insertIntoPublicGraph(lib, insertsOnPublic); - await insertIntoSpecificGraphs(lib, insertsOnGraphs); - - } -} - -function separateBetweenTargetAndPublic(changes, context) { - const changesOnPublic = new Set(); - const changesOnGraphs = {}; - for(let change of changes) { - const subject = change.subject; - const contextTriples = context.filter((context) => context.subject === subject); - const graphTriple = contextTriples.find( - (context) => context.predicate === '' - && context.object !== `<${LANDING_ZONE_GRAPH}>` - ) - const otherContextTriples = context.filter( - (context) => context.predicate !== '' - && context.predicate !== '' - && !(context.subject === subject && context.predicate === '') - ); - if(graphTriple) { - const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(change.object !== '') { - if(!changesOnGraphs[graph]) { - changesOnGraphs[graph] = new Set(); - } - changesOnGraphs[graph].add(`${change.subject} ${change.predicate} ${change.object}.`) - } - for(let triple of otherContextTriples) { - if(triple.object === '') continue; - if(!changesOnGraphs[graph]) { - changesOnGraphs[graph] = new Set(); - } - changesOnGraphs[graph].add(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - const typeTriple = contextTriples.find( - (context) => context.subject === subject - && context.predicate === '' - ); - if(!typeTriple) continue; - const type = typeTriple.object; - if(publicTypes.includes(type)) { - changesOnPublic.add(`${change.subject} ${change.predicate} ${change.object}.`) - for(let triple of otherContextTriples) { - changesOnPublic.add(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - } - return {public: changesOnPublic, target: changesOnGraphs} -} - -module.exports = { - dispatch -}; \ No newline at end of file diff --git a/config/clb-consumer/contact-data/initial-sync-dispatching.js b/config/clb-consumer/contact-data/initial-sync-dispatching.js deleted file mode 100644 index d3b037f..0000000 --- a/config/clb-consumer/contact-data/initial-sync-dispatching.js +++ /dev/null @@ -1,69 +0,0 @@ -const { batchedDbUpdate, moveDataAfterInitialSync } = require('./util'); -const { - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - LANDING_ZONE_GRAPH, -} = require('./config'); - -const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; - -/** -* Dispatch the fetched information to a target graph. -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjects } data - The fetched quad information, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, muAuthSudo } = lib; - const { termObjects } = data; - - // Steps: - // 1. Write the triples to the landing zone graph - without any mapping or filtering - // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) - - if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { - console.warn(`Service configured to skip MU_AUTH!`); - } - console.log(`Using ${endpoint} to insert triples`); - - if (termObjects.length) { - const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - - await batchedDbUpdate( - muAuthSudo.updateSudo, - LANDING_ZONE_GRAPH, - originalInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } -} - -async function onFinishInitialIngest(lib,data) { - const { muAuthSudo } = lib; - - console.log(`!! On-finish triggered !!`); - - await moveDataAfterInitialSync(muAuthSudo.updateSudo, endpoint) -} - -module.exports = { - dispatch, - onFinishInitialIngest -}; \ No newline at end of file diff --git a/config/clb-consumer/contact-data/util.js b/config/clb-consumer/contact-data/util.js deleted file mode 100644 index 1e47cc5..0000000 --- a/config/clb-consumer/contact-data/util.js +++ /dev/null @@ -1,337 +0,0 @@ -const { - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - LANDING_ZONE_GRAPH, - BATCH_SIZE, -} = require('./config'); - -const prefixes = ` -PREFIX besluit: -PREFIX skos: -PREFIX lblodgeneriek: -PREFIX org: -PREFIX code: -PREFIX adms: -PREFIX generiek: -PREFIX ere: -PREFIX organisatie: -PREFIX mu: -PREFIX euvoc: -PREFIX prov: -PREFIX schema: -PREFIX locn: -PREFIX foaf: -PREFIX ext: -` - -async function batchedDbUpdate( - muUpdate, - graph, - triples, - extraHeaders, - endpoint, - batchSize, - maxAttempts, - sleepBetweenBatches = 1000, - sleepTimeOnFail = 1000, - operation = 'INSERT' -) { - for (let i = 0; i < triples.length; i += batchSize) { - console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); - - const batch = triples.slice(i, i + batchSize).join('\n'); - - const insertCall = async () => { - await muUpdate(` -${operation} DATA { -GRAPH <${graph}> { -${batch} -} -} -`, extraHeaders, endpoint); - }; - - await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); - - console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); - await new Promise(r => setTimeout(r, sleepBetweenBatches)); - } -} - -async function operationWithRetry(callback, - attempt, - maxAttempts, - sleepTimeOnFail) { - try { - if (typeof callback === "function") - return await callback(); - else // Catch error from promise - not how I would do it normally, but allows re use of existing code. - return await callback; - } - catch (e) { - console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); - console.log(`Error: ${e}`); - console.log(`Sleeping ${sleepTimeOnFail} ms`); - - if (attempt >= maxAttempts) { - console.log(`Max attempts reached for ${callback.toString()}, giving up`); - throw e; - } - - await new Promise(r => setTimeout(r, sleepTimeOnFail)); - return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); - } -} - - -async function insertIntoPublicGraph(lib, statements) { - console.log(`Inserting ${statements.length} statements into public graph`); - - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - 'http://mu.semte.ch/graphs/public', - Array.from(statements), - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'INSERT'); -} - -async function insertIntoSpecificGraphs(lib, statementsWithGraphs) { - - for( let graph in statementsWithGraphs) { - console.log(`Inserting ${statementsWithGraphs[graph].length} statements into ${graph} graph`); - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - graph, - Array.from(statementsWithGraphs[graph]), - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'INSERT'); - } - -} - -async function deleteFromPublicGraph(lib, statements) { - console.log(`Deleting ${statements.length} statements from public graph`); - - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - 'http://mu.semte.ch/graphs/public', - Array.from(statements), - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'DELETE'); -} - -async function deleteFromSpecificGraphs(lib, statementsWithGraphs) { - - - for( let graph in statementsWithGraphs) { - console.log(`Deleting ${statementsWithGraphs[graph].length} statements from ${graph} graph`); - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - graph, - Array.from(statementsWithGraphs[graph]), - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'DELETE'); - } - -} - - -async function moveDataAfterInitialSync(muUpdate, endpoint) { - // Move primary sites admin units - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?adminUnit org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } - } WHERE { - GRAPH { - ?adminUnit a ?type. - } - GRAPH <${LANDING_ZONE_GRAPH}> { - ?adminUnit org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - } - } - `, undefined, endpoint) - - // Move sites administrative units - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?adminUnit org:hasSite ?site. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } - } WHERE { - GRAPH { - ?adminUnit a ?type. - } - GRAPH <${LANDING_ZONE_GRAPH}> { - ?adminUnit org:hasSite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - } - } - `, undefined, endpoint) - - // Move primary sites worships - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?adminUnit org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } - } WHERE { - GRAPH { - ?adminUnit a ?type. - } - GRAPH <${LANDING_ZONE_GRAPH}> { - ?adminUnit org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - } - } - `, undefined, endpoint) - - // Move sites worships - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?adminUnit org:hasSite ?site. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } - } WHERE { - GRAPH { - ?adminUnit a ?type. - } - GRAPH <${LANDING_ZONE_GRAPH}> { - ?adminUnit org:hasSite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - } - } - `, undefined, endpoint) - - // Move sites worships - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?type a . - ?type ?a ?b. - } - } WHERE { - GRAPH <${LANDING_ZONE_GRAPH}> { - ?type a . - ?type ?a ?b. - } - } - `, undefined, endpoint) -} - -module.exports = { - batchedDbUpdate, - moveDataAfterInitialSync, - insertIntoPublicGraph, - deleteFromPublicGraph, - insertIntoSpecificGraphs, - deleteFromSpecificGraphs, -}; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1be9310..a540619 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -470,37 +470,6 @@ services: - triplestore:virtuoso logging: *default-logging -################################################################################ - # CLB CONSUMERS - ################################################################################ - clb-contact-data-consumer: - image: lblod/delta-consumer:0.0.18 - environment: - DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" - DCR_SERVICE_NAME: "clb-contact-data-consumer" - DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/ContactGegevensCacheGraphDump" - DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactData" - DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactDataDeltaFileSyncing" - DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/clb-contact-data-consumer" - DCR_DISABLE_INITIAL_SYNC: "false" - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" - DCR_SYNC_FILES_PATH: "/sync/contact-data/files" - BATCH_SIZE: 1000 - DCR_KEEP_DELTA_FILES: "true" - DCR_ENABLE_DELTA_CONTEXT: "true" - DCR_LANDING_ZONE_DATABASE: "triplestore" - DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/clb-contact-data" - volumes: - - ./config/clb-consumer/contact-data:/config/triples-dispatching/custom-dispatching - - ./data/files/consumer-files/clb-contact-data:/consumer-files/ - restart: always - labels: - - "logging=true" - links: - - db:database - - triplestore:virtuoso - logging: *default-logging - ################################################################################ # DELTA CONSUMERS: END ################################################################################