diff --git a/config/clb-consumer/contact-data/config.js b/config/clb-consumer/contact-data/config.js deleted file mode 100644 index 052d8f7..0000000 --- a/config/clb-consumer/contact-data/config.js +++ /dev/null @@ -1,30 +0,0 @@ -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 1000; -const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; -const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; -const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; -const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); -const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); -const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 100); -const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); -const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); - -// Graphs -const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/clb-contact-data`; - -// ENDPOINTS -const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'database'; -const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; - -module.exports = { - BATCH_SIZE, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MAX_DB_RETRY_ATTEMPTS, - MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - LANDING_ZONE_GRAPH, - LANDING_ZONE_DATABASE_ENDPOINT -}; \ No newline at end of file diff --git a/config/clb-consumer/contact-data/delta-context-config.js b/config/clb-consumer/contact-data/delta-context-config.js deleted file mode 100644 index edbe30b..0000000 --- a/config/clb-consumer/contact-data/delta-context-config.js +++ /dev/null @@ -1,268 +0,0 @@ - -const PREFIXES = ` -PREFIX besluit: - PREFIX skos: - PREFIX lblodgeneriek: - PREFIX org: - PREFIX code: - PREFIX adms: - PREFIX generiek: - PREFIX ere: - PREFIX organisatie: - PREFIX mu: - PREFIX euvoc: - PREFIX prov: - PREFIX schema: - PREFIX locn: - PREFIX ext: ` - -const contextConfig = { - addTypes: { - scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both - exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta - }, - contextQueries: [ - { - trigger: { // subjectType or predicateValue - subjectType: "org:Site" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - ${subject} ext:goesInGraph ?graphThree. - ${subject} ext:goesInGraph ?graphFour. - } WHERE { - OPTIONAL { - GRAPH { - ?adminUnitOne org:hasPrimarySite ${subject}. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ?adminUnitTwo org:hasSite ${subject}. - } - BIND( AS ?graphTwo) - } - OPTIONAL { - GRAPH { - ?adminUnitThree org:hasPrimarySite ${subject}. - } - BIND( AS ?graphThree) - } - OPTIONAL { - GRAPH { - ?adminUnitFour org:hasSite ${subject}. - } - BIND( AS ?graphFour) - } - }` - }, - { - trigger: { // subjectType or predicateValue - subjectType: "schema:ContactPoint" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - } WHERE { - OPTIONAL { - GRAPH { - ?siteOne org:siteAddress ${subject}. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ?siteTwo org:siteAddress ${subject}. - } - BIND( AS ?graphTwo) - } - }` - }, - { - trigger: { // subjectType or predicateValue - subjectType: "locn:Address" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - } WHERE { - OPTIONAL { - GRAPH { - ?siteOne organisatie:bestaatUit ${subject}. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ?siteTwo organisatie:bestaatUit ${subject}. - } - BIND( AS ?graphTwo) - } - }` - }, - //Predicate triggers to solve out of order deltas - { - trigger: { // subjectType or predicateValue - predicateValue: "org:hasSite" - }, - //We assume that only administrative units are in the shared graph of OP due to the need of sharing the prefLabel with worships services - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } WHERE { - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphTwo) - } - ${subject} org:hasSite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - }` - }, - { - trigger: { // subjectType or predicateValue - predicateValue: "org:hasPrimarySite" - }, - //We assume that only administrative units are in the shared graph of OP due to the need of sharing the prefLabel with worships services - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ${subject} ext:goesInGraph ?graphOne. - ${subject} ext:goesInGraph ?graphTwo. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } WHERE { - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphOne) - } - OPTIONAL { - GRAPH { - ${subject} a ?type. - } - BIND( AS ?graphTwo) - } - ${subject} org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - }` - }, - { - trigger: { // subjectType or predicateValue - predicateValue: "org:siteAddress" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ?contact ?pContact ?oContact. - ?addressContact ?pAddressContact ?oAddressContact. - } WHERE { - ${subject} org:siteAddress ?contact. - ?contact ?pContact ?oContact. - OPTIONAL { - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - }` - }, - { - trigger: { // subjectType or predicateValue - predicateValue: "locn:address" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ?addressContact ?pAddressContact ?oAddressContact. - } WHERE { - ${subject} locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - }` - }, - { - trigger: { // subjectType or predicateValue - predicateValue: "organisatie:bestaatUit" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ?address ?pAddress ?oAddress. - } WHERE { - ${subject} organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - }` - } - ] -} - -module.exports = { - contextConfig, - PREFIXES -}; \ No newline at end of file diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js deleted file mode 100644 index 5f8c603..0000000 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ /dev/null @@ -1,104 +0,0 @@ -const { - deleteFromPublicGraph, - insertIntoPublicGraph, - deleteFromSpecificGraphs, - insertIntoSpecificGraphs -} = require('./util'); - -const { - LANDING_ZONE_GRAPH -} = require('./config') - -const publicTypes = [ - '' -] - -/** -* Dispatch the fetched information to a target graph. -* Note: will be ADDED to it's own graph. -* We take only care of adding them, not updating triples, this is a TODO -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; - - // Both arrays are the same length, so we can zip them together for easier processing - zippedChangeSets = termObjectChangeSets.map((o, i) => ({ - original: o, - withContext: termObjectChangeSetsWithContext && termObjectChangeSetsWithContext[i] - })); - - console.log(`Received ${zippedChangeSets.length} change sets`) - - for (let { original, withContext } of zippedChangeSets) { - - const {public: insertsOnPublic, target: insertsOnGraphs} = separateBetweenTargetAndPublic(original.inserts, withContext.inserts) - const {public: deletesOnPublic, target: deletesOnGraphs} = separateBetweenTargetAndPublic(original.deletes, withContext.deletes) - - await deleteFromPublicGraph(lib, deletesOnPublic); - await deleteFromSpecificGraphs(lib, deletesOnGraphs); - await insertIntoPublicGraph(lib, insertsOnPublic); - await insertIntoSpecificGraphs(lib, insertsOnGraphs); - - } -} - -function separateBetweenTargetAndPublic(changes, context) { - const changesOnPublic = new Set(); - const changesOnGraphs = {}; - for(let change of changes) { - const subject = change.subject; - const contextTriples = context.filter((context) => context.subject === subject); - const graphTriple = contextTriples.find( - (context) => context.predicate === '' - && context.object !== `<${LANDING_ZONE_GRAPH}>` - ) - const otherContextTriples = context.filter( - (context) => context.predicate !== '' - && context.predicate !== '' - && !(context.subject === subject && context.predicate === '') - ); - if(graphTriple) { - const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(change.object !== '') { - if(!changesOnGraphs[graph]) { - changesOnGraphs[graph] = new Set(); - } - changesOnGraphs[graph].add(`${change.subject} ${change.predicate} ${change.object}.`) - } - for(let triple of otherContextTriples) { - if(triple.object === '') continue; - if(!changesOnGraphs[graph]) { - changesOnGraphs[graph] = new Set(); - } - changesOnGraphs[graph].add(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - const typeTriple = contextTriples.find( - (context) => context.subject === subject - && context.predicate === '' - ); - if(!typeTriple) continue; - const type = typeTriple.object; - if(publicTypes.includes(type)) { - changesOnPublic.add(`${change.subject} ${change.predicate} ${change.object}.`) - for(let triple of otherContextTriples) { - changesOnPublic.add(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - } - return {public: changesOnPublic, target: changesOnGraphs} -} - -module.exports = { - dispatch -}; \ No newline at end of file diff --git a/config/clb-consumer/contact-data/initial-sync-dispatching.js b/config/clb-consumer/contact-data/initial-sync-dispatching.js deleted file mode 100644 index d3b037f..0000000 --- a/config/clb-consumer/contact-data/initial-sync-dispatching.js +++ /dev/null @@ -1,69 +0,0 @@ -const { batchedDbUpdate, moveDataAfterInitialSync } = require('./util'); -const { - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - LANDING_ZONE_GRAPH, -} = require('./config'); - -const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; - -/** -* Dispatch the fetched information to a target graph. -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjects } data - The fetched quad information, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, muAuthSudo } = lib; - const { termObjects } = data; - - // Steps: - // 1. Write the triples to the landing zone graph - without any mapping or filtering - // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) - - if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { - console.warn(`Service configured to skip MU_AUTH!`); - } - console.log(`Using ${endpoint} to insert triples`); - - if (termObjects.length) { - const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - - await batchedDbUpdate( - muAuthSudo.updateSudo, - LANDING_ZONE_GRAPH, - originalInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } -} - -async function onFinishInitialIngest(lib,data) { - const { muAuthSudo } = lib; - - console.log(`!! On-finish triggered !!`); - - await moveDataAfterInitialSync(muAuthSudo.updateSudo, endpoint) -} - -module.exports = { - dispatch, - onFinishInitialIngest -}; \ No newline at end of file diff --git a/config/clb-consumer/contact-data/util.js b/config/clb-consumer/contact-data/util.js deleted file mode 100644 index 1e47cc5..0000000 --- a/config/clb-consumer/contact-data/util.js +++ /dev/null @@ -1,337 +0,0 @@ -const { - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - LANDING_ZONE_GRAPH, - BATCH_SIZE, -} = require('./config'); - -const prefixes = ` -PREFIX besluit: -PREFIX skos: -PREFIX lblodgeneriek: -PREFIX org: -PREFIX code: -PREFIX adms: -PREFIX generiek: -PREFIX ere: -PREFIX organisatie: -PREFIX mu: -PREFIX euvoc: -PREFIX prov: -PREFIX schema: -PREFIX locn: -PREFIX foaf: -PREFIX ext: -` - -async function batchedDbUpdate( - muUpdate, - graph, - triples, - extraHeaders, - endpoint, - batchSize, - maxAttempts, - sleepBetweenBatches = 1000, - sleepTimeOnFail = 1000, - operation = 'INSERT' -) { - for (let i = 0; i < triples.length; i += batchSize) { - console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); - - const batch = triples.slice(i, i + batchSize).join('\n'); - - const insertCall = async () => { - await muUpdate(` -${operation} DATA { -GRAPH <${graph}> { -${batch} -} -} -`, extraHeaders, endpoint); - }; - - await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); - - console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); - await new Promise(r => setTimeout(r, sleepBetweenBatches)); - } -} - -async function operationWithRetry(callback, - attempt, - maxAttempts, - sleepTimeOnFail) { - try { - if (typeof callback === "function") - return await callback(); - else // Catch error from promise - not how I would do it normally, but allows re use of existing code. - return await callback; - } - catch (e) { - console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); - console.log(`Error: ${e}`); - console.log(`Sleeping ${sleepTimeOnFail} ms`); - - if (attempt >= maxAttempts) { - console.log(`Max attempts reached for ${callback.toString()}, giving up`); - throw e; - } - - await new Promise(r => setTimeout(r, sleepTimeOnFail)); - return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); - } -} - - -async function insertIntoPublicGraph(lib, statements) { - console.log(`Inserting ${statements.length} statements into public graph`); - - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - 'http://mu.semte.ch/graphs/public', - Array.from(statements), - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'INSERT'); -} - -async function insertIntoSpecificGraphs(lib, statementsWithGraphs) { - - for( let graph in statementsWithGraphs) { - console.log(`Inserting ${statementsWithGraphs[graph].length} statements into ${graph} graph`); - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - graph, - Array.from(statementsWithGraphs[graph]), - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'INSERT'); - } - -} - -async function deleteFromPublicGraph(lib, statements) { - console.log(`Deleting ${statements.length} statements from public graph`); - - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - 'http://mu.semte.ch/graphs/public', - Array.from(statements), - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'DELETE'); -} - -async function deleteFromSpecificGraphs(lib, statementsWithGraphs) { - - - for( let graph in statementsWithGraphs) { - console.log(`Deleting ${statementsWithGraphs[graph].length} statements from ${graph} graph`); - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - graph, - Array.from(statementsWithGraphs[graph]), - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'DELETE'); - } - -} - - -async function moveDataAfterInitialSync(muUpdate, endpoint) { - // Move primary sites admin units - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?adminUnit org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } - } WHERE { - GRAPH { - ?adminUnit a ?type. - } - GRAPH <${LANDING_ZONE_GRAPH}> { - ?adminUnit org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - } - } - `, undefined, endpoint) - - // Move sites administrative units - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?adminUnit org:hasSite ?site. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } - } WHERE { - GRAPH { - ?adminUnit a ?type. - } - GRAPH <${LANDING_ZONE_GRAPH}> { - ?adminUnit org:hasSite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - } - } - `, undefined, endpoint) - - // Move primary sites worships - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?adminUnit org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } - } WHERE { - GRAPH { - ?adminUnit a ?type. - } - GRAPH <${LANDING_ZONE_GRAPH}> { - ?adminUnit org:hasPrimarySite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - } - } - `, undefined, endpoint) - - // Move sites worships - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?adminUnit org:hasSite ?site. - ?site ?pSite ?oSite. - ?contact ?pContact ?oContact. - ?address ?pAddress ?oAddress. - ?addressContact ?pAddressContact ?oAddressContact. - } - } WHERE { - GRAPH { - ?adminUnit a ?type. - } - GRAPH <${LANDING_ZONE_GRAPH}> { - ?adminUnit org:hasSite ?site. - ?site ?pSite ?oSite. - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - ?contact locn:address ?addressContact. - ?addressContact ?pAddressContact ?oAddressContact. - } - UNION - { - ?site org:siteAddress ?contact. - ?contact ?pContact ?oContact. - } - UNION - { - ?site organisatie:bestaatUit ?address. - ?address ?pAddress ?oAddress. - } - } - } - `, undefined, endpoint) - - // Move sites worships - await muUpdate(` - ${prefixes} - INSERT { - GRAPH { - ?type a . - ?type ?a ?b. - } - } WHERE { - GRAPH <${LANDING_ZONE_GRAPH}> { - ?type a . - ?type ?a ?b. - } - } - `, undefined, endpoint) -} - -module.exports = { - batchedDbUpdate, - moveDataAfterInitialSync, - insertIntoPublicGraph, - deleteFromPublicGraph, - insertIntoSpecificGraphs, - deleteFromSpecificGraphs, -}; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 1be9310..a540619 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -470,37 +470,6 @@ services: - triplestore:virtuoso logging: *default-logging -################################################################################ - # CLB CONSUMERS - ################################################################################ - clb-contact-data-consumer: - image: lblod/delta-consumer:0.0.18 - environment: - DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" - DCR_SERVICE_NAME: "clb-contact-data-consumer" - DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/ContactGegevensCacheGraphDump" - DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactData" - DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactDataDeltaFileSyncing" - DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/clb-contact-data-consumer" - DCR_DISABLE_INITIAL_SYNC: "false" - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" - DCR_SYNC_FILES_PATH: "/sync/contact-data/files" - BATCH_SIZE: 1000 - DCR_KEEP_DELTA_FILES: "true" - DCR_ENABLE_DELTA_CONTEXT: "true" - DCR_LANDING_ZONE_DATABASE: "triplestore" - DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/clb-contact-data" - volumes: - - ./config/clb-consumer/contact-data:/config/triples-dispatching/custom-dispatching - - ./data/files/consumer-files/clb-contact-data:/consumer-files/ - restart: always - labels: - - "logging=true" - links: - - db:database - - triplestore:virtuoso - logging: *default-logging - ################################################################################ # DELTA CONSUMERS: END ################################################################################