From 93672567173a9c457d925e00716a031b190853ec Mon Sep 17 00:00:00 2001 From: Oscar Rodriguez Villalobos Date: Thu, 4 Apr 2024 10:35:14 +0200 Subject: [PATCH 01/25] Add legal and alternative names to producer --- .../organizations-public-info/export.json | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/config/delta-producer/organizations-public-info/export.json b/config/delta-producer/organizations-public-info/export.json index 100be72..2494b05 100644 --- a/config/delta-producer/organizations-public-info/export.json +++ b/config/delta-producer/organizations-public-info/export.json @@ -26,7 +26,9 @@ "http://data.lblod.info/vocabularies/erediensten/denominatie", "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", - "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/ns/regorg#legalName", + "http://www.w3.org/2004/02/skos/core#altLabel" ] }, { @@ -50,7 +52,9 @@ "http://www.w3.org/ns/org#linkedTo", "http://www.w3.org/ns/org#hasSubOrganization", "http://www.w3.org/ns/org#classification", - "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/ns/regorg#legalName", + "http://www.w3.org/2004/02/skos/core#altLabel" ] }, { @@ -84,7 +88,9 @@ "http://www.w3.org/ns/org#resultedFrom", "http://www.w3.org/ns/regorg#orgStatus", "https://data.lblod.info/vocabularies/generiek/geplandeEindDatum", - "http://www.w3.org/ns/org#purpose" + "http://www.w3.org/ns/org#purpose", + "http://www.w3.org/ns/regorg#legalName", + "http://www.w3.org/2004/02/skos/core#altLabel" ] }, { @@ -98,7 +104,9 @@ "http://www.w3.org/2004/02/skos/core#prefLabel", "http://www.w3.org/ns/org#linkedTo", "http://data.lblod.info/vocabularies/erediensten/typeEredienst", - "http://www.w3.org/ns/adms#identifier" + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#legalName", + "http://www.w3.org/2004/02/skos/core#altLabel" ] }, { From 675a984cc421ed830e3245255af20234b3c36d95 Mon Sep 17 00:00:00 2001 From: Oscar Rodriguez Villalobos Date: Thu, 4 Apr 2024 14:54:28 +0200 Subject: [PATCH 02/25] alt label was already present --- .../organizations-public-info/export.json | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/config/delta-producer/organizations-public-info/export.json b/config/delta-producer/organizations-public-info/export.json index 2494b05..eceaa7a 100644 --- a/config/delta-producer/organizations-public-info/export.json +++ b/config/delta-producer/organizations-public-info/export.json @@ -13,6 +13,7 @@ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/regorg#orgStatus", "http://www.w3.org/ns/org#hasPrimarySite", @@ -26,9 +27,7 @@ "http://data.lblod.info/vocabularies/erediensten/denominatie", "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", - "http://data.lblod.info/vocabularies/erediensten/typeEredienst", - "http://www.w3.org/ns/regorg#legalName", - "http://www.w3.org/2004/02/skos/core#altLabel" + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" ] }, { @@ -42,6 +41,7 @@ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/regorg#orgStatus", "http://www.w3.org/ns/org#hasPrimarySite", @@ -52,9 +52,7 @@ "http://www.w3.org/ns/org#linkedTo", "http://www.w3.org/ns/org#hasSubOrganization", "http://www.w3.org/ns/org#classification", - "http://data.lblod.info/vocabularies/erediensten/typeEredienst", - "http://www.w3.org/ns/regorg#legalName", - "http://www.w3.org/2004/02/skos/core#altLabel" + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" ] }, { @@ -76,6 +74,7 @@ "http://www.opengis.net/ont/geosparql#sfWithin", "http://www.w3.org/2002/07/owl#sameAs", "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#prefLabel", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/org#changedBy", @@ -88,9 +87,7 @@ "http://www.w3.org/ns/org#resultedFrom", "http://www.w3.org/ns/regorg#orgStatus", "https://data.lblod.info/vocabularies/generiek/geplandeEindDatum", - "http://www.w3.org/ns/org#purpose", - "http://www.w3.org/ns/regorg#legalName", - "http://www.w3.org/2004/02/skos/core#altLabel" + "http://www.w3.org/ns/org#purpose" ] }, { From 53c5bd12753ba726887a29c309de92b56fe4b02d Mon Sep 17 00:00:00 2001 From: mirdono Date: Thu, 4 Apr 2024 14:24:23 +0200 Subject: [PATCH 03/25] feat: export legal and alternative names via producers --- config/delta-producer/organizations/export.json | 4 ++++ config/delta-producer/public/export.json | 5 +++++ config/delta-producer/worship-posts/export.json | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/config/delta-producer/organizations/export.json b/config/delta-producer/organizations/export.json index 2c703bd..941d65a 100644 --- a/config/delta-producer/organizations/export.json +++ b/config/delta-producer/organizations/export.json @@ -127,6 +127,7 @@ ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/regorg#orgStatus", @@ -155,6 +156,7 @@ ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/regorg#orgStatus", @@ -191,6 +193,7 @@ "http://www.w3.org/2002/07/owl#sameAs", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/org#changedBy", "http://www.w3.org/ns/org#classification", @@ -214,6 +217,7 @@ ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/regorg#orgStatus", diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index f0c3e85..7756c6a 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -15,6 +15,7 @@ "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/regorg#orgStatus", @@ -46,6 +47,7 @@ "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/regorg#orgStatus", @@ -84,6 +86,7 @@ "http://www.w3.org/2002/07/owl#sameAs", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/ns/adms#identifier", "http://www.w3.org/ns/org#changedBy", "http://www.w3.org/ns/org#classification", @@ -105,6 +108,8 @@ "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", + "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/org#linkedTo", "http://data.lblod.info/vocabularies/erediensten/typeEredienst", "http://www.w3.org/ns/adms#identifier" diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index 0f540bd..28eef61 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -22,6 +22,7 @@ "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/regorg#orgStatus", "http://data.lblod.info/vocabularies/erediensten/typeEredienst", @@ -42,6 +43,7 @@ "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/regorg#orgStatus", "http://data.lblod.info/vocabularies/erediensten/typeEredienst", @@ -61,6 +63,7 @@ "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/regorg#orgStatus", "http://data.lblod.info/vocabularies/erediensten/typeEredienst", @@ -121,6 +124,7 @@ "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/regorg#legalName", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/regorg#orgStatus", "http://www.w3.org/ns/org#hasPrimarySite", From 8e090ca8727937b82713157774731eb55dbb63d3 Mon Sep 17 00:00:00 2001 From: mirdono Date: Fri, 5 Apr 2024 16:03:37 +0200 Subject: [PATCH 04/25] chore: bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index eb8023f..d766e34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -181,7 +181,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.23.0 + image: lblod/frontend-organization-portal:1.23.1-0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 6b89b32e995c954abf8c0aabb31d544b4b363c70 Mon Sep 17 00:00:00 2001 From: mirdono Date: Mon, 8 Apr 2024 13:45:34 +0200 Subject: [PATCH 05/25] chore: bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d766e34..8218f18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -181,7 +181,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.23.1-0 + image: lblod/frontend-organization-portal:1.23.1 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 0ad11a22895fd907c6800f8710884a55e4739f0b Mon Sep 17 00:00:00 2001 From: mirdono Date: Wed, 10 Apr 2024 16:18:02 +0200 Subject: [PATCH 06/25] chore: bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8218f18..45de472 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -181,7 +181,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.23.1 + image: lblod/frontend-organization-portal:1.23.2-0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 857919804d6dbdeb4b3f95de7316d9ce042c6a97 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Thu, 11 Apr 2024 10:22:07 +0200 Subject: [PATCH 07/25] used regex for filtering empty literals --- .../2024/20240327131000-delete-all-empty-literals.sparql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/migrations/2024/20240327131000-delete-all-empty-literals.sparql b/config/migrations/2024/20240327131000-delete-all-empty-literals.sparql index 48a6dfd..8ae247b 100644 --- a/config/migrations/2024/20240327131000-delete-all-empty-literals.sparql +++ b/config/migrations/2024/20240327131000-delete-all-empty-literals.sparql @@ -14,7 +14,7 @@ WHERE { graph ?g { ?s ?p ?o - Filter(?o = " " || ?o = "") - + FILTER (isLiteral(?o)) + FILTER regex(str(?o), "^\\s*$") } } \ No newline at end of file From c0fad917f1c9ca4b25d7bbaeb2d0c2591785e337 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Thu, 11 Apr 2024 12:38:22 +0200 Subject: [PATCH 08/25] leave out structured identifiers from deletion --- .../2024/20240316100500-delete-all-empty-uris.sparql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/migrations/2024/20240316100500-delete-all-empty-uris.sparql b/config/migrations/2024/20240316100500-delete-all-empty-uris.sparql index 722fc7f..5f95b8e 100644 --- a/config/migrations/2024/20240316100500-delete-all-empty-uris.sparql +++ b/config/migrations/2024/20240316100500-delete-all-empty-uris.sparql @@ -15,7 +15,8 @@ WHERE { GRAPH ?g { ?s ?type ; ?uuid . - + + FILTER (?type != ) FILTER NOT EXISTS { FILTER (?p != ) FILTER (?p != ) From 64044b2ce09bf593fb1e310787089902f0f8027e Mon Sep 17 00:00:00 2001 From: Oscar Rodriguez Villalobos Date: Fri, 16 Feb 2024 13:09:48 +0100 Subject: [PATCH 09/25] Added first version of the clb consumer --- config/contact-data/config.js | 32 ++ config/contact-data/delta-context-config.js | 256 +++++++++++++ config/contact-data/delta-sync-dispatching.js | 122 +++++++ .../contact-data/initial-sync-dispatching.js | 69 ++++ config/contact-data/util.js | 337 ++++++++++++++++++ docker-compose.yml | 31 ++ 6 files changed, 847 insertions(+) create mode 100644 config/contact-data/config.js create mode 100644 config/contact-data/delta-context-config.js create mode 100644 config/contact-data/delta-sync-dispatching.js create mode 100644 config/contact-data/initial-sync-dispatching.js create mode 100644 config/contact-data/util.js diff --git a/config/contact-data/config.js b/config/contact-data/config.js new file mode 100644 index 0000000..7c4b8f0 --- /dev/null +++ b/config/contact-data/config.js @@ -0,0 +1,32 @@ +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 1000; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); +const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); +const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 100); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); + +// Graphs +const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/organizations`; +const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/op`; + +// ENDPOINTS +const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'database'; +const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; + +module.exports = { + BATCH_SIZE, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MAX_DB_RETRY_ATTEMPTS, + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + LANDING_ZONE_GRAPH, + TARGET_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT +}; \ No newline at end of file diff --git a/config/contact-data/delta-context-config.js b/config/contact-data/delta-context-config.js new file mode 100644 index 0000000..d2d40f5 --- /dev/null +++ b/config/contact-data/delta-context-config.js @@ -0,0 +1,256 @@ + +const PREFIXES = ` +PREFIX besluit: + PREFIX skos: + PREFIX lblodgeneriek: + PREFIX org: + PREFIX code: + PREFIX adms: + PREFIX generiek: + PREFIX ere: + PREFIX organisatie: + PREFIX mu: + PREFIX euvoc: + PREFIX prov: + PREFIX schema: + PREFIX locn: + PREFIX ext: ` + +const contextConfig = { + addTypes: { + scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both + exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta + }, + contextQueries: [ + { + trigger: { // subjectType or predicateValue + subjectType: "org:Site" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. + ${subject} ext:goesInGraph ?graphThree. + ${subject} ext:goesInGraph ?graphFour. + } WHERE { + OPTIONAL { + GRAPH { + ?adminUnitOne org:hasPrimarySite ${subject}; + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ?adminUnitTwo org:hasSite ${subject}; + } + BIND( AS ?graphTwo) + } + OPTIONAL { + GRAPH { + ?adminUnitThree org:hasPrimarySite ${subject}; + } + BIND( AS ?graphThree) + } + OPTIONAL { + GRAPH { + ?adminUnitFour org:hasSite ${subject}; + } + BIND( AS ?graphFour) + } + }` + }, + { + trigger: { // subjectType or predicateValue + subjectType: "schema:ContactPoint" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. + } WHERE { + OPTIONAL { + GRAPH { + ?siteOne org:siteAddress ${subject}. + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ?siteTwo org:siteAddress ${subject}. + } + BIND( AS ?graphTwo) + } + }` + }, + { + trigger: { // subjectType or predicateValue + subjectType: "locn:Address" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. + } WHERE { + OPTIONAL { + GRAPH { + ?siteOne organisatie:bestaatUit ${subject}. + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ?siteTwo organisatie:bestaatUit ${subject}. + } + BIND( AS ?graphTwo) + } + }` + }, + //Predicate triggers to solve out of order deltas + { + trigger: { // subjectType or predicateValue + predicateValue: "org:hasSite" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:contextDataGoesInGraph ?graphOne. + ${subject} ext:contextDataGoesInGraph ?graphTwo. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } WHERE { + OPTIONAL { + GRAPH { + ${subject} a ?type. + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ${subject} a ?type. + } + BIND( AS ?graphTwo) + } + ${subject} org:hasSite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph) + }` + }, + { + trigger: { // subjectType or predicateValue + predicateValue: "org:hasPrimarySite" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ${subject} ext:contextDataGoesInGraph ?graphOne. + ${subject} ext:contextDataGoesInGraph ?graphTwo. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } WHERE { + OPTIONAL { + GRAPH { + ${subject} a ?type. + } + BIND( AS ?graphOne) + } + OPTIONAL { + GRAPH { + ${subject} a ?type. + } + BIND( AS ?graphTwo) + } + ${subject} org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph) + }` + }, + { + trigger: { // subjectType or predicateValue + predicateValue: "org:siteAddress" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?contact ?pContact ?oContact. + ?addressContact ?pAddressContact ?oAddressContact. + } WHERE { + ${subject} org:siteAddress ?contact. + ?contact ?pContact ?oContact. + OPTIONAL { + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + }` + }, + { + trigger: { // subjectType or predicateValue + predicateValue: "locn:address" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?addressContact ?pAddressContact ?oAddressContact. + } WHERE { + ${subject} locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + }` + }, + { + trigger: { // subjectType or predicateValue + predicateValue: "organisatie:bestaatUit" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?address ?pAddress ?oAddress. + } WHERE { + ${subject} organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + }` + } + ] +} + +module.exports = { + contextConfig, + PREFIXES +}; \ No newline at end of file diff --git a/config/contact-data/delta-sync-dispatching.js b/config/contact-data/delta-sync-dispatching.js new file mode 100644 index 0000000..3bf9577 --- /dev/null +++ b/config/contact-data/delta-sync-dispatching.js @@ -0,0 +1,122 @@ +const { + deleteFromPublicGraph, + insertIntoPublicGraph, + deleteFromSpecificGraphs, + insertIntoSpecificGraphs +} = require('./util'); + +const { + LANDING_ZONE_GRAPH +} = require('./config') +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ + +const publicTypes = [ + '' +] +async function dispatch(lib, data) { + const { mu, fetch } = lib; + let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; + + // Both arrays are the same length, so we can zip them together for easier processing + zippedChangeSets = termObjectChangeSets.map((o, i) => ({ + original: o, + withContext: termObjectChangeSetsWithContext && termObjectChangeSetsWithContext[i] + })); + + console.log(`Received ${zippedChangeSets.length} change sets`) + + for (let { original, withContext } of zippedChangeSets) { + const insertsOnPublic = []; + const insertsOnGraphs = {}; + for(let insert of original.inserts) { + const subject = insert.subject; + const contextTriples = withContext.inserts.filter((context) => context.subject === subject); + const graphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) + const otherContextTriples = withContext.inserts.filter((context) => context.predicate !== '' && context.predicate !== '' && !(context.subject === subject && context.predicate === '')); + if(graphTriple) { + const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" + if(!insertsOnGraphs[graph]) { + insertsOnGraphs[graph] = [`${insert.subject} ${insert.predicate} ${insert.object}.`] + } else { + insertsOnGraphs[graph].push(`${insert.subject} ${insert.predicate} ${insert.object}.`) + } + for(let triple of otherContextTriples) { + insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === ''); + if(typeTriple) { + const type = typeTriple.object; + if(publicTypes.includes(type)) { + insertsOnPublic.push(`${insert.subject} ${insert.predicate} ${insert.object}.`) + //For the case where the subject is an admin unit but everything else must go in private graph + const contextGraphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) + for(let triple of otherContextTriples) { + if(contextGraphTriple) { + const graph = contextGraphTriple.object.slice(1,-1); + if(!insertsOnGraphs[graph]) { + insertsOnGraphs[graph] = [`${triple.subject} ${triple.predicate} ${triple.object}.`] + } else { + insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } else { + insertsOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + } + } + } + const deletesOnPublic = []; + const deletesOnGraphs = {}; + for(let deletion of original.deletes) { + const subject = deletion.subject; + const contextTriples = withContext.deletes.filter((context) => context.subject === subject); + const graphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) + const otherContextTriples = withContext.deletes.filter((context) => context.predicate !== '' && context.predicate !== '' && !(context.subject === subject && context.predicate === '')); + if(graphTriple) { + const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" + if(!deletesOnGraphs[graph]) { + deletesOnGraphs[graph] = [`${deletion.subject} ${deletion.predicate} ${deletion.object}.`] + } else { + deletesOnGraphs[graph].push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`) + } + for(let triple of otherContextTriples) { + deletesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === ''); + if(!typeTriple) continue; + const type = typeTriple.object; + if(publicTypes.includes(type)) { + deletesOnPublic.push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`) + for(let triple of otherContextTriples) { + deletesOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + } + + await deleteFromPublicGraph(lib, deletesOnPublic); + await deleteFromSpecificGraphs(lib, deletesOnGraphs); + await insertIntoPublicGraph(lib, insertsOnPublic); + await insertIntoSpecificGraphs(lib, insertsOnGraphs); + + } +} + +module.exports = { + dispatch +}; \ No newline at end of file diff --git a/config/contact-data/initial-sync-dispatching.js b/config/contact-data/initial-sync-dispatching.js new file mode 100644 index 0000000..d2adc14 --- /dev/null +++ b/config/contact-data/initial-sync-dispatching.js @@ -0,0 +1,69 @@ +const { batchedDbUpdate, moveDataAfterInitialSync } = require('./util'); +const { + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + LANDING_ZONE_GRAPH, +} = require('./config'); + +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo } = lib; + const { termObjects } = data; + + // Steps: + // 1. Write the triples to the landing zone graph - withouth any mapping or filtering + // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + await batchedDbUpdate( + muAuthSudo.updateSudo, + LANDING_ZONE_GRAPH, + originalInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } +} + +async function onFinishInitialIngest(lib,data) { + const { muAuthSudo } = lib; + + console.log(`!! On-finish triggered !!`); + + await moveDataAfterInitialSync(muAuthSudo.updateSudo, endpoint) +} + +module.exports = { + dispatch, + onFinishInitialIngest +}; \ No newline at end of file diff --git a/config/contact-data/util.js b/config/contact-data/util.js new file mode 100644 index 0000000..1913027 --- /dev/null +++ b/config/contact-data/util.js @@ -0,0 +1,337 @@ +const { + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + LANDING_ZONE_GRAPH, + BATCH_SIZE, +} = require('./config'); + +const prefixes = ` +PREFIX besluit: +PREFIX skos: +PREFIX lblodgeneriek: +PREFIX org: +PREFIX code: +PREFIX adms: +PREFIX generiek: +PREFIX ere: +PREFIX organisatie: +PREFIX mu: +PREFIX euvoc: +PREFIX prov: +PREFIX schema: +PREFIX locn: +PREFIX foaf: +PREFIX ext: +` + +async function batchedDbUpdate( + muUpdate, + graph, + triples, + extraHeaders, + endpoint, + batchSize, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, + operation = 'INSERT' +) { + for (let i = 0; i < triples.length; i += batchSize) { + console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); + + const batch = triples.slice(i, i + batchSize).join('\n'); + + const insertCall = async () => { + await muUpdate(` +${operation} DATA { +GRAPH <${graph}> { +${batch} +} +} +`, extraHeaders, endpoint); + }; + + await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); + + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function operationWithRetry(callback, + attempt, + maxAttempts, + sleepTimeOnFail) { + try { + if (typeof callback === "function") + return await callback(); + else // Catch error from promise - not how I would do it normally, but allows re use of existing code. + return await callback; + } + catch (e) { + console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); + console.log(`Error: ${e}`); + console.log(`Sleeping ${sleepTimeOnFail} ms`); + + if (attempt >= maxAttempts) { + console.log(`Max attempts reached for ${callback.toString()}, giving up`); + throw e; + } + + await new Promise(r => setTimeout(r, sleepTimeOnFail)); + return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); + } +} + + +async function insertIntoPublicGraph(lib, statements) { + console.log(`Inserting ${statements.length} statements into public graph`); + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + 'http://mu.semte.ch/graphs/public', + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); +} + +async function insertIntoSpecificGraphs(lib, statementsWithGraphs) { + + for( let graph in statementsWithGraphs) { + console.log(`Inserting ${statementsWithGraphs[graph].length} statements into ${graph} graph`); + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + graph, + statementsWithGraphs[graph], + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); + } + +} + +async function deleteFromPublicGraph(lib, statements) { + console.log(`Deleting ${statements.length} statements from public graph`); + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + 'http://mu.semte.ch/graphs/public', + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); +} + +async function deleteFromSpecificGraphs(lib, statementsWithGraphs) { + + + for( let graph in statementsWithGraphs) { + console.log(`Deleting ${statementsWithGraphs[graph].length} statements from ${graph} graph`); + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + graph, + statementsWithGraphs[graph], + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); + } + +} + + +async function moveDataAfterInitialSync(muUpdate, endpoint) { + // Move primary sites admin units + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?adminUnit org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } + } WHERE { + GRAPH { + ?adminUnit a ?type. + } + GRAPH <${LANDING_ZONE_GRAPH}> { + ?adminUnit org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + } + } + `, undefined, endpoint) + + // Move sites administrative units + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?adminUnit org:hasSite ?site. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } + } WHERE { + GRAPH { + ?adminUnit a ?type. + } + GRAPH <${LANDING_ZONE_GRAPH}> { + ?adminUnit org:hasSite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + } + } + `, undefined, endpoint) + + // Move primary sites worships + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?adminUnit org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } + } WHERE { + GRAPH { + ?adminUnit a ?type. + } + GRAPH <${LANDING_ZONE_GRAPH}> { + ?adminUnit org:hasPrimarySite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + } + } + `, undefined, endpoint) + + // Move sites worships + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?adminUnit org:hasSite ?site. + ?site ?pSite ?oSite. + ?contact ?pContact ?oContact. + ?address ?pAddress ?oAddress. + ?addressContact ?pAddressContact ?oAddressContact. + } + } WHERE { + GRAPH { + ?adminUnit a ?type. + } + GRAPH <${LANDING_ZONE_GRAPH}> { + ?adminUnit org:hasSite ?site. + ?site ?pSite ?oSite. + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + ?contact locn:address ?addressContact. + ?addressContact ?pAddressContact ?oAddressContact. + } + UNION + { + ?site org:siteAddress ?contact. + ?contact ?pContact ?oContact. + } + UNION + { + ?site organisatie:bestaatUit ?address. + ?address ?pAddress ?oAddress. + } + } + } + `, undefined, endpoint) + + // Move sites worships + await muUpdate(` + ${prefixes} + INSERT { + GRAPH { + ?type a . + ?type ?a ?b. + } + } WHERE { + GRAPH <${LANDING_ZONE_GRAPH}> { + ?type a . + ?type ?a ?b. + } + } + `, undefined, endpoint) +} + +module.exports = { + batchedDbUpdate, + moveDataAfterInitialSync, + insertIntoPublicGraph, + deleteFromPublicGraph, + insertIntoSpecificGraphs, + deleteFromSpecificGraphs, +}; \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0dbf73a..0ab9682 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -633,6 +633,37 @@ services: - triplestore:virtuoso logging: *default-logging +################################################################################ + # CLB CONSUMERS + ################################################################################ + clb-contact-data-consumer: + image: lblod/delta-consumer:0.0.17-rc.7 + environment: + DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" + DCR_SERVICE_NAME: "clb-contact-data-consumer" + DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/ContactGegevensCacheGraphDump" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactData" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/contactDataDeltaFileSyncing" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/clb-contact-data-consumer" + DCR_DISABLE_INITIAL_SYNC: "false" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" + DCR_SYNC_FILES_PATH: "/sync/contact-data/files" + BATCH_SIZE: 1000 + DCR_KEEP_DELTA_FILES: "true" + DCR_ENABLE_DELTA_CONTEXT: "true" + DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/clb-contact-data" + volumes: + - ./config/clb-consumer/contact-data:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/contact-data:/consumer-files/ + restart: always + labels: + - "logging=true" + links: + - db:database + - triplestore:virtuoso + logging: *default-logging + ################################################################################ # DELTA CONSUMERS: END ################################################################################ From d569d1536006d7d174f8dfcc9b7ed8cfac38d00c Mon Sep 17 00:00:00 2001 From: Oscar Rodriguez Villalobos Date: Wed, 21 Feb 2024 12:03:20 +0100 Subject: [PATCH 10/25] bug fixes --- config/contact-data/delta-context-config.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/contact-data/delta-context-config.js b/config/contact-data/delta-context-config.js index d2d40f5..c2097aa 100644 --- a/config/contact-data/delta-context-config.js +++ b/config/contact-data/delta-context-config.js @@ -36,25 +36,25 @@ const contextConfig = { } WHERE { OPTIONAL { GRAPH { - ?adminUnitOne org:hasPrimarySite ${subject}; + ?adminUnitOne org:hasPrimarySite ${subject}. } BIND( AS ?graphOne) } OPTIONAL { GRAPH { - ?adminUnitTwo org:hasSite ${subject}; + ?adminUnitTwo org:hasSite ${subject}. } BIND( AS ?graphTwo) } OPTIONAL { GRAPH { - ?adminUnitThree org:hasPrimarySite ${subject}; + ?adminUnitThree org:hasPrimarySite ${subject}. } BIND( AS ?graphThree) } OPTIONAL { GRAPH { - ?adminUnitFour org:hasSite ${subject}; + ?adminUnitFour org:hasSite ${subject}. } BIND( AS ?graphFour) } From 02b0e825f118ab71d9e406452846e4cb9f77b444 Mon Sep 17 00:00:00 2001 From: Oscar Date: Thu, 22 Feb 2024 16:01:27 +0100 Subject: [PATCH 11/25] bug fixes --- config/{ => clb-consumer}/contact-data/config.js | 0 config/{ => clb-consumer}/contact-data/delta-context-config.js | 0 .../{ => clb-consumer}/contact-data/delta-sync-dispatching.js | 1 + .../{ => clb-consumer}/contact-data/initial-sync-dispatching.js | 0 config/{ => clb-consumer}/contact-data/util.js | 0 docker-compose.yml | 2 +- 6 files changed, 2 insertions(+), 1 deletion(-) rename config/{ => clb-consumer}/contact-data/config.js (100%) rename config/{ => clb-consumer}/contact-data/delta-context-config.js (100%) rename config/{ => clb-consumer}/contact-data/delta-sync-dispatching.js (99%) rename config/{ => clb-consumer}/contact-data/initial-sync-dispatching.js (100%) rename config/{ => clb-consumer}/contact-data/util.js (100%) diff --git a/config/contact-data/config.js b/config/clb-consumer/contact-data/config.js similarity index 100% rename from config/contact-data/config.js rename to config/clb-consumer/contact-data/config.js diff --git a/config/contact-data/delta-context-config.js b/config/clb-consumer/contact-data/delta-context-config.js similarity index 100% rename from config/contact-data/delta-context-config.js rename to config/clb-consumer/contact-data/delta-context-config.js diff --git a/config/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js similarity index 99% rename from config/contact-data/delta-sync-dispatching.js rename to config/clb-consumer/contact-data/delta-sync-dispatching.js index 3bf9577..320bed6 100644 --- a/config/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -39,6 +39,7 @@ async function dispatch(lib, data) { console.log(`Received ${zippedChangeSets.length} change sets`) + for (let { original, withContext } of zippedChangeSets) { const insertsOnPublic = []; const insertsOnGraphs = {}; diff --git a/config/contact-data/initial-sync-dispatching.js b/config/clb-consumer/contact-data/initial-sync-dispatching.js similarity index 100% rename from config/contact-data/initial-sync-dispatching.js rename to config/clb-consumer/contact-data/initial-sync-dispatching.js diff --git a/config/contact-data/util.js b/config/clb-consumer/contact-data/util.js similarity index 100% rename from config/contact-data/util.js rename to config/clb-consumer/contact-data/util.js diff --git a/docker-compose.yml b/docker-compose.yml index 0ab9682..531d1cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -655,7 +655,7 @@ services: DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/clb-contact-data" volumes: - ./config/clb-consumer/contact-data:/config/triples-dispatching/custom-dispatching - - ./data/files/consumer-files/contact-data:/consumer-files/ + - ./data/files/consumer-files/clb-contact-data:/consumer-files/ restart: always labels: - "logging=true" From 1bf7b59905bf58171ab6ee1a7b3e62776fc2b679 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 10:27:21 +0100 Subject: [PATCH 12/25] Delete target graph as it's not used --- config/clb-consumer/contact-data/config.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/clb-consumer/contact-data/config.js b/config/clb-consumer/contact-data/config.js index 7c4b8f0..966cb4c 100644 --- a/config/clb-consumer/contact-data/config.js +++ b/config/clb-consumer/contact-data/config.js @@ -9,7 +9,6 @@ const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_T const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); // Graphs -const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/organizations`; const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/op`; // ENDPOINTS @@ -27,6 +26,5 @@ module.exports = { SLEEP_TIME_AFTER_FAILED_DB_OPERATION, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, LANDING_ZONE_GRAPH, - TARGET_GRAPH, LANDING_ZONE_DATABASE_ENDPOINT }; \ No newline at end of file From 0b1d8f3b12a02b1488496adb3fec50498afb46e2 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 10:31:01 +0100 Subject: [PATCH 13/25] avoid code between function and documentation --- .../clb-consumer/contact-data/delta-sync-dispatching.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index 320bed6..8263e47 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -8,6 +8,11 @@ const { const { LANDING_ZONE_GRAPH } = require('./config') + +const publicTypes = [ + '' +] + /** * Dispatch the fetched information to a target graph. * Note: will be ADDED to it's own graph. @@ -23,10 +28,6 @@ const { * ] * @return {void} Nothing */ - -const publicTypes = [ - '' -] async function dispatch(lib, data) { const { mu, fetch } = lib; let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; From c0e802d1f38495862729a8ff6cdb2da1cb8df9fb Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 10:32:33 +0100 Subject: [PATCH 14/25] Remove unused graph bindings --- config/clb-consumer/contact-data/delta-context-config.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-context-config.js b/config/clb-consumer/contact-data/delta-context-config.js index c2097aa..a5c078d 100644 --- a/config/clb-consumer/contact-data/delta-context-config.js +++ b/config/clb-consumer/contact-data/delta-context-config.js @@ -153,7 +153,6 @@ const contextConfig = { ?site organisatie:bestaatUit ?address. ?address ?pAddress ?oAddress. } - BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph) }` }, { @@ -200,7 +199,6 @@ const contextConfig = { ?site organisatie:bestaatUit ?address. ?address ?pAddress ?oAddress. } - BIND(IRI(CONCAT("http://mu.semte.ch/graphs/organizations/", ?uuid)) AS ?graph) }` }, { From ce93e787e6ee6f4669a3ae7d563fa3521fcbbe24 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 10:42:59 +0100 Subject: [PATCH 15/25] typo cleanup and update consumer --- config/clb-consumer/contact-data/delta-sync-dispatching.js | 1 - config/clb-consumer/contact-data/initial-sync-dispatching.js | 2 +- docker-compose.yml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index 8263e47..7171c21 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -29,7 +29,6 @@ const publicTypes = [ * @return {void} Nothing */ async function dispatch(lib, data) { - const { mu, fetch } = lib; let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; // Both arrays are the same length, so we can zip them together for easier processing diff --git a/config/clb-consumer/contact-data/initial-sync-dispatching.js b/config/clb-consumer/contact-data/initial-sync-dispatching.js index d2adc14..d3b037f 100644 --- a/config/clb-consumer/contact-data/initial-sync-dispatching.js +++ b/config/clb-consumer/contact-data/initial-sync-dispatching.js @@ -30,7 +30,7 @@ async function dispatch(lib, data) { const { termObjects } = data; // Steps: - // 1. Write the triples to the landing zone graph - withouth any mapping or filtering + // 1. Write the triples to the landing zone graph - without any mapping or filtering // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { diff --git a/docker-compose.yml b/docker-compose.yml index 531d1cb..64bb30d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -637,7 +637,7 @@ services: # CLB CONSUMERS ################################################################################ clb-contact-data-consumer: - image: lblod/delta-consumer:0.0.17-rc.7 + image: lblod/delta-consumer:0.0.18 environment: DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" DCR_SERVICE_NAME: "clb-contact-data-consumer" From c483546709dd9dbd615b3d1db42342838e888188 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 5 Mar 2024 13:14:28 +0100 Subject: [PATCH 16/25] improve readability of the code --- .../contact-data/delta-sync-dispatching.js | 114 +++++++----------- 1 file changed, 45 insertions(+), 69 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index 7171c21..a85ea43 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -39,76 +39,10 @@ async function dispatch(lib, data) { console.log(`Received ${zippedChangeSets.length} change sets`) - for (let { original, withContext } of zippedChangeSets) { - const insertsOnPublic = []; - const insertsOnGraphs = {}; - for(let insert of original.inserts) { - const subject = insert.subject; - const contextTriples = withContext.inserts.filter((context) => context.subject === subject); - const graphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) - const otherContextTriples = withContext.inserts.filter((context) => context.predicate !== '' && context.predicate !== '' && !(context.subject === subject && context.predicate === '')); - if(graphTriple) { - const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(!insertsOnGraphs[graph]) { - insertsOnGraphs[graph] = [`${insert.subject} ${insert.predicate} ${insert.object}.`] - } else { - insertsOnGraphs[graph].push(`${insert.subject} ${insert.predicate} ${insert.object}.`) - } - for(let triple of otherContextTriples) { - insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === ''); - if(typeTriple) { - const type = typeTriple.object; - if(publicTypes.includes(type)) { - insertsOnPublic.push(`${insert.subject} ${insert.predicate} ${insert.object}.`) - //For the case where the subject is an admin unit but everything else must go in private graph - const contextGraphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) - for(let triple of otherContextTriples) { - if(contextGraphTriple) { - const graph = contextGraphTriple.object.slice(1,-1); - if(!insertsOnGraphs[graph]) { - insertsOnGraphs[graph] = [`${triple.subject} ${triple.predicate} ${triple.object}.`] - } else { - insertsOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } else { - insertsOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - } - } - } - const deletesOnPublic = []; - const deletesOnGraphs = {}; - for(let deletion of original.deletes) { - const subject = deletion.subject; - const contextTriples = withContext.deletes.filter((context) => context.subject === subject); - const graphTriple = contextTriples.find((context) => context.predicate === '' && context.object !== `<${LANDING_ZONE_GRAPH}>`) - const otherContextTriples = withContext.deletes.filter((context) => context.predicate !== '' && context.predicate !== '' && !(context.subject === subject && context.predicate === '')); - if(graphTriple) { - const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(!deletesOnGraphs[graph]) { - deletesOnGraphs[graph] = [`${deletion.subject} ${deletion.predicate} ${deletion.object}.`] - } else { - deletesOnGraphs[graph].push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`) - } - for(let triple of otherContextTriples) { - deletesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - const typeTriple = contextTriples.find((context) => context.subject === subject && context.predicate === ''); - if(!typeTriple) continue; - const type = typeTriple.object; - if(publicTypes.includes(type)) { - deletesOnPublic.push(`${deletion.subject} ${deletion.predicate} ${deletion.object}.`) - for(let triple of otherContextTriples) { - deletesOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) - } - } - } + + const {public: insertsOnPublic, target: insertsOnGraphs} = separateBetweenTargetAndPublic(original.inserts, withContext.inserts) + const {public: deletesOnPublic, target: deletesOnGraphs} = separateBetweenTargetAndPublic(original.deletes, withContext.deletes) await deleteFromPublicGraph(lib, deletesOnPublic); await deleteFromSpecificGraphs(lib, deletesOnGraphs); @@ -118,6 +52,48 @@ async function dispatch(lib, data) { } } +function separateBetweenTargetAndPublic(changes, context) { + const changesOnPublic = []; + const changesOnGraphs = {}; + for(let change of changes) { + const subject = change.subject; + const contextTriples = context.filter((context) => context.subject === subject); + const graphTriple = contextTriples.find( + (context) => context.predicate === '' + && context.object !== `<${LANDING_ZONE_GRAPH}>` + ) + const otherContextTriples = context.filter( + (context) => context.predicate !== '' + && context.predicate !== '' + && !(context.subject === subject && context.predicate === '') + ); + if(graphTriple) { + const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" + if(!changesOnGraphs[graph]) { + changesOnGraphs[graph] = [`${change.subject} ${change.predicate} ${change.object}.`] + } else { + changesOnGraphs[graph].push(`${change.subject} ${change.predicate} ${change.object}.`) + } + for(let triple of otherContextTriples) { + changesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + const typeTriple = contextTriples.find( + (context) => context.subject === subject + && context.predicate === '' + ); + if(!typeTriple) continue; + const type = typeTriple.object; + if(publicTypes.includes(type)) { + changesOnPublic.push(`${change.subject} ${change.predicate} ${change.object}.`) + for(let triple of otherContextTriples) { + changesOnPublic.push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } + } + } + return {public: changesOnPublic, target: changesOnGraphs} +} + module.exports = { dispatch }; \ No newline at end of file From c34ff821254fd19d70b1af90208846c7fe16bb6f Mon Sep 17 00:00:00 2001 From: Oscar Date: Mon, 11 Mar 2024 12:41:35 +0100 Subject: [PATCH 17/25] changed default landing zone graph --- config/clb-consumer/contact-data/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/clb-consumer/contact-data/config.js b/config/clb-consumer/contact-data/config.js index 966cb4c..052d8f7 100644 --- a/config/clb-consumer/contact-data/config.js +++ b/config/clb-consumer/contact-data/config.js @@ -9,7 +9,7 @@ const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_T const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); // Graphs -const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/op`; +const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/clb-contact-data`; // ENDPOINTS const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'database'; From 3052bf831512f7408aed13b681b513cea6709443 Mon Sep 17 00:00:00 2001 From: Oscar Date: Thu, 28 Mar 2024 11:25:32 +0100 Subject: [PATCH 18/25] fix bug site and primary site was not being added to the correct graph --- config/clb-consumer/contact-data/delta-context-config.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-context-config.js b/config/clb-consumer/contact-data/delta-context-config.js index a5c078d..b15b10e 100644 --- a/config/clb-consumer/contact-data/delta-context-config.js +++ b/config/clb-consumer/contact-data/delta-context-config.js @@ -116,8 +116,8 @@ const contextConfig = { queryTemplate: (subject) => ` ${PREFIXES} CONSTRUCT { - ${subject} ext:contextDataGoesInGraph ?graphOne. - ${subject} ext:contextDataGoesInGraph ?graphTwo. + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. ?site ?pSite ?oSite. ?contact ?pContact ?oContact. ?address ?pAddress ?oAddress. @@ -162,8 +162,8 @@ const contextConfig = { queryTemplate: (subject) => ` ${PREFIXES} CONSTRUCT { - ${subject} ext:contextDataGoesInGraph ?graphOne. - ${subject} ext:contextDataGoesInGraph ?graphTwo. + ${subject} ext:goesInGraph ?graphOne. + ${subject} ext:goesInGraph ?graphTwo. ?site ?pSite ?oSite. ?contact ?pContact ?oContact. ?address ?pAddress ?oAddress. From 188d7f0e18d739b627104ee610115cd163cac177 Mon Sep 17 00:00:00 2001 From: Oscar Date: Mon, 8 Apr 2024 10:35:02 +0200 Subject: [PATCH 19/25] Fix bugs (not tested) --- .../contact-data/delta-sync-dispatching.js | 13 ++++++++----- docker-compose.yml | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index a85ea43..95941e9 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -69,12 +69,15 @@ function separateBetweenTargetAndPublic(changes, context) { ); if(graphTriple) { const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(!changesOnGraphs[graph]) { - changesOnGraphs[graph] = [`${change.subject} ${change.predicate} ${change.object}.`] - } else { - changesOnGraphs[graph].push(`${change.subject} ${change.predicate} ${change.object}.`) - } + if(!change.object === '') { + if(!changesOnGraphs[graph]) { + changesOnGraphs[graph] = [`${change.subject} ${change.predicate} ${change.object}.`] + } else { + changesOnGraphs[graph].push(`${change.subject} ${change.predicate} ${change.object}.`) + } + } for(let triple of otherContextTriples) { + if(triple.object === '') continue; changesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) } } diff --git a/docker-compose.yml b/docker-compose.yml index 64bb30d..a9bdabf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -637,7 +637,7 @@ services: # CLB CONSUMERS ################################################################################ clb-contact-data-consumer: - image: lblod/delta-consumer:0.0.18 + image: lblod/delta-consumer:0.0.24 environment: DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" DCR_SERVICE_NAME: "clb-contact-data-consumer" From 41f5b74b8ff3601f2a46e1765845707344b63807 Mon Sep 17 00:00:00 2001 From: Oscar Date: Wed, 10 Apr 2024 10:00:59 +0200 Subject: [PATCH 20/25] revert to old version --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a9bdabf..64bb30d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -637,7 +637,7 @@ services: # CLB CONSUMERS ################################################################################ clb-contact-data-consumer: - image: lblod/delta-consumer:0.0.24 + image: lblod/delta-consumer:0.0.18 environment: DCR_SYNC_BASE_URL: "https://dev.contactgegevens-loket.lblod.info/" DCR_SERVICE_NAME: "clb-contact-data-consumer" From 9e3f365ee381b626a0cee2bc694ed38eaf841f47 Mon Sep 17 00:00:00 2001 From: Oscar Date: Thu, 11 Apr 2024 14:56:28 +0200 Subject: [PATCH 21/25] fixed bug with object not present in triggering triple --- .../contact-data/delta-sync-dispatching.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/config/clb-consumer/contact-data/delta-sync-dispatching.js b/config/clb-consumer/contact-data/delta-sync-dispatching.js index 95941e9..982980d 100644 --- a/config/clb-consumer/contact-data/delta-sync-dispatching.js +++ b/config/clb-consumer/contact-data/delta-sync-dispatching.js @@ -69,7 +69,7 @@ function separateBetweenTargetAndPublic(changes, context) { ); if(graphTriple) { const graph = graphTriple.object.slice(1,-1); // We have to slice it to remove the "<" and ">" - if(!change.object === '') { + if(change.object !== '') { if(!changesOnGraphs[graph]) { changesOnGraphs[graph] = [`${change.subject} ${change.predicate} ${change.object}.`] } else { @@ -77,8 +77,12 @@ function separateBetweenTargetAndPublic(changes, context) { } } for(let triple of otherContextTriples) { - if(triple.object === '') continue; - changesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + if(triple.object === '') continue; + if(!changesOnGraphs[graph]) { + changesOnGraphs[graph] = [`${triple.subject} ${triple.predicate} ${triple.object}.`] + } else { + changesOnGraphs[graph].push(`${triple.subject} ${triple.predicate} ${triple.object}.`) + } } } const typeTriple = contextTriples.find( From df5d249246633c585fcdd746dd6e3cb5332866f1 Mon Sep 17 00:00:00 2001 From: mirdono Date: Thu, 11 Apr 2024 17:38:07 +0200 Subject: [PATCH 22/25] chore: frontend bump --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 45de472..0e42538 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -181,7 +181,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.23.2-0 + image: lblod/frontend-organization-portal:1.23.2 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 43523ec37dcc20c9de7a009169539935718d2deb Mon Sep 17 00:00:00 2001 From: Laurent Deketelaere Date: Thu, 11 Apr 2024 17:57:30 +0200 Subject: [PATCH 23/25] chore: Bump sync-wegwijs-organization-service to v2.0.0 --- config/resources/domain.json | 4 ++++ docker-compose.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/config/resources/domain.json b/config/resources/domain.json index 66dc367..ab21178 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -646,6 +646,10 @@ "start-date": { "type": "date", "predicate": "ext:startDate" + }, + "modified": { + "type": "date", + "predicate": "dc_terms:modified" } }, "relationships": { diff --git a/docker-compose.yml b/docker-compose.yml index 101385b..f8c219e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -299,7 +299,7 @@ services: restart: always logging: *default-logging kbo-data-sync: - image: lblod/sync-ovo-numbers-service:1.0.2 + image: lblod/sync-wegwijs-organization-service:2.0.0 links: - db:database labels: From 48b5a82fdaeecfdefcc090e6a7daa406ab3808c0 Mon Sep 17 00:00:00 2001 From: mirdono Date: Mon, 15 Apr 2024 11:21:02 +0200 Subject: [PATCH 24/25] fix: correct type for classification to match resource configuration --- ...sentative-body-classification-code-type.sparql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 config/migrations/2024/20240415085335-correct-representative-body-classification-code-type.sparql diff --git a/config/migrations/2024/20240415085335-correct-representative-body-classification-code-type.sparql b/config/migrations/2024/20240415085335-correct-representative-body-classification-code-type.sparql new file mode 100644 index 0000000..39b0fdc --- /dev/null +++ b/config/migrations/2024/20240415085335-correct-representative-body-classification-code-type.sparql @@ -0,0 +1,15 @@ +PREFIX ext: + +DELETE { + GRAPH { + ?s a ext:RepresentatieveOrgaanClassificatieCode . + } +} INSERT { + GRAPH { + ?s a ext:RepresentatiefOrgaanClassificatieCode . + } +} WHERE { + GRAPH { + ?s a ext:RepresentatieveOrgaanClassificatieCode . + } +} From 95e737873a3b39543f680cfb983123157be3643e Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Mon, 15 Apr 2024 11:52:42 +0200 Subject: [PATCH 25/25] removed migration for empty URIs --- ...0240316100500-delete-all-empty-uris.sparql | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 config/migrations/2024/20240316100500-delete-all-empty-uris.sparql diff --git a/config/migrations/2024/20240316100500-delete-all-empty-uris.sparql b/config/migrations/2024/20240316100500-delete-all-empty-uris.sparql deleted file mode 100644 index 5f95b8e..0000000 --- a/config/migrations/2024/20240316100500-delete-all-empty-uris.sparql +++ /dev/null @@ -1,26 +0,0 @@ -DELETE { - GRAPH ?g { - ?s ?type ; - ?uuid . - } -} -WHERE { - VALUES ?g { - - - - - - } - GRAPH ?g { - ?s ?type ; - ?uuid . - - FILTER (?type != ) - FILTER NOT EXISTS { - FILTER (?p != ) - FILTER (?p != ) - ?s ?p ?o . - } - } -}