From f120638741ce4e911eed398a5666d123b075d7ba Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Fri, 17 Nov 2023 09:54:53 +0100 Subject: [PATCH 01/21] phone number migration OP-1464 --- ...20231112161533-update-phone-numbers.sparql | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 config/migrations/2023/20231112161533-update-phone-numbers.sparql diff --git a/config/migrations/2023/20231112161533-update-phone-numbers.sparql b/config/migrations/2023/20231112161533-update-phone-numbers.sparql new file mode 100644 index 0000000..61556c9 --- /dev/null +++ b/config/migrations/2023/20231112161533-update-phone-numbers.sparql @@ -0,0 +1,38 @@ +DELETE { + ?s ?o . +} +INSERT { + GRAPH { + ?s ?newPhoneNumber . + } +} +WHERE { + GRAPH { + ?s ?o . + } + + FILTER (SUBSTR(?o, 1, 1) != "+" AND STRLEN(STR(?o)) > 4) + BIND(REPLACE(STR(?o),"[. - /]","") AS ?removedSpaces) . + BIND (CONCAT('tel:+32', SUBSTR(?removedSpaces, 2)) as ?newPhoneNumber) +} + +DELETE { + ?s ?o . +} +INSERT { + GRAPH { + ?s ?newPhoneNumber . + } +} +WHERE { + GRAPH { + ?s ?o . + } + + FILTER (SUBSTR(?o, 1, 1) = "+") + BIND(REPLACE(STR(?o),"[. - /]","") AS ?removedSpaces) . + BIND (CONCAT('tel:', ?removedSpaces, 2) as ?newPhoneNumber) +} + + + From 57e09e9b470fec06d949fb958c647bea36330822 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Fri, 17 Nov 2023 11:41:06 +0100 Subject: [PATCH 02/21] migration OK --- .../20231112161533-update-phone-numbers.sparql | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/config/migrations/2023/20231112161533-update-phone-numbers.sparql b/config/migrations/2023/20231112161533-update-phone-numbers.sparql index 61556c9..ce1c0ed 100644 --- a/config/migrations/2023/20231112161533-update-phone-numbers.sparql +++ b/config/migrations/2023/20231112161533-update-phone-numbers.sparql @@ -1,13 +1,15 @@ DELETE { + GRAPH ?g { ?s ?o . + } } INSERT { - GRAPH { + GRAPH ?g { ?s ?newPhoneNumber . } } WHERE { - GRAPH { + GRAPH ?g { ?s ?o . } @@ -15,17 +17,19 @@ WHERE { BIND(REPLACE(STR(?o),"[. - /]","") AS ?removedSpaces) . BIND (CONCAT('tel:+32', SUBSTR(?removedSpaces, 2)) as ?newPhoneNumber) } - +; DELETE { + GRAPH ?g { ?s ?o . + } } INSERT { - GRAPH { + GRAPH ?g { ?s ?newPhoneNumber . } } WHERE { - GRAPH { + GRAPH ?g { ?s ?o . } From 30f4341e01cc6dfb9f3a4fc55f40462ea199903e Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Mon, 20 Nov 2023 11:03:31 +0100 Subject: [PATCH 03/21] fixnig remarks --- ...20231112161533-update-phone-numbers.sparql | 66 +++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/config/migrations/2023/20231112161533-update-phone-numbers.sparql b/config/migrations/2023/20231112161533-update-phone-numbers.sparql index ce1c0ed..d7ac17e 100644 --- a/config/migrations/2023/20231112161533-update-phone-numbers.sparql +++ b/config/migrations/2023/20231112161533-update-phone-numbers.sparql @@ -1,42 +1,86 @@ DELETE { - GRAPH ?g { + GRAPH { ?s ?o . } } INSERT { - GRAPH ?g { + GRAPH { ?s ?newPhoneNumber . } } WHERE { - GRAPH ?g { + GRAPH { ?s ?o . } - FILTER (SUBSTR(?o, 1, 1) != "+" AND STRLEN(STR(?o)) > 4) - BIND(REPLACE(STR(?o),"[. - /]","") AS ?removedSpaces) . + FILTER (SUBSTR(?o, 1, 1) != "+" AND STRLEN(STR(?o)) > 4 and !regex(?o, ".*[a-zA-Z].*", "i")) + BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . BIND (CONCAT('tel:+32', SUBSTR(?removedSpaces, 2)) as ?newPhoneNumber) } + ; + DELETE { - GRAPH ?g { + GRAPH { ?s ?o . } } INSERT { - GRAPH ?g { + GRAPH { ?s ?newPhoneNumber . } } WHERE { - GRAPH ?g { + GRAPH { ?s ?o . } - FILTER (SUBSTR(?o, 1, 1) = "+") - BIND(REPLACE(STR(?o),"[. - /]","") AS ?removedSpaces) . - BIND (CONCAT('tel:', ?removedSpaces, 2) as ?newPhoneNumber) + FILTER (SUBSTR(?o, 1, 1) = "+" and !regex(?o, ".*[a-zA-Z].*", "i")) + BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . + BIND (CONCAT('tel:', ?removedSpaces) as ?newPhoneNumber) } +; +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?newPhoneNumber . + } +} +WHERE { + GRAPH { + ?s ?o . + } + + FILTER (SUBSTR(?o, 1, 1) != "+" AND STRLEN(STR(?o)) > 4 and !regex(?o, ".*[a-zA-Z].*", "i")) + BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . + BIND (CONCAT('tel:+32', SUBSTR(?removedSpaces, 2)) as ?newPhoneNumber) +} + +; + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?newPhoneNumber . + } +} +WHERE { + GRAPH { + ?s ?o . + } + + FILTER (SUBSTR(?o, 1, 1) = "+" and !regex(?o, ".*[a-zA-Z].*", "i")) + BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . + BIND (CONCAT('tel:', ?removedSpaces) as ?newPhoneNumber) +} From 866cff52cc58448737873cf002c72288e5d98ac9 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Tue, 21 Nov 2023 11:09:07 +0100 Subject: [PATCH 04/21] fixed weird + issue --- .../2023/20231112161533-update-phone-numbers.sparql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/migrations/2023/20231112161533-update-phone-numbers.sparql b/config/migrations/2023/20231112161533-update-phone-numbers.sparql index d7ac17e..78d38ac 100644 --- a/config/migrations/2023/20231112161533-update-phone-numbers.sparql +++ b/config/migrations/2023/20231112161533-update-phone-numbers.sparql @@ -35,9 +35,9 @@ WHERE { ?s ?o . } - FILTER (SUBSTR(?o, 1, 1) = "+" and !regex(?o, ".*[a-zA-Z].*", "i")) + FILTER (STRSTARTS(?o, "+") and !regex(?o, ".*[a-zA-Z].*", "i")) BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . - BIND (CONCAT('tel:', ?removedSpaces) as ?newPhoneNumber) + BIND (CONCAT('tel:+', ?removedSpaces) as ?newPhoneNumber) } ; @@ -79,8 +79,8 @@ WHERE { ?s ?o . } - FILTER (SUBSTR(?o, 1, 1) = "+" and !regex(?o, ".*[a-zA-Z].*", "i")) + FILTER (STRSTARTS(?o, "+") and !regex(?o, ".*[a-zA-Z].*", "i")) BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . - BIND (CONCAT('tel:', ?removedSpaces) as ?newPhoneNumber) + BIND (CONCAT('tel:+', ?removedSpaces) as ?newPhoneNumber) } From 2fd9ce3774cbdd5aa9420c7a6c674143226fdea9 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 28 Nov 2023 16:49:58 +0100 Subject: [PATCH 05/21] Sync with Sharepoint list Organisaties (#304) * Set up deltas for sync with sharepoint * add service * add dev environment variables * Extract config to the stack * Adjust the config and mappings * Filter out worship services * Add contacten sp list service * Add configuration for contact list * Add config files for contacten sync * Wire sharepoint sync on main graph instead of producer * Add env variables * Add op sync initiator * Minor updates * Improve config, add job controller rules and more * Adjust rules * Remove unused producer changes * Don't restart initiator on dev setup * Sync OP status to a dedicated column * Add ABB_ID and split bus and house numbers * Remove config file we will not use * Add NIS and secondary phone number, rename columns * Remove isInitialMatchingMapping, not used anymore * Match on URI instead of UUID * Adapt field names to new lists * Specify matching field in config * Set version of op-sync-with-sharepoint --------- Co-authored-by: Nordine Bittich --- config/delta/rules.js | 55 +++++++++ config/jobs-controller/config.json | 18 +++ .../sharepoint/sync/organizations/config.json | 108 ++++++++++++++++++ docker-compose.dev.yml | 4 + docker-compose.yml | 31 +++++ 5 files changed, 216 insertions(+) create mode 100644 config/sharepoint/sync/organizations/config.json diff --git a/config/delta/rules.js b/config/delta/rules.js index b6b75ea..a5d3e12 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -162,6 +162,61 @@ export default [ optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } }, + // organizations-sync-with-sharepoint/delta : jobs come from the jobs graph, data from the two main graphs + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/jobs' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/shared' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/administrative-unit' + } + }, + callback: { + url: 'http://organizations-sync-with-sharepoint/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + } /* Commenting ldes rules until ready to go to qa/prod. In the meantime, we override the configuration on dev. { diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 5817d84..e22a768 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -126,5 +126,23 @@ "nextIndex": "0" } ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/sp-sync/initialSyncing/organizations": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/sp-sync/initialSyncing", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/sp-sync/healingOperation/organizations": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/sp-sync/healing", + "nextIndex": "0" + } + ] } } diff --git a/config/sharepoint/sync/organizations/config.json b/config/sharepoint/sync/organizations/config.json new file mode 100644 index 0000000..265d0cb --- /dev/null +++ b/config/sharepoint/sync/organizations/config.json @@ -0,0 +1,108 @@ +{ + "sourceGraphs": [ + "http://mu.semte.ch/graphs/administrative-unit", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/public" + ], + "objects": [ + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "mappings": [ + { + "op": ["http://www.w3.org/2004/02/skos/core#prefLabel"], + "sl": "Title" + }, + { + "op": ["http://www.w3.org/ns/org#classification", "http://www.w3.org/2004/02/skos/core#prefLabel"], + "sl": "OP_Type" + }, + { + "op": ["http://www.w3.org/ns/regorg#orgStatus", "http://www.w3.org/2004/02/skos/core#prefLabel"], + "sl": "OP_Status" + }, + { + "op": ["http://mu.semte.ch/vocabularies/core/uuid"], + "sl": "OP_Uuid" + }, + { + "op": ["http://mu.semte.ch/vocabularies/core/uuid", "^http://mu.semte.ch/vocabularies/core/uuid"], + "sl": "OP_Uri", + "isMatchingField": "true" + }, + { + "op": ["http://data.vlaanderen.be/ns/besluit#werkingsgebied", "http://www.w3.org/2004/02/skos/core#exactMatch", "http://www.w3.org/2004/02/skos/core#notation"], + "sl": "OP_NIScode" + } + ], + "pathToMatchingUri": "?s a . BIND (?s as ?matchingUri)" + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "mappings": [ + { + "op": ["https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"], + "sl": "OP_KBO" + } + ], + "pathToMatchingUri": "?matchingUri a ;\n\t ?kboIdentifier .\n ?kboIdentifier ?kboLabel ;\n\t ?s . FILTER (?kboLabel IN (\"KBO nummer\"@nl, \"KBO nummer\"))" + }, + { + "type": "http://www.w3.org/ns/locn#Address", + "mappings": [ + { + "op": ["http://www.w3.org/ns/locn#thoroughfare"], + "sl": "OP_Straat" + }, + { + "op": ["https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer"], + "sl": "OP_Huisnummer" + }, + { + "op": ["https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer"], + "sl": "OP_Busnummer" + }, + { + "op": ["http://www.w3.org/ns/locn#postCode"], + "sl": "OP_Postcode" + }, + { + "op": ["https://data.vlaanderen.be/ns/adres#gemeentenaam"], + "sl": "OP_Gemeente" + }, + { + "op": ["http://www.w3.org/ns/locn#adminUnitL2"], + "sl": "OP_Provincie" + } + ], + "pathToMatchingUri": "?matchingUri a ;\n\t / ?s ." + }, + { + "type": "http://schema.org/ContactPoint", + "mappings": [ + { + "op": ["http://schema.org/telephone"], + "sl": "OP_Primair_Telefoonnummer" + }, + { + "op": ["http://xmlns.com/foaf/0.1/page"], + "sl": "OP_Website" + }, + { + "op": ["http://schema.org/email"], + "sl": "OP_Email" + } + ], + "pathToMatchingUri": "?matchingUri a ;\n\t / ?s . ?s \"Primary\"" + }, + { + "type": "http://schema.org/ContactPoint", + "mappings": [ + { + "op": ["http://schema.org/telephone"], + "sl": "OP_Secundair_Telefoonnummer" + } + ], + "pathToMatchingUri": "?matchingUri a ;\n\t / ?s . ?s \"Secondary\"" + } + ] +} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 9f092e4..f4ed919 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -65,6 +65,10 @@ services: restart: "no" delta-producer-dump-file-publisher: restart: "no" + organizations-sync-with-sharepoint-jobs-initiator: + restart: "no" + organizations-sync-with-sharepoint: + restart: "no" construct-administrative-unit-relationships: restart: "no" delta-producer-bg-jobs-initiator-worship-posts: diff --git a/docker-compose.yml b/docker-compose.yml index 5943890..8ac55b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -222,6 +222,37 @@ services: - "logging=true" restart: always logging: *default-logging + organizations-sync-with-sharepoint-jobs-initiator: + image: lblod/delta-producer-background-jobs-initiator:0.4.3 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/sp-sync/initialSyncing/organizations" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/sp-sync/healingOperation/organizations" + INITIAL_PUBLICATION_GRAPH_SYNC_TASK_OPERATION: "http://redpencil.data.gift/id/jobs/concept/TaskOperation/sp-sync/initialSyncing" + HEALING_TASK_OPERATION: "http://redpencil.data.gift/id/jobs/concept/TaskOperation/sp-sync/healing" + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. + ENABLE_DUMP_FILE_CREATION: "false" + links: + - db:database + labels: + - "logging=true" + restart: always + logging: *default-logging + organizations-sync-with-sharepoint: + image: lblod/op-sync-with-sharepoint-service:0.1.0 + environment: + INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/sp-sync/initialSyncing/organizations" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/sp-sync/healingOperation/organizations" + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" + links: + - db:database + volumes: + - ./config/sharepoint/sync/organizations/:/config/ + labels: + - "logging=true" + restart: always + logging: *default-logging construct-administrative-unit-relationships: image: lblod/construct-administrative-unit-relationships-service:0.1.5 environment: From a462b53d0609c597e892796944377f4e3e241255 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 28 Nov 2023 19:22:55 +0100 Subject: [PATCH 06/21] Bump op-sync-with-sharepoint-service --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8ac55b5..882655b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -240,7 +240,7 @@ services: restart: always logging: *default-logging organizations-sync-with-sharepoint: - image: lblod/op-sync-with-sharepoint-service:0.1.0 + image: lblod/op-sync-with-sharepoint-service:0.1.2 environment: INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/sp-sync/initialSyncing/organizations" HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/sp-sync/healingOperation/organizations" From 674f27bff8402d53ab3ffcf1956ca42bbd83cc69 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Wed, 29 Nov 2023 09:57:47 +0100 Subject: [PATCH 07/21] saving progress for pr review --- ...20231112161533-update-phone-numbers.sparql | 6 +-- config/reasoner/dl2op/main/phone_number.n3 | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 config/reasoner/dl2op/main/phone_number.n3 diff --git a/config/migrations/2023/20231112161533-update-phone-numbers.sparql b/config/migrations/2023/20231112161533-update-phone-numbers.sparql index 78d38ac..64b78ca 100644 --- a/config/migrations/2023/20231112161533-update-phone-numbers.sparql +++ b/config/migrations/2023/20231112161533-update-phone-numbers.sparql @@ -12,9 +12,8 @@ WHERE { GRAPH { ?s ?o . } - - FILTER (SUBSTR(?o, 1, 1) != "+" AND STRLEN(STR(?o)) > 4 and !regex(?o, ".*[a-zA-Z].*", "i")) BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . + FILTER (SUBSTR(?removedSpaces, 1, 1) != "+" AND STRLEN(STR(?removedSpaces)) > 8 AND STRLEN(STR(?removedSpaces)) < 11 and !regex(?o, ".*[a-zA-Z].*", "i")) BIND (CONCAT('tel:+32', SUBSTR(?removedSpaces, 2)) as ?newPhoneNumber) } @@ -34,9 +33,8 @@ WHERE { GRAPH { ?s ?o . } - - FILTER (STRSTARTS(?o, "+") and !regex(?o, ".*[a-zA-Z].*", "i")) BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . + FILTER (STRSTARTS(?o, "+") and !regex(?o, ".*[a-zA-Z].*", "i")) BIND (CONCAT('tel:+', ?removedSpaces) as ?newPhoneNumber) } diff --git a/config/reasoner/dl2op/main/phone_number.n3 b/config/reasoner/dl2op/main/phone_number.n3 new file mode 100644 index 0000000..db784d8 --- /dev/null +++ b/config/reasoner/dl2op/main/phone_number.n3 @@ -0,0 +1,46 @@ +PREFIX string: +PREFIX e: +PREFIX schema: +#:s :p +# "+32488888888", +# "0477777777", +# "0477.77.77.00", +# "0", +# " +32 445 55 99 91", +# "tel:+3144887755", +# "0032 4 58 44 66 87", +# "00 32 0 498 77 7744". +{ + + + ?s schema:telephone ?number. + + # number not matches regex + + ?number string:notMatches """[a-zA-Z]""". + + # Remove all non digits + ( ?number """[^+\\d]""" "" ) string:replace ?scrape_result. + + # Replace leading 00 with + + ( ?scrape_result """^00""" "+") string:replace ?replaced_zero_zero. + + ( ?replaced_zero_zero """^0""" "+32") string:replace ?default_be. + + ( ?default_be """\\+320""" "+32" ) string:replace ?result. + + # ?result string:matches """^+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""". + ?result string:matches """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""". + + # concat with tel: + ("tel:" ?result) string:concatenation ?parsedNumber . + + { + ?s schema:telephone ?number. + } e:becomes { + ?s schema:telephone ?parsedNumber. + }. + +} => { + ?s schema:telephone ?parsedNumber. +}. From 672b4bce6a119fff0eea7626c09015105b497c33 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Thu, 30 Nov 2023 10:49:47 +0100 Subject: [PATCH 08/21] reasoner buf fixed --- config/reasoner/dl2op/main/phone_number.n3 | 23 +++++----------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/config/reasoner/dl2op/main/phone_number.n3 b/config/reasoner/dl2op/main/phone_number.n3 index db784d8..4acfd1f 100644 --- a/config/reasoner/dl2op/main/phone_number.n3 +++ b/config/reasoner/dl2op/main/phone_number.n3 @@ -1,19 +1,7 @@ PREFIX string: PREFIX e: -PREFIX schema: -#:s :p -# "+32488888888", -# "0477777777", -# "0477.77.77.00", -# "0", -# " +32 445 55 99 91", -# "tel:+3144887755", -# "0032 4 58 44 66 87", -# "00 32 0 498 77 7744". { - - - ?s schema:telephone ?number. + :s schema:telephone ?number. # number not matches regex @@ -32,15 +20,14 @@ PREFIX schema: # ?result string:matches """^+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""". ?result string:matches """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""". - # concat with tel: - ("tel:" ?result) string:concatenation ?parsedNumber . + ( "tel:" ?result ) string:concatenation ?parsed_result . { - ?s schema:telephone ?number. + :s schema:telephone ?number. } e:becomes { - ?s schema:telephone ?parsedNumber. + :s schema:telephone ?parsed_result . }. } => { - ?s schema:telephone ?parsedNumber. + :s schema:telephone ?parsed_result . }. From 122b51ad0ec64f711fdd3fb76f80e703d928f80f Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Thu, 30 Nov 2023 11:12:02 +0100 Subject: [PATCH 09/21] migration check --- ...02311121615322-update-phone-numbers.sparql | 202 ++++++++++++++++++ ...20231112161533-update-phone-numbers.sparql | 84 -------- 2 files changed, 202 insertions(+), 84 deletions(-) create mode 100644 config/migrations/2023/202311121615322-update-phone-numbers.sparql delete mode 100644 config/migrations/2023/20231112161533-update-phone-numbers.sparql diff --git a/config/migrations/2023/202311121615322-update-phone-numbers.sparql b/config/migrations/2023/202311121615322-update-phone-numbers.sparql new file mode 100644 index 0000000..5e56d80 --- /dev/null +++ b/config/migrations/2023/202311121615322-update-phone-numbers.sparql @@ -0,0 +1,202 @@ +#fix nrs starting with 0032 in Administrative-unit + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?fixedTel . + } +} +WHERE { + GRAPH { + ?s ?o . + } + FILTER(!regex(?o, ".*[a-zA-Z].*", "i")) . + BIND(REPLACE(STR(?o),"[. -/();:=+]","") AS ?removedSpaces) . + FILTER(STRSTARTS(?removedSpaces, "0032")) . + BIND(CONCAT("32", SUBSTR(?removedSpaces, 5)) as ?newTel ) . + FILTER(STRLEN(?newTel) > 9 and STRLEN(?newTel) < 12) . + BIND(CONCAT("+", ?newTel) AS ?fixedTel) . +} +; + +#fix nrs starting with 0032 in worship-service + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?fixedTel . + } +} +WHERE { + GRAPH { + ?s ?o . + } + + FILTER(!regex(?o, ".*[a-zA-Z].*", "i")) . + BIND(REPLACE(STR(?o),"[. -/();:=+]","") AS ?removedSpaces) . + FILTER(STRSTARTS(?removedSpaces, "0032")) . + BIND(CONCAT("32", SUBSTR(?removedSpaces, 5)) as ?newTel ) . + FILTER(STRLEN(?newTel) > 9 and STRLEN(?newTel) < 12) . + BIND(CONCAT("+", ?newTel) AS ?fixedTel) . +} +; + + +#fix nrs starting with 320 in administrative-unit + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?fixedTel . + } +} +WHERE { + GRAPH { + ?s ?o . + } + FILTER(!regex(?o, ".*[a-zA-Z].*", "i")) . + BIND(REPLACE(STR(?o),"[. -/();:=+]","") AS ?removedSpaces) . + FILTER(STRSTARTS(?removedSpaces, "320")) . + BIND(CONCAT("32", SUBSTR(?removedSpaces, 4)) as ?newTel ) . + FILTER(STRLEN(?newTel) > 9 and STRLEN(?newTel) < 12) . + BIND(CONCAT("+", ?newTel) AS ?fixedTel) . +} +; +#fix nrs starting with 320 in worship-service + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?fixedTel . + } +} +WHERE { + GRAPH { + ?s ?o . + } + + FILTER(!regex(?o, ".*[a-zA-Z].*", "i")) . + BIND(REPLACE(STR(?o),"[. -/();:=+]","") AS ?removedSpaces) . + FILTER(STRSTARTS(?removedSpaces, "0032")) . + BIND(CONCAT("+32", SUBSTR(?removedSpaces, 5)) as ?newTel ) . + FILTER(STRLEN(?newTel) > 9 and STRLEN(?newTel) < 12) . + BIND(CONCAT("+", ?newTel) AS ?fixedTel) . +} +; + +#fix nrs starting with 0 in administrative-unit + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?fixedTel . + } +} +WHERE { + GRAPH { + ?s ?o . + } + FILTER(!regex(?o, ".*[a-zA-Z].*", "i")) . + BIND(REPLACE(STR(?o),"[. -/();:=+]","") AS ?removedSpaces) . + FILTER(STRSTARTS(?removedSpaces, "0")) . + BIND(CONCAT("32", SUBSTR(?removedSpaces, 2)) as ?newTel ) . + FILTER(STRLEN(?newTel) > 9 and STRLEN(?newTel) < 12) . + BIND(CONCAT("+", ?newTel) AS ?fixedTel) . +} +; + + +#fix nrs starting with 03/04 in worship-service + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?fixedTel . + } +} +WHERE { + GRAPH { + ?s ?o . + } + + FILTER(!regex(?o, ".*[a-zA-Z].*", "i")) . + BIND(REPLACE(STR(?o),"[. -/();:=+]","") AS ?removedSpaces) . + FILTER(STRSTARTS(?removedSpaces, "0")) . + BIND(CONCAT("32", SUBSTR(?removedSpaces, 2)) as ?newTel ) . + FILTER(STRLEN(?newTel) > 9 and STRLEN(?newTel) < 12) . + BIND(CONCAT("+", ?newTel) AS ?fixedTel) . + +} +; + +#convert all nrs starting with 32 in administrative-unit + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?convertedTel . + } +} +WHERE { + GRAPH { + ?s ?o . + } + FILTER(!regex(?o, ".*[a-zA-Z].*", "i")) . + BIND(REPLACE(STR(?o),"[. -/();:=+]","") AS ?removedSpaces) . + FILTER(STRSTARTS(?removedSpaces, "32") AND STRLEN(?removedSpaces) > 9 and STRLEN(?removedSpaces) < 12) . + BIND(CONCAT("tel:+", ?removedSpaces) as ?convertedTel) . +} +; + + +#convert all nrs starting with 32 in worship-service + +DELETE { + GRAPH { + ?s ?o . + } +} +INSERT { + GRAPH { + ?s ?convertedTel . + } +} +WHERE { + GRAPH { + ?s ?o . + } + + FILTER(!regex(?o, ".*[a-zA-Z].*", "i")) . + BIND(REPLACE(STR(?o),"[. -/();:=+]","") AS ?removedSpaces) . + FILTER(STRSTARTS(?removedSpaces, "32") AND STRLEN(?removedSpaces) > 9 and STRLEN(?removedSpaces) < 12) . + BIND(CONCAT("tel:+", ?removedSpaces) as ?convertedTel) . +} +; \ No newline at end of file diff --git a/config/migrations/2023/20231112161533-update-phone-numbers.sparql b/config/migrations/2023/20231112161533-update-phone-numbers.sparql deleted file mode 100644 index 64b78ca..0000000 --- a/config/migrations/2023/20231112161533-update-phone-numbers.sparql +++ /dev/null @@ -1,84 +0,0 @@ -DELETE { - GRAPH { - ?s ?o . - } -} -INSERT { - GRAPH { - ?s ?newPhoneNumber . - } -} -WHERE { - GRAPH { - ?s ?o . - } - BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . - FILTER (SUBSTR(?removedSpaces, 1, 1) != "+" AND STRLEN(STR(?removedSpaces)) > 8 AND STRLEN(STR(?removedSpaces)) < 11 and !regex(?o, ".*[a-zA-Z].*", "i")) - BIND (CONCAT('tel:+32', SUBSTR(?removedSpaces, 2)) as ?newPhoneNumber) -} - -; - -DELETE { - GRAPH { - ?s ?o . - } -} -INSERT { - GRAPH { - ?s ?newPhoneNumber . - } -} -WHERE { - GRAPH { - ?s ?o . - } - BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . - FILTER (STRSTARTS(?o, "+") and !regex(?o, ".*[a-zA-Z].*", "i")) - BIND (CONCAT('tel:+', ?removedSpaces) as ?newPhoneNumber) -} - -; - -DELETE { - GRAPH { - ?s ?o . - } -} -INSERT { - GRAPH { - ?s ?newPhoneNumber . - } -} -WHERE { - GRAPH { - ?s ?o . - } - - FILTER (SUBSTR(?o, 1, 1) != "+" AND STRLEN(STR(?o)) > 4 and !regex(?o, ".*[a-zA-Z].*", "i")) - BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . - BIND (CONCAT('tel:+32', SUBSTR(?removedSpaces, 2)) as ?newPhoneNumber) -} - -; - -DELETE { - GRAPH { - ?s ?o . - } -} -INSERT { - GRAPH { - ?s ?newPhoneNumber . - } -} -WHERE { - GRAPH { - ?s ?o . - } - - FILTER (STRSTARTS(?o, "+") and !regex(?o, ".*[a-zA-Z].*", "i")) - BIND(REPLACE(STR(?o),"[. -/()]","") AS ?removedSpaces) . - BIND (CONCAT('tel:+', ?removedSpaces) as ?newPhoneNumber) -} - From 4e8af93f236905918f30bd6381c6e44264c126cd Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Thu, 30 Nov 2023 11:43:07 +0100 Subject: [PATCH 10/21] reasoner changes --- config/reasoner/dl2op/main/phone_number.n3 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/reasoner/dl2op/main/phone_number.n3 b/config/reasoner/dl2op/main/phone_number.n3 index 4acfd1f..b6a7a2d 100644 --- a/config/reasoner/dl2op/main/phone_number.n3 +++ b/config/reasoner/dl2op/main/phone_number.n3 @@ -1,7 +1,8 @@ PREFIX string: PREFIX e: +PREFIX schema: { - :s schema:telephone ?number. + ?s schema:telephone ?number. # number not matches regex From d34e28ec0ea3fa1a5d1d46f89be06fee491b4469 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 30 Nov 2023 13:18:50 +0100 Subject: [PATCH 11/21] Add report generation service + orgs report (#331) * Add report generation service + orgs report * Adapt configuration * Add download-exports service to the stack * Update gitignore * Bump report generation service * Remove unnecessary import * Add dispatcher rule for download exports --- .gitignore | 1 + config/dispatcher/dispatcher.ex | 8 ++ config/reports/index.js | 5 ++ config/reports/organizations-report.js | 115 +++++++++++++++++++++++++ config/reports/utils.js | 29 +++++++ docker-compose.yml | 24 ++++++ 6 files changed, 182 insertions(+) create mode 100644 config/reports/index.js create mode 100644 config/reports/organizations-report.js create mode 100644 config/reports/utils.js diff --git a/.gitignore b/.gitignore index caed09e..ccfbba6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ !/data/db/toLoad !/data/db/toLoad/* /data/files/* +!/data/files/exports/.gitkeep /data/temp/* docker-compose.override.yml .idea diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 4ef9420..cce1b56 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -373,6 +373,14 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-worship-posts/files/" end + ################################################################# + # Exports + ################################################################# + + get "/download-exports/*path" do + Proxy.forward conn, path, "http://download-exports/" + end + ############################################################### # frontend layer ############################################################### diff --git a/config/reports/index.js b/config/reports/index.js new file mode 100644 index 0000000..3a124aa --- /dev/null +++ b/config/reports/index.js @@ -0,0 +1,5 @@ +import OrganizationsReport from './organizations-report' + +export default [ + OrganizationsReport, +]; diff --git a/config/reports/organizations-report.js b/config/reports/organizations-report.js new file mode 100644 index 0000000..d8e666e --- /dev/null +++ b/config/reports/organizations-report.js @@ -0,0 +1,115 @@ +import { generateReportFromData, batchedQuery } from '../helpers.js'; +import { PREFIXES } from './utils' + +export default { + cronPattern: '0 00 23 * * *', + name: 'organizations', + execute: async () => { + const reportData = { + title: 'organizations', + description: 'List of the Organizations', + filePrefix: 'exports/organizations' + }; + + console.log('Generating organizations report'); + + const queryStringPart1 = ` + ${PREFIXES} + + SELECT DISTINCT ?bestuur ?label ?classification ?typeEredienst + WHERE { + VALUES ?type { + org:Organization + besluit:Bestuurseenheid + } + + GRAPH ?g { + ?bestuur a ?type . + OPTIONAL { ?bestuur skos:prefLabel ?label. } + } + + OPTIONAL { + ?bestuur org:classification ?classificationUri . + ?classificationUri skos:prefLabel ?classification. + } + + OPTIONAL { + ?bestuur ere:typeEredienst ?typeEredienstUri. + ?typeEredienstUri skos:prefLabel ?typeEredienst. + } + + FILTER (?g IN ( , , )) + } + `; + + const queryResponsePart1 = await batchedQuery(queryStringPart1); + + const dataPart1 = queryResponsePart1.results.bindings.reduce( (acc, row) => { + acc[getSafeValue(row, 'bestuur')] = { + bestuur: getSafeValue(row, 'bestuur'), + label: getSafeValue(row, 'label'), + classification: getSafeValue(row, 'classification'), + typeEredienst: getSafeValue(row, 'typeEredienst'), + }; + return acc; + }, {}); + + const queryStringPart2 = ` + ${PREFIXES} + + SELECT DISTINCT ?bestuur ?provincie + WHERE { + VALUES ?type { + org:Organization + besluit:Bestuurseenheid + } + + GRAPH ?g { + ?bestuur a ?type . + } + + GRAPH ?h { + OPTIONAL { + ?bestuur org:hasPrimarySite ?primarySite. + + OPTIONAL { + ?primarySite organisatie:bestaatUit ?sa. + OPTIONAL { ?sa locn:adminUnitL2 ?provincie. } + } + } + } + + FILTER (?g IN ( , , )) + FILTER (?h IN ( , , )) + } + `; + + const queryResponsePart2 = await batchedQuery(queryStringPart2); + const dataPart2 = queryResponsePart2.results.bindings.reduce( (acc, row) => { + let dataPart = { + bestuur: getSafeValue(row, 'bestuur'), + provincie: getSafeValue(row, 'provincie') + }; + + acc[getSafeValue(row, 'bestuur')] = Object.assign(dataPart, dataPart1[getSafeValue(row, 'bestuur')]); + return acc; + }, {}); + + await generateReportFromData(Object.values(dataPart2), [ + 'bestuur', + 'label', + 'classification', + 'typeEredienst', + 'provincie' + ], reportData); + } +}; + +function getSafeValue(entry, property){ + return entry[property] ? wrapInQuote(entry[property].value) : null; +} + +// Some values might contain commas; wrapping them in escapes quotes doesn't disrupt the columns. +function wrapInQuote(value) { + return `\"${value}\"`; +} diff --git a/config/reports/utils.js b/config/reports/utils.js new file mode 100644 index 0000000..fdda6e4 --- /dev/null +++ b/config/reports/utils.js @@ -0,0 +1,29 @@ +export const PREFIXES = ` + PREFIX adms: + PREFIX adres: + PREFIX besluit: + PREFIX ch: + PREFIX code: + PREFIX dbpedia: + PREFIX dc_terms: + PREFIX ere: + PREFIX euro: + PREFIX euvoc: + PREFIX ext: + PREFIX foaf: + PREFIX generiek: + PREFIX lblodlg: + PREFIX locn: + PREFIX mandaat: + PREFIX org: + PREFIX organisatie: + PREFIX person: + PREFIX persoon: + PREFIX prov: + PREFIX rdfs: + PREFIX regorg: + PREFIX schema: + PREFIX skos: + PREFIX vcard: + PREFIX xsd: +`; diff --git a/docker-compose.yml b/docker-compose.yml index 882655b..e6b5e9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -270,6 +270,30 @@ services: - "logging=true" restart: always logging: *default-logging + report-generation: + image: lblod/loket-report-generation-service:0.7.0 + environment: + DEFAULT_GRAPH: "http://mu.semte.ch/graphs/shared" + ONLY_KEEP_LATEST_REPORT: "true" + links: + - db:database + volumes: + - ./data/files:/share + - ./config/reports:/config + labels: + - "logging=true" + restart: always + logging: *default-logging + download-exports: + image: lblod/download-exports-service:0.1.0 + links: + - db:database + volumes: + - ./data/files:/share + labels: + - "logging=true" + restart: always + logging: *default-logging sync-ovo-numbers: image: lblod/sync-ovo-numbers-service:0.1.0 links: From 78baadce4fc11606a4ed0ab7f00724a1a7299503 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Thu, 30 Nov 2023 13:42:39 +0100 Subject: [PATCH 12/21] changed :s to ?s --- config/reasoner/dl2op/main/phone_number.n3 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/reasoner/dl2op/main/phone_number.n3 b/config/reasoner/dl2op/main/phone_number.n3 index b6a7a2d..e04eb47 100644 --- a/config/reasoner/dl2op/main/phone_number.n3 +++ b/config/reasoner/dl2op/main/phone_number.n3 @@ -24,11 +24,11 @@ PREFIX schema: ( "tel:" ?result ) string:concatenation ?parsed_result . { - :s schema:telephone ?number. + ?s schema:telephone ?number. } e:becomes { - :s schema:telephone ?parsed_result . + ?s schema:telephone ?parsed_result . }. } => { - :s schema:telephone ?parsed_result . + ?s schema:telephone ?parsed_result . }. From a944075616b0ae01f03a4ace149f3e6fbf576f43 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Thu, 30 Nov 2023 18:30:48 +0100 Subject: [PATCH 13/21] updated prefixes --- .../reasoner/dl2op/main-info/phone_number.n3 | 54 +++++++++++++++++++ config/reasoner/dl2op/main/phone_number.n3 | 19 +++++++ 2 files changed, 73 insertions(+) create mode 100644 config/reasoner/dl2op/main-info/phone_number.n3 diff --git a/config/reasoner/dl2op/main-info/phone_number.n3 b/config/reasoner/dl2op/main-info/phone_number.n3 new file mode 100644 index 0000000..0325fd5 --- /dev/null +++ b/config/reasoner/dl2op/main-info/phone_number.n3 @@ -0,0 +1,54 @@ +PREFIX string: +PREFIX e: +PREFIX schema: +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: + +{ + ?s schema:telephone ?number. + + # number not matches regex + + ?number string:notMatches """[a-zA-Z]""". + + # Remove all non digits + ( ?number """[^+\\d]""" "" ) string:replace ?scrape_result. + + # Replace leading 00 with + + ( ?scrape_result """^00""" "+") string:replace ?replaced_zero_zero. + + ( ?replaced_zero_zero """^0""" "+32") string:replace ?default_be. + + ( ?default_be """\\+320""" "+32" ) string:replace ?result. + + # ?result string:matches """^+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""". + ?result string:matches """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""". + + ( "tel:" ?result ) string:concatenation ?parsed_result . + + { + ?s schema:telephone ?number. + } e:becomes { + ?s schema:telephone ?parsed_result . + }. + +} => { + ?s schema:telephone ?parsed_result . +}. diff --git a/config/reasoner/dl2op/main/phone_number.n3 b/config/reasoner/dl2op/main/phone_number.n3 index e04eb47..f83f5ad 100644 --- a/config/reasoner/dl2op/main/phone_number.n3 +++ b/config/reasoner/dl2op/main/phone_number.n3 @@ -1,6 +1,25 @@ PREFIX string: PREFIX e: PREFIX schema: +PREFIX : <#> +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX crypto: +PREFIX e: +PREFIX ex: +PREFIX foaf: +PREFIX func: +PREFIX generiek: +PREFIX log: +PREFIX mandaat: +PREFIX mu: +PREFIX org: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX skos: +PREFIX string: +PREFIX xsd: { ?s schema:telephone ?number. From d5c6a3f66607ece37cd1c2dc0a4f402d8648342e Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Thu, 30 Nov 2023 18:36:54 +0100 Subject: [PATCH 14/21] added rules to maininfo --- config/reasoner/dl2op/main-info/phone_number.n3 | 1 - 1 file changed, 1 deletion(-) diff --git a/config/reasoner/dl2op/main-info/phone_number.n3 b/config/reasoner/dl2op/main-info/phone_number.n3 index 0325fd5..f83f5ad 100644 --- a/config/reasoner/dl2op/main-info/phone_number.n3 +++ b/config/reasoner/dl2op/main-info/phone_number.n3 @@ -20,7 +20,6 @@ PREFIX persoon_loket: PREFIX skos: PREFIX string: PREFIX xsd: - { ?s schema:telephone ?number. From 4dfe1953a700df2c528fb929058957852bcdf929 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 4 Dec 2023 09:34:07 +0100 Subject: [PATCH 15/21] Bump reasoner service --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b9d8a59..3e11570 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -232,7 +232,7 @@ services: restart: always logging: *default-logging reasoner: - image: eyereasoner/reasoning-service:increased-stack-limit + image: eyereasoner/reasoning-service:1.0.3 volumes: - ./config/reasoner:/config labels: From bf425b7dbb168365523400435e3f390399c09964 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 4 Dec 2023 10:24:35 +0100 Subject: [PATCH 16/21] Bump reasoner service --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3e11570..90cc857 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -232,7 +232,7 @@ services: restart: always logging: *default-logging reasoner: - image: eyereasoner/reasoning-service:1.0.3 + image: eyereasoner/reasoning-service:1.0.4 volumes: - ./config/reasoner:/config labels: From 562f463af33e2995d00f4d2d27299ad578db14e1 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 4 Dec 2023 11:51:52 +0100 Subject: [PATCH 17/21] Add missing rule for contact point of leidinggevenden --- config/reasoner/dl2op/main/query.n3q | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/config/reasoner/dl2op/main/query.n3q b/config/reasoner/dl2op/main/query.n3q index b72ce01..1db4e37 100644 --- a/config/reasoner/dl2op/main/query.n3q +++ b/config/reasoner/dl2op/main/query.n3q @@ -26,6 +26,16 @@ ?p ?o . }. +# For functionarissen, the contact point is on the bestuursfunctie and not on the agent in position + +{ + ?s a ; + ?o . + } => { + ?s a ; + ?o . +}. + # Functionarissen should be linked to an igs, agb, apb, gemeente, district, provincie or ocmw only (FOR NOW) { @@ -225,8 +235,6 @@ ?p ?o . }. - - { ?s a ; ?p ?o . From 7cbf418fcc7e2006048a61f88fc5c3c340ba2250 Mon Sep 17 00:00:00 2001 From: Elias Papanikitas Date: Mon, 4 Dec 2023 16:36:36 +0100 Subject: [PATCH 18/21] fixed typo --- ...-numbers.sparql => 20231112161532-update-phone-numbers.sparql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename config/migrations/2023/{202311121615322-update-phone-numbers.sparql => 20231112161532-update-phone-numbers.sparql} (100%) diff --git a/config/migrations/2023/202311121615322-update-phone-numbers.sparql b/config/migrations/2023/20231112161532-update-phone-numbers.sparql similarity index 100% rename from config/migrations/2023/202311121615322-update-phone-numbers.sparql rename to config/migrations/2023/20231112161532-update-phone-numbers.sparql From 79d2dda77a7e2b8054ecee002f2d376c147e9425 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 6 Dec 2023 14:09:38 +0100 Subject: [PATCH 19/21] Add publication triplestore and organizations public info producer (#356) --- .../dump-file-publisher/config.json | 24 ++ .../organizations-public-info/export.json | 349 ++++++++++++++++++ config/delta/rules.js | 17 + config/dispatcher/dispatcher.ex | 8 + config/jobs-controller/config.json | 32 ++ .../virtuoso-production.ini | 253 +++++++++++++ config/publication-triplestore/virtuoso.ini | 253 +++++++++++++ docker-compose.dev.yml | 4 + docker-compose.yml | 65 +++- 9 files changed, 1004 insertions(+), 1 deletion(-) create mode 100644 config/delta-producer/organizations-public-info/export.json create mode 100644 config/publication-triplestore/virtuoso-production.ini create mode 100755 config/publication-triplestore/virtuoso.ini diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 42c183f..83076a1 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -3,6 +3,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true @@ -11,14 +12,34 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "fileBaseName": "dump-organizations-public-info", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", + "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas", + "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "fileBaseName": "dump-organizations-public-info", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", + "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas", + "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas", + "cleanupOldDumps": true + }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://mu.semte.ch/graphs/public", "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true @@ -27,6 +48,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://mu.semte.ch/graphs/public", "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true @@ -35,6 +57,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", "fileBaseName": "dump-worship-posts", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://mu.semte.ch/graphs/public", "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true @@ -43,6 +66,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", "fileBaseName": "dump-worship-posts", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://mu.semte.ch/graphs/public", "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true diff --git a/config/delta-producer/organizations-public-info/export.json b/config/delta-producer/organizations-public-info/export.json new file mode 100644 index 0000000..f3bb667 --- /dev/null +++ b/config/delta-producer/organizations-public-info/export.json @@ -0,0 +1,349 @@ +{ + "export": [ + { + "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/denominatie", + "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", + "http://data.lblod.info/vocabularies/erediensten/denominatie", + "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://data.vlaanderen.be/ns/besluit#werkingsgebied", + "http://www.opengis.net/ont/geosparql#sfWithin", + "http://www.w3.org/2002/07/owl#sameAs", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/regorg#orgStatus" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/org#linkedTo", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/ns/adms#identifier" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://data.lblod.info/vocabularies/erediensten/typebetrokkenheid", + "http://www.w3.org/ns/org#organization", + "http://data.lblod.info/vocabularies/erediensten/financieringspercentage" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role", + "http://data.lblod.info/vocabularies/erediensten/functie" + ] + }, + { + "type": "http://www.w3.org/ns/org#Site", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#siteAddress", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://data.lblod.info/vocabularies/erediensten/vestigingstype" + ] + }, + { + "type": "http://www.w3.org/ns/locn#Address", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer", + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer", + "http://www.w3.org/ns/locn#thoroughfare", + "http://www.w3.org/ns/locn#postCode", + "https://data.vlaanderen.be/ns/adres#gemeentenaam", + "http://www.w3.org/ns/locn#adminUnitL2", + "https://data.vlaanderen.be/ns/adres#land", + "http://www.w3.org/ns/locn#fullAddress", + "https://data.vlaanderen.be/ns/adres#verwijstNaar", + "http://purl.org/dc/terms/source" + ] + }, + { + "type": "http://schema.org/ContactPoint", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/locn#address", + "http://xmlns.com/foaf/0.1/page", + "http://schema.org/faxNumber", + "http://schema.org/contactType", + "http://xmlns.com/foaf/0.1/page", + "http://schema.org/email", + "http://schema.org/telephone" + ] + }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ] + }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ], + "additionalFilter": "?ro ?subject ; a ." + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ] + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ], + "additionalFilter": "?ro / ?subject ; a ." + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#classification", + "http://data.vlaanderen.be/ns/mandaat#bindingEinde", + "http://data.vlaanderen.be/ns/mandaat#bindingStart", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://www.w3.org/ns/org#hasPost" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://www.w3.org/ns/org#ChangeEvent", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://purl.org/dc/terms/date", + "http://purl.org/dc/terms/description", + "http://data.lblod.info/vocabularies/contacthub/typeWijziging", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#changedBy", + "http://lblod.data.gift/vocabularies/organisatie/veranderingsgebeurtenisResultaat", + "http://www.w3.org/ns/org#originalOrganization", + "http://www.w3.org/ns/org#resultingOrganization", + "http://data.europa.eu/m8g/hasFormalFramework" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + } + ] +} diff --git a/config/delta/rules.js b/config/delta/rules.js index a5d3e12..3a24e37 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -105,6 +105,23 @@ export default [ ] } }, + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-organizations-public-info/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, { match: { }, diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index cce1b56..7c1c085 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -357,6 +357,14 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/" end + ################################################################# + # DELTA: organizations-public-info + ################################################################# + + get "/sync/organizations-public-info/files/*path" do + Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations-public-info/files/" + end + ################################################################# # DELTA: public ################################################################# diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index e22a768..4115501 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -63,6 +63,38 @@ } ] }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextIndex": "0" + }, + { + "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "1" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "0" + } + ] + }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { "tasksConfiguration": [ { diff --git a/config/publication-triplestore/virtuoso-production.ini b/config/publication-triplestore/virtuoso-production.ini new file mode 100644 index 0000000..c074796 --- /dev/null +++ b/config/publication-triplestore/virtuoso-production.ini @@ -0,0 +1,253 @@ +; +; virtuoso.ini +; +; Configuration file for the OpenLink Virtuoso VDBMS Server +; +; To learn more about this product, or any other product in our +; portfolio, please check out our web site at: +; +; http://virtuoso.openlinksw.com/ +; +; or contact us at: +; +; general.information@openlinksw.com +; +; If you have any technical questions, please contact our support +; staff at: +; +; technical.support@openlinksw.com +; +; +; Database setup +; +[Database] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.db +ErrorLogFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.log +LockFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.lck +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.trx +xa_persistent_file = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.pxa +ErrorLogLevel = 7 +FileExtend = 200 +MaxCheckpointRemap = 17000 ;;niels 1/4 of the number of buffers (assuming db uses most of the available buffers, you can check with `status()` in isql-v +Striping = 0 +TempStorage = TempDatabase +Syslog = 1 ; write to syslog in case of failures where writing to log is not possible + +[TempDatabase] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.db +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.trx +MaxCheckpointRemap = 2000 +Striping = 0 + +; +; Server parameters +; +[Parameters] +ServerPort = 1111 +LiteMode = 0 +DisableUnixSocket = 1 +DisableTcpSocket = 0 +;SSLServerPort = 2111 +;SSLCertificate = cert.pem +;SSLPrivateKey = pk.pem +;X509ClientVerify = 0 +;X509ClientVerifyDepth = 0 +;X509ClientVerifyCAFile = ca.pem +MaxClientConnections = 10 +CheckpointInterval = 60 +TraceOn = errors ; write traces +O_DIRECT = 0 +CaseMode = 2 +MaxStaticCursorRows = 5000 +CheckpointAuditTrail = 0 +AllowOSCalls = 0 +SchedulerInterval = 10 +DirsAllowed = ., /usr/local/virtuoso-opensource/share/virtuoso/vad +ThreadCleanupInterval = 10 +ThreadThreshold = 10 +ResourcesCleanupInterval = 10 +FreeTextBatchSize = 100000 +SingleCPU = 0 +VADInstallDir = /usr/local/virtuoso-opensource/share/virtuoso/vad/ +PrefixResultNames = 0 +RdfFreeTextRulesSize = 100 +IndexTreeMaps = 256 +MaxMemPoolSize = 200000000 +PrefixResultNames = 0 +MacSpotlight = 0 +MaxQueryMem = 4G ; memory allocated to query processor ;;Niels said to 4g +VectorSize = 1000 ; initial parallel query vector (array of query operations) size +MaxVectorSize = 1000000 ; query vector size threshold. +AdjustVectorSize = 0 +ThreadsPerQuery = 4 +AsyncQueueMaxThreads = 10 +TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient consumption is 450MB (roll forward + roll back) +;; +;; When running with large data sets, one should configure the Virtuoso +;; process to use between 2/3 to 3/5 of free system memory and to stripe +;; storage on all available disks. +;; +;; Uncomment next two lines if there is 2 GB system memory free +;;NumberOfBuffers = 170000 +;;MaxDirtyBuffers = 130000 + +;; Uncomment next two lines if there is 4 GB system memory free +;;NumberOfBuffers = 340000 +;;MaxDirtyBuffers = 250000 +;; Uncomment next two lines if there is 8 GB system memory free +NumberOfBuffers = 680000 +MaxDirtyBuffers = 500000 +;; Uncomment next two lines if there is 16 GB system memory free +;NumberOfBuffers = 1360000 +;MaxDirtyBuffers = 1000000 +;; Uncomment next two lines if there is 32 GB system memory free +;NumberOfBuffers = 2720000 +;MaxDirtyBuffers = 2000000 +;; Uncomment next two lines if there is 48 GB system memory free +;NumberOfBuffers = 4000000 +;MaxDirtyBuffers = 3000000 +;; Uncomment next two lines if there is 64 GB system memory free +;NumberOfBuffers = 5450000 +;MaxDirtyBuffers = 4000000 +;; +;; Note the default settings will take very little memory +;; but will not result in very good performance +;; +;NumberOfBuffers = 10000 +;MaxDirtyBuffers = 6000 + +;; Introduced after bug with export service +MaxSortedTopRows = 1000000 ; 1 million! + +[HTTPServer] +ServerPort = 8890 +ServerRoot = /usr/local/virtuoso-opensource/var/lib/virtuoso/vsp +MaxClientConnections = 10 +DavRoot = DAV +EnabledDavVSP = 0 +HTTPProxyEnabled = 0 +TempASPXDir = 0 +DefaultMailServer = localhost:25 +ServerThreads = 10 +MaxKeepAlives = 10 +KeepAliveTimeout = 10 +MaxCachedProxyConnections = 10 +ProxyConnectionCacheTimeout = 15 +HTTPThreadSize = 280000 +HttpPrintWarningsInOutput = 0 +Charset = UTF-8 +;;HTTPLogFile = http.log +MaintenancePage = atomic.html +EnabledGzipContent = 1 + +[AutoRepair] +BadParentLinks = 0 + +[Client] +SQL_PREFETCH_ROWS = 100 +SQL_PREFETCH_BYTES = 16000 +SQL_QUERY_TIMEOUT = 0 +SQL_TXN_TIMEOUT = 0 +;SQL_NO_CHAR_C_ESCAPE = 1 +;SQL_UTF8_EXECS = 0 +;SQL_NO_SYSTEM_TABLES = 0 +;SQL_BINARY_TIMESTAMP = 1 +;SQL_ENCRYPTION_ON_PASSWORD = -1 + +[VDB] +ArrayOptimization = 0 +NumArrayParameters = 10 +VDBDisconnectTimeout = 1000 +KeepConnectionOnFixedThread = 0 + +[Replication] +ServerName = db-D602566B774E +ServerEnable = 1 +QueueMax = 50000 + +; +; Striping setup +; +; These parameters have only effect when Striping is set to 1 in the +; [Database] section, in which case the DatabaseFile parameter is ignored. +; +; With striping, the database is spawned across multiple segments +; where each segment can have multiple stripes. +; +; Format of the lines below: +; Segment = , [, .. ] +; +; must be ordered from 1 up. +; +; The is the total size of the segment which is equally divided +; across all stripes forming the segment. Its specification can be in +; gigabytes (g), megabytes (m), kilobytes (k) or in database blocks +; (b, the default) +; +; Note that the segment size must be a multiple of the database page size +; which is currently 8k. Also, the segment size must be divisible by the +; number of stripe files forming the segment. +; +; The example below creates a 200 meg database striped on two segments +; with two stripes of 50 meg and one of 100 meg. +; +; You can always add more segments to the configuration, but once +; added, do not change the setup. +; +[Striping] +Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +Segment2 = 100M, db-seg2-1.db +;... +;[TempStriping] +;Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +;Segment2 = 100M, db-seg2-1.db +;... +;[Ucms] +;UcmPath = +;Ucm1 = +;Ucm2 = +;... + +[Zero Config] +ServerName = virtuoso (D602566B774E) +;ServerDSN = ZDSN +;SSLServerName = +;SSLServerDSN = + +[Mono] +;MONO_TRACE = Off +;MONO_PATH = +;MONO_ROOT = +;MONO_CFG_DIR = +;virtclr.dll = + +[URIQA] +DynamicLocal = 0 +DefaultHost = localhost:8890 + +[SPARQL] +;ExternalQuerySource = 1 +;ExternalXsltSource = 1 +;DefaultGraph = http://localhost:8890/dataspace +;ImmutableGraphs = http://localhost:8890/dataspace +ResultSetMaxRows = 1000000 ; 1 million +;MaxQueryCostEstimationTime = 4000 ; in seconds +MaxQueryExecutionTime = 1200 ; in seconds +DefaultQuery = select distinct ?Concept where {[] a ?Concept} LIMIT 100 +DeferInferenceRulesInit = 0 ; controls inference rules loading +;PingService = http://rpc.pingthesemanticweb.com/ + +[Plugins] +LoadPath = /usr/local/virtuoso-opensource/lib/virtuoso/hosting +;Load1 = plain, wikiv +;Load2 = plain, mediawiki +;Load3 = plain, creolewiki +;Load4 = plain, im +;Load5 = plain, wbxml2 +;Load6 = plain, hslookup +;Load7 = attach, libphp5.so +;Load8 = Hosting, hosting_php.so +;Load9 = Hosting,hosting_perl.so +;Load10 = Hosting,hosting_python.so +;Load11 = Hosting,hosting_ruby.so +;Load12 = msdtc,msdtc_sample diff --git a/config/publication-triplestore/virtuoso.ini b/config/publication-triplestore/virtuoso.ini new file mode 100755 index 0000000..fb33835 --- /dev/null +++ b/config/publication-triplestore/virtuoso.ini @@ -0,0 +1,253 @@ +; +; virtuoso.ini +; +; Configuration file for the OpenLink Virtuoso VDBMS Server +; +; To learn more about this product, or any other product in our +; portfolio, please check out our web site at: +; +; http://virtuoso.openlinksw.com/ +; +; or contact us at: +; +; general.information@openlinksw.com +; +; If you have any technical questions, please contact our support +; staff at: +; +; technical.support@openlinksw.com +; +; +; Database setup +; +[Database] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.db +ErrorLogFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.log +LockFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.lck +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.trx +xa_persistent_file = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.pxa +ErrorLogLevel = 7 +FileExtend = 200 +MaxCheckpointRemap = 8500 ;;niels 1/4 of the number of buffers (assuming db uses most of the available buffers, you can check with `status()` in isql-v +Striping = 0 +TempStorage = TempDatabase +Syslog = 1 ; write to syslog in case of failures where writing to log is not possible + +[TempDatabase] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.db +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.trx +MaxCheckpointRemap = 2000 +Striping = 0 + +; +; Server parameters +; +[Parameters] +ServerPort = 1111 +LiteMode = 0 +DisableUnixSocket = 1 +DisableTcpSocket = 0 +;SSLServerPort = 2111 +;SSLCertificate = cert.pem +;SSLPrivateKey = pk.pem +;X509ClientVerify = 0 +;X509ClientVerifyDepth = 0 +;X509ClientVerifyCAFile = ca.pem +MaxClientConnections = 10 +CheckpointInterval = 60 +TraceOn = errors ; write traces +O_DIRECT = 0 +CaseMode = 2 +MaxStaticCursorRows = 5000 +CheckpointAuditTrail = 0 +AllowOSCalls = 0 +SchedulerInterval = 10 +DirsAllowed = ., /usr/local/virtuoso-opensource/share/virtuoso/vad +ThreadCleanupInterval = 10 +ThreadThreshold = 10 +ResourcesCleanupInterval = 10 +FreeTextBatchSize = 100000 +SingleCPU = 0 +VADInstallDir = /usr/local/virtuoso-opensource/share/virtuoso/vad/ +PrefixResultNames = 0 +RdfFreeTextRulesSize = 100 +IndexTreeMaps = 256 +MaxMemPoolSize = 200000000 +PrefixResultNames = 0 +MacSpotlight = 0 +MaxQueryMem = 2G +VectorSize = 1000 ; initial parallel query vector (array of query operations) size +MaxVectorSize = 1000000 ; query vector size threshold. +AdjustVectorSize = 0 +ThreadsPerQuery = 4 +AsyncQueueMaxThreads = 10 +TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient consumption is 450MB (roll forward + roll back) +;; +;; When running with large data sets, one should configure the Virtuoso +;; process to use between 2/3 to 3/5 of free system memory and to stripe +;; storage on all available disks. +;; +;; Uncomment next two lines if there is 2 GB system memory free +;;NumberOfBuffers = 170000 +;;MaxDirtyBuffers = 130000 + +;; Uncomment next two lines if there is 4 GB system memory free +NumberOfBuffers = 340000 +MaxDirtyBuffers = 250000 +;; Uncomment next two lines if there is 8 GB system memory free +;NumberOfBuffers = 680000 +;MaxDirtyBuffers = 500000 +;; Uncomment next two lines if there is 16 GB system memory free +;NumberOfBuffers = 1360000 +;MaxDirtyBuffers = 1000000 +;; Uncomment next two lines if there is 32 GB system memory free +;NumberOfBuffers = 2720000 +;MaxDirtyBuffers = 2000000 +;; Uncomment next two lines if there is 48 GB system memory free +;NumberOfBuffers = 4000000 +;MaxDirtyBuffers = 3000000 +;; Uncomment next two lines if there is 64 GB system memory free +;NumberOfBuffers = 5450000 +;MaxDirtyBuffers = 4000000 +;; +;; Note the default settings will take very little memory +;; but will not result in very good performance +;; +;NumberOfBuffers = 10000 +;MaxDirtyBuffers = 6000 + +;; Introduced after bug with export service +MaxSortedTopRows = 1000000 ; 1 million! + +[HTTPServer] +ServerPort = 8890 +ServerRoot = /usr/local/virtuoso-opensource/var/lib/virtuoso/vsp +MaxClientConnections = 10 +DavRoot = DAV +EnabledDavVSP = 0 +HTTPProxyEnabled = 0 +TempASPXDir = 0 +DefaultMailServer = localhost:25 +ServerThreads = 10 +MaxKeepAlives = 10 +KeepAliveTimeout = 10 +MaxCachedProxyConnections = 10 +ProxyConnectionCacheTimeout = 15 +HTTPThreadSize = 280000 +HttpPrintWarningsInOutput = 0 +Charset = UTF-8 +;;HTTPLogFile = http.log +MaintenancePage = atomic.html +EnabledGzipContent = 1 + +[AutoRepair] +BadParentLinks = 0 + +[Client] +SQL_PREFETCH_ROWS = 100 +SQL_PREFETCH_BYTES = 16000 +SQL_QUERY_TIMEOUT = 0 +SQL_TXN_TIMEOUT = 0 +;SQL_NO_CHAR_C_ESCAPE = 1 +;SQL_UTF8_EXECS = 0 +;SQL_NO_SYSTEM_TABLES = 0 +;SQL_BINARY_TIMESTAMP = 1 +;SQL_ENCRYPTION_ON_PASSWORD = -1 + +[VDB] +ArrayOptimization = 0 +NumArrayParameters = 10 +VDBDisconnectTimeout = 1000 +KeepConnectionOnFixedThread = 0 + +[Replication] +ServerName = db-D602566B774E +ServerEnable = 1 +QueueMax = 50000 + +; +; Striping setup +; +; These parameters have only effect when Striping is set to 1 in the +; [Database] section, in which case the DatabaseFile parameter is ignored. +; +; With striping, the database is spawned across multiple segments +; where each segment can have multiple stripes. +; +; Format of the lines below: +; Segment = , [, .. ] +; +; must be ordered from 1 up. +; +; The is the total size of the segment which is equally divided +; across all stripes forming the segment. Its specification can be in +; gigabytes (g), megabytes (m), kilobytes (k) or in database blocks +; (b, the default) +; +; Note that the segment size must be a multiple of the database page size +; which is currently 8k. Also, the segment size must be divisible by the +; number of stripe files forming the segment. +; +; The example below creates a 200 meg database striped on two segments +; with two stripes of 50 meg and one of 100 meg. +; +; You can always add more segments to the configuration, but once +; added, do not change the setup. +; +[Striping] +Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +Segment2 = 100M, db-seg2-1.db +;... +;[TempStriping] +;Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +;Segment2 = 100M, db-seg2-1.db +;... +;[Ucms] +;UcmPath = +;Ucm1 = +;Ucm2 = +;... + +[Zero Config] +ServerName = virtuoso (D602566B774E) +;ServerDSN = ZDSN +;SSLServerName = +;SSLServerDSN = + +[Mono] +;MONO_TRACE = Off +;MONO_PATH = +;MONO_ROOT = +;MONO_CFG_DIR = +;virtclr.dll = + +[URIQA] +DynamicLocal = 0 +DefaultHost = localhost:8890 + +[SPARQL] +;ExternalQuerySource = 1 +;ExternalXsltSource = 1 +;DefaultGraph = http://localhost:8890/dataspace +;ImmutableGraphs = http://localhost:8890/dataspace +ResultSetMaxRows = 1000000 ; 1 million +;MaxQueryCostEstimationTime = 4000 ; in seconds +MaxQueryExecutionTime = 1200 ; in seconds +DefaultQuery = select distinct ?Concept where {[] a ?Concept} LIMIT 100 +DeferInferenceRulesInit = 0 ; controls inference rules loading +;PingService = http://rpc.pingthesemanticweb.com/ + +[Plugins] +LoadPath = /usr/local/virtuoso-opensource/lib/virtuoso/hosting +;Load1 = plain, wikiv +;Load2 = plain, mediawiki +;Load3 = plain, creolewiki +;Load4 = plain, im +;Load5 = plain, wbxml2 +;Load6 = plain, hslookup +;Load7 = attach, libphp5.so +;Load8 = Hosting, hosting_php.so +;Load9 = Hosting,hosting_perl.so +;Load10 = Hosting,hosting_python.so +;Load11 = Hosting,hosting_ruby.so +;Load12 = msdtc,msdtc_sample \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f4ed919..0906798 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -63,6 +63,10 @@ services: restart: "no" file: restart: "no" + publication-triplestore: + restart: "no" + ports: + - "8891:8890" delta-producer-dump-file-publisher: restart: "no" organizations-sync-with-sharepoint-jobs-initiator: diff --git a/docker-compose.yml b/docker-compose.yml index 3b4de11..ce52526 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -302,11 +302,22 @@ services: - "logging=true" restart: always logging: *default-logging + ################################################################################ # DELTA GENERAL ################################################################################ + publication-triplestore: + image: redpencil/virtuoso:1.2.0-rc.1 + environment: + SPARQL_UPDATE: "true" + DEFAULT_GRAPH: "http://mu.semte.ch/application" + volumes: + - ./data/publication-triplestore:/data + - ./config/publication-triplestore/virtuoso.ini:/data/virtuoso.ini + restart: always + logging: *default-logging delta-producer-dump-file-publisher: - image: lblod/delta-producer-dump-file-publisher:0.9.1 + image: lblod/delta-producer-dump-file-publisher:0.10.2 volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share @@ -314,6 +325,7 @@ services: - db:database restart: always logging: *default-logging + ################################################################################ # DELTA ORGANIZATIONS: START ################################################################################ @@ -363,6 +375,57 @@ services: # DELTA ORGANIZATIONS: END ################################################################################ + ################################################################################ + # DELTA ORGANIZATIONS PUBLIC INFO: START + ################################################################################ + delta-producer-bg-jobs-initiator-organizations-public-info: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-organizations-public-info: + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + RELATIVE_FILE_PATH: "deltas/organizations" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" + PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + volumes: + - ./config/delta-producer/organizations-public-info:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + logging: *default-logging + ################################################################################ + # DELTA ORGANIZATIONS PUBLIC INFO: END + ################################################################################ + ################################################################################ # DELTA PUBLIC INFORMATION: START ################################################################################ From ddb13e210df2cd4ebdd0ecfd1ad9d8a11bc4b1e9 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 6 Dec 2023 14:50:57 +0100 Subject: [PATCH 20/21] Fix relative file path of producer --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ce52526..fe457b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -399,7 +399,7 @@ services: environment: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/organizations" + RELATIVE_FILE_PATH: "deltas/organizations-public-info" PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" From 8c2cd9c959d19162d627e3a058b9256bf4e54a3b Mon Sep 17 00:00:00 2001 From: mirdono Date: Thu, 7 Dec 2023 10:43:00 +0100 Subject: [PATCH 21/21] Chore: bumped construct-administrative-unit-relationships-service to v0.1.6 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index fe457b9..e328563 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -254,7 +254,7 @@ services: restart: always logging: *default-logging construct-administrative-unit-relationships: - image: lblod/construct-administrative-unit-relationships-service:0.1.5 + image: lblod/construct-administrative-unit-relationships-service:0.1.6 environment: START_DATE_WORSHIP_GOVERNING_BODY: "2023-04-01T00:00:00" END_DATE_WORSHIP_GOVERNING_BODY: "2026-03-31T00:00:00"