From 8a7eb8ede017af0d39c8211a56885bcc678871b7 Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 09:44:08 +0100 Subject: [PATCH 01/23] adding migrations --- ...flush-existing-leidinggevenden-data.sparql | 132 ++++++++++++++++++ ...flush-existing-leidinggevenden-data.sparql | 132 ++++++++++++++++++ ...flush-existing-leidinggevenden-data.sparql | 132 ++++++++++++++++++ ...flush-existing-leidinggevenden-data.sparql | 132 ++++++++++++++++++ ...flush-existing-leidinggevenden-data.sparql | 132 ++++++++++++++++++ .../20250113151947-flush-landing-graph.sparql | 1 + .../20250113152654-flush-old-jobs.sparql | 50 +++++++ 7 files changed, 711 insertions(+) create mode 100644 config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql create mode 100644 config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql create mode 100644 config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql create mode 100644 config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql create mode 100644 config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql create mode 100644 config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql create mode 100644 config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113152654-flush-old-jobs.sparql diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql new file mode 100644 index 0000000..301b53e --- /dev/null +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql @@ -0,0 +1,132 @@ +PREFIX foaf: + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } +} WHERE { + + VALUES ?p { + + + foaf:givenName + foaf:familyName + + } + + ?functionaris a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + ?contactpoint a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?bestuursfunctie ?contactpoint. + } + +} WHERE { + GRAPH { + ?bestuursfunctie a ; + ?contactpoint. + } +} \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql new file mode 100644 index 0000000..301b53e --- /dev/null +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql @@ -0,0 +1,132 @@ +PREFIX foaf: + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } +} WHERE { + + VALUES ?p { + + + foaf:givenName + foaf:familyName + + } + + ?functionaris a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + ?contactpoint a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?bestuursfunctie ?contactpoint. + } + +} WHERE { + GRAPH { + ?bestuursfunctie a ; + ?contactpoint. + } +} \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql new file mode 100644 index 0000000..301b53e --- /dev/null +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql @@ -0,0 +1,132 @@ +PREFIX foaf: + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } +} WHERE { + + VALUES ?p { + + + foaf:givenName + foaf:familyName + + } + + ?functionaris a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + ?contactpoint a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?bestuursfunctie ?contactpoint. + } + +} WHERE { + GRAPH { + ?bestuursfunctie a ; + ?contactpoint. + } +} \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql new file mode 100644 index 0000000..301b53e --- /dev/null +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql @@ -0,0 +1,132 @@ +PREFIX foaf: + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } +} WHERE { + + VALUES ?p { + + + foaf:givenName + foaf:familyName + + } + + ?functionaris a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + ?contactpoint a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?bestuursfunctie ?contactpoint. + } + +} WHERE { + GRAPH { + ?bestuursfunctie a ; + ?contactpoint. + } +} \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql new file mode 100644 index 0000000..301b53e --- /dev/null +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql @@ -0,0 +1,132 @@ +PREFIX foaf: + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } +} WHERE { + + VALUES ?p { + + + foaf:givenName + foaf:familyName + + } + + ?functionaris a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + ?contactpoint a ; + ?s. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?s a ; + ?p ?o. + } + +} WHERE { + + VALUES ?p { + + + + + + + + } + + ?bestuursfunctie a ; + ?contactpoint. + + GRAPH { + ?s a ; + ?p ?o. + } +} + +; + +DELETE { + GRAPH { + ?bestuursfunctie ?contactpoint. + } + +} WHERE { + GRAPH { + ?bestuursfunctie a ; + ?contactpoint. + } +} \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql new file mode 100644 index 0000000..2848ff2 --- /dev/null +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql @@ -0,0 +1 @@ +CLEAR GRAPH \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113152654-flush-old-jobs.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113152654-flush-old-jobs.sparql new file mode 100644 index 0000000..d98649a --- /dev/null +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113152654-flush-old-jobs.sparql @@ -0,0 +1,50 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?job ?jobP ?jobO. + ?task ?taskP ?taskO. + ?container ?containerP ?containerO. + } +} +WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + { + ?job a ; + task:operation ?operation; + ?jobP ?jobO. + + ?task dct:isPartOf ?job; + task:resultsContainer ?container; + ?taskP ?taskO. + + ?container a nfo:DataContainer; + ?containerP ?containerO. + } + UNION + { + ?job a ; + task:operation ?operation; + ?jobP ?jobO. + + ?task dct:isPartOf ?job; + ?taskP ?taskO. + } + } +} \ No newline at end of file From 8750fddb2dd2d8ef49f14473fd50b19cd54bea2d Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 10:23:41 +0100 Subject: [PATCH 02/23] added mapping rules --- .../queries/dl2op/data-cleanup/phone.rq | 23 +++++++++++++++++++ .../queries/dl2op/inheritance/Agent.rq | 14 +++++++++++ .../dl2op/inheritance/AgentInPositie.rq | 12 ++++++++++ .../queries/dl2op/inverse/holds-to-held-by.rq | 8 +++++++ .../mapping/queries/dl2op/pass/address.sparql | 6 +++++ .../queries/dl2op/pass/bestuursfunctie.sparql | 7 ++++++ .../queries/dl2op/pass/contactpoint.sparql | 7 ++++++ .../queries/dl2op/pass/functionaris.sparql | 6 +++++ .../mapping/queries/dl2op/pass/person.sparql | 11 +++++++++ .../mapping/queries/dl2op/predicate/names.rq | 22 ++++++++++++++++++ 10 files changed, 116 insertions(+) create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inverse/holds-to-held-by.rq create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/bestuursfunctie.sparql create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq new file mode 100644 index 0000000..3ffc95a --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq @@ -0,0 +1,23 @@ +PREFIX schema: + +CONSTRUCT { + ?s schema:telephone ?cleanedPhoneNumber . +} WHERE { + ?s schema:telephone ?phoneNumber . + + # Remove all non digits + BIND(REPLACE(?phoneNumber, """[^+\\d]""", "") AS ?scrapeResult) + + # Replace leading 00 with + + BIND(REPLACE(?scrapeResult, """^00""", "+") AS ?replacedZeroZero) + + BIND(REPLACE(?replacedZeroZero, """^0""", "+32") AS ?defaultBe) + + BIND(REPLACE(?defaultBe, """\\+320""", "+32") AS ?extendedDefaultBe) + + BIND(REPLACE(?extendedDefaultBe, """^320""", "+32") AS ?finalResult) + + FILTER(REGEX(?finalResult, """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""")) + + BIND(CONCAT("tel:", ?finalResult) as ?cleanedPhoneNumber) +} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq new file mode 100644 index 0000000..8252a85 --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq @@ -0,0 +1,14 @@ +PREFIX dc_terms: +PREFIX lblodlg: +PREFIX person: + +CONSTRUCT { + ?s a dc_terms:Agent . +} WHERE { + VALUES ?type { + lblodlg:Functionaris + person:Person + } + + ?s a ?type . +} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq new file mode 100644 index 0000000..2505191 --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq @@ -0,0 +1,12 @@ +PREFIX ch: +PREFIX lblodlg: + +CONSTRUCT { + ?s a ch:AgentInPositie . +} WHERE { + VALUES ?type { + lblodlg:Functionaris + } + + ?s a ?type . +} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inverse/holds-to-held-by.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inverse/holds-to-held-by.rq new file mode 100644 index 0000000..0b373ed --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inverse/holds-to-held-by.rq @@ -0,0 +1,8 @@ +PREFIX org: + +CONSTRUCT { + ?o org:heldBy ?s. +} +WHERE { + ?s org:holds ?o. +} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql new file mode 100644 index 0000000..7d72f5a --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql @@ -0,0 +1,6 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. +} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/bestuursfunctie.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/bestuursfunctie.sparql new file mode 100644 index 0000000..238cd07 --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/bestuursfunctie.sparql @@ -0,0 +1,7 @@ +CONSTRUCT { + ?s ?contactpoint. +} +WHERE { + ?s a . + ?s ?contactpoint. +} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql new file mode 100644 index 0000000..839fbd4 --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql @@ -0,0 +1,7 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + FILTER (?p NOT IN ()) +} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql new file mode 100644 index 0000000..c339689 --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql @@ -0,0 +1,6 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. +} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql new file mode 100644 index 0000000..60d28ac --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql @@ -0,0 +1,11 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + + FILTER (?p IN ( + , + + )) +} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq new file mode 100644 index 0000000..cd33077 --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq @@ -0,0 +1,22 @@ +PREFIX foaf: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX person: + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) + +# First name: persoon:gebruikteVoornaam => foaf:givenName +# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam + +# We do both in one query with an OPTIONAL because they need to happen simultaneously +# not to end up in a loop + +CONSTRUCT { + ?s foaf:givenName ?gebruikteVoornaam . + ?s persoon:gebruikteVoornaam ?name . +} WHERE { + ?s a person:Person . + + OPTIONAL { ?s persoon_loket:gebruikteVoornaam ?gebruikteVoornaam . } + OPTIONAL { ?s foaf:name ?name . } +} From 19e3423202188342a0a9fb6f9ef0f8ece2d8c62f Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 10:23:52 +0100 Subject: [PATCH 03/23] updated migrations --- ...flush-existing-leidinggevenden-data.sparql | 107 ---------------- ...flush-existing-leidinggevenden-data.sparql | 107 ---------------- ...flush-existing-leidinggevenden-data.sparql | 95 -------------- ...flush-existing-leidinggevenden-data.sparql | 106 +--------------- ...flush-existing-leidinggevenden-data.sparql | 119 ------------------ 5 files changed, 1 insertion(+), 533 deletions(-) diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql index 301b53e..e93d26b 100644 --- a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql @@ -22,111 +22,4 @@ DELETE { ?s a ; ?p ?o. } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - ?contactpoint a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?bestuursfunctie ?contactpoint. - } - -} WHERE { - GRAPH { - ?bestuursfunctie a ; - ?contactpoint. - } } \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql index 301b53e..d8c4d26 100644 --- a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql @@ -1,31 +1,5 @@ PREFIX foaf: -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } -} WHERE { - - VALUES ?p { - - - foaf:givenName - foaf:familyName - - } - - ?functionaris a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - DELETE { GRAPH { ?s a ; @@ -48,85 +22,4 @@ DELETE { ?s a ; ?p ?o. } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - ?contactpoint a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?bestuursfunctie ?contactpoint. - } - -} WHERE { - GRAPH { - ?bestuursfunctie a ; - ?contactpoint. - } } \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql index 301b53e..455cadb 100644 --- a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql @@ -1,57 +1,5 @@ PREFIX foaf: -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } -} WHERE { - - VALUES ?p { - - - foaf:givenName - foaf:familyName - - } - - ?functionaris a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - DELETE { GRAPH { ?s a ; @@ -86,47 +34,4 @@ DELETE { ?s a ; ?p ?o. } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?bestuursfunctie ?contactpoint. - } - -} WHERE { - GRAPH { - ?bestuursfunctie a ; - ?contactpoint. - } } \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql index 301b53e..9fb0993 100644 --- a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql @@ -1,95 +1,5 @@ PREFIX foaf: -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } -} WHERE { - - VALUES ?p { - - - foaf:givenName - foaf:familyName - - } - - ?functionaris a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - ?contactpoint a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - DELETE { GRAPH { ?s a ; @@ -109,24 +19,10 @@ DELETE { } ?bestuursfunctie a ; - ?contactpoint. + ?s. GRAPH { ?s a ; ?p ?o. } -} - -; - -DELETE { - GRAPH { - ?bestuursfunctie ?contactpoint. - } - -} WHERE { - GRAPH { - ?bestuursfunctie a ; - ?contactpoint. - } } \ No newline at end of file diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql index 301b53e..a60160b 100644 --- a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql +++ b/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql @@ -1,124 +1,5 @@ PREFIX foaf: -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } -} WHERE { - - VALUES ?p { - - - foaf:givenName - foaf:familyName - - } - - ?functionaris a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - ?contactpoint a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - GRAPH { - ?s a ; - ?p ?o. - } -} - -; - DELETE { GRAPH { ?bestuursfunctie ?contactpoint. From 631ba827ee2246c25b4edcc7936fdd0e1c3c7988 Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 10:24:04 +0100 Subject: [PATCH 04/23] update doc --- CHANGELOG.md | 38 ++++++++++++++++++++++++++++++++++++++ README.md | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 012a7a9..9eba479 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,42 @@ # Changelog +## 1.29.0 +### Backend + - Upgraded `leidinggevenden-consumer` (OP-3533, OP-3511) + +### Deploy notes +``` +drc down; +``` +Update `docker-compose.override.yml` to: +``` + leidinggevenden-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_LANDING_ZONE_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +Then: +``` +drc up -d migrations +drc up -d db leidinggevenden-consumer +# Wait until success of the previous step +``` +Then, update `docker-compose.override.yml` to: +``` + leidinggevenden-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint + DCR_LANDING_ZONE_DATABASE: "database" + DCR_REMAPPING_DATABASE: "database" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +``` +drc up -d +``` + ## 1.28.2 (2025-01-13) ### Backend - Added "Opdrachthoudende vereniging met private deelname" in delta public (DL-6368) diff --git a/README.md b/README.md index a59e0e3..08a8e67 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,44 @@ Backend for the organization portal application, based on the mu.semte.ch micros You can shut down using `docker-compose stop` and remove everything using `docker-compose rm`. +#### Setup consumers +The application depends on external data. You'll have to perform extra steps if you want this data to be available. +These are multiple data sets, we'll try to be exhaustive on how to ingest these. Even though the procedures are similar. + +#### Setup http://data.lblod.info/vocabularies/leidinggevenden/Functionaris +Note: this is assuming you have never ran this before. +``` +drc down; +``` +Update `docker-compose.override.yml` to: +``` + leidinggevenden-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_LANDING_ZONE_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +Then: +``` +drc up -d migrations +drc up -d db leidinggevenden-consumer +# Wait until success of the previous step +``` +Then, update `docker-compose.override.yml` to: +``` + leidinggevenden-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint + DCR_LANDING_ZONE_DATABASE: "database" + DCR_REMAPPING_DATABASE: "database" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +``` +drc up -d +``` ### Setting up the delta-producers related services To make sure the app can share data, producers need to be set up. There is an intial sync, that is potentially very expensive, and must be started manually From aaf87ee32fa724252cd7844caaf267504bf93e57 Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 10:24:15 +0100 Subject: [PATCH 05/23] update consumer-config --- docker-compose.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0ecd92d..763f16f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -401,24 +401,26 @@ services: # LOKET LEIDINGGEVENDEN CONSUMERS ################################################################################ leidinggevenden-consumer: - image: lblod/delta-consumer:0.0.24 + image: lblod/delta-consumer:0.0.27 environment: DCR_SERVICE_NAME: "leidinggevenden-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API + DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files" DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/LeidinggevendenCacheGraphDump" DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden" DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden" DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/leidinggevenden-consumer" DCR_DISABLE_INITIAL_SYNC: "true" - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" - DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files" DCR_KEEP_DELTA_FILES: "true" - BATCH_SIZE: 1000 - DCR_ENABLE_DELTA_CONTEXT: "true" - DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_DELTA_JOBS_RETENTION_PERIOD: "30" + DCR_ENABLE_TRIPLE_REMAPPING: "true" DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/leidinggevenden" + DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/administrative-unit" + DCR_BATCH_SIZE: 1000 + SUDO_QUERY_oRETRY_FOR_HTTP_STATUS_CODES: "404,500,503" + SUDO_QUERY_RETRY: "true" volumes: - - ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching + - ./config/loket-consumer/leidinggevenden/mapping:/config/mapping - ./data/files/consumer-files/leidinggevenden:/consumer-files/ restart: always labels: From 347844e341f37a394ecf60de423c7b419274d795 Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 11:19:19 +0100 Subject: [PATCH 06/23] update the filter so empty strings are not ingested --- .../mapping/queries/dl2op/data-cleanup/phone.rq | 4 ++++ .../leidinggevenden/mapping/queries/dl2op/pass/address.sparql | 4 ++++ .../mapping/queries/dl2op/pass/contactpoint.sparql | 4 ++++ .../mapping/queries/dl2op/pass/functionaris.sparql | 4 ++++ .../leidinggevenden/mapping/queries/dl2op/pass/person.sparql | 4 ++++ .../leidinggevenden/mapping/queries/dl2op/predicate/names.rq | 1 + 6 files changed, 21 insertions(+) diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq index 3ffc95a..f43d0a8 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq @@ -20,4 +20,8 @@ CONSTRUCT { FILTER(REGEX(?finalResult, """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""")) BIND(CONCAT("tel:", ?finalResult) as ?cleanedPhoneNumber) + + FILTER NOT EXISTS { + ?s schema:telephone "" . + } } diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql index 7d72f5a..41f7523 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql @@ -3,4 +3,8 @@ CONSTRUCT { } WHERE { ?s a ; ?p ?o. + + FILTER NOT EXISTS { + ?s ?p "" . + } } \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql index 839fbd4..d87a2ef 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql @@ -4,4 +4,8 @@ CONSTRUCT { ?s a ; ?p ?o. FILTER (?p NOT IN ()) + + FILTER NOT EXISTS { + ?s ?p "" . + } } \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql index c339689..e753a79 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql @@ -3,4 +3,8 @@ CONSTRUCT { } WHERE { ?s a ; ?p ?o. + + FILTER NOT EXISTS { + ?s ?p "" . + } } \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql index 60d28ac..3433dba 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql @@ -8,4 +8,8 @@ CONSTRUCT { , )) + + FILTER NOT EXISTS { + ?s ?p "" . + } } \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq index cd33077..eee2571 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq @@ -19,4 +19,5 @@ CONSTRUCT { OPTIONAL { ?s persoon_loket:gebruikteVoornaam ?gebruikteVoornaam . } OPTIONAL { ?s foaf:name ?name . } + } From a6571eefa523e6157a1d4e345c149a8ca6a711d1 Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 12:09:07 +0100 Subject: [PATCH 07/23] Current consumer is VERY picky on how you write the mapping-query See: https://github.com/lblod/delta-consumer/blob/v0.0.27/lib/delta-sparql-mapping.js#L267 make sure you write first the graph or "triple"-pattern and filter after that. For now. --- .../leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq | 3 ++- .../mapping/queries/dl2op/inheritance/AgentInPositie.rq | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq index 8252a85..66d5eb2 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq @@ -5,10 +5,11 @@ PREFIX person: CONSTRUCT { ?s a dc_terms:Agent . } WHERE { + ?s a ?type . + VALUES ?type { lblodlg:Functionaris person:Person } - ?s a ?type . } diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq index 2505191..e3a97b3 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq @@ -4,9 +4,8 @@ PREFIX lblodlg: CONSTRUCT { ?s a ch:AgentInPositie . } WHERE { + ?s a ?type . VALUES ?type { lblodlg:Functionaris } - - ?s a ?type . } From dba1ada5e26b5f512272b14f0bf70d58fce9d22c Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 14:42:54 +0100 Subject: [PATCH 08/23] Updated mapping rules --- .../mapping/queries/dl2op/pass/person.sparql | 3 ++- .../dl2op/predicate/person-foaf-name.sparql | 21 +++++++++++++++++++ ...es.rq => person-gebruikte-voornaam.sparql} | 10 ++++----- 3 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-foaf-name.sparql rename config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/{names.rq => person-gebruikte-voornaam.sparql} (77%) diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql index 3433dba..02b21ea 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql @@ -6,7 +6,8 @@ CONSTRUCT { FILTER (?p IN ( , - + , + )) FILTER NOT EXISTS { diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-foaf-name.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-foaf-name.sparql new file mode 100644 index 0000000..0347ec9 --- /dev/null +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-foaf-name.sparql @@ -0,0 +1,21 @@ +PREFIX foaf: +PREFIX persoon: +PREFIX persoon_loket: +PREFIX person: + +# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) + +# First name: persoon:gebruikteVoornaam => foaf:givenName +# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam + +# We do both in one query with an OPTIONAL because they need to happen simultaneously +# not to end up in a loop + +# Part 1 + +CONSTRUCT { + ?s persoon:gebruikteVoornaam ?name . +} WHERE { + ?s a person:Person; + foaf:name ?name . +} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-gebruikte-voornaam.sparql similarity index 77% rename from config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq rename to config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-gebruikte-voornaam.sparql index eee2571..87c6463 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/names.rq +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-gebruikte-voornaam.sparql @@ -11,13 +11,11 @@ PREFIX person: # We do both in one query with an OPTIONAL because they need to happen simultaneously # not to end up in a loop +# Part 2 + CONSTRUCT { ?s foaf:givenName ?gebruikteVoornaam . - ?s persoon:gebruikteVoornaam ?name . } WHERE { - ?s a person:Person . - - OPTIONAL { ?s persoon_loket:gebruikteVoornaam ?gebruikteVoornaam . } - OPTIONAL { ?s foaf:name ?name . } - + ?s a person:Person ; + persoon_loket:gebruikteVoornaam ?gebruikteVoornaam . } From a001f6b9e4a0ec4d8473a7aaa1e15267c8a9440a Mon Sep 17 00:00:00 2001 From: felix Date: Fri, 17 Jan 2025 15:12:52 +0100 Subject: [PATCH 09/23] Remove old dispatching rules --- .../leidinggevenden-dispatching/config.js | 32 ---- .../delta-context-config.js | 72 -------- .../delta-sync-dispatching.js | 85 --------- .../initial-sync-dispatching.js | 85 --------- .../leidinggevenden-dispatching/util.js | 168 ------------------ 5 files changed, 442 deletions(-) delete mode 100644 config/loket-consumer/leidinggevenden-dispatching/config.js delete mode 100644 config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js delete mode 100644 config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js delete mode 100644 config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js delete mode 100644 config/loket-consumer/leidinggevenden-dispatching/util.js diff --git a/config/loket-consumer/leidinggevenden-dispatching/config.js b/config/loket-consumer/leidinggevenden-dispatching/config.js deleted file mode 100644 index b3ebc2a..0000000 --- a/config/loket-consumer/leidinggevenden-dispatching/config.js +++ /dev/null @@ -1,32 +0,0 @@ -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; -const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; -const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; -const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql'; -const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); -const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); -const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); -const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); -const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); - -// Graphs -const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`; -const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`; - -// ENDPOINTS -const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db'; -const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; - -module.exports = { - BATCH_SIZE, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MAX_DB_RETRY_ATTEMPTS, - MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - LANDING_ZONE_GRAPH, - TARGET_GRAPH, - LANDING_ZONE_DATABASE_ENDPOINT -}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js deleted file mode 100644 index 973462f..0000000 --- a/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js +++ /dev/null @@ -1,72 +0,0 @@ - -const PREFIXES = ` -PREFIX org: -PREFIX mandaat: -PREFIX lblodlg: -PREFIX besluit: ` - -const contextConfig = { - addTypes: { - scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both - exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta - }, - contextQueries: [ - // Case Leidnggevenden - // They can be triggered when being inserted or when their link to a bestuursfunctie is added - { - trigger: { // subjectType or predicateValue - subjectType: "lblodlg:Functionaris" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ?functionaris org:holds ?bestuursfunctie ; - ?p ?o . - ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; - mandaat:isTijdspecialisatieVan ?orgaan . - ?orgaan besluit:bestuurt ?eenheid . - ?eenheid besluit:classificatie ?classification . - } WHERE { - GRAPH { - VALUES ?functionaris { ${subject} } - ?functionaris org:holds ?bestuursfunctie ; - ?p ?o . - ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; - mandaat:isTijdspecialisatieVan ?orgaan . - ?orgaan besluit:bestuurt ?eenheid . - ?eenheid besluit:classificatie ?classification . - } - }` - }, - { - trigger: { - predicateValue: "org:holds" - }, - queryTemplate: (subject) => ` - ${PREFIXES} - CONSTRUCT { - ?functionaris org:holds ?bestuursfunctie ; - ?p ?o . - ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; - mandaat:isTijdspecialisatieVan ?orgaan . - ?orgaan besluit:bestuurt ?eenheid . - ?eenheid besluit:classificatie ?classification . - } WHERE { - GRAPH { - VALUES ?functionaris { ${subject} } - ?functionaris org:holds ?bestuursfunctie ; - ?p ?o . - ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; - mandaat:isTijdspecialisatieVan ?orgaan . - ?orgaan besluit:bestuurt ?eenheid . - ?eenheid besluit:classificatie ?classification . - } - }` - } - ] -} - -module.exports = { - contextConfig, - PREFIXES -}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js deleted file mode 100644 index b309f35..0000000 --- a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js +++ /dev/null @@ -1,85 +0,0 @@ -const { - transformStatements, - deleteFromTargetGraph, - insertIntoTargetGraph, -} = require('./util'); - -/** -* Dispatch the fetched information to a target graph. -* Note: will be ADDED to it's own graph. -* We take only care of adding them, not updating triples, this is a TODO -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, fetch } = lib; - let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; - - // Both arrays are the same length, so we can zip them together for easier processing - zippedChangeSets = termObjectChangeSets.map((o, i) => ({ - original: o, - withContext: termObjectChangeSetsWithContext[i] - })); - - console.log(`Received ${zippedChangeSets.length} change sets`) - - for (let { original, withContext } of zippedChangeSets) { - let originalDeletes, originalInserts, insertsWithContext, deletesWithContext; - try { - originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - } catch (e) { - console.log(`Error while processing change set: ${e}`); - console.log(`Original change set: ${JSON.stringify(original)}`); - throw e; - } - - // Extra context needed for mapping from DL to OP model and filtering based on type. - try { - insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - } catch (e) { - console.log(`Error while processing change set with context: ${e}`); - console.log(`Change set with context: ${JSON.stringify(withContext)}`); - throw e; - } - - if (originalDeletes.length) { - // Map deletes from DL to OP model - const transformedDeletes = await transformStatements(fetch, deletesWithContext); - - if (!transformedDeletes.length) { - console.log(`Warn: Delete statements mapped to empty result.`); - console.log(`Input: ${deletesWithContext}`); - console.log(`Output: ${transformedDeletes}`); - } else { - await deleteFromTargetGraph(lib, transformedDeletes); - } - } - - if (originalInserts.length) { - // Map inserts from DL to OP model - const transformedInserts = await transformStatements(fetch, insertsWithContext); - - if (!transformedInserts.length) { - console.log(`Warn: Insert statements mapped to empty result.`); - console.log(`Input: ${insertsWithContext}`); - console.log(`Output: ${transformedInserts}`); - } else { - await insertIntoTargetGraph(lib, transformedInserts); - } - } - } -} - -module.exports = { - dispatch -}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js deleted file mode 100644 index d5d672f..0000000 --- a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js +++ /dev/null @@ -1,85 +0,0 @@ -const { batchedDbUpdate, transformLandingZoneGraph } = require('./util'); -const { - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - TARGET_GRAPH, - LANDING_ZONE_GRAPH, - LANDING_ZONE_DATABASE_ENDPOINT -} = require('./config'); - -const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; - -/** -* Dispatch the fetched information to a target graph. -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjects } data - The fetched quad information, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, muAuthSudo } = lib; - const { termObjects } = data; - - // Steps: - // 1. Write the triples to the landing zone graph - withouth any mapping or filtering - // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) - - if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { - console.warn(`Service configured to skip MU_AUTH!`); - } - console.log(`Using ${endpoint} to insert triples`); - - if (termObjects.length) { - const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - - await batchedDbUpdate( - muAuthSudo.updateSudo, - LANDING_ZONE_GRAPH, - originalInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - LANDING_ZONE_DATABASE_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } -} - -async function onFinishInitialIngest(lib) { - const { muAuthSudo, fetch } = lib; - - console.log(`!! On-finish triggered !!`); - - const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint); - - console.log(`Transformed ${transformedInsertTriples.length} triples`); - - await batchedDbUpdate( - muAuthSudo.updateSudo, - TARGET_GRAPH, - transformedInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); -} - -module.exports = { - dispatch, - onFinishInitialIngest -}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/util.js b/config/loket-consumer/leidinggevenden-dispatching/util.js deleted file mode 100644 index 5eced4a..0000000 --- a/config/loket-consumer/leidinggevenden-dispatching/util.js +++ /dev/null @@ -1,168 +0,0 @@ -const { - MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - TARGET_GRAPH, - LANDING_ZONE_GRAPH, - BATCH_SIZE, -} = require('./config'); - -async function batchedDbUpdate( - muUpdate, - graph, - triples, - extraHeaders, - endpoint, - batchSize, - maxAttempts, - sleepBetweenBatches = 1000, - sleepTimeOnFail = 1000, - operation = 'INSERT' -) { - - for (let i = 0; i < triples.length; i += batchSize) { - console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); - - const batch = triples.slice(i, i + batchSize).join('\n'); - - const insertCall = async () => { - await muUpdate(` -${operation} DATA { -GRAPH <${graph}> { -${batch} -} -} -`, extraHeaders, endpoint); - }; - - await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); - - console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); - await new Promise(r => setTimeout(r, sleepBetweenBatches)); - } -} - -async function operationWithRetry(callback, - attempt, - maxAttempts, - sleepTimeOnFail) { - try { - if (typeof callback === "function") - return await callback(); - else // Catch error from promise - not how I would do it normally, but allows re use of existing code. - return await callback; - } - catch (e) { - console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); - console.log(`Error: ${e}`); - console.log(`Sleeping ${sleepTimeOnFail} ms`); - - if (attempt >= maxAttempts) { - console.log(`Max attempts reached for ${callback.toString()}, giving up`); - throw e; - } - - await new Promise(r => setTimeout(r, sleepTimeOnFail)); - return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); - } -} - -/** - * Splits an array into two parts, a part that passes and a part that fails a predicate function. - * Credits: https://github.com/benjay10 - * @public - * @function partition - * @param {Array} arr - Array to be partitioned - * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. - * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively - */ -function partition(arr, fn) { - let passes = [], fails = []; - arr.forEach((item) => (fn(item) ? passes : fails).push(item)); - return { passes, fails }; -} - -/** - * Send triples to reasoning service for conversion - * - */ -async function transformTriples(fetch, triples, mapping) { - return await operationWithRetry(mainConversion(fetch, triples, mapping), 0, - MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); -} - -async function mainConversion(fetch, triples, mapping) { - let formdata = new URLSearchParams(); - formdata.append("data", triples); - - let requestOptions = { - method: 'POST', - body: formdata, - redirect: 'follow' - }; - - const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) - return await response.text(); -} - -async function transformStatements(fetch, triples, mapping = 'main') { - console.log(`Received ${JSON.stringify(triples)} to transform`); - const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping); - statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : []; - console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`); - return statements; -} - -async function deleteFromTargetGraph(lib, statements) { - console.log(`Deleting ${statements.length} statements from target graph`); - console.log(`Statements: ${JSON.stringify(statements)}`) - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - TARGET_GRAPH, - statements, - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'DELETE'); -} - -async function insertIntoTargetGraph(lib, statements) { - console.log(`Inserting ${statements.length} statements into target graph`); - console.log(`Statements: ${JSON.stringify(statements)}`) - - await batchedDbUpdate( - lib.muAuthSudo.updateSudo, - TARGET_GRAPH, - statements, - {}, - process.env.MU_SPARQL_ENDPOINT, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - 'INSERT'); -} - -async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') { - console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`); - - const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`); - const text = await response.text(); - const statements = text.replace(/\n{2,}/g, '').split('\n'); - - return statements; -} - -module.exports = { - batchedDbUpdate, - partition, - transformStatements, - transformLandingZoneGraph, - deleteFromTargetGraph, - insertIntoTargetGraph -}; From 3bc47a7fb0d9be8c4ea94c84bd0f7843c0df0cde Mon Sep 17 00:00:00 2001 From: mirdono Date: Tue, 14 Jan 2025 10:32:56 +0100 Subject: [PATCH 10/23] fix: merger change event between St.-Amandus and Sint-Martinus The worship service St.-Amandus was not included as `orginalOrganization` in the merger change event somehow. As a result it was not listed as involved organisation in the change event details. Furthermore, its status was not correctly updated to "Not active". --- ...ange-event-sint_martinus-st_amandus.sparql | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 config/migrations-triggering-indexing/20250114093158-fix-merger-change-event-sint_martinus-st_amandus.sparql diff --git a/config/migrations-triggering-indexing/20250114093158-fix-merger-change-event-sint_martinus-st_amandus.sparql b/config/migrations-triggering-indexing/20250114093158-fix-merger-change-event-sint_martinus-st_amandus.sparql new file mode 100644 index 0000000..cafa7f5 --- /dev/null +++ b/config/migrations-triggering-indexing/20250114093158-fix-merger-change-event-sint_martinus-st_amandus.sparql @@ -0,0 +1,23 @@ +PREFIX regorg: +PREFIX org: + +DELETE { + GRAPH { + ?stAmandus regorg:orgStatus ?oldStatus. + } +} INSERT { + GRAPH { + ?stAmandus regorg:orgStatus ?notActiveStatus. + + ?mergerChangeEvent org:originalOrganization ?stAmandus. + } +} WHERE { + GRAPH { + ?stAmandus regorg:orgStatus ?oldStatus. + } + VALUES ?stAmandus { + + } + BIND ( AS ?mergerChangeEvent) + BIND ( AS ?notActiveStatus) +} From 0e8885f4dd0c857e8d611e9a170609ed3385e3af Mon Sep 17 00:00:00 2001 From: mirdono Date: Fri, 17 Jan 2025 12:03:03 +0100 Subject: [PATCH 11/23] chore: update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b533746..e823bc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Changelog +## 1.28.5 (TODO) +### Backend +- Datafix: correct merger change events for worship organisations (OP-3534) +### Deploy notes +- `drc restart migrations-triggering-indexing; drc logs -ft --tail=200 migrations-triggering-indexing` + ## 1.28.4 (2025-01-16) ### Backend #### Data From 5eca8a2e1d74a821c7d1976adf9a0d1fb89b2676 Mon Sep 17 00:00:00 2001 From: mirdono Date: Tue, 21 Jan 2025 10:11:46 +0100 Subject: [PATCH 12/23] chore: bump frontend --- CHANGELOG.md | 3 +++ docker-compose.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e823bc3..ea88372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,12 @@ # Changelog ## 1.28.5 (TODO) +### Frontend +- Bump to [v1.28.1](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.1) ### Backend - Datafix: correct merger change events for worship organisations (OP-3534) ### Deploy notes - `drc restart migrations-triggering-indexing; drc logs -ft --tail=200 migrations-triggering-indexing` +- `drc pull frontend; drc up -d frontend` ## 1.28.4 (2025-01-16) ### Backend diff --git a/docker-compose.yml b/docker-compose.yml index 0ecd92d..ae03806 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.28.0 + image: lblod/frontend-organization-portal:1.28.1 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 62769f855c201cc9b20d1bd34c0b83dbb572c8a5 Mon Sep 17 00:00:00 2001 From: mirdono Date: Tue, 21 Jan 2025 12:07:29 +0100 Subject: [PATCH 13/23] chore: bump frontend --- CHANGELOG.md | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea88372..f3ead38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## 1.28.5 (TODO) ### Frontend -- Bump to [v1.28.1](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.1) +- Bump to [v1.28.2](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.2) ### Backend - Datafix: correct merger change events for worship organisations (OP-3534) ### Deploy notes diff --git a/docker-compose.yml b/docker-compose.yml index ae03806..4928d01 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.28.1 + image: lblod/frontend-organization-portal:1.28.2 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 261ba383b62ecd64b90733b1958a847ead388e16 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Jan 2025 15:37:24 +0100 Subject: [PATCH 14/23] Update doc --- CHANGELOG.md | 9 +++++---- README.md | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fcc7b0..499a1c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ Update `docker-compose.override.yml` to: leidinggevenden-consumer: environment: DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint - DCR_LANDING_ZONE_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso - DCR_REMAPPING_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso DCR_DISABLE_DELTA_INGEST: "false" DCR_DISABLE_INITIAL_SYNC: "false" ``` @@ -28,13 +28,14 @@ Then, update `docker-compose.override.yml` to: leidinggevenden-consumer: environment: DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint - DCR_LANDING_ZONE_DATABASE: "database" - DCR_REMAPPING_DATABASE: "database" + DCR_LANDING_ZONE_DATABASE: "db" + DCR_REMAPPING_DATABASE: "db" DCR_DISABLE_DELTA_INGEST: "false" DCR_DISABLE_INITIAL_SYNC: "false" ``` ``` drc up -d +sh scripts/reset-elastic.sh ``` ## 1.28.4 (2025-01-16) ### Backend diff --git a/README.md b/README.md index 08a8e67..0b9011f 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,8 @@ Update `docker-compose.override.yml` to: leidinggevenden-consumer: environment: DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint - DCR_LANDING_ZONE_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso - DCR_REMAPPING_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso DCR_DISABLE_DELTA_INGEST: "false" DCR_DISABLE_INITIAL_SYNC: "false" ``` @@ -41,13 +41,14 @@ Then, update `docker-compose.override.yml` to: leidinggevenden-consumer: environment: DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint - DCR_LANDING_ZONE_DATABASE: "database" - DCR_REMAPPING_DATABASE: "database" + DCR_LANDING_ZONE_DATABASE: "db" + DCR_REMAPPING_DATABASE: "db" DCR_DISABLE_DELTA_INGEST: "false" DCR_DISABLE_INITIAL_SYNC: "false" ``` ``` drc up -d +sh scripts/reset-elastic.sh ``` ### Setting up the delta-producers related services From 7834352617a22203e3ce16a1f885c43deb40c66a Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Jan 2025 15:37:39 +0100 Subject: [PATCH 15/23] Fix phone mapping --- .../mapping/queries/dl2op/data-cleanup/phone.rq | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq index f43d0a8..7e761c1 100644 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq +++ b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq @@ -17,11 +17,5 @@ CONSTRUCT { BIND(REPLACE(?extendedDefaultBe, """^320""", "+32") AS ?finalResult) - FILTER(REGEX(?finalResult, """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$""")) - - BIND(CONCAT("tel:", ?finalResult) as ?cleanedPhoneNumber) - - FILTER NOT EXISTS { - ?s schema:telephone "" . - } + BIND(IF(REGEX(?finalResult, """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$"""), CONCAT("tel:", ?finalResult), ?phoneNumber) AS ?cleanedPhoneNumber) } From 0faa8100c8052968e7b1290ff7f5cf50dd9d8044 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Jan 2025 15:37:48 +0100 Subject: [PATCH 16/23] Bump consumer to v0.1.0 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 763f16f..e4b4399 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -401,7 +401,7 @@ services: # LOKET LEIDINGGEVENDEN CONSUMERS ################################################################################ leidinggevenden-consumer: - image: lblod/delta-consumer:0.0.27 + image: lblod/delta-consumer:0.1.0 environment: DCR_SERVICE_NAME: "leidinggevenden-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API From 9972451d0e3c983306295cb8e29214dc33309170 Mon Sep 17 00:00:00 2001 From: felix Date: Wed, 22 Jan 2025 16:09:25 +0100 Subject: [PATCH 17/23] fix typo --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e4b4399..630e58f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -417,7 +417,7 @@ services: DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/leidinggevenden" DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/administrative-unit" DCR_BATCH_SIZE: 1000 - SUDO_QUERY_oRETRY_FOR_HTTP_STATUS_CODES: "404,500,503" + SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" SUDO_QUERY_RETRY: "true" volumes: - ./config/loket-consumer/leidinggevenden/mapping:/config/mapping From 3b812946e48f824fadc4d035d885f480b7bdce16 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Jan 2025 16:11:59 +0100 Subject: [PATCH 18/23] Reorganize migrations and split jobs removal --- ...flush-existing-leidinggevenden-data.sparql | 0 ...flush-existing-leidinggevenden-data.sparql | 0 ...flush-existing-leidinggevenden-data.sparql | 0 ...flush-existing-leidinggevenden-data.sparql | 0 ...flush-existing-leidinggevenden-data.sparql | 0 .../20250113151947-flush-landing-graph.sparql | 0 ...leidinggevenden-consumer-containers.sparql | 37 +++++++++++++++++++ ...ush-leidinggevenden-consumer-tasks.sparql} | 32 ++++------------ ...flush-leidinggevenden-consumer-jobs.sparql | 32 ++++++++++++++++ ...leidinggevenden-consumer-containers.sparql | 37 +++++++++++++++++++ ...lush-leidinggevenden-consumer-tasks.sparql | 34 +++++++++++++++++ ...flush-leidinggevenden-consumer-jobs.sparql | 32 ++++++++++++++++ ...leidinggevenden-consumer-containers.sparql | 34 +++++++++++++++++ ...lush-leidinggevenden-consumer-tasks.sparql | 31 ++++++++++++++++ ...flush-leidinggevenden-consumer-jobs.sparql | 29 +++++++++++++++ 15 files changed, 274 insertions(+), 24 deletions(-) rename config/migrations/2025/{20250113-update-leidinggevenden-consumer => 20250113000000-update-leidinggevenden-consumer}/20250113151927-flush-existing-leidinggevenden-data.sparql (100%) rename config/migrations/2025/{20250113-update-leidinggevenden-consumer => 20250113000000-update-leidinggevenden-consumer}/20250113151928-flush-existing-leidinggevenden-data.sparql (100%) rename config/migrations/2025/{20250113-update-leidinggevenden-consumer => 20250113000000-update-leidinggevenden-consumer}/20250113151929-flush-existing-leidinggevenden-data.sparql (100%) rename config/migrations/2025/{20250113-update-leidinggevenden-consumer => 20250113000000-update-leidinggevenden-consumer}/20250113151930-flush-existing-leidinggevenden-data.sparql (100%) rename config/migrations/2025/{20250113-update-leidinggevenden-consumer => 20250113000000-update-leidinggevenden-consumer}/20250113151931-flush-existing-leidinggevenden-data.sparql (100%) rename config/migrations/2025/{20250113-update-leidinggevenden-consumer => 20250113000000-update-leidinggevenden-consumer}/20250113151947-flush-landing-graph.sparql (100%) create mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152654-flush-leidinggevenden-consumer-containers.sparql rename config/migrations/2025/{20250113-update-leidinggevenden-consumer/20250113152654-flush-old-jobs.sparql => 20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152655-flush-leidinggevenden-consumer-tasks.sparql} (64%) create mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152656-flush-leidinggevenden-consumer-jobs.sparql create mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152657-flush-leidinggevenden-consumer-containers.sparql create mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152658-flush-leidinggevenden-consumer-tasks.sparql create mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152659-flush-leidinggevenden-consumer-jobs.sparql create mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152660-flush-leidinggevenden-consumer-containers.sparql create mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152661-flush-leidinggevenden-consumer-tasks.sparql create mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152662-flush-leidinggevenden-consumer-jobs.sparql diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql similarity index 100% rename from config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql rename to config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql similarity index 100% rename from config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql rename to config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql similarity index 100% rename from config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql rename to config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql similarity index 100% rename from config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql rename to config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql similarity index 100% rename from config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql rename to config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql similarity index 100% rename from config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql rename to config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152654-flush-leidinggevenden-consumer-containers.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152654-flush-leidinggevenden-consumer-containers.sparql new file mode 100644 index 0000000..c92f096 --- /dev/null +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152654-flush-leidinggevenden-consumer-containers.sparql @@ -0,0 +1,37 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?container ?containerP ?containerO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation; + dct:created ?created. + + FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) + + ?task dct:isPartOf ?job; + task:resultsContainer ?container. + + ?container a nfo:DataContainer; + ?containerP ?containerO. + } +} diff --git a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113152654-flush-old-jobs.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152655-flush-leidinggevenden-consumer-tasks.sparql similarity index 64% rename from config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113152654-flush-old-jobs.sparql rename to config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152655-flush-leidinggevenden-consumer-tasks.sparql index d98649a..94475f6 100644 --- a/config/migrations/2025/20250113-update-leidinggevenden-consumer/20250113152654-flush-old-jobs.sparql +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152655-flush-leidinggevenden-consumer-tasks.sparql @@ -13,38 +13,22 @@ PREFIX dbpedia: DELETE { GRAPH ?g { - ?job ?jobP ?jobO. ?task ?taskP ?taskO. - ?container ?containerP ?containerO. } -} -WHERE { +} WHERE { VALUES ?operation { } GRAPH ?g { - { - ?job a ; - task:operation ?operation; - ?jobP ?jobO. + ?job a ; + dct:created ?created; + task:operation ?operation. - ?task dct:isPartOf ?job; - task:resultsContainer ?container; + FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) + + ?task dct:isPartOf ?job; ?taskP ?taskO. - - ?container a nfo:DataContainer; - ?containerP ?containerO. - } - UNION - { - ?job a ; - task:operation ?operation; - ?jobP ?jobO. - - ?task dct:isPartOf ?job; - ?taskP ?taskO. - } } -} \ No newline at end of file +} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152656-flush-leidinggevenden-consumer-jobs.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152656-flush-leidinggevenden-consumer-jobs.sparql new file mode 100644 index 0000000..52ef4d4 --- /dev/null +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152656-flush-leidinggevenden-consumer-jobs.sparql @@ -0,0 +1,32 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?job ?jobP ?jobO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + dct:created ?created; + task:operation ?operation; + ?jobP ?jobO. + + FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) + } +} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152657-flush-leidinggevenden-consumer-containers.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152657-flush-leidinggevenden-consumer-containers.sparql new file mode 100644 index 0000000..842e847 --- /dev/null +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152657-flush-leidinggevenden-consumer-containers.sparql @@ -0,0 +1,37 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?container ?containerP ?containerO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation; + dct:created ?created. + + FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) + + ?task dct:isPartOf ?job; + task:resultsContainer ?container. + + ?container a nfo:DataContainer; + ?containerP ?containerO. + } +} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152658-flush-leidinggevenden-consumer-tasks.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152658-flush-leidinggevenden-consumer-tasks.sparql new file mode 100644 index 0000000..7847429 --- /dev/null +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152658-flush-leidinggevenden-consumer-tasks.sparql @@ -0,0 +1,34 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?task ?taskP ?taskO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + dct:created ?created; + task:operation ?operation. + + FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) + + ?task dct:isPartOf ?job; + ?taskP ?taskO. + } +} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152659-flush-leidinggevenden-consumer-jobs.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152659-flush-leidinggevenden-consumer-jobs.sparql new file mode 100644 index 0000000..4f33643 --- /dev/null +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152659-flush-leidinggevenden-consumer-jobs.sparql @@ -0,0 +1,32 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?job ?jobP ?jobO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + dct:created ?created; + task:operation ?operation; + ?jobP ?jobO. + + FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) + } +} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152660-flush-leidinggevenden-consumer-containers.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152660-flush-leidinggevenden-consumer-containers.sparql new file mode 100644 index 0000000..4d90259 --- /dev/null +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152660-flush-leidinggevenden-consumer-containers.sparql @@ -0,0 +1,34 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?container ?containerP ?containerO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation. + + ?task dct:isPartOf ?job; + task:resultsContainer ?container. + + ?container a nfo:DataContainer; + ?containerP ?containerO. + } +} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152661-flush-leidinggevenden-consumer-tasks.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152661-flush-leidinggevenden-consumer-tasks.sparql new file mode 100644 index 0000000..1564173 --- /dev/null +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152661-flush-leidinggevenden-consumer-tasks.sparql @@ -0,0 +1,31 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?task ?taskP ?taskO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation. + + ?task dct:isPartOf ?job; + ?taskP ?taskO. + } +} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152662-flush-leidinggevenden-consumer-jobs.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152662-flush-leidinggevenden-consumer-jobs.sparql new file mode 100644 index 0000000..32ae341 --- /dev/null +++ b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152662-flush-leidinggevenden-consumer-jobs.sparql @@ -0,0 +1,29 @@ +PREFIX mu: +PREFIX task: +PREFIX dct: +PREFIX prov: +PREFIX nie: +PREFIX ext: +PREFIX oslc: +PREFIX cogs: +PREFIX adms: +PREFIX nfo: +PREFIX dbpedia: + + +DELETE { + GRAPH ?g { + ?job ?jobP ?jobO. + } +} WHERE { + VALUES ?operation { + + + } + + GRAPH ?g { + ?job a ; + task:operation ?operation; + ?jobP ?jobO. + } +} From b358278067cbbc5b3b7a8640ed1484cd47bba1e0 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Jan 2025 16:31:44 +0100 Subject: [PATCH 19/23] Revert "Chore/update leidinggevenden consumer" --- CHANGELOG.md | 39 +--- README.md | 39 ---- .../leidinggevenden-dispatching/config.js | 32 ++++ .../delta-context-config.js | 72 ++++++++ .../delta-sync-dispatching.js | 85 +++++++++ .../initial-sync-dispatching.js | 85 +++++++++ .../leidinggevenden-dispatching/util.js | 168 ++++++++++++++++++ .../queries/dl2op/data-cleanup/phone.rq | 21 --- .../queries/dl2op/inheritance/Agent.rq | 15 -- .../dl2op/inheritance/AgentInPositie.rq | 11 -- .../queries/dl2op/inverse/holds-to-held-by.rq | 8 - .../mapping/queries/dl2op/pass/address.sparql | 10 -- .../queries/dl2op/pass/bestuursfunctie.sparql | 7 - .../queries/dl2op/pass/contactpoint.sparql | 11 -- .../queries/dl2op/pass/functionaris.sparql | 10 -- .../mapping/queries/dl2op/pass/person.sparql | 16 -- .../dl2op/predicate/person-foaf-name.sparql | 21 --- .../person-gebruikte-voornaam.sparql | 21 --- ...flush-existing-leidinggevenden-data.sparql | 25 --- ...flush-existing-leidinggevenden-data.sparql | 25 --- ...flush-existing-leidinggevenden-data.sparql | 37 ---- ...flush-existing-leidinggevenden-data.sparql | 28 --- ...flush-existing-leidinggevenden-data.sparql | 13 -- .../20250113151947-flush-landing-graph.sparql | 1 - ...leidinggevenden-consumer-containers.sparql | 37 ---- ...lush-leidinggevenden-consumer-tasks.sparql | 34 ---- ...flush-leidinggevenden-consumer-jobs.sparql | 32 ---- ...leidinggevenden-consumer-containers.sparql | 37 ---- ...lush-leidinggevenden-consumer-tasks.sparql | 34 ---- ...flush-leidinggevenden-consumer-jobs.sparql | 32 ---- ...leidinggevenden-consumer-containers.sparql | 34 ---- ...lush-leidinggevenden-consumer-tasks.sparql | 31 ---- ...flush-leidinggevenden-consumer-jobs.sparql | 29 --- docker-compose.yml | 16 +- 34 files changed, 451 insertions(+), 665 deletions(-) create mode 100644 config/loket-consumer/leidinggevenden-dispatching/config.js create mode 100644 config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js create mode 100644 config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js create mode 100644 config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js create mode 100644 config/loket-consumer/leidinggevenden-dispatching/util.js delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inverse/holds-to-held-by.rq delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/bestuursfunctie.sparql delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-foaf-name.sparql delete mode 100644 config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-gebruikte-voornaam.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152654-flush-leidinggevenden-consumer-containers.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152655-flush-leidinggevenden-consumer-tasks.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152656-flush-leidinggevenden-consumer-jobs.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152657-flush-leidinggevenden-consumer-containers.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152658-flush-leidinggevenden-consumer-tasks.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152659-flush-leidinggevenden-consumer-jobs.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152660-flush-leidinggevenden-consumer-containers.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152661-flush-leidinggevenden-consumer-tasks.sparql delete mode 100644 config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152662-flush-leidinggevenden-consumer-jobs.sparql diff --git a/CHANGELOG.md b/CHANGELOG.md index 16a6c2f..f3ead38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,47 +1,12 @@ # Changelog - ## 1.28.5 (TODO) ### Frontend - Bump to [v1.28.2](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.2) ### Backend - Datafix: correct merger change events for worship organisations (OP-3534) -#### Consumer -- Upgraded `leidinggevenden-consumer` [OP-3533], [OP-3511] ### Deploy notes -``` -drc down; -``` -Update `docker-compose.override.yml` to: -``` - leidinggevenden-consumer: - environment: - DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint - DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso - DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso - DCR_DISABLE_DELTA_INGEST: "false" - DCR_DISABLE_INITIAL_SYNC: "false" -``` -Then: -``` -drc up -d migrations migrations-triggering-indexing; drc logs -ft --tail=200 migrations migrations-triggering-indexing -drc up -d db leidinggevenden-consumer -# Wait until success of the previous step -``` -Then, update `docker-compose.override.yml` to: -``` - leidinggevenden-consumer: - environment: - DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint - DCR_LANDING_ZONE_DATABASE: "db" - DCR_REMAPPING_DATABASE: "db" - DCR_DISABLE_DELTA_INGEST: "false" - DCR_DISABLE_INITIAL_SYNC: "false" -``` -``` -drc pull frontend -drc up -d -sh scripts/reset-elastic.sh -``` +- `drc restart migrations-triggering-indexing; drc logs -ft --tail=200 migrations-triggering-indexing` +- `drc pull frontend; drc up -d frontend` ## 1.28.4 (2025-01-16) ### Backend diff --git a/README.md b/README.md index 0b9011f..a59e0e3 100644 --- a/README.md +++ b/README.md @@ -11,45 +11,6 @@ Backend for the organization portal application, based on the mu.semte.ch micros You can shut down using `docker-compose stop` and remove everything using `docker-compose rm`. -#### Setup consumers -The application depends on external data. You'll have to perform extra steps if you want this data to be available. -These are multiple data sets, we'll try to be exhaustive on how to ingest these. Even though the procedures are similar. - -#### Setup http://data.lblod.info/vocabularies/leidinggevenden/Functionaris -Note: this is assuming you have never ran this before. -``` -drc down; -``` -Update `docker-compose.override.yml` to: -``` - leidinggevenden-consumer: - environment: - DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint - DCR_LANDING_ZONE_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso - DCR_REMAPPING_DATABASE: "triplestore" # for the initial sync, we go directly to virtuoso - DCR_DISABLE_DELTA_INGEST: "false" - DCR_DISABLE_INITIAL_SYNC: "false" -``` -Then: -``` -drc up -d migrations -drc up -d db leidinggevenden-consumer -# Wait until success of the previous step -``` -Then, update `docker-compose.override.yml` to: -``` - leidinggevenden-consumer: - environment: - DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # choose the correct endpoint - DCR_LANDING_ZONE_DATABASE: "db" - DCR_REMAPPING_DATABASE: "db" - DCR_DISABLE_DELTA_INGEST: "false" - DCR_DISABLE_INITIAL_SYNC: "false" -``` -``` -drc up -d -sh scripts/reset-elastic.sh -``` ### Setting up the delta-producers related services To make sure the app can share data, producers need to be set up. There is an intial sync, that is potentially very expensive, and must be started manually diff --git a/config/loket-consumer/leidinggevenden-dispatching/config.js b/config/loket-consumer/leidinggevenden-dispatching/config.js new file mode 100644 index 0000000..b3ebc2a --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/config.js @@ -0,0 +1,32 @@ +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql'; +const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); +const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); +const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); + +// Graphs +const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`; +const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`; + +// ENDPOINTS +const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db'; +const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; + +module.exports = { + BATCH_SIZE, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MAX_DB_RETRY_ATTEMPTS, + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + LANDING_ZONE_GRAPH, + TARGET_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT +}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js new file mode 100644 index 0000000..973462f --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js @@ -0,0 +1,72 @@ + +const PREFIXES = ` +PREFIX org: +PREFIX mandaat: +PREFIX lblodlg: +PREFIX besluit: ` + +const contextConfig = { + addTypes: { + scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both + exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta + }, + contextQueries: [ + // Case Leidnggevenden + // They can be triggered when being inserted or when their link to a bestuursfunctie is added + { + trigger: { // subjectType or predicateValue + subjectType: "lblodlg:Functionaris" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?functionaris org:holds ?bestuursfunctie ; + ?p ?o . + ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } WHERE { + GRAPH { + VALUES ?functionaris { ${subject} } + ?functionaris org:holds ?bestuursfunctie ; + ?p ?o . + ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } + }` + }, + { + trigger: { + predicateValue: "org:holds" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?functionaris org:holds ?bestuursfunctie ; + ?p ?o . + ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } WHERE { + GRAPH { + VALUES ?functionaris { ${subject} } + ?functionaris org:holds ?bestuursfunctie ; + ?p ?o . + ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } + }` + } + ] +} + +module.exports = { + contextConfig, + PREFIXES +}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js new file mode 100644 index 0000000..b309f35 --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js @@ -0,0 +1,85 @@ +const { + transformStatements, + deleteFromTargetGraph, + insertIntoTargetGraph, +} = require('./util'); + +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, fetch } = lib; + let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; + + // Both arrays are the same length, so we can zip them together for easier processing + zippedChangeSets = termObjectChangeSets.map((o, i) => ({ + original: o, + withContext: termObjectChangeSetsWithContext[i] + })); + + console.log(`Received ${zippedChangeSets.length} change sets`) + + for (let { original, withContext } of zippedChangeSets) { + let originalDeletes, originalInserts, insertsWithContext, deletesWithContext; + try { + originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + } catch (e) { + console.log(`Error while processing change set: ${e}`); + console.log(`Original change set: ${JSON.stringify(original)}`); + throw e; + } + + // Extra context needed for mapping from DL to OP model and filtering based on type. + try { + insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + } catch (e) { + console.log(`Error while processing change set with context: ${e}`); + console.log(`Change set with context: ${JSON.stringify(withContext)}`); + throw e; + } + + if (originalDeletes.length) { + // Map deletes from DL to OP model + const transformedDeletes = await transformStatements(fetch, deletesWithContext); + + if (!transformedDeletes.length) { + console.log(`Warn: Delete statements mapped to empty result.`); + console.log(`Input: ${deletesWithContext}`); + console.log(`Output: ${transformedDeletes}`); + } else { + await deleteFromTargetGraph(lib, transformedDeletes); + } + } + + if (originalInserts.length) { + // Map inserts from DL to OP model + const transformedInserts = await transformStatements(fetch, insertsWithContext); + + if (!transformedInserts.length) { + console.log(`Warn: Insert statements mapped to empty result.`); + console.log(`Input: ${insertsWithContext}`); + console.log(`Output: ${transformedInserts}`); + } else { + await insertIntoTargetGraph(lib, transformedInserts); + } + } + } +} + +module.exports = { + dispatch +}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js new file mode 100644 index 0000000..d5d672f --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js @@ -0,0 +1,85 @@ +const { batchedDbUpdate, transformLandingZoneGraph } = require('./util'); +const { + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + TARGET_GRAPH, + LANDING_ZONE_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT +} = require('./config'); + +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo } = lib; + const { termObjects } = data; + + // Steps: + // 1. Write the triples to the landing zone graph - withouth any mapping or filtering + // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + await batchedDbUpdate( + muAuthSudo.updateSudo, + LANDING_ZONE_GRAPH, + originalInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + LANDING_ZONE_DATABASE_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } +} + +async function onFinishInitialIngest(lib) { + const { muAuthSudo, fetch } = lib; + + console.log(`!! On-finish triggered !!`); + + const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint); + + console.log(`Transformed ${transformedInsertTriples.length} triples`); + + await batchedDbUpdate( + muAuthSudo.updateSudo, + TARGET_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); +} + +module.exports = { + dispatch, + onFinishInitialIngest +}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/util.js b/config/loket-consumer/leidinggevenden-dispatching/util.js new file mode 100644 index 0000000..5eced4a --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/util.js @@ -0,0 +1,168 @@ +const { + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + TARGET_GRAPH, + LANDING_ZONE_GRAPH, + BATCH_SIZE, +} = require('./config'); + +async function batchedDbUpdate( + muUpdate, + graph, + triples, + extraHeaders, + endpoint, + batchSize, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, + operation = 'INSERT' +) { + + for (let i = 0; i < triples.length; i += batchSize) { + console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); + + const batch = triples.slice(i, i + batchSize).join('\n'); + + const insertCall = async () => { + await muUpdate(` +${operation} DATA { +GRAPH <${graph}> { +${batch} +} +} +`, extraHeaders, endpoint); + }; + + await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); + + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function operationWithRetry(callback, + attempt, + maxAttempts, + sleepTimeOnFail) { + try { + if (typeof callback === "function") + return await callback(); + else // Catch error from promise - not how I would do it normally, but allows re use of existing code. + return await callback; + } + catch (e) { + console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); + console.log(`Error: ${e}`); + console.log(`Sleeping ${sleepTimeOnFail} ms`); + + if (attempt >= maxAttempts) { + console.log(`Max attempts reached for ${callback.toString()}, giving up`); + throw e; + } + + await new Promise(r => setTimeout(r, sleepTimeOnFail)); + return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); + } +} + +/** + * Splits an array into two parts, a part that passes and a part that fails a predicate function. + * Credits: https://github.com/benjay10 + * @public + * @function partition + * @param {Array} arr - Array to be partitioned + * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. + * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively + */ +function partition(arr, fn) { + let passes = [], fails = []; + arr.forEach((item) => (fn(item) ? passes : fails).push(item)); + return { passes, fails }; +} + +/** + * Send triples to reasoning service for conversion + * + */ +async function transformTriples(fetch, triples, mapping) { + return await operationWithRetry(mainConversion(fetch, triples, mapping), 0, + MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); +} + +async function mainConversion(fetch, triples, mapping) { + let formdata = new URLSearchParams(); + formdata.append("data", triples); + + let requestOptions = { + method: 'POST', + body: formdata, + redirect: 'follow' + }; + + const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) + return await response.text(); +} + +async function transformStatements(fetch, triples, mapping = 'main') { + console.log(`Received ${JSON.stringify(triples)} to transform`); + const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping); + statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : []; + console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`); + return statements; +} + +async function deleteFromTargetGraph(lib, statements) { + console.log(`Deleting ${statements.length} statements from target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + TARGET_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); +} + +async function insertIntoTargetGraph(lib, statements) { + console.log(`Inserting ${statements.length} statements into target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + TARGET_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); +} + +async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') { + console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`); + + const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`); + const text = await response.text(); + const statements = text.replace(/\n{2,}/g, '').split('\n'); + + return statements; +} + +module.exports = { + batchedDbUpdate, + partition, + transformStatements, + transformLandingZoneGraph, + deleteFromTargetGraph, + insertIntoTargetGraph +}; diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq deleted file mode 100644 index 7e761c1..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/data-cleanup/phone.rq +++ /dev/null @@ -1,21 +0,0 @@ -PREFIX schema: - -CONSTRUCT { - ?s schema:telephone ?cleanedPhoneNumber . -} WHERE { - ?s schema:telephone ?phoneNumber . - - # Remove all non digits - BIND(REPLACE(?phoneNumber, """[^+\\d]""", "") AS ?scrapeResult) - - # Replace leading 00 with + - BIND(REPLACE(?scrapeResult, """^00""", "+") AS ?replacedZeroZero) - - BIND(REPLACE(?replacedZeroZero, """^0""", "+32") AS ?defaultBe) - - BIND(REPLACE(?defaultBe, """\\+320""", "+32") AS ?extendedDefaultBe) - - BIND(REPLACE(?extendedDefaultBe, """^320""", "+32") AS ?finalResult) - - BIND(IF(REGEX(?finalResult, """^\\+((32\\d{8})|(324\\d{8})|((?!32)\\d+))$"""), CONCAT("tel:", ?finalResult), ?phoneNumber) AS ?cleanedPhoneNumber) -} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq deleted file mode 100644 index 66d5eb2..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/Agent.rq +++ /dev/null @@ -1,15 +0,0 @@ -PREFIX dc_terms: -PREFIX lblodlg: -PREFIX person: - -CONSTRUCT { - ?s a dc_terms:Agent . -} WHERE { - ?s a ?type . - - VALUES ?type { - lblodlg:Functionaris - person:Person - } - -} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq deleted file mode 100644 index e3a97b3..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inheritance/AgentInPositie.rq +++ /dev/null @@ -1,11 +0,0 @@ -PREFIX ch: -PREFIX lblodlg: - -CONSTRUCT { - ?s a ch:AgentInPositie . -} WHERE { - ?s a ?type . - VALUES ?type { - lblodlg:Functionaris - } -} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inverse/holds-to-held-by.rq b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inverse/holds-to-held-by.rq deleted file mode 100644 index 0b373ed..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/inverse/holds-to-held-by.rq +++ /dev/null @@ -1,8 +0,0 @@ -PREFIX org: - -CONSTRUCT { - ?o org:heldBy ?s. -} -WHERE { - ?s org:holds ?o. -} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql deleted file mode 100644 index 41f7523..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/address.sparql +++ /dev/null @@ -1,10 +0,0 @@ -CONSTRUCT { - ?s ?p ?o. -} WHERE { - ?s a ; - ?p ?o. - - FILTER NOT EXISTS { - ?s ?p "" . - } -} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/bestuursfunctie.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/bestuursfunctie.sparql deleted file mode 100644 index 238cd07..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/bestuursfunctie.sparql +++ /dev/null @@ -1,7 +0,0 @@ -CONSTRUCT { - ?s ?contactpoint. -} -WHERE { - ?s a . - ?s ?contactpoint. -} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql deleted file mode 100644 index d87a2ef..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/contactpoint.sparql +++ /dev/null @@ -1,11 +0,0 @@ -CONSTRUCT { - ?s ?p ?o. -} WHERE { - ?s a ; - ?p ?o. - FILTER (?p NOT IN ()) - - FILTER NOT EXISTS { - ?s ?p "" . - } -} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql deleted file mode 100644 index e753a79..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/functionaris.sparql +++ /dev/null @@ -1,10 +0,0 @@ -CONSTRUCT { - ?s ?p ?o. -} WHERE { - ?s a ; - ?p ?o. - - FILTER NOT EXISTS { - ?s ?p "" . - } -} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql deleted file mode 100644 index 02b21ea..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/pass/person.sparql +++ /dev/null @@ -1,16 +0,0 @@ -CONSTRUCT { - ?s ?p ?o. -} WHERE { - ?s a ; - ?p ?o. - - FILTER (?p IN ( - , - , - - )) - - FILTER NOT EXISTS { - ?s ?p "" . - } -} \ No newline at end of file diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-foaf-name.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-foaf-name.sparql deleted file mode 100644 index 0347ec9..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-foaf-name.sparql +++ /dev/null @@ -1,21 +0,0 @@ -PREFIX foaf: -PREFIX persoon: -PREFIX persoon_loket: -PREFIX person: - -# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) - -# First name: persoon:gebruikteVoornaam => foaf:givenName -# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam - -# We do both in one query with an OPTIONAL because they need to happen simultaneously -# not to end up in a loop - -# Part 1 - -CONSTRUCT { - ?s persoon:gebruikteVoornaam ?name . -} WHERE { - ?s a person:Person; - foaf:name ?name . -} diff --git a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-gebruikte-voornaam.sparql b/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-gebruikte-voornaam.sparql deleted file mode 100644 index 87c6463..0000000 --- a/config/loket-consumer/leidinggevenden/mapping/queries/dl2op/predicate/person-gebruikte-voornaam.sparql +++ /dev/null @@ -1,21 +0,0 @@ -PREFIX foaf: -PREFIX persoon: -PREFIX persoon_loket: -PREFIX person: - -# DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) - -# First name: persoon:gebruikteVoornaam => foaf:givenName -# Alternatieve voornaam: foaf:name => persoon:gebruikteVoornaam - -# We do both in one query with an OPTIONAL because they need to happen simultaneously -# not to end up in a loop - -# Part 2 - -CONSTRUCT { - ?s foaf:givenName ?gebruikteVoornaam . -} WHERE { - ?s a person:Person ; - persoon_loket:gebruikteVoornaam ?gebruikteVoornaam . -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql deleted file mode 100644 index e93d26b..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151927-flush-existing-leidinggevenden-data.sparql +++ /dev/null @@ -1,25 +0,0 @@ -PREFIX foaf: - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } -} WHERE { - - VALUES ?p { - - - foaf:givenName - foaf:familyName - - } - - ?functionaris a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} \ No newline at end of file diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql deleted file mode 100644 index d8c4d26..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151928-flush-existing-leidinggevenden-data.sparql +++ /dev/null @@ -1,25 +0,0 @@ -PREFIX foaf: - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - GRAPH { - ?s a ; - ?p ?o. - } -} \ No newline at end of file diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql deleted file mode 100644 index 455cadb..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151929-flush-existing-leidinggevenden-data.sparql +++ /dev/null @@ -1,37 +0,0 @@ -PREFIX foaf: - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - - - - - - - } - - ?bestuursfunctie a ; - ?contactpoint. - - ?contactpoint a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} \ No newline at end of file diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql deleted file mode 100644 index 9fb0993..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151930-flush-existing-leidinggevenden-data.sparql +++ /dev/null @@ -1,28 +0,0 @@ -PREFIX foaf: - -DELETE { - GRAPH { - ?s a ; - ?p ?o. - } - -} WHERE { - - VALUES ?p { - - - - - - - - } - - ?bestuursfunctie a ; - ?s. - - GRAPH { - ?s a ; - ?p ?o. - } -} \ No newline at end of file diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql deleted file mode 100644 index a60160b..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151931-flush-existing-leidinggevenden-data.sparql +++ /dev/null @@ -1,13 +0,0 @@ -PREFIX foaf: - -DELETE { - GRAPH { - ?bestuursfunctie ?contactpoint. - } - -} WHERE { - GRAPH { - ?bestuursfunctie a ; - ?contactpoint. - } -} \ No newline at end of file diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql deleted file mode 100644 index 2848ff2..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113151947-flush-landing-graph.sparql +++ /dev/null @@ -1 +0,0 @@ -CLEAR GRAPH \ No newline at end of file diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152654-flush-leidinggevenden-consumer-containers.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152654-flush-leidinggevenden-consumer-containers.sparql deleted file mode 100644 index c92f096..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152654-flush-leidinggevenden-consumer-containers.sparql +++ /dev/null @@ -1,37 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?container ?containerP ?containerO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - task:operation ?operation; - dct:created ?created. - - FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) - - ?task dct:isPartOf ?job; - task:resultsContainer ?container. - - ?container a nfo:DataContainer; - ?containerP ?containerO. - } -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152655-flush-leidinggevenden-consumer-tasks.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152655-flush-leidinggevenden-consumer-tasks.sparql deleted file mode 100644 index 94475f6..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152655-flush-leidinggevenden-consumer-tasks.sparql +++ /dev/null @@ -1,34 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?task ?taskP ?taskO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - dct:created ?created; - task:operation ?operation. - - FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) - - ?task dct:isPartOf ?job; - ?taskP ?taskO. - } -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152656-flush-leidinggevenden-consumer-jobs.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152656-flush-leidinggevenden-consumer-jobs.sparql deleted file mode 100644 index 52ef4d4..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2023/20250113152656-flush-leidinggevenden-consumer-jobs.sparql +++ /dev/null @@ -1,32 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?job ?jobP ?jobO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - dct:created ?created; - task:operation ?operation; - ?jobP ?jobO. - - FILTER(?created < "2024-01-01T00:00:00"^^xsd:dateTime) - } -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152657-flush-leidinggevenden-consumer-containers.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152657-flush-leidinggevenden-consumer-containers.sparql deleted file mode 100644 index 842e847..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152657-flush-leidinggevenden-consumer-containers.sparql +++ /dev/null @@ -1,37 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?container ?containerP ?containerO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - task:operation ?operation; - dct:created ?created. - - FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) - - ?task dct:isPartOf ?job; - task:resultsContainer ?container. - - ?container a nfo:DataContainer; - ?containerP ?containerO. - } -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152658-flush-leidinggevenden-consumer-tasks.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152658-flush-leidinggevenden-consumer-tasks.sparql deleted file mode 100644 index 7847429..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152658-flush-leidinggevenden-consumer-tasks.sparql +++ /dev/null @@ -1,34 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?task ?taskP ?taskO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - dct:created ?created; - task:operation ?operation. - - FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) - - ?task dct:isPartOf ?job; - ?taskP ?taskO. - } -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152659-flush-leidinggevenden-consumer-jobs.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152659-flush-leidinggevenden-consumer-jobs.sparql deleted file mode 100644 index 4f33643..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2024/20250113152659-flush-leidinggevenden-consumer-jobs.sparql +++ /dev/null @@ -1,32 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?job ?jobP ?jobO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - dct:created ?created; - task:operation ?operation; - ?jobP ?jobO. - - FILTER(?created < "2025-01-01T00:00:00"^^xsd:dateTime) - } -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152660-flush-leidinggevenden-consumer-containers.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152660-flush-leidinggevenden-consumer-containers.sparql deleted file mode 100644 index 4d90259..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152660-flush-leidinggevenden-consumer-containers.sparql +++ /dev/null @@ -1,34 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?container ?containerP ?containerO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - task:operation ?operation. - - ?task dct:isPartOf ?job; - task:resultsContainer ?container. - - ?container a nfo:DataContainer; - ?containerP ?containerO. - } -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152661-flush-leidinggevenden-consumer-tasks.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152661-flush-leidinggevenden-consumer-tasks.sparql deleted file mode 100644 index 1564173..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152661-flush-leidinggevenden-consumer-tasks.sparql +++ /dev/null @@ -1,31 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?task ?taskP ?taskO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - task:operation ?operation. - - ?task dct:isPartOf ?job; - ?taskP ?taskO. - } -} diff --git a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152662-flush-leidinggevenden-consumer-jobs.sparql b/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152662-flush-leidinggevenden-consumer-jobs.sparql deleted file mode 100644 index 32ae341..0000000 --- a/config/migrations/2025/20250113000000-update-leidinggevenden-consumer/20250113152654-flush-old-jobs/2025/20250113152662-flush-leidinggevenden-consumer-jobs.sparql +++ /dev/null @@ -1,29 +0,0 @@ -PREFIX mu: -PREFIX task: -PREFIX dct: -PREFIX prov: -PREFIX nie: -PREFIX ext: -PREFIX oslc: -PREFIX cogs: -PREFIX adms: -PREFIX nfo: -PREFIX dbpedia: - - -DELETE { - GRAPH ?g { - ?job ?jobP ?jobO. - } -} WHERE { - VALUES ?operation { - - - } - - GRAPH ?g { - ?job a ; - task:operation ?operation; - ?jobP ?jobO. - } -} diff --git a/docker-compose.yml b/docker-compose.yml index 5914f08..4928d01 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -401,26 +401,24 @@ services: # LOKET LEIDINGGEVENDEN CONSUMERS ################################################################################ leidinggevenden-consumer: - image: lblod/delta-consumer:0.1.0 + image: lblod/delta-consumer:0.0.24 environment: DCR_SERVICE_NAME: "leidinggevenden-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API - DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files" DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/LeidinggevendenCacheGraphDump" DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/leidinggevenden" DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/leidinggevenden" DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/leidinggevenden-consumer" DCR_DISABLE_INITIAL_SYNC: "true" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" + DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files" DCR_KEEP_DELTA_FILES: "true" - DCR_DELTA_JOBS_RETENTION_PERIOD: "30" - DCR_ENABLE_TRIPLE_REMAPPING: "true" + BATCH_SIZE: 1000 + DCR_ENABLE_DELTA_CONTEXT: "true" + DCR_LANDING_ZONE_DATABASE: "triplestore" DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/leidinggevenden" - DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/administrative-unit" - DCR_BATCH_SIZE: 1000 - SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" - SUDO_QUERY_RETRY: "true" volumes: - - ./config/loket-consumer/leidinggevenden/mapping:/config/mapping + - ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching - ./data/files/consumer-files/leidinggevenden:/consumer-files/ restart: always labels: From e10c825f75a159094abc94076d5eb4b789538139 Mon Sep 17 00:00:00 2001 From: mirdono Date: Tue, 21 Jan 2025 20:35:17 +0100 Subject: [PATCH 20/23] fix: index municipality names as keywords instead of regular text Index municipality names as keywords instead of as plain text. This prevents them from being split at word separators. Thereby avoiding that searches for a municipality name, as used by the frontend's municipality filter, return organisations whose municipality is a partial match. For example, previously the results when searching for "Puurs" also included the organisations located in "Sint-Amands-Puurs". The latter was tokenized into three terms, therefore also matching plain "Puurs". By treating the municipality name as keyword this split non longer happens. --- config/search/config.json | 10 +++++----- config/search/dev/config.json | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/config/search/config.json b/config/search/config.json index 643458d..3a5d67e 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -223,7 +223,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -385,7 +385,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -540,7 +540,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -727,7 +727,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -880,7 +880,7 @@ } }, "municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", diff --git a/config/search/dev/config.json b/config/search/dev/config.json index bdb3394..ab1902c 100644 --- a/config/search/dev/config.json +++ b/config/search/dev/config.json @@ -195,7 +195,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -357,7 +357,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -512,7 +512,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -700,7 +700,7 @@ } }, "organization_municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", @@ -853,7 +853,7 @@ } }, "municipality": { - "type": "text", + "type": "keyword", "fields": { "field": { "type": "keyword", From eb8790f40bdf3ea319e66d877402a127d1d6953e Mon Sep 17 00:00:00 2001 From: mirdono Date: Wed, 22 Jan 2025 09:24:33 +0100 Subject: [PATCH 21/23] chore: add changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3ead38..4b5e906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,11 @@ - Bump to [v1.28.2](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.2) ### Backend - Datafix: correct merger change events for worship organisations (OP-3534) +- Avoid municipality filter to include partial matches (OP-3509) ### Deploy notes - `drc restart migrations-triggering-indexing; drc logs -ft --tail=200 migrations-triggering-indexing` - `drc pull frontend; drc up -d frontend` +- Perform a full re-index by executing `./scripts/reset-elastic.sh` ## 1.28.4 (2025-01-16) ### Backend From 43e7cb0daccb1cb519201c6836525410ab65c6c7 Mon Sep 17 00:00:00 2001 From: mirdono Date: Wed, 22 Jan 2025 20:26:33 +0100 Subject: [PATCH 22/23] chore: bump frontend --- CHANGELOG.md | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b5e906..d8b51a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## 1.28.5 (TODO) ### Frontend -- Bump to [v1.28.2](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.2) +- Bump to [v1.28.3](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.3) ### Backend - Datafix: correct merger change events for worship organisations (OP-3534) - Avoid municipality filter to include partial matches (OP-3509) diff --git a/docker-compose.yml b/docker-compose.yml index 4928d01..b9557ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.28.2 + image: lblod/frontend-organization-portal:1.28.3 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 7316df1ab2e4e02f28cdd8accf41ca466298ffd7 Mon Sep 17 00:00:00 2001 From: mirdono Date: Thu, 23 Jan 2025 16:47:32 +0100 Subject: [PATCH 23/23] chore: added release date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8b51a4..7b7a5d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -## 1.28.5 (TODO) +## 1.28.5 (2025-01-23) ### Frontend - Bump to [v1.28.3](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.3) ### Backend