From a538b42abd730e8d4059efc816c703700290cba7 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Tue, 14 Feb 2023 11:40:17 +0100 Subject: [PATCH 01/17] add initial ldes services --- config/delta/rules.js | 12 +++ config/dispatcher/dispatcher.ex | 4 + config/ldes-delta-pusher/dispatch.ts | 13 ++++ data/ldes-feed/organizations/1.ttl | 111 +++++++++++++++++++++++++++ docker-compose.dev.yml | 2 +- docker-compose.yml | 27 +++++++ 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 config/ldes-delta-pusher/dispatch.ts create mode 100644 data/ldes-feed/organizations/1.ttl diff --git a/config/delta/rules.js b/config/delta/rules.js index 2b2899f..68657a5 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -195,5 +195,17 @@ export default [ ignoreFromSelf: true, optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] } + }, + { + match: { + + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + } } ] diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 1a8eb1b..763fc83 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -286,6 +286,10 @@ defmodule Dispatcher do forward conn, path, "http://construct-administrative-unit-relationships/create-relationships/" end + get "/ldes/*path", %{ layer: :api_services } do + forward conn, path, "http://ldes-backend/" + end + ################################################################# # ERROR REPORT ################################################################# diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts new file mode 100644 index 0000000..2c1bf3c --- /dev/null +++ b/config/ldes-delta-pusher/dispatch.ts @@ -0,0 +1,13 @@ +import { moveTriples } from "../support"; +import { Changeset } from "../types"; + +export default async function dispatch(changesets: Changeset[]) { + for (const changeset of changesets) { + await moveTriples([ + { + inserts: changeset.inserts, + deletes: changeset.deletes, + }, + ]); + } +} \ No newline at end of file diff --git a/data/ldes-feed/organizations/1.ttl b/data/ldes-feed/organizations/1.ttl new file mode 100644 index 0000000..c942bf9 --- /dev/null +++ b/data/ldes-feed/organizations/1.ttl @@ -0,0 +1,111 @@ + + . + + . + + <./1> . + +<./1> . + + . + + . + + . + + "63EB64180A219366CF7BB44B" . + + . + + "2023-02-14T10:36:08.610Z"^^ . + + . + + "63eb642dabf18a000e000001" . + + . + + "2023-02-14T10:36:29.642Z"^^ . + + . + + "2023-02-14T10:36:29+00:00" . + + . + + "ABBOrganisatiePortaalGebruiker-editeerder" . + + . + + "63eb64b3abf18a000e000002" . + + . + + "2023-02-14T10:38:43.728Z"^^ . + + . + + "2023-02-14T10:38:43+00:00" . + + . + + "ABBOrganisatiePortaalGebruiker-beheerder" . + + . + + . + + "ee75bd30-ac53-11ed-bc10-e3552d6f40d6" . + + . + + "2023-02-14T10:40:00.146Z"^^ . + + "2023-02-14T10:40:00.131Z" . + + . + + "2023-02-14T10:40:00.131Z" . + + . + + . + + . + + . + + "ee771cc0-ac53-11ed-bc10-e3552d6f40d6" . + + . + + "2023-02-14T10:40:00.154Z"^^ . + + "2023-02-14T10:40:00.140Z" . + + . + + "2023-02-14T10:40:00.140Z" . + + . + + . + + "0" . + + . + + . + + "2023-02-14T10:40:01.204Z"^^ . + + . + + . + + . + + . + + "2023-02-14T10:40:01.224Z"^^ . + diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5d68e6d..7a53405 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -76,4 +76,4 @@ services: delta-producer-bg-jobs-initiator-worship-posts: restart: "no" delta-producer-pub-graph-maintainer-worship-posts: - restart: "no" + restart: "no" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6d1f4cb..b5ebcfc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -214,6 +214,9 @@ services: image: eyereasoner/reasoning-service:1.0.1 volumes: - ./config/reasoner:/config + + + ################################################################################ # DELTA GENERAL ################################################################################ @@ -554,6 +557,30 @@ services: # DELTA CONSUMERS: END ################################################################################ + ################################################################################ + # LDES PRODUCERS + ################################################################################ + + ldes-delta-pusher: + image: redpencil/ldes-delta-pusher:latest + environment: + LDES_ENDPOINT: 'http://fragmentation-producer/organizations' + volumes: + - ./config/ldes-delta-pusher/:/config/ + + fragmentation-producer: + image: redpencil/fragmentation-producer:0.1.3 + environment: + FOLDER_DEPTH: 1 + PAGE_RESOURCES_COUNT: 50 + LDES_STREAM_PREFIX: 'http://data.lblod.info/streams/op/' + TIME_TREE_RELATION_PATH: 'http://www.w3.org/ns/prov#generatedAtTime' + CACHE_SIZE: 10 + DATA_FOLDER: '/data' + ENABLE_AUTH: 'false' + volumes: + - './data/ldes-feed/:/data/' + ################################################################################ # MU SEARCH ################################################################################ From 335501ede6a650d2f322fd2e08319425471170f9 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Tue, 14 Feb 2023 12:34:57 +0100 Subject: [PATCH 02/17] use custom dispatch method to send organization details to ldes endpoint --- .gitignore | 1 + config/ldes-delta-pusher/dispatch.ts | 28 +++++-- data/ldes-feed/organizations/1.ttl | 111 --------------------------- docker-compose.yml | 6 +- 4 files changed, 27 insertions(+), 119 deletions(-) delete mode 100644 data/ldes-feed/organizations/1.ttl diff --git a/.gitignore b/.gitignore index d6b6a17..f5c1878 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ docker-compose.override.yml *.iml /data/elasticsearch/** +/data/ldes-feed/* /tmp .DS_STORE .env diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts index 2c1bf3c..ba852e3 100644 --- a/config/ldes-delta-pusher/dispatch.ts +++ b/config/ldes-delta-pusher/dispatch.ts @@ -1,13 +1,29 @@ import { moveTriples } from "../support"; import { Changeset } from "../types"; +import { query, sparqlEscapeUri } from 'mu'; export default async function dispatch(changesets: Changeset[]) { for (const changeset of changesets) { - await moveTriples([ - { - inserts: changeset.inserts, - deletes: changeset.deletes, - }, - ]); + const subjects = new Set(changeset.inserts.map((insert) => insert.subject.value)); + for (const subject of subjects) { + const { results: { bindings } } = await query(` + PREFIX org: + CONSTRUCT { + ${sparqlEscapeUri(subject)} a org:Organization; + ?p ?o. + } + WHERE { + ${sparqlEscapeUri(subject)} a org:Organization; + ?p ?o. + } + `); + if(bindings.length){ + await moveTriples([ + { + inserts: bindings.map(({ s, p, o}) => { return { subject: s, predicate: p, object: o} }), + } + ]) + } + } } } \ No newline at end of file diff --git a/data/ldes-feed/organizations/1.ttl b/data/ldes-feed/organizations/1.ttl deleted file mode 100644 index c942bf9..0000000 --- a/data/ldes-feed/organizations/1.ttl +++ /dev/null @@ -1,111 +0,0 @@ - - . - - . - - <./1> . - -<./1> . - - . - - . - - . - - "63EB64180A219366CF7BB44B" . - - . - - "2023-02-14T10:36:08.610Z"^^ . - - . - - "63eb642dabf18a000e000001" . - - . - - "2023-02-14T10:36:29.642Z"^^ . - - . - - "2023-02-14T10:36:29+00:00" . - - . - - "ABBOrganisatiePortaalGebruiker-editeerder" . - - . - - "63eb64b3abf18a000e000002" . - - . - - "2023-02-14T10:38:43.728Z"^^ . - - . - - "2023-02-14T10:38:43+00:00" . - - . - - "ABBOrganisatiePortaalGebruiker-beheerder" . - - . - - . - - "ee75bd30-ac53-11ed-bc10-e3552d6f40d6" . - - . - - "2023-02-14T10:40:00.146Z"^^ . - - "2023-02-14T10:40:00.131Z" . - - . - - "2023-02-14T10:40:00.131Z" . - - . - - . - - . - - . - - "ee771cc0-ac53-11ed-bc10-e3552d6f40d6" . - - . - - "2023-02-14T10:40:00.154Z"^^ . - - "2023-02-14T10:40:00.140Z" . - - . - - "2023-02-14T10:40:00.140Z" . - - . - - . - - "0" . - - . - - . - - "2023-02-14T10:40:01.204Z"^^ . - - . - - . - - . - - . - - "2023-02-14T10:40:01.224Z"^^ . - diff --git a/docker-compose.yml b/docker-compose.yml index b5ebcfc..e259310 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -564,11 +564,13 @@ services: ldes-delta-pusher: image: redpencil/ldes-delta-pusher:latest environment: - LDES_ENDPOINT: 'http://fragmentation-producer/organizations' + LDES_ENDPOINT: 'http://ldes-backend/organizations' volumes: - ./config/ldes-delta-pusher/:/config/ + links: + - db:database - fragmentation-producer: + ldes-backend: image: redpencil/fragmentation-producer:0.1.3 environment: FOLDER_DEPTH: 1 From 7774283905ec199f8ee9a6b9ff0d98423c70de53 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Tue, 14 Feb 2023 12:57:32 +0100 Subject: [PATCH 03/17] update LDES dispatch query to take contact info into account --- config/ldes-delta-pusher/dispatch.ts | 42 ++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts index ba852e3..42feb68 100644 --- a/config/ldes-delta-pusher/dispatch.ts +++ b/config/ldes-delta-pusher/dispatch.ts @@ -7,14 +7,40 @@ export default async function dispatch(changesets: Changeset[]) { const subjects = new Set(changeset.inserts.map((insert) => insert.subject.value)); for (const subject of subjects) { const { results: { bindings } } = await query(` - PREFIX org: - CONSTRUCT { - ${sparqlEscapeUri(subject)} a org:Organization; - ?p ?o. - } - WHERE { - ${sparqlEscapeUri(subject)} a org:Organization; - ?p ?o. + PREFIX skos: + PREFIX code: + PREFIX org: + PREFIX adms: + + CONSTRUCT { + ?organization a org:Organization; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } WHERE { + { + VALUES ?organization { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } UNION { + VALUES ?site { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } UNION { + VALUES ?siteAddress { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } } `); if(bindings.length){ From 96175d3de2c92c32fb4fbb77587dcbebde2c7177 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Wed, 3 May 2023 09:08:56 +0200 Subject: [PATCH 04/17] update ldes-backend image to 0.3.1 --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e259310..093e056 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -571,7 +571,7 @@ services: - db:database ldes-backend: - image: redpencil/fragmentation-producer:0.1.3 + image: redpencil/fragmentation-producer:0.3.1 environment: FOLDER_DEPTH: 1 PAGE_RESOURCES_COUNT: 50 @@ -580,6 +580,7 @@ services: CACHE_SIZE: 10 DATA_FOLDER: '/data' ENABLE_AUTH: 'false' + restart: always volumes: - './data/ldes-feed/:/data/' From e887c6ffeb0de56c148d7b8979afa49258c5c02d Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Wed, 3 May 2023 09:09:19 +0200 Subject: [PATCH 05/17] add gracePeriod to ldes-delta-pusher rule --- config/delta/rules.js | 1 + 1 file changed, 1 insertion(+) diff --git a/config/delta/rules.js b/config/delta/rules.js index 68657a5..0c98f11 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -206,6 +206,7 @@ export default [ }, options: { resourceFormat: 'v0.0.1', + gracePeriod: 1000, } } ] From a94ddafb98cc49f8987a23b299ae512115dacf41 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Wed, 3 May 2023 09:09:55 +0200 Subject: [PATCH 06/17] add organization name to data to be pushed to ldes feed --- config/ldes-delta-pusher/dispatch.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts index 42feb68..9ff7ab3 100644 --- a/config/ldes-delta-pusher/dispatch.ts +++ b/config/ldes-delta-pusher/dispatch.ts @@ -14,6 +14,7 @@ export default async function dispatch(changesets: Changeset[]) { CONSTRUCT { ?organization a org:Organization; + skos:prefLabel ?prefLabel; adms:identifier ?identifier; org:hasPrimarySite ?site. ?site org:siteAddress ?siteAddress. @@ -22,6 +23,7 @@ export default async function dispatch(changesets: Changeset[]) { { VALUES ?organization { ${sparqlEscapeUri(subject)} } ?organization a org:Organization; + skos:prefLabel ?prefLabel; adms:identifier ?identifier; org:hasPrimarySite ?site. ?site org:siteAddress ?siteAddress. @@ -29,6 +31,7 @@ export default async function dispatch(changesets: Changeset[]) { } UNION { VALUES ?site { ${sparqlEscapeUri(subject)} } ?organization a org:Organization; + skos:prefLabel ?prefLabel; adms:identifier ?identifier; org:hasPrimarySite ?site. ?site org:siteAddress ?siteAddress. @@ -36,6 +39,7 @@ export default async function dispatch(changesets: Changeset[]) { } UNION { VALUES ?siteAddress { ${sparqlEscapeUri(subject)} } ?organization a org:Organization; + skos:prefLabel ?prefLabel; adms:identifier ?identifier; org:hasPrimarySite ?site. ?site org:siteAddress ?siteAddress. From ed321540bacaa7922d0295f7bd4a752d885c1a12 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Thu, 4 May 2023 09:11:04 +0200 Subject: [PATCH 07/17] update fragmentation-producer to 0.3.2 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 093e056..09028ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -571,7 +571,7 @@ services: - db:database ldes-backend: - image: redpencil/fragmentation-producer:0.3.1 + image: redpencil/fragmentation-producer:0.3.2 environment: FOLDER_DEPTH: 1 PAGE_RESOURCES_COUNT: 50 From a7334e161df865cc2579ee5e57b85d224a3cef2f Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Mon, 8 May 2023 10:12:56 +0200 Subject: [PATCH 08/17] limit pushes to ldes-feed to 3 main graphs --- config/delta/rules.js | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/config/delta/rules.js b/config/delta/rules.js index e262b64..84dbd3c 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -200,7 +200,42 @@ export default [ }, { match: { - + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/shared', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/worship-service', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/administrative-unit', + } }, callback: { url: 'http://ldes-delta-pusher/publish', From 8a779c5d9f7959063215a174e5003599471d2ebd Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Mon, 8 May 2023 10:25:58 +0200 Subject: [PATCH 09/17] remove unnecessary spaces from compose file --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b94e41f..afda03b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -231,9 +231,6 @@ services: image: eyereasoner/reasoning-service:1.0.2 volumes: - ./config/reasoner:/config - - - ################################################################################ # DELTA GENERAL ################################################################################ From c62c0aa4a94cfed5101c8cd9e5147c0b8af3587c Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Mon, 8 May 2023 11:44:19 +0200 Subject: [PATCH 10/17] add migration to add org:Organization type to all existing organizations --- ...0230508093745-add-organization-type.sparql | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 config/migrations/2023/20230508093745-add-organization-type.sparql diff --git a/config/migrations/2023/20230508093745-add-organization-type.sparql b/config/migrations/2023/20230508093745-add-organization-type.sparql new file mode 100644 index 0000000..a4e5dee --- /dev/null +++ b/config/migrations/2023/20230508093745-add-organization-type.sparql @@ -0,0 +1,24 @@ +PREFIX org: +PREFIX besluit: +PREFIX ere: +INSERT { + GRAPH ?g { + ?org a org:Organization . + } +} WHERE { + VALUES ?g { + + + + } + GRAPH ?g { + VALUES ?type { + besluit:Bestuurseenheid + ere:RepresentatiefOrgaan + ere:EredienstBestuurseenheid + ere:BestuurVanDeEredienst + ere:CentraalBestuurVanDeEredienst + } + ?org a ?type . + } +} \ No newline at end of file From b518a5ea7988a116380c3e197eb1ff01b1b92d68 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Tue, 9 May 2023 16:08:24 +0200 Subject: [PATCH 11/17] adjust change event (#276) --- .../20230509142503-adjust-change-event.sparql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 config/migrations/2023/20230509142503-adjust-change-event.sparql diff --git a/config/migrations/2023/20230509142503-adjust-change-event.sparql b/config/migrations/2023/20230509142503-adjust-change-event.sparql new file mode 100644 index 0000000..2044e8c --- /dev/null +++ b/config/migrations/2023/20230509142503-adjust-change-event.sparql @@ -0,0 +1,16 @@ +delete { + graph ?g { + . + } +} +insert { + graph ?g { + . + + } +} +where { + graph ?g { + . + } +} From d01168989f5ead1f06aa0c58765234983ba33591 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Wed, 10 May 2023 11:12:22 +0200 Subject: [PATCH 12/17] feat/delete-dup-change-events (#275) --- ...142145-move-decisions-change-events.sparql | 75 +++++++++++++++++++ ...2148-delete-duplicate-change-events.sparql | 57 ++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 config/migrations/2023/20230509142145-move-decisions-change-events.sparql create mode 100644 config/migrations/2023/20230509142148-delete-duplicate-change-events.sparql diff --git a/config/migrations/2023/20230509142145-move-decisions-change-events.sparql b/config/migrations/2023/20230509142145-move-decisions-change-events.sparql new file mode 100644 index 0000000..8c4b441 --- /dev/null +++ b/config/migrations/2023/20230509142145-move-decisions-change-events.sparql @@ -0,0 +1,75 @@ + +PREFIX rdf: +PREFIX rdfs: +PREFIX skos: +PREFIX org: +PREFIX mu: +PREFIX xsd: +PREFIX dct: +PREFIX skos: +PREFIX concepts: +PREFIX bestuur: +PREFIX org: +PREFIX rdf: +PREFIX contacthub: +PREFIX xsd: +PREFIX ce: +PREFIX cer: +PREFIX m8g: +PREFIX besluit: +PREFIX eli: +PREFIX foaf: + + +DELETE { + graph { + ?decision ?decisionP ?decisionO. + } +} +INSERT { + graph { + ?decision ?decisionP ?decisionO. + } +} +WHERE { + graph { + ?changeEvent a ; + ?decision. + } + graph { + ?decision ?decisionP ?decisionO. + + } + + +} +; + + +DELETE { + graph { + ?decisionActivity ?p ?o. + } +} +INSERT { + graph { + ?decisionActivity ?p ?o. + } +} +WHERE { + graph { + ?decision a . + } + graph { + ?decisionActivity a ; + ?decision; + ?p ?o. + } + + +} + + + + + diff --git a/config/migrations/2023/20230509142148-delete-duplicate-change-events.sparql b/config/migrations/2023/20230509142148-delete-duplicate-change-events.sparql new file mode 100644 index 0000000..f3297b1 --- /dev/null +++ b/config/migrations/2023/20230509142148-delete-duplicate-change-events.sparql @@ -0,0 +1,57 @@ + +PREFIX rdf: +PREFIX rdfs: +PREFIX skos: +PREFIX org: +PREFIX mu: +PREFIX xsd: +PREFIX dct: +PREFIX skos: +PREFIX concepts: +PREFIX bestuur: +PREFIX org: +PREFIX rdf: +PREFIX contacthub: +PREFIX xsd: +PREFIX ce: +PREFIX cer: +PREFIX m8g: +PREFIX besluit: +PREFIX eli: +PREFIX foaf: + + +DELETE { + graph ?g { + ?changeEvent ?p ?o. + ?changeEventResult ?cP ?cO. + ?decision ?decisionP ?decisionO. + ?decisionActivity ?decisionActivityP ?desitionActivityO. + } +} +WHERE { + graph ?g { + ?changeEvent a ; + ?changeEventResult; + mu:uuid ?uuidChangeEvent; + ?p ?o. + ?changeEventResult a ; + mu:uuid ?uuidChangeEventResult; + ?cP ?cO. + + optional { + ?changeEvent ?decision. + ?decision ?decisionP ?decisionO. + optional { + ?decisionActivity a ; + ?decision; + ?decisionActivityP ?desitionActivityO. + } + + } + + filter(?uuidChangeEvent in ("63343865036B579F0A26FF5C", "63343E99036B579F0A26FF6F", "633446D6036B579F0A26FF87","63345772036B579F0A26FF93")) + + } +} + From cc4a84228130f723b807d521d6afd12f770d16df Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 10 May 2023 11:15:27 +0200 Subject: [PATCH 13/17] Remove deprecated delta-producer-json-diff-file-publisher service (#274) --- config/delta/rules.js | 36 ------------------------- config/dispatcher/dispatcher.ex | 13 ++------- docker-compose.dev.yml | 6 +++-- docker-compose.yml | 48 ++++++++------------------------- 4 files changed, 17 insertions(+), 86 deletions(-) diff --git a/config/delta/rules.js b/config/delta/rules.js index b502977..1594ec9 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -105,24 +105,6 @@ export default [ ] } }, - { - match: { - graph: { - type: 'uri', - value: 'http://redpencil.data.gift/id/deltas/producer/organizations' - } - }, - callback: { - url: 'http://delta-producer-json-diff-publisher-organizations/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] - } - }, { match: { }, @@ -140,24 +122,6 @@ export default [ ] } }, - { - match: { - graph: { - type: 'uri', - value: 'http://redpencil.data.gift/id/deltas/producer/public' - } - }, - callback: { - url: 'http://delta-producer-json-diff-publisher-public/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] - } - }, { match: { // anything diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index c4128ef..3225002 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -337,21 +337,12 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://resource/distributions/" end - - ################################################################# - # DELTA: administrative-units - ################################################################# - - get "/sync/administrative-units/files/*path" do - Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-administrative-units/files/" - end - ################################################################# # DELTA: organizations ################################################################# get "/sync/organizations/files/*path" do - Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-organizations/files/" + Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/" end ################################################################# @@ -359,7 +350,7 @@ defmodule Dispatcher do ################################################################# get "/sync/public/files/*path" do - Proxy.forward conn, path, "http://delta-producer-json-diff-publisher-public/files/" + Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-public/files/" end ################################################################# diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 52c53e8..aaa82ee 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -55,8 +55,6 @@ services: restart: "no" delta-producer-pub-graph-maintainer-organizations: restart: "no" - delta-producer-json-diff-publisher-organizations: - restart: "no" search: restart: "no" volumes: @@ -73,6 +71,10 @@ services: restart: "no" delta-producer-pub-graph-maintainer-worship-posts: restart: "no" + delta-producer-bg-jobs-initiator-public: + restart: "no" + delta-producer-pub-graph-maintainer-public: + restart: "no" # use this to test acmidm # tlsproxy: # image: caddy:2 diff --git a/docker-compose.yml b/docker-compose.yml index df6529a..aa736f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -263,7 +263,7 @@ services: restart: always logging: *default-logging delta-producer-pub-graph-maintainer-organizations: - image: lblod/delta-producer-publication-graph-maintainer:0.7.0 + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 environment: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" @@ -275,6 +275,11 @@ services: USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" volumes: - ./config/delta-producer/organizations:/config - ./data/files/:/share @@ -283,24 +288,6 @@ services: - triplestore:virtuoso restart: always logging: *default-logging - delta-producer-json-diff-publisher-organizations: - image: lblod/delta-producer-json-diff-file-publisher:0.2.3 - environment: - MAX_BODY_SIZE: "50mb" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/organizations" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - volumes: - - ./data/files:/share - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS: END ################################################################################ @@ -337,6 +324,11 @@ services: USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" volumes: - ./config/delta-producer/public:/config - ./data/files/:/share @@ -345,24 +337,6 @@ services: - triplestore:virtuoso restart: always logging: *default-logging - delta-producer-json-diff-publisher-public: - image: lblod/delta-producer-json-diff-file-publisher:0.2.3 - environment: - MAX_BODY_SIZE: "50mb" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/public" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - volumes: - - ./data/files:/share - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging ################################################################################ # DELTA PUBLIC INFORMATION: END ################################################################################ From 22cc8ad09eef8ded1c16cb2726a3198b68a8b084 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Wed, 10 May 2023 16:28:45 +0200 Subject: [PATCH 14/17] bump resource --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d27cd28..4adfd51 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ services: restart: always logging: *default-logging resource: - image: semtech/mu-cl-resources:1.22.0 + image: semtech/mu-cl-resources:feature-fix-false-boolean-primitive environment: CACHE_CLEAR_PATH: "http://cache/.mu/clear-keys" links: From 2dbc29adeb7403ebd0311618d52c0fab22561b1c Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Fri, 12 May 2023 14:19:44 +0200 Subject: [PATCH 15/17] bump mu-cl-resource --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4adfd51..6f4557e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ services: restart: always logging: *default-logging resource: - image: semtech/mu-cl-resources:feature-fix-false-boolean-primitive + image: semtech/mu-cl-resources:1.22.1 environment: CACHE_CLEAR_PATH: "http://cache/.mu/clear-keys" links: From 21532f17c067ceab8856c4188cbb7c3da09b631f Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Sun, 14 May 2023 12:44:38 +0200 Subject: [PATCH 16/17] autocommit --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6f4557e..a28b026 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -181,7 +181,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.17.0-0 + image: lblod/frontend-organization-portal:1.17.0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 3a23aa3c2deb58b3d540261c4adde55e39286bfc Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 15 May 2023 18:16:54 +0200 Subject: [PATCH 17/17] Fix/limit positions to types we already onboarded rebased (#278) * rebased * Fix n3q query for mandatarissen --------- Co-authored-by: Claire Lovisa --- .../delta-context-config.js | 65 ++++++++++- .../delta-context-config.js | 65 ++++++++++- ...e-not-yet-onboarded-functionarissen.sparql | 21 ++++ config/reasoner/dl2op/main/query.n3q | 102 ++++++++++++++++++ docker-compose.yml | 2 +- 5 files changed, 250 insertions(+), 5 deletions(-) create mode 100644 config/migrations/2023/20230515111900-remove-not-yet-onboarded-functionarissen.sparql diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js index a8df81c..973462f 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js +++ b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js @@ -1,11 +1,72 @@ + +const PREFIXES = ` +PREFIX org: +PREFIX mandaat: +PREFIX lblodlg: +PREFIX besluit: ` + const contextConfig = { addTypes: { scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta }, - contextQueries: [] + contextQueries: [ + // Case Leidnggevenden + // They can be triggered when being inserted or when their link to a bestuursfunctie is added + { + trigger: { // subjectType or predicateValue + subjectType: "lblodlg:Functionaris" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?functionaris org:holds ?bestuursfunctie ; + ?p ?o . + ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } WHERE { + GRAPH { + VALUES ?functionaris { ${subject} } + ?functionaris org:holds ?bestuursfunctie ; + ?p ?o . + ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } + }` + }, + { + trigger: { + predicateValue: "org:holds" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?functionaris org:holds ?bestuursfunctie ; + ?p ?o . + ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } WHERE { + GRAPH { + VALUES ?functionaris { ${subject} } + ?functionaris org:holds ?bestuursfunctie ; + ?p ?o . + ?orgaanInTime lblodlg:heeftBestuursfunctie ?bestuursfunctie ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } + }` + } + ] } module.exports = { - contextConfig + contextConfig, + PREFIXES }; diff --git a/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js b/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js index a8df81c..c32c2a9 100644 --- a/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js +++ b/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js @@ -1,11 +1,72 @@ + +const PREFIXES = ` +PREFIX org: +PREFIX mandaat: +PREFIX lblodlg: +PREFIX besluit: ` + const contextConfig = { addTypes: { scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta }, - contextQueries: [] + contextQueries: [ + // Case Mandatarissen + // They can be triggered when being inserted or when their link to a mandate is added + { + trigger: { // subjectType or predicateValue + subjectType: "mandaat:Mandataris" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?mandataris org:holds ?mandate ; + ?p ?o . + ?orgaanInTime org:hasPost ?mandate ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } WHERE { + GRAPH { + VALUES ?mandataris { ${subject} } + ?mandataris org:holds ?mandate ; + ?p ?o . + ?orgaanInTime org:hasPost ?mandate ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } + }` + }, + { + trigger: { + predicateValue: "org:holds" + }, + queryTemplate: (subject) => ` + ${PREFIXES} + CONSTRUCT { + ?mandataris org:holds ?mandate ; + ?p ?o . + ?orgaanInTime org:hasPost ?mandate ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } WHERE { + GRAPH { + VALUES ?mandataris { ${subject} } + ?mandataris org:holds ?mandate ; + ?p ?o . + ?orgaanInTime org:hasPost ?mandate ; + mandaat:isTijdspecialisatieVan ?orgaan . + ?orgaan besluit:bestuurt ?eenheid . + ?eenheid besluit:classificatie ?classification . + } + }` + } + ] } module.exports = { - contextConfig + contextConfig, + PREFIXES }; diff --git a/config/migrations/2023/20230515111900-remove-not-yet-onboarded-functionarissen.sparql b/config/migrations/2023/20230515111900-remove-not-yet-onboarded-functionarissen.sparql new file mode 100644 index 0000000..a3ee318 --- /dev/null +++ b/config/migrations/2023/20230515111900-remove-not-yet-onboarded-functionarissen.sparql @@ -0,0 +1,21 @@ +DELETE { + GRAPH { + ?functionaris ?pfunctionaris ?ofunctionaris . + } +} WHERE { + GRAPH { + ?functionaris a ; + ?bestuursfunctie ; + ?pfunctionaris ?ofunctionaris . + } + + ?orgaan ?bestuursfunctie ; + // ?classification . + + FILTER(?classification NOT IN ( + , + , + , + + )) +} diff --git a/config/reasoner/dl2op/main/query.n3q b/config/reasoner/dl2op/main/query.n3q index ed0415d..5998591 100644 --- a/config/reasoner/dl2op/main/query.n3q +++ b/config/reasoner/dl2op/main/query.n3q @@ -1,5 +1,7 @@ # Filtering classes to ingest +@prefix log: . + { ?s a ; ?p ?o . @@ -24,17 +26,117 @@ ?p ?o . }. +# Functionarissen should be linked to a gemeente, district, provincie or ocmw only (FOR NOW) + { ?s a ; + ?bestuursfunctie ; ?p ?o . + + ?orgaanInTime ?bestuursfunctie ; + ?orgaan . + ?orgaan ?bestuur . + ?bestuur . } => { ?s a ; ?p ?o . }. { + ?s a ; + ?bestuursfunctie ; + ?p ?o . + + ?orgaanInTime ?bestuursfunctie ; + ?orgaan . + ?orgaan ?bestuur . + ?bestuur . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?bestuursfunctie ; + ?p ?o . + + ?orgaanInTime ?bestuursfunctie ; + ?orgaan . + ?orgaan ?bestuur . + ?bestuur . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?bestuursfunctie ; + ?p ?o . + + ?orgaanInTime ?bestuursfunctie ; + ?orgaan . + ?orgaan ?bestuur . + ?bestuur . + } => { + ?s a ; + ?p ?o . +}. + +# Mandatarissen should be linked to a gemeente, district, provincie or ocmw only (FOR NOW) + +{ + ?s a ; + ?mandate ; + ?p ?o . + + ?orgaanInTime ?mandate ; + ?orgaan . + ?orgaan ?bestuur . + ?bestuur . + } => { ?s a ; ?p ?o . +}. + +{ + ?s a ; + ?mandate ; + ?p ?o . + + ?orgaanInTime ?mandate ; + ?orgaan . + ?orgaan ?bestuur . + ?bestuur . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?mandate ; + ?p ?o . + + ?orgaanInTime ?mandate ; + ?orgaan . + ?orgaan ?bestuur . + ?bestuur . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?mandate ; + ?p ?o . + + ?orgaanInTime ?mandate ; + ?orgaan . + ?orgaan ?bestuur . + ?bestuur . } => { ?s a ; ?p ?o . diff --git a/docker-compose.yml b/docker-compose.yml index a28b026..369444b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -228,7 +228,7 @@ services: restart: always logging: *default-logging reasoner: - image: eyereasoner/reasoning-service:1.0.2 + image: eyereasoner/reasoning-service:increased-stack-limit volumes: - ./config/reasoner:/config ################################################################################