From a538b42abd730e8d4059efc816c703700290cba7 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Tue, 14 Feb 2023 11:40:17 +0100 Subject: [PATCH 01/10] add initial ldes services --- config/delta/rules.js | 12 +++ config/dispatcher/dispatcher.ex | 4 + config/ldes-delta-pusher/dispatch.ts | 13 ++++ data/ldes-feed/organizations/1.ttl | 111 +++++++++++++++++++++++++++ docker-compose.dev.yml | 2 +- docker-compose.yml | 27 +++++++ 6 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 config/ldes-delta-pusher/dispatch.ts create mode 100644 data/ldes-feed/organizations/1.ttl diff --git a/config/delta/rules.js b/config/delta/rules.js index 2b2899f..68657a5 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -195,5 +195,17 @@ export default [ ignoreFromSelf: true, optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] } + }, + { + match: { + + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + } } ] diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 1a8eb1b..763fc83 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -286,6 +286,10 @@ defmodule Dispatcher do forward conn, path, "http://construct-administrative-unit-relationships/create-relationships/" end + get "/ldes/*path", %{ layer: :api_services } do + forward conn, path, "http://ldes-backend/" + end + ################################################################# # ERROR REPORT ################################################################# diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts new file mode 100644 index 0000000..2c1bf3c --- /dev/null +++ b/config/ldes-delta-pusher/dispatch.ts @@ -0,0 +1,13 @@ +import { moveTriples } from "../support"; +import { Changeset } from "../types"; + +export default async function dispatch(changesets: Changeset[]) { + for (const changeset of changesets) { + await moveTriples([ + { + inserts: changeset.inserts, + deletes: changeset.deletes, + }, + ]); + } +} \ No newline at end of file diff --git a/data/ldes-feed/organizations/1.ttl b/data/ldes-feed/organizations/1.ttl new file mode 100644 index 0000000..c942bf9 --- /dev/null +++ b/data/ldes-feed/organizations/1.ttl @@ -0,0 +1,111 @@ + + . + + . + + <./1> . + +<./1> . + + . + + . + + . + + "63EB64180A219366CF7BB44B" . + + . + + "2023-02-14T10:36:08.610Z"^^ . + + . + + "63eb642dabf18a000e000001" . + + . + + "2023-02-14T10:36:29.642Z"^^ . + + . + + "2023-02-14T10:36:29+00:00" . + + . + + "ABBOrganisatiePortaalGebruiker-editeerder" . + + . + + "63eb64b3abf18a000e000002" . + + . + + "2023-02-14T10:38:43.728Z"^^ . + + . + + "2023-02-14T10:38:43+00:00" . + + . + + "ABBOrganisatiePortaalGebruiker-beheerder" . + + . + + . + + "ee75bd30-ac53-11ed-bc10-e3552d6f40d6" . + + . + + "2023-02-14T10:40:00.146Z"^^ . + + "2023-02-14T10:40:00.131Z" . + + . + + "2023-02-14T10:40:00.131Z" . + + . + + . + + . + + . + + "ee771cc0-ac53-11ed-bc10-e3552d6f40d6" . + + . + + "2023-02-14T10:40:00.154Z"^^ . + + "2023-02-14T10:40:00.140Z" . + + . + + "2023-02-14T10:40:00.140Z" . + + . + + . + + "0" . + + . + + . + + "2023-02-14T10:40:01.204Z"^^ . + + . + + . + + . + + . + + "2023-02-14T10:40:01.224Z"^^ . + diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5d68e6d..7a53405 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -76,4 +76,4 @@ services: delta-producer-bg-jobs-initiator-worship-posts: restart: "no" delta-producer-pub-graph-maintainer-worship-posts: - restart: "no" + restart: "no" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 6d1f4cb..b5ebcfc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -214,6 +214,9 @@ services: image: eyereasoner/reasoning-service:1.0.1 volumes: - ./config/reasoner:/config + + + ################################################################################ # DELTA GENERAL ################################################################################ @@ -554,6 +557,30 @@ services: # DELTA CONSUMERS: END ################################################################################ + ################################################################################ + # LDES PRODUCERS + ################################################################################ + + ldes-delta-pusher: + image: redpencil/ldes-delta-pusher:latest + environment: + LDES_ENDPOINT: 'http://fragmentation-producer/organizations' + volumes: + - ./config/ldes-delta-pusher/:/config/ + + fragmentation-producer: + image: redpencil/fragmentation-producer:0.1.3 + environment: + FOLDER_DEPTH: 1 + PAGE_RESOURCES_COUNT: 50 + LDES_STREAM_PREFIX: 'http://data.lblod.info/streams/op/' + TIME_TREE_RELATION_PATH: 'http://www.w3.org/ns/prov#generatedAtTime' + CACHE_SIZE: 10 + DATA_FOLDER: '/data' + ENABLE_AUTH: 'false' + volumes: + - './data/ldes-feed/:/data/' + ################################################################################ # MU SEARCH ################################################################################ From 335501ede6a650d2f322fd2e08319425471170f9 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Tue, 14 Feb 2023 12:34:57 +0100 Subject: [PATCH 02/10] use custom dispatch method to send organization details to ldes endpoint --- .gitignore | 1 + config/ldes-delta-pusher/dispatch.ts | 28 +++++-- data/ldes-feed/organizations/1.ttl | 111 --------------------------- docker-compose.yml | 6 +- 4 files changed, 27 insertions(+), 119 deletions(-) delete mode 100644 data/ldes-feed/organizations/1.ttl diff --git a/.gitignore b/.gitignore index d6b6a17..f5c1878 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ docker-compose.override.yml *.iml /data/elasticsearch/** +/data/ldes-feed/* /tmp .DS_STORE .env diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts index 2c1bf3c..ba852e3 100644 --- a/config/ldes-delta-pusher/dispatch.ts +++ b/config/ldes-delta-pusher/dispatch.ts @@ -1,13 +1,29 @@ import { moveTriples } from "../support"; import { Changeset } from "../types"; +import { query, sparqlEscapeUri } from 'mu'; export default async function dispatch(changesets: Changeset[]) { for (const changeset of changesets) { - await moveTriples([ - { - inserts: changeset.inserts, - deletes: changeset.deletes, - }, - ]); + const subjects = new Set(changeset.inserts.map((insert) => insert.subject.value)); + for (const subject of subjects) { + const { results: { bindings } } = await query(` + PREFIX org: + CONSTRUCT { + ${sparqlEscapeUri(subject)} a org:Organization; + ?p ?o. + } + WHERE { + ${sparqlEscapeUri(subject)} a org:Organization; + ?p ?o. + } + `); + if(bindings.length){ + await moveTriples([ + { + inserts: bindings.map(({ s, p, o}) => { return { subject: s, predicate: p, object: o} }), + } + ]) + } + } } } \ No newline at end of file diff --git a/data/ldes-feed/organizations/1.ttl b/data/ldes-feed/organizations/1.ttl deleted file mode 100644 index c942bf9..0000000 --- a/data/ldes-feed/organizations/1.ttl +++ /dev/null @@ -1,111 +0,0 @@ - - . - - . - - <./1> . - -<./1> . - - . - - . - - . - - "63EB64180A219366CF7BB44B" . - - . - - "2023-02-14T10:36:08.610Z"^^ . - - . - - "63eb642dabf18a000e000001" . - - . - - "2023-02-14T10:36:29.642Z"^^ . - - . - - "2023-02-14T10:36:29+00:00" . - - . - - "ABBOrganisatiePortaalGebruiker-editeerder" . - - . - - "63eb64b3abf18a000e000002" . - - . - - "2023-02-14T10:38:43.728Z"^^ . - - . - - "2023-02-14T10:38:43+00:00" . - - . - - "ABBOrganisatiePortaalGebruiker-beheerder" . - - . - - . - - "ee75bd30-ac53-11ed-bc10-e3552d6f40d6" . - - . - - "2023-02-14T10:40:00.146Z"^^ . - - "2023-02-14T10:40:00.131Z" . - - . - - "2023-02-14T10:40:00.131Z" . - - . - - . - - . - - . - - "ee771cc0-ac53-11ed-bc10-e3552d6f40d6" . - - . - - "2023-02-14T10:40:00.154Z"^^ . - - "2023-02-14T10:40:00.140Z" . - - . - - "2023-02-14T10:40:00.140Z" . - - . - - . - - "0" . - - . - - . - - "2023-02-14T10:40:01.204Z"^^ . - - . - - . - - . - - . - - "2023-02-14T10:40:01.224Z"^^ . - diff --git a/docker-compose.yml b/docker-compose.yml index b5ebcfc..e259310 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -564,11 +564,13 @@ services: ldes-delta-pusher: image: redpencil/ldes-delta-pusher:latest environment: - LDES_ENDPOINT: 'http://fragmentation-producer/organizations' + LDES_ENDPOINT: 'http://ldes-backend/organizations' volumes: - ./config/ldes-delta-pusher/:/config/ + links: + - db:database - fragmentation-producer: + ldes-backend: image: redpencil/fragmentation-producer:0.1.3 environment: FOLDER_DEPTH: 1 From 7774283905ec199f8ee9a6b9ff0d98423c70de53 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Tue, 14 Feb 2023 12:57:32 +0100 Subject: [PATCH 03/10] update LDES dispatch query to take contact info into account --- config/ldes-delta-pusher/dispatch.ts | 42 ++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts index ba852e3..42feb68 100644 --- a/config/ldes-delta-pusher/dispatch.ts +++ b/config/ldes-delta-pusher/dispatch.ts @@ -7,14 +7,40 @@ export default async function dispatch(changesets: Changeset[]) { const subjects = new Set(changeset.inserts.map((insert) => insert.subject.value)); for (const subject of subjects) { const { results: { bindings } } = await query(` - PREFIX org: - CONSTRUCT { - ${sparqlEscapeUri(subject)} a org:Organization; - ?p ?o. - } - WHERE { - ${sparqlEscapeUri(subject)} a org:Organization; - ?p ?o. + PREFIX skos: + PREFIX code: + PREFIX org: + PREFIX adms: + + CONSTRUCT { + ?organization a org:Organization; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } WHERE { + { + VALUES ?organization { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } UNION { + VALUES ?site { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } UNION { + VALUES ?siteAddress { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } } `); if(bindings.length){ From 96175d3de2c92c32fb4fbb77587dcbebde2c7177 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Wed, 3 May 2023 09:08:56 +0200 Subject: [PATCH 04/10] update ldes-backend image to 0.3.1 --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e259310..093e056 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -571,7 +571,7 @@ services: - db:database ldes-backend: - image: redpencil/fragmentation-producer:0.1.3 + image: redpencil/fragmentation-producer:0.3.1 environment: FOLDER_DEPTH: 1 PAGE_RESOURCES_COUNT: 50 @@ -580,6 +580,7 @@ services: CACHE_SIZE: 10 DATA_FOLDER: '/data' ENABLE_AUTH: 'false' + restart: always volumes: - './data/ldes-feed/:/data/' From e887c6ffeb0de56c148d7b8979afa49258c5c02d Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Wed, 3 May 2023 09:09:19 +0200 Subject: [PATCH 05/10] add gracePeriod to ldes-delta-pusher rule --- config/delta/rules.js | 1 + 1 file changed, 1 insertion(+) diff --git a/config/delta/rules.js b/config/delta/rules.js index 68657a5..0c98f11 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -206,6 +206,7 @@ export default [ }, options: { resourceFormat: 'v0.0.1', + gracePeriod: 1000, } } ] From a94ddafb98cc49f8987a23b299ae512115dacf41 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Wed, 3 May 2023 09:09:55 +0200 Subject: [PATCH 06/10] add organization name to data to be pushed to ldes feed --- config/ldes-delta-pusher/dispatch.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts index 42feb68..9ff7ab3 100644 --- a/config/ldes-delta-pusher/dispatch.ts +++ b/config/ldes-delta-pusher/dispatch.ts @@ -14,6 +14,7 @@ export default async function dispatch(changesets: Changeset[]) { CONSTRUCT { ?organization a org:Organization; + skos:prefLabel ?prefLabel; adms:identifier ?identifier; org:hasPrimarySite ?site. ?site org:siteAddress ?siteAddress. @@ -22,6 +23,7 @@ export default async function dispatch(changesets: Changeset[]) { { VALUES ?organization { ${sparqlEscapeUri(subject)} } ?organization a org:Organization; + skos:prefLabel ?prefLabel; adms:identifier ?identifier; org:hasPrimarySite ?site. ?site org:siteAddress ?siteAddress. @@ -29,6 +31,7 @@ export default async function dispatch(changesets: Changeset[]) { } UNION { VALUES ?site { ${sparqlEscapeUri(subject)} } ?organization a org:Organization; + skos:prefLabel ?prefLabel; adms:identifier ?identifier; org:hasPrimarySite ?site. ?site org:siteAddress ?siteAddress. @@ -36,6 +39,7 @@ export default async function dispatch(changesets: Changeset[]) { } UNION { VALUES ?siteAddress { ${sparqlEscapeUri(subject)} } ?organization a org:Organization; + skos:prefLabel ?prefLabel; adms:identifier ?identifier; org:hasPrimarySite ?site. ?site org:siteAddress ?siteAddress. From ed321540bacaa7922d0295f7bd4a752d885c1a12 Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Thu, 4 May 2023 09:11:04 +0200 Subject: [PATCH 07/10] update fragmentation-producer to 0.3.2 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 093e056..09028ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -571,7 +571,7 @@ services: - db:database ldes-backend: - image: redpencil/fragmentation-producer:0.3.1 + image: redpencil/fragmentation-producer:0.3.2 environment: FOLDER_DEPTH: 1 PAGE_RESOURCES_COUNT: 50 From a7334e161df865cc2579ee5e57b85d224a3cef2f Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Mon, 8 May 2023 10:12:56 +0200 Subject: [PATCH 08/10] limit pushes to ldes-feed to 3 main graphs --- config/delta/rules.js | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/config/delta/rules.js b/config/delta/rules.js index e262b64..84dbd3c 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -200,7 +200,42 @@ export default [ }, { match: { - + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/shared', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/worship-service', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/administrative-unit', + } }, callback: { url: 'http://ldes-delta-pusher/publish', From 8a779c5d9f7959063215a174e5003599471d2ebd Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Mon, 8 May 2023 10:25:58 +0200 Subject: [PATCH 09/10] remove unnecessary spaces from compose file --- docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b94e41f..afda03b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -231,9 +231,6 @@ services: image: eyereasoner/reasoning-service:1.0.2 volumes: - ./config/reasoner:/config - - - ################################################################################ # DELTA GENERAL ################################################################################ From c62c0aa4a94cfed5101c8cd9e5147c0b8af3587c Mon Sep 17 00:00:00 2001 From: Elena Poelman Date: Mon, 8 May 2023 11:44:19 +0200 Subject: [PATCH 10/10] add migration to add org:Organization type to all existing organizations --- ...0230508093745-add-organization-type.sparql | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 config/migrations/2023/20230508093745-add-organization-type.sparql diff --git a/config/migrations/2023/20230508093745-add-organization-type.sparql b/config/migrations/2023/20230508093745-add-organization-type.sparql new file mode 100644 index 0000000..a4e5dee --- /dev/null +++ b/config/migrations/2023/20230508093745-add-organization-type.sparql @@ -0,0 +1,24 @@ +PREFIX org: +PREFIX besluit: +PREFIX ere: +INSERT { + GRAPH ?g { + ?org a org:Organization . + } +} WHERE { + VALUES ?g { + + + + } + GRAPH ?g { + VALUES ?type { + besluit:Bestuurseenheid + ere:RepresentatiefOrgaan + ere:EredienstBestuurseenheid + ere:BestuurVanDeEredienst + ere:CentraalBestuurVanDeEredienst + } + ?org a ?type . + } +} \ No newline at end of file