diff --git a/.gitignore b/.gitignore index ff36b53..caed09e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ docker-compose.override.yml *.iml /data/elasticsearch/** +/data/ldes-feed/* /tmp .DS_STORE .env diff --git a/config/delta/rules.js b/config/delta/rules.js index 1594ec9..8127167 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -161,5 +161,53 @@ export default [ ignoreFromSelf: true, optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/shared', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/worship-service', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/administrative-unit', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } } ] diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 3225002..24d7df3 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -305,6 +305,10 @@ defmodule Dispatcher do forward conn, path, "http://construct-administrative-unit-relationships/create-relationships/" end + get "/ldes/*path", %{ layer: :api_services } do + forward conn, path, "http://ldes-backend/" + end + ################################################################# # ERROR REPORT ################################################################# diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts new file mode 100644 index 0000000..9ff7ab3 --- /dev/null +++ b/config/ldes-delta-pusher/dispatch.ts @@ -0,0 +1,59 @@ +import { moveTriples } from "../support"; +import { Changeset } from "../types"; +import { query, sparqlEscapeUri } from 'mu'; + +export default async function dispatch(changesets: Changeset[]) { + for (const changeset of changesets) { + const subjects = new Set(changeset.inserts.map((insert) => insert.subject.value)); + for (const subject of subjects) { + const { results: { bindings } } = await query(` + PREFIX skos: + PREFIX code: + PREFIX org: + PREFIX adms: + + CONSTRUCT { + ?organization a org:Organization; + skos:prefLabel ?prefLabel; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } WHERE { + { + VALUES ?organization { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + skos:prefLabel ?prefLabel; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } UNION { + VALUES ?site { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + skos:prefLabel ?prefLabel; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } UNION { + VALUES ?siteAddress { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + skos:prefLabel ?prefLabel; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } + } + `); + if(bindings.length){ + await moveTriples([ + { + inserts: bindings.map(({ s, p, o}) => { return { subject: s, predicate: p, object: o} }), + } + ]) + } + } + } +} \ No newline at end of file diff --git a/config/migrations/2023/20230508093745-add-organization-type.sparql b/config/migrations/2023/20230508093745-add-organization-type.sparql new file mode 100644 index 0000000..a4e5dee --- /dev/null +++ b/config/migrations/2023/20230508093745-add-organization-type.sparql @@ -0,0 +1,24 @@ +PREFIX org: +PREFIX besluit: +PREFIX ere: +INSERT { + GRAPH ?g { + ?org a org:Organization . + } +} WHERE { + VALUES ?g { + + + + } + GRAPH ?g { + VALUES ?type { + besluit:Bestuurseenheid + ere:RepresentatiefOrgaan + ere:EredienstBestuurseenheid + ere:BestuurVanDeEredienst + ere:CentraalBestuurVanDeEredienst + } + ?org a ?type . + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index aa736f9..d27cd28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -490,6 +490,33 @@ services: # DELTA CONSUMERS: END ################################################################################ + ################################################################################ + # LDES PRODUCERS + ################################################################################ + + ldes-delta-pusher: + image: redpencil/ldes-delta-pusher:latest + environment: + LDES_ENDPOINT: 'http://ldes-backend/organizations' + volumes: + - ./config/ldes-delta-pusher/:/config/ + links: + - db:database + + ldes-backend: + image: redpencil/fragmentation-producer:0.3.2 + environment: + FOLDER_DEPTH: 1 + PAGE_RESOURCES_COUNT: 50 + LDES_STREAM_PREFIX: 'http://data.lblod.info/streams/op/' + TIME_TREE_RELATION_PATH: 'http://www.w3.org/ns/prov#generatedAtTime' + CACHE_SIZE: 10 + DATA_FOLDER: '/data' + ENABLE_AUTH: 'false' + restart: always + volumes: + - './data/ldes-feed/:/data/' + ################################################################################ # MU SEARCH ################################################################################