diff --git a/.gitignore b/.gitignore index ff36b53..caed09e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ docker-compose.override.yml *.iml /data/elasticsearch/** +/data/ldes-feed/* /tmp .DS_STORE .env diff --git a/config/delta/rules.js b/config/delta/rules.js index b502977..8127167 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -105,24 +105,6 @@ export default [ ] } }, - { - match: { - graph: { - type: 'uri', - value: 'http://redpencil.data.gift/id/deltas/producer/organizations' - } - }, - callback: { - url: 'http://delta-producer-json-diff-publisher-organizations/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] - } - }, { match: { }, @@ -140,24 +122,6 @@ export default [ ] } }, - { - match: { - graph: { - type: 'uri', - value: 'http://redpencil.data.gift/id/deltas/producer/public' - } - }, - callback: { - url: 'http://delta-producer-json-diff-publisher-public/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] - } - }, { match: { // anything @@ -197,5 +161,53 @@ export default [ ignoreFromSelf: true, optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/shared', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/worship-service', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://mu.semte.ch/graphs/administrative-unit', + } + }, + callback: { + url: 'http://ldes-delta-pusher/publish', + method: 'POST', + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + } } ] diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index c4128ef..24d7df3 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -305,6 +305,10 @@ defmodule Dispatcher do forward conn, path, "http://construct-administrative-unit-relationships/create-relationships/" end + get "/ldes/*path", %{ layer: :api_services } do + forward conn, path, "http://ldes-backend/" + end + ################################################################# # ERROR REPORT ################################################################# @@ -337,21 +341,12 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://resource/distributions/" end - - ################################################################# - # DELTA: administrative-units - ################################################################# - - get "/sync/administrative-units/files/*path" do - Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-administrative-units/files/" - end - ################################################################# # DELTA: organizations ################################################################# get "/sync/organizations/files/*path" do - Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-organizations/files/" + Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/" end ################################################################# @@ -359,7 +354,7 @@ defmodule Dispatcher do ################################################################# get "/sync/public/files/*path" do - Proxy.forward conn, path, "http://delta-producer-json-diff-publisher-public/files/" + Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-public/files/" end ################################################################# diff --git a/config/ldes-delta-pusher/dispatch.ts b/config/ldes-delta-pusher/dispatch.ts new file mode 100644 index 0000000..9ff7ab3 --- /dev/null +++ b/config/ldes-delta-pusher/dispatch.ts @@ -0,0 +1,59 @@ +import { moveTriples } from "../support"; +import { Changeset } from "../types"; +import { query, sparqlEscapeUri } from 'mu'; + +export default async function dispatch(changesets: Changeset[]) { + for (const changeset of changesets) { + const subjects = new Set(changeset.inserts.map((insert) => insert.subject.value)); + for (const subject of subjects) { + const { results: { bindings } } = await query(` + PREFIX skos: + PREFIX code: + PREFIX org: + PREFIX adms: + + CONSTRUCT { + ?organization a org:Organization; + skos:prefLabel ?prefLabel; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } WHERE { + { + VALUES ?organization { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + skos:prefLabel ?prefLabel; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } UNION { + VALUES ?site { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + skos:prefLabel ?prefLabel; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } UNION { + VALUES ?siteAddress { ${sparqlEscapeUri(subject)} } + ?organization a org:Organization; + skos:prefLabel ?prefLabel; + adms:identifier ?identifier; + org:hasPrimarySite ?site. + ?site org:siteAddress ?siteAddress. + ?siteAddress ?p ?o. + } + } + `); + if(bindings.length){ + await moveTriples([ + { + inserts: bindings.map(({ s, p, o}) => { return { subject: s, predicate: p, object: o} }), + } + ]) + } + } + } +} \ No newline at end of file diff --git a/config/migrations/2023/20230508093745-add-organization-type.sparql b/config/migrations/2023/20230508093745-add-organization-type.sparql new file mode 100644 index 0000000..a4e5dee --- /dev/null +++ b/config/migrations/2023/20230508093745-add-organization-type.sparql @@ -0,0 +1,24 @@ +PREFIX org: +PREFIX besluit: +PREFIX ere: +INSERT { + GRAPH ?g { + ?org a org:Organization . + } +} WHERE { + VALUES ?g { + + + + } + GRAPH ?g { + VALUES ?type { + besluit:Bestuurseenheid + ere:RepresentatiefOrgaan + ere:EredienstBestuurseenheid + ere:BestuurVanDeEredienst + ere:CentraalBestuurVanDeEredienst + } + ?org a ?type . + } +} \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 52c53e8..aaa82ee 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -55,8 +55,6 @@ services: restart: "no" delta-producer-pub-graph-maintainer-organizations: restart: "no" - delta-producer-json-diff-publisher-organizations: - restart: "no" search: restart: "no" volumes: @@ -73,6 +71,10 @@ services: restart: "no" delta-producer-pub-graph-maintainer-worship-posts: restart: "no" + delta-producer-bg-jobs-initiator-public: + restart: "no" + delta-producer-pub-graph-maintainer-public: + restart: "no" # use this to test acmidm # tlsproxy: # image: caddy:2 diff --git a/docker-compose.yml b/docker-compose.yml index 53f1ba9..01357ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -263,7 +263,7 @@ services: restart: always logging: *default-logging delta-producer-pub-graph-maintainer-organizations: - image: lblod/delta-producer-publication-graph-maintainer:0.7.0 + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 environment: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" @@ -275,6 +275,11 @@ services: USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" volumes: - ./config/delta-producer/organizations:/config - ./data/files/:/share @@ -283,24 +288,6 @@ services: - triplestore:virtuoso restart: always logging: *default-logging - delta-producer-json-diff-publisher-organizations: - image: lblod/delta-producer-json-diff-file-publisher:0.2.3 - environment: - MAX_BODY_SIZE: "50mb" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/organizations" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - volumes: - - ./data/files:/share - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging ################################################################################ # DELTA ORGANIZATIONS: END ################################################################################ @@ -337,6 +324,11 @@ services: USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" volumes: - ./config/delta-producer/public:/config - ./data/files/:/share @@ -345,24 +337,6 @@ services: - triplestore:virtuoso restart: always logging: *default-logging - delta-producer-json-diff-publisher-public: - image: lblod/delta-producer-json-diff-file-publisher:0.2.3 - environment: - MAX_BODY_SIZE: "50mb" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/public" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - volumes: - - ./data/files:/share - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging ################################################################################ # DELTA PUBLIC INFORMATION: END ################################################################################ @@ -516,6 +490,33 @@ services: # DELTA CONSUMERS: END ################################################################################ + ################################################################################ + # LDES PRODUCERS + ################################################################################ + + ldes-delta-pusher: + image: redpencil/ldes-delta-pusher:latest + environment: + LDES_ENDPOINT: 'http://ldes-backend/organizations' + volumes: + - ./config/ldes-delta-pusher/:/config/ + links: + - db:database + + ldes-backend: + image: redpencil/fragmentation-producer:0.3.2 + environment: + FOLDER_DEPTH: 1 + PAGE_RESOURCES_COUNT: 50 + LDES_STREAM_PREFIX: 'http://data.lblod.info/streams/op/' + TIME_TREE_RELATION_PATH: 'http://www.w3.org/ns/prov#generatedAtTime' + CACHE_SIZE: 10 + DATA_FOLDER: '/data' + ENABLE_AUTH: 'false' + restart: always + volumes: + - './data/ldes-feed/:/data/' + ################################################################################ # MU SEARCH ################################################################################