Merge branch 'master' into hotfix/1.17.x

2023-07-13 10:17:04 +02:00
parent cc49c6c5b7 3a23aa3c2d
commit f4ea1e7307
7 changed files with 180 additions and 86 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ docker-compose.override.yml
 *.iml

 /data/elasticsearch/**
+/data/ldes-feed/*
 /tmp
 .DS_STORE
 .env
--- a/config/delta/rules.js
+++ b/config/delta/rules.js
@@ -105,24 +105,6 @@ export default [
      ]
    }
  },
-  {
-    match: {
-      graph: {
-        type: 'uri',
-        value: 'http://redpencil.data.gift/id/deltas/producer/organizations'
-      }
-    },
-    callback: {
-      url: 'http://delta-producer-json-diff-publisher-organizations/delta',
-      method: 'POST'
-    },
-    options: {
-      resourceFormat: 'v0.0.1',
-      gracePeriod: 1000,
-      ignoreFromSelf: true,
-      optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"]
-    }
-  },
  {
    match: {
    },
@@ -140,24 +122,6 @@ export default [
      ]
    }
  },
-  {
-    match: {
-      graph: {
-        type: 'uri',
-        value: 'http://redpencil.data.gift/id/deltas/producer/public'
-      }
-    },
-    callback: {
-      url: 'http://delta-producer-json-diff-publisher-public/delta',
-      method: 'POST'
-    },
-    options: {
-      resourceFormat: 'v0.0.1',
-      gracePeriod: 1000,
-      ignoreFromSelf: true,
-      optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"]
-    }
-  },
  {
    match: {
      // anything
@@ -197,5 +161,53 @@ export default [
      ignoreFromSelf: true,
      optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"]
    }
+  },
+  {
+    match: {
+      graph: {
+        type: 'uri',
+        value: 'http://mu.semte.ch/graphs/shared',
+      }
+    },
+    callback: {
+      url: 'http://ldes-delta-pusher/publish',
+      method: 'POST',
+    },
+    options: {
+      resourceFormat: 'v0.0.1',
+      gracePeriod: 1000,
+    }
+  },
+  {
+    match: {
+      graph: {
+        type: 'uri',
+        value: 'http://mu.semte.ch/graphs/worship-service',
+      }
+    },
+    callback: {
+      url: 'http://ldes-delta-pusher/publish',
+      method: 'POST',
+    },
+    options: {
+      resourceFormat: 'v0.0.1',
+      gracePeriod: 1000,
+    }
+  },
+  {
+    match: {
+      graph: {
+        type: 'uri',
+        value: 'http://mu.semte.ch/graphs/administrative-unit',
+      }
+    },
+    callback: {
+      url: 'http://ldes-delta-pusher/publish',
+      method: 'POST',
+    },
+    options: {
+      resourceFormat: 'v0.0.1',
+      gracePeriod: 1000,
+    }
  }
 ]
--- a/config/dispatcher/dispatcher.ex
+++ b/config/dispatcher/dispatcher.ex
@@ -305,6 +305,10 @@ defmodule Dispatcher do
    forward conn, path, "http://construct-administrative-unit-relationships/create-relationships/"
  end

+  get "/ldes/*path", %{ layer: :api_services } do
+    forward conn, path, "http://ldes-backend/"
+  end
+
  #################################################################
  # ERROR REPORT
  #################################################################
@@ -337,21 +341,12 @@ defmodule Dispatcher do
    Proxy.forward conn, path, "http://resource/distributions/"
  end

-
-  #################################################################
-  #  DELTA: administrative-units
-  #################################################################
-
-  get "/sync/administrative-units/files/*path" do
-    Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-administrative-units/files/"
-  end
-
  #################################################################
  #  DELTA: organizations
  #################################################################

  get "/sync/organizations/files/*path" do
-    Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-organizations/files/"
+    Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/"
  end

  #################################################################
@@ -359,7 +354,7 @@ defmodule Dispatcher do
  #################################################################

  get "/sync/public/files/*path" do
-    Proxy.forward conn, path, "http://delta-producer-json-diff-publisher-public/files/"
+    Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-public/files/"
  end

  #################################################################
--- a/config/ldes-delta-pusher/dispatch.ts
+++ b/config/ldes-delta-pusher/dispatch.ts
@@ -0,0 +1,59 @@
+import { moveTriples } from "../support";
+import { Changeset } from "../types";
+import { query, sparqlEscapeUri } from 'mu';
+
+export default async function dispatch(changesets: Changeset[]) {
+	for (const changeset of changesets) {
+		const subjects = new Set(changeset.inserts.map((insert) => insert.subject.value));
+		for (const subject of subjects) {
+			const { results: { bindings } } = await query(`
+        PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
+        PREFIX code: <http://telegraphis.net/ontology/measurement/code#>
+        PREFIX org: <http://www.w3.org/ns/org#>
+        PREFIX adms: <http://www.w3.org/ns/adms#>
+  
+        CONSTRUCT {
+          ?organization a org:Organization;
+                        skos:prefLabel ?prefLabel;
+                        adms:identifier ?identifier;
+                        org:hasPrimarySite ?site.
+          ?site org:siteAddress ?siteAddress.
+          ?siteAddress ?p ?o.
+        } WHERE {
+          {
+            VALUES ?organization { ${sparqlEscapeUri(subject)} }
+            ?organization a org:Organization;
+                          skos:prefLabel ?prefLabel;
+                          adms:identifier ?identifier;
+                          org:hasPrimarySite ?site.
+            ?site org:siteAddress ?siteAddress.
+            ?siteAddress ?p ?o.
+          } UNION {
+            VALUES ?site { ${sparqlEscapeUri(subject)} }
+            ?organization a org:Organization;
+                          skos:prefLabel ?prefLabel;
+                          adms:identifier ?identifier;
+                          org:hasPrimarySite ?site.
+            ?site org:siteAddress ?siteAddress.
+            ?siteAddress ?p ?o.
+          } UNION {
+            VALUES ?siteAddress { ${sparqlEscapeUri(subject)} }
+            ?organization a org:Organization;
+                          skos:prefLabel ?prefLabel;
+                          adms:identifier ?identifier;
+                          org:hasPrimarySite ?site.
+            ?site org:siteAddress ?siteAddress.
+            ?siteAddress ?p ?o.
+          } 
+        }
+			`);
+      if(bindings.length){
+        await moveTriples([
+          {
+            inserts: bindings.map(({ s, p, o}) => { return { subject: s, predicate: p, object: o} }),
+          }
+        ])
+      }
+		}
+	}
+}
--- a/config/migrations/2023/20230508093745-add-organization-type.sparql
+++ b/config/migrations/2023/20230508093745-add-organization-type.sparql
@@ -0,0 +1,24 @@
+PREFIX org: <http://www.w3.org/ns/org#>
+PREFIX besluit: <http://data.vlaanderen.be/ns/besluit#>
+PREFIX ere: <http://data.lblod.info/vocabularies/erediensten/>
+INSERT {
+  GRAPH ?g {
+    ?org a org:Organization .
+  }
+} WHERE {
+  VALUES ?g {
+    <http://mu.semte.ch/graphs/shared>
+    <http://mu.semte.ch/graphs/worship-service>
+    <http://mu.semte.ch/graphs/administrative-unit>
+  }
+  GRAPH ?g {
+    VALUES ?type { 
+      besluit:Bestuurseenheid 
+      ere:RepresentatiefOrgaan 
+      ere:EredienstBestuurseenheid 
+      ere:BestuurVanDeEredienst 
+      ere:CentraalBestuurVanDeEredienst 
+    }
+    ?org a ?type .
+  }
+}
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -55,8 +55,6 @@ services:
    restart: "no"
  delta-producer-pub-graph-maintainer-organizations:
    restart: "no"
-  delta-producer-json-diff-publisher-organizations:
-    restart: "no"
  search:
    restart: "no"
    volumes:
@@ -73,6 +71,10 @@ services:
    restart: "no"
  delta-producer-pub-graph-maintainer-worship-posts:
    restart: "no"
+  delta-producer-bg-jobs-initiator-public:
+    restart: "no"
+  delta-producer-pub-graph-maintainer-public:
+    restart: "no"
  # use this to test acmidm
  # tlsproxy:
  #   image: caddy:2
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -263,7 +263,7 @@ services:
    restart: always
    logging: *default-logging
  delta-producer-pub-graph-maintainer-organizations:
-    image: lblod/delta-producer-publication-graph-maintainer:0.7.0
+    image: lblod/delta-producer-publication-graph-maintainer:0.15.4
    environment:
      MAX_BODY_SIZE: "50mb"
      JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
@@ -275,6 +275,11 @@ services:
      USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
      SKIP_MU_AUTH_INITIAL_SYNC: "true"
      HEALING_PATCH_GRAPH_BATCH_SIZE: 500
+      SERVE_DELTA_FILES: "true"
+      PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations"
+      DELTA_INTERVAL_MS: 10000
+      PRETTY_PRINT_DIFF_JSON: "true"
+      ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
    volumes:
      - ./config/delta-producer/organizations:/config
      - ./data/files/:/share
@@ -283,24 +288,6 @@ services:
      - triplestore:virtuoso
    restart: always
    logging: *default-logging
-  delta-producer-json-diff-publisher-organizations:
-    image: lblod/delta-producer-json-diff-file-publisher:0.2.3
-    environment:
-      MAX_BODY_SIZE: "50mb"
-      ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
-      RELATIVE_FILE_PATH: "deltas/organizations"
-      PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations"
-      PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations"
-      DELTA_INTERVAL_MS: 10000
-      PRETTY_PRINT_DIFF_JSON: "true"
-    volumes:
-      - ./data/files:/share
-    labels:
-      - "logging=true"
-    links:
-      - db:database
-    restart: always
-    logging: *default-logging
  ################################################################################
  # DELTA ORGANIZATIONS: END
  ################################################################################
@@ -337,6 +324,11 @@ services:
      USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
      SKIP_MU_AUTH_INITIAL_SYNC: "true"
      HEALING_PATCH_GRAPH_BATCH_SIZE: 500
+      SERVE_DELTA_FILES: "true"
+      PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public"
+      DELTA_INTERVAL_MS: 10000
+      PRETTY_PRINT_DIFF_JSON: "true"
+      ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
    volumes:
      - ./config/delta-producer/public:/config
      - ./data/files/:/share
@@ -345,24 +337,6 @@ services:
      - triplestore:virtuoso
    restart: always
    logging: *default-logging
-  delta-producer-json-diff-publisher-public:
-    image: lblod/delta-producer-json-diff-file-publisher:0.2.3
-    environment:
-      MAX_BODY_SIZE: "50mb"
-      ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
-      RELATIVE_FILE_PATH: "deltas/public"
-      PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public"
-      PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public"
-      DELTA_INTERVAL_MS: 10000
-      PRETTY_PRINT_DIFF_JSON: "true"
-    volumes:
-      - ./data/files:/share
-    labels:
-      - "logging=true"
-    links:
-      - db:database
-    restart: always
-    logging: *default-logging
  ################################################################################
  # DELTA PUBLIC INFORMATION: END
  ################################################################################
@@ -516,6 +490,33 @@ services:
  # DELTA CONSUMERS: END
  ################################################################################

+  ################################################################################
+  # LDES PRODUCERS
+  ################################################################################
+
+  ldes-delta-pusher:
+    image: redpencil/ldes-delta-pusher:latest
+    environment:
+      LDES_ENDPOINT: 'http://ldes-backend/organizations'
+    volumes:
+      - ./config/ldes-delta-pusher/:/config/
+    links:
+      - db:database
+
+  ldes-backend:
+    image: redpencil/fragmentation-producer:0.3.2
+    environment:
+      FOLDER_DEPTH: 1
+      PAGE_RESOURCES_COUNT: 50
+      LDES_STREAM_PREFIX: 'http://data.lblod.info/streams/op/'
+      TIME_TREE_RELATION_PATH: 'http://www.w3.org/ns/prov#generatedAtTime'
+      CACHE_SIZE: 10
+      DATA_FOLDER: '/data'
+      ENABLE_AUTH: 'false'
+    restart: always
+    volumes:
+      - './data/ldes-feed/:/data/'
+
  ################################################################################
  # MU SEARCH
  ################################################################################