Merge pull request #417 from lblod/fix/adminitrative-unit-dump-publisher

Fix prefLabel missing in Kalliope dump after edition
2024-04-29 14:40:43 +02:00
parent 0616d24ad0 e7129dc41d
commit 26d5a23a3c
8 changed files with 77 additions and 45 deletions
--- a/config/delta-producer/background-job-initiator/config.json
+++ b/config/delta-producer/background-job-initiator/config.json
@@ -0,0 +1,12 @@
+[
+    {
+        "name": "organizations",
+        "jobsGraph": "http://mu.semte.ch/graphs/jobs",
+        "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations",
+        "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations",
+        "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations",
+        "cronPatternDumpJob": "42 * * * *",
+        "cronPatternHealingJob": "0 0 4 * * *",
+        "startInitialSync": true
+    }
+]
--- a/config/delta-producer/dump-file-publisher/config.json
+++ b/config/delta-producer/dump-file-publisher/config.json
@@ -3,7 +3,7 @@
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
    "fileBaseName": "dump-organizations",
-    "publicationGraphEndpoint": "http://triplestore:8890/sparql",
+    "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql",
    "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
    "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
    "cleanupOldDumps": true
@@ -12,7 +12,7 @@
    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
    "fileBaseName": "dump-organizations",
-    "publicationGraphEndpoint": "http://triplestore:8890/sparql",
+    "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql",
    "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
    "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
    "cleanupOldDumps": true
--- a/config/delta-producer/publication-graph-maintainer/config.json
+++ b/config/delta-producer/publication-graph-maintainer/config.json
@@ -0,0 +1,25 @@
+{
+    "organizations": {
+        "deltaInterval": 10000,
+        "deltaPath": "/organizations/delta",
+        "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations",
+        "exportConfigPath": "/config/organizations/export.json",
+        "filesPath": "/organizations/files",
+        "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations",
+        "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations",
+        "jobsGraph": "http://mu.semte.ch/graphs/jobs",
+        "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
+        "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations",
+        "queuePollInterval": 3000,
+        "relativeFilePath": "/deltas/organizations",
+        "serveDeltaFiles": true,
+        "skipMuAuthInitialSync": true,
+        "skipMuAuthHealing": true,
+        "useFileDiff": true,
+        "healingInitialBatchSizeInsert": 1000,
+        "useVirtuosoForExpensiveSelects": true,
+        "errorGraph": "http://mu.semte.ch/graphs/jobs",
+        "loginPath": "/organizations/login",
+        "logOutgoingDelta": true
+    }
+}
--- a/config/delta/rules.js
+++ b/config/delta/rules.js
@@ -92,7 +92,7 @@ export default [
    match: {
    },
    callback: {
-      url: 'http://delta-producer-pub-graph-maintainer-organizations/delta',
+      url: 'http://delta-producer-publication-graph-maintainer/organizations/delta',
      method: 'POST'
    },
    options: {
--- a/config/dispatcher/dispatcher.ex
+++ b/config/dispatcher/dispatcher.ex
@@ -361,7 +361,7 @@ defmodule Dispatcher do
  #################################################################

  get "/sync/organizations/files/*path" do
-    Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/"
+    Proxy.forward conn, path, "http://delta-producer-publication-graph-maintainer/files/"
  end

  #################################################################
--- a/config/migrations/2024/20240429105000-clear-producer-organizations-graph.sparql
+++ b/config/migrations/2024/20240429105000-clear-producer-organizations-graph.sparql
@@ -0,0 +1,3 @@
+# The producer organizations graph is now in the publication-triplestore
+# Lets remove it from the main triplestore
+CLEAR GRAPH <http://redpencil.data.gift/id/deltas/producer/organizations>
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -51,9 +51,9 @@ services:
    restart: "no"
  uri-info:
    restart: "no"
-  delta-producer-bg-jobs-initiator-organizations:
+  delta-producer-background-jobs-initiator:
    restart: "no"
-  delta-producer-pub-graph-maintainer-organizations:
+  delta-producer-publication-graph-maintainer:
    restart: "no"
  search:
    restart: "no"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -206,10 +206,10 @@ services:
    image: semtech/mu-file-service:3.2.0
    volumes:
      - ./data/files:/share
-    labels:
-      - "logging=true"
    links:
      - db:database
+    labels:
+      - "logging=true"
    restart: always
    logging: *default-logging
  jobs-controller:
@@ -264,6 +264,8 @@ services:
      END_DATE_WORSHIP_GOVERNING_BODY: "2026-03-31T00:00:00"
    links:
      - db:database
+    labels:
+      - "logging=true"
    restart: always
    logging: *default-logging
  reasoner:
@@ -326,6 +328,8 @@ services:
    volumes:
      - ./data/publication-triplestore:/data
      - ./config/publication-triplestore/virtuoso.ini:/data/virtuoso.ini
+    labels:
+      - "logging=true"
    restart: always
    logging: *default-logging
  delta-producer-dump-file-publisher:
@@ -335,57 +339,39 @@ services:
      - ./data/files:/share
    links:
      - db:database
-    restart: always
-    logging: *default-logging
-
-  ################################################################################
-  # DELTA ORGANIZATIONS: START
-  ################################################################################
-  delta-producer-bg-jobs-initiator-organizations:
-    image: lblod/delta-producer-background-jobs-initiator:0.3.1
-    environment:
-      JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
-      DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations"
-      INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations"
-      HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations"
-      CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour
-      CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM
-      START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts.
    labels:
      - "logging=true"
-    links:
-      - db:database
    restart: always
    logging: *default-logging
-  delta-producer-pub-graph-maintainer-organizations:
-    image: lblod/delta-producer-publication-graph-maintainer:0.15.4
+  delta-producer-background-jobs-initiator:
+    image: lblod/delta-producer-background-jobs-initiator:1.0.1
+    volumes:
+      - ./config/delta-producer/background-job-initiator:/config
+    links:
+      - db:database
+    labels:
+      - "logging=true"
+    restart: always
+    logging: *default-logging
+  delta-producer-publication-graph-maintainer:
+    image: lblod/delta-producer-publication-graph-maintainer:1.1.1-rc.1
    environment:
      MAX_BODY_SIZE: "50mb"
-      JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
-      RELATIVE_FILE_PATH: "deltas/organizations"
-      PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations"
-      HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations"
-      INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations"
-      QUEUE_POLL_INTERVAL: 3000
-      USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
-      SKIP_MU_AUTH_INITIAL_SYNC: "true"
-      HEALING_PATCH_GRAPH_BATCH_SIZE: 500
-      SERVE_DELTA_FILES: "true"
-      PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations"
-      DELTA_INTERVAL_MS: 10000
      PRETTY_PRINT_DIFF_JSON: "true"
-      ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
+      PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql"
+      PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql"
+      SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503"
+      CONFIG_SERVICES_JSON_PATH: '/config/publication-graph-maintainer/config.json'
    volumes:
-      - ./config/delta-producer/organizations:/config
+      - ./config/delta-producer:/config
      - ./data/files/:/share
    links:
      - db:database
      - triplestore:virtuoso
+    labels:
+      - "logging=true"
    restart: always
    logging: *default-logging
-  ################################################################################
-  # DELTA ORGANIZATIONS: END
-  ################################################################################

  ################################################################################
  # DELTA ORGANIZATIONS PUBLIC INFO: START
@@ -433,6 +419,8 @@ services:
      - db:database
      - triplestore:virtuoso
    restart: always
+    labels:
+      - "logging=true"
    logging: *default-logging
  ################################################################################
  # DELTA ORGANIZATIONS PUBLIC INFO: END
@@ -482,6 +470,8 @@ services:
      - db:database
      - triplestore:virtuoso
    restart: always
+    labels:
+      - "logging=true"
    logging: *default-logging
  ################################################################################
  # DELTA PUBLIC INFORMATION: END
@@ -531,6 +521,8 @@ services:
      - db:database
      - triplestore:virtuoso
    restart: always
+    labels:
+      - "logging=true"
    logging: *default-logging
  ################################################################################
  # DELTA WORSHIP POSTS: END