Consume mandatarissen, leidinggevenden and worship positions from Loket (#239)

* loket filter * cleanup previously migrated data before sync * increase mu-auth timouts * split up deletion migration to avoid timouts * identical utls added catch statements * deletes before inserts * Adapt migrations to splitted graphs * Adapt consumers to the delta-consumer with contexts * Bump consumers and cache * Resolve cache not getting cleared * Flush ministers of non active worship services too --------- Co-authored-by: Claire Lovisa <lovisa.claire@gmail.com>
2023-04-21 11:57:15 +02:00
parent 3aadf0960e
commit fefb103cb5
35 changed files with 1329 additions and 699 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -35,7 +35,7 @@ services:
    restart: always
    logging: *default-logging
  db:
-    image: semtech/mu-authorization:feature-service-roam-r1
+    image: semtech/mu-authorization:feature-service-roam-r1.1
    environment:
      MU_SPARQL_ENDPOINT: "http://triplestore:8890/sparql"
      LOG_OUTGOING_SPARQL_QUERIES: "true"
@@ -43,6 +43,8 @@ services:
      LOG_ACCESS_RIGHTS: "true"
      LOG_OUTGOING_SPARQL_QUERY_ROUNDTRIP: "true"
      ERROR_ON_UNWRITTEN_DATA: "true"
+      QUERY_MAX_PROCESSING_TIME: "300000"
+      QUERY_MAX_EXECUTION_TIME: "300000"
    volumes:
      - ./config/authorization:/config
    labels:
@@ -70,7 +72,7 @@ services:
    restart: always
    logging: *default-logging
  resource:
-    image: semtech/mu-cl-resources:1.21.1
+    image: semtech/mu-cl-resources:1.22.0
    environment:
      CACHE_CLEAR_PATH: "http://cache/.mu/clear-keys"
    links:
@@ -82,7 +84,7 @@ services:
    restart: always
    logging: *default-logging
  cache:
-    image: semtech/mu-cache:2.0.1
+    image: semtech/mu-cache:2.0.2
    links:
      - resource:backend
    labels:
@@ -226,7 +228,7 @@ services:
    restart: always
    logging: *default-logging
  reasoner:
-    image: eyereasoner/reasoning-service:1.0.1
+    image: eyereasoner/reasoning-service:1.0.2
    volumes:
      - ./config/reasoner:/config
  ################################################################################
@@ -418,25 +420,28 @@ services:
  # DELTA CONSUMERS
  ################################################################################
  ################################################################################
-  # LOKET MANDATEN CONSUMERS
+  # LOKET MANDATARISSEN CONSUMERS
  ################################################################################
-  mandaten-consumer:
-    image: lblod/delta-consumer:0.0.14
+  mandatarissen-consumer:
+    image: lblod/delta-consumer:0.0.17-rc.7
    environment:
-      DCR_SERVICE_NAME: "mandatendatabank-consumer"
+      DCR_SERVICE_NAME: "mandatarissen-consumer"
      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
      DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/MandatarissenCacheGraphDump"
      DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen"
      DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen"
-      DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatendatabank-consumer"
+      DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatarissen-consumer"
      DCR_DISABLE_INITIAL_SYNC: "true"
      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
+      BATCH_SIZE: 1000
      DCR_SYNC_FILES_PATH: "/sync/mandatarissen/files"
-      FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
-      KEEP_DELTA_FILES: "true"
+      DCR_KEEP_DELTA_FILES: "true"
+      DCR_ENABLE_DELTA_CONTEXT: "true"
+      DCR_LANDING_ZONE_DATABASE: "triplestore"
+      DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/mandatarissen"
    volumes:
-      - ./config/loket-consumer/mandate-dispatching:/config/triples-dispatching/custom-dispatching
-      - ./data/files/consumer-files/mandaten:/consumer-files/
+      - ./config/loket-consumer/mandatarissen-dispatching:/config/triples-dispatching/custom-dispatching
+      - ./data/files/consumer-files/mandatarissen:/consumer-files/
    restart: always
    labels:
      - "logging=true"
@@ -449,7 +454,7 @@ services:
  # LOKET LEIDINGGEVENDEN CONSUMERS
  ################################################################################
  leidinggevenden-consumer:
-    image: lblod/delta-consumer:0.0.14
+    image: lblod/delta-consumer:0.0.17-rc.7
    environment:
      DCR_SERVICE_NAME: "leidinggevenden-consumer"
      DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API
@@ -460,8 +465,11 @@ services:
      DCR_DISABLE_INITIAL_SYNC: "true"
      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
      DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files"
-      FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
-      KEEP_DELTA_FILES: "true"
+      DCR_KEEP_DELTA_FILES: "true"
+      BATCH_SIZE: 1000
+      DCR_ENABLE_DELTA_CONTEXT: "true"
+      DCR_LANDING_ZONE_DATABASE: "triplestore"
+      DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/leidinggevenden"
    volumes:
      - ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching
      - ./data/files/consumer-files/leidinggevenden:/consumer-files/
@@ -477,22 +485,24 @@ services:
  # LOKET WORSHIP SENSITIVE CONSUMERS
  ################################################################################
  worship-services-sensitive-consumer:
-    image: lblod/delta-consumer:0.0.14
+    image: lblod/delta-consumer:0.0.17-rc.7
    environment:
      DCR_SYNC_BASE_URL: "https://loket.lblod.info/"
      DCR_SERVICE_NAME: "worship-services-sensitive-consumer"
-      DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files"
      DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump"
      DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive"
      DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing"
      DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer"
      DCR_DISABLE_INITIAL_SYNC: "true"
-      FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data"
+      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true"
+      DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files"
+      BATCH_SIZE: 1000
      DCR_KEEP_DELTA_FILES: "true"
-      DCR_DELTA_FILE_FOLDER: "/consumer-files"
-      BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false"
+      DCR_ENABLE_DELTA_CONTEXT: "true"
+      DCR_LANDING_ZONE_DATABASE: "triplestore"
+      DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/worship-services-sensitive"
    volumes:
-      - ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching
+      - ./config/loket-consumer/worship-services-sensitive-dispatching:/config/triples-dispatching/custom-dispatching
      - ./data/files/consumer-files/worship-services-sensitive:/consumer-files/
    restart: always
    labels: