Merge branch 'development' into fix/publisher-config

2023-01-09 13:07:30 +01:00
parent 0e64b2589c 607384293d
commit fef430002b
7 changed files with 213 additions and 49 deletions
--- a/config/delta-producer/dump-file-publisher/config.json
+++ b/config/delta-producer/dump-file-publisher/config.json
@@ -46,5 +46,21 @@
    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
    "cleanupOldDumps": true
+  },
+  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" : {
+    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump",
+    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts",
+    "fileBaseName": "dump-worship-posts",
+    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
+    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
+    "cleanupOldDumps": true
+  },
+  "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" : {
+    "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump",
+    "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts",
+    "fileBaseName": "dump-worship-posts",
+    "targetDcatGraph": "http://mu.semte.ch/graphs/public",
+    "targetFilesGraph": "http://mu.semte.ch/graphs/public",
+    "cleanupOldDumps": true
  }
 }
--- a/config/delta-producer/worship-posts/export.json
+++ b/config/delta-producer/worship-posts/export.json
@@ -0,0 +1,47 @@
+{
+  "export": [
+    {
+      "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/organisatieportaal"
+      ],
+      "properties": [
+        "http://mu.semte.ch/vocabularies/core/uuid",
+        "http://www.w3.org/ns/org#role"
+      ],
+      "additionalFilter": "?orgaan <http://www.w3.org/ns/org#hasPost> ?subject ; <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan>/<http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (<http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>, <http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>))"
+    },
+    {
+      "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
+      "strictTypeExport": "true",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/organisatieportaal"
+      ],
+      "properties": [
+        "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor"
+      ]
+    },
+    {
+      "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
+      "strictTypeExport": "true",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/organisatieportaal"
+      ],
+      "properties": [
+        "http://www.w3.org/ns/org#hasPost"
+      ],
+      "additionalFilter": "?subject <https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan>/<http://data.vlaanderen.be/ns/besluit#bestuurt> ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (<http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst>, <http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst>))"
+    },
+    {
+      "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar",
+      "graphsFilter": [
+        "http://mu.semte.ch/graphs/organisatieportaal"
+      ],
+      "properties": [
+        "http://mu.semte.ch/vocabularies/core/uuid",
+        "http://www.w3.org/ns/org#role",
+        "http://data.lblod.info/vocabularies/erediensten/functie"
+      ]
+    }
+  ]
+}
--- a/config/delta/rules.js
+++ b/config/delta/rules.js
@@ -156,6 +156,41 @@ export default [
      optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
    }
  },
+  {
+    match: {
+    },
+    callback: {
+      url: 'http://delta-producer-pub-graph-maintainer-worship-posts/delta',
+      method: 'POST'
+    },
+    options: {
+      resourceFormat: 'v0.0.1',
+      gracePeriod: 1000,
+      ignoreFromSelf: true,
+      optOutMuScopeIds: [
+                          "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
+                          "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
+                        ]
+    }
+  },
+  {
+    match: {
+      graph: {
+        type: 'uri',
+        value: 'http://redpencil.data.gift/id/deltas/producer/worship-posts'
+      }
+    },
+    callback: {
+      url: 'http://delta-producer-json-diff-publisher-worship-posts/delta',
+      method: 'POST'
+    },
+    options: {
+      resourceFormat: 'v0.0.1',
+      gracePeriod: 1000,
+      ignoreFromSelf: true,
+      optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
+    }
+  },
  {
    match: {
      predicate: {
--- a/config/search/config.json
+++ b/config/search/config.json
@@ -141,8 +141,8 @@
    {
      "type": "mandatory",
      "on_path": "mandatories",
-      "rdf_type": "http://data.vlaanderen.be/ns/mandaat#Mandataris",
-      "sub_types": [
+      "rdf_type": [
+        "http://data.vlaanderen.be/ns/mandaat#Mandataris",
        "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris"
      ],
      "properties": {
@@ -150,11 +150,11 @@
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://xmlns.com/foaf/0.1/givenName"
        ],
-        "id": [
+        "person_id": [
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
-        "uuid": [
+        "person_uuid": [
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
@@ -312,11 +312,11 @@
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://xmlns.com/foaf/0.1/givenName"
        ],
-        "id": [
+        "person_id": [
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
-        "uuid": [
+        "person_uuid": [
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
@@ -468,8 +468,8 @@
    {
      "type": "minister",
      "on_path": "ministers",
-      "rdf_type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar",
-      "sub_types": [
+      "rdf_type": [
+        "http://data.lblod.info/vocabularies/erediensten/RolBedienaar",
        "http://data.lblod.info/vocabularies/contacthub/AgentInPositie"
      ],
      "properties": {
@@ -477,11 +477,11 @@
          "http://www.w3.org/ns/org#heldBy",
          "http://xmlns.com/foaf/0.1/givenName"
        ],
-        "id": [
+        "person_id": [
          "http://www.w3.org/ns/org#heldBy",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
-        "uuid": [
+        "person_uuid": [
          "http://www.w3.org/ns/org#heldBy",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
@@ -546,7 +546,7 @@
              }
            }
          },
-          "id": {
+          "person_id": {
            "type": "text",
            "fields": {
              "field": {
@@ -638,11 +638,11 @@
          }
        },
        {
-          "name": "id",
+          "name": "person_id",
          "mappings": {
-            "minister": "id",
-            "agent": "id",
-            "mandatory": "id"
+            "minister": "person_id",
+            "agent": "person_id",
+            "mandatory": "person_id"
          }
        },
        {
@@ -814,8 +814,8 @@
    {
      "type": "unit",
      "on_path": "units",
-      "rdf_type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
-      "sub_types": [
+      "rdf_type": [
+        "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
        "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid",
        "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst",
        "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst"
--- a/config/search/dev/config.json
+++ b/config/search/dev/config.json
@@ -1,7 +1,7 @@
 {
  "automatic_index_updates": true,
  "persist_indexes": true,
-  "number_of_threads": 11,
+  "number_of_threads": 8,
  "eager_indexing_groups": [
    [
      {
@@ -46,27 +46,27 @@
      }
    ]
  ],
-  "default_settings" : {
+  "default_settings": {
    "analysis": {
      "normalizer": {
        "custom_sort_normalizer": {
-            "type": "custom",
-            "char_filter": [],
-            "filter": [
-                "lowercase",
-                "trim",
-                "asciifolding"
-            ]
+          "type": "custom",
+          "char_filter": [],
+          "filter": [
+            "lowercase",
+            "trim",
+            "asciifolding"
+          ]
        }
-    }
+      }
    }
  },
  "types": [
    {
      "type": "mandatory",
      "on_path": "mandatories",
-      "rdf_type": "http://data.vlaanderen.be/ns/mandaat#Mandataris",
-      "sub_types": [
+      "rdf_type": [
+        "http://data.vlaanderen.be/ns/mandaat#Mandataris",
        "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris"
      ],
      "properties": {
@@ -74,11 +74,11 @@
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://xmlns.com/foaf/0.1/givenName"
        ],
-        "id": [
+        "person_id": [
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
-        "uuid": [
+        "person_uuid": [
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
@@ -153,7 +153,7 @@
              }
            }
          },
-          "id": {
+          "person_id": {
            "type": "text",
            "fields": {
              "field": {
@@ -236,11 +236,11 @@
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://xmlns.com/foaf/0.1/givenName"
        ],
-        "id": [
+        "person_id": [
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
-        "uuid": [
+        "person_uuid": [
          "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
@@ -315,7 +315,7 @@
              }
            }
          },
-          "id": {
+          "person_id": {
            "type": "text",
            "fields": {
              "field": {
@@ -392,8 +392,8 @@
    {
      "type": "minister",
      "on_path": "ministers",
-      "rdf_type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar",
-      "sub_types": [
+      "rdf_type": [
+        "http://data.lblod.info/vocabularies/erediensten/RolBedienaar",
        "http://data.lblod.info/vocabularies/contacthub/AgentInPositie"
      ],
      "properties": {
@@ -401,11 +401,11 @@
          "http://www.w3.org/ns/org#heldBy",
          "http://xmlns.com/foaf/0.1/givenName"
        ],
-        "id": [
+        "person_id": [
          "http://www.w3.org/ns/org#heldBy",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
-        "uuid": [
+        "person_uuid": [
          "http://www.w3.org/ns/org#heldBy",
          "http://mu.semte.ch/vocabularies/core/uuid"
        ],
@@ -470,7 +470,7 @@
              }
            }
          },
-          "id": {
+          "person_id": {
            "type": "text",
            "fields": {
              "field": {
@@ -562,11 +562,11 @@
          }
        },
        {
-          "name": "id",
+          "name": "person_id",
          "mappings": {
-            "minister": "id",
-            "agent": "id",
-            "mandatory": "id"
+            "minister": "person_id",
+            "agent": "person_id",
+            "mandatory": "person_id"
          }
        },
        {
@@ -661,7 +661,7 @@
              }
            }
          },
-          "id": {
+          "person_id": {
            "type": "text",
            "fields": {
              "field": {
@@ -739,8 +739,8 @@
    {
      "type": "unit",
      "on_path": "units",
-      "rdf_type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
-      "sub_types": [
+      "rdf_type": [
+        "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
        "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid",
        "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst",
        "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst"
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -71,3 +71,7 @@ services:
    restart: "no"
  construct-administrative-unit-relationships:
    restart: "no"
+  delta-producer-bg-jobs-initiator-worship-posts:
+    restart: "no"
+  delta-producer-pub-graph-maintainer-worship-posts:
+    restart: "no"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -403,6 +403,68 @@ services:
  # DELTA PUBLIC INFORMATION: END
  ################################################################################

+  ################################################################################
+  # DELTA WORSHIP POSTS: START
+  ################################################################################
+  delta-producer-bg-jobs-initiator-worship-posts:
+    image: lblod/delta-producer-background-jobs-initiator:0.3.1
+    environment:
+      JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal"
+      DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts"
+      INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts"
+      HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts"
+      CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours
+      CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM
+      START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts.
+    labels:
+      - "logging=true"
+    links:
+      - db:database
+    restart: always
+    logging: *default-logging
+  delta-producer-pub-graph-maintainer-worship-posts:
+    image: lblod/delta-producer-publication-graph-maintainer:0.15.1
+    environment:
+      MAX_BODY_SIZE: "50mb"
+      JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal"
+      RELATIVE_FILE_PATH: "deltas/worship-posts"
+      PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts"
+      HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts"
+      INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts"
+      QUEUE_POLL_INTERVAL: 3000
+      USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
+      SKIP_MU_AUTH_INITIAL_SYNC: "true"
+      HEALING_PATCH_GRAPH_BATCH_SIZE: 500
+    volumes:
+      - ./config/delta-producer/worship-posts:/config
+      - ./data/files/:/share
+    links:
+      - db:database
+      - triplestore:virtuoso
+    restart: always
+    logging: *default-logging
+  delta-producer-json-diff-publisher-worship-posts:
+    image: lblod/delta-producer-json-diff-file-publisher:0.2.3
+    environment:
+      MAX_BODY_SIZE: "50mb"
+      ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal"
+      RELATIVE_FILE_PATH: "deltas/worship-posts"
+      PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts"
+      PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts"
+      DELTA_INTERVAL_MS: 10000
+      PRETTY_PRINT_DIFF_JSON: "true"
+    volumes:
+      - ./data/files:/share
+    labels:
+      - "logging=true"
+    links:
+      - db:database
+    restart: always
+    logging: *default-logging
+  ################################################################################
+  # DELTA WORSHIP POSTS: END
+  ################################################################################
+
  ################################################################################
  # DELTA CONSUMERS
  ################################################################################
@@ -465,10 +527,10 @@ services:
  ################################################################################

  ################################################################################
-  # MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH
+  # MU SEARCH 
  ################################################################################
  search:
-    image: nbittich/mu-search:wip-inheritance-alpha2 # Bump to fixed version when merged and release to the original repo
+    image: semtech/mu-search:0.9.0-beta.1 
    links:
      - db:database
    restart: always