From d4f9352c61838c6ef5e16c765374c9e6845e985c Mon Sep 17 00:00:00 2001 From: Riad Abdallah Date: Thu, 3 Oct 2024 16:38:57 +0200 Subject: [PATCH 01/14] Move streams to consolidated publication graph maintainer --- .../publication-graph-maintainer/config.json | 116 ++++++++++++++---- 1 file changed, 92 insertions(+), 24 deletions(-) diff --git a/config/delta-producer/publication-graph-maintainer/config.json b/config/delta-producer/publication-graph-maintainer/config.json index a7d0a14..3bfcd68 100644 --- a/config/delta-producer/publication-graph-maintainer/config.json +++ b/config/delta-producer/publication-graph-maintainer/config.json @@ -1,25 +1,93 @@ { - "organizations": { - "deltaInterval": 10000, - "deltaPath": "/organizations/delta", - "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations", - "exportConfigPath": "/config/organizations/export.json", - "filesPath": "/organizations/files", - "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations", - "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations", - "jobsGraph": "http://mu.semte.ch/graphs/jobs", - "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", - "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations", - "queuePollInterval": 3000, - "relativeFilePath": "/deltas/organizations", - "serveDeltaFiles": true, - "skipMuAuthInitialSync": true, - "skipMuAuthHealing": true, - "useFileDiff": true, - "healingInitialBatchSizeInsert": 1000, - "useVirtuosoForExpensiveSelects": true, - "errorGraph": "http://mu.semte.ch/graphs/jobs", - "loginPath": "/organizations/login", - "logOutgoingDelta": true - } -} \ No newline at end of file + "organizations": { + "deltaInterval": 10000, + "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations", + "exportConfigPath": "/config/organizations/export.json", + "filesPath": "/organizations/files", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", + "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations", + "queuePollInterval": 3000, + "relativeFilePath": "/deltas/organizations", + "serveDeltaFiles": true, + "skipMuAuthInitialSync": true, + "skipMuAuthHealing": true, + "useFileDiff": true, + "healingInitialBatchSizeInsert": 1000, + "useVirtuosoForExpensiveSelects": true, + "errorGraph": "http://mu.semte.ch/graphs/jobs", + "loginPath": "/organizations/login", + "logOutgoingDelta": true + }, + "public": { + "deltaInterval": 10000, + "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-public", + "exportConfigPath": "/config/public/export.json", + "filesPath": "/public/files", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/public", + "queuePollInterval": 3000, + "relativeFilePath": "/deltas/public", + "serveDeltaFiles": true, + "skipMuAuthInitialSync": true, + "skipMuAuthHealing": true, + "useFileDiff": true, + "healingInitialBatchSizeInsert": 1000, + "healingPatchPublicationGraphTaskOperation": 500, + "useVirtuosoForExpensiveSelects": true, + "errorGraph": "http://mu.semte.ch/graphs/jobs", + "loginPath": "/public/login", + "logOutgoingDelta": true + }, + "public-info": { + "deltaInterval": 10000, + "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-public-info", + "exportConfigPath": "/config/organizations-public-info/export.json", + "filesPath": "/public-info/files", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public-info", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public-info", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/public-info", + "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/public-info", + "queuePollInterval": 3000, + "relativeFilePath": "/deltas/public-info", + "serveDeltaFiles": true, + "skipMuAuthInitialSync": true, + "skipMuAuthHealing": true, + "useFileDiff": true, + "healingInitialBatchSizeInsert": 1000, + "healingPatchPublicationGraphTaskOperation": 500, + "useVirtuosoForExpensiveSelects": true, + "errorGraph": "http://mu.semte.ch/graphs/jobs", + "loginPath": "/public-info/login", + "logOutgoingDelta": true + }, + "worship-posts": { + "deltaInterval": 10000, + "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-worship-posts", + "exportConfigPath": "/config/worship-posts/export.json", + "filesPath": "/worship-posts/files", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", + "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts", + "queuePollInterval": 3000, + "relativeFilePath": "/deltas/worship-posts", + "serveDeltaFiles": true, + "skipMuAuthInitialSync": true, + "skipMuAuthHealing": true, + "useFileDiff": true, + "healingInitialBatchSizeInsert": 1000, + "healingPatchPublicationGraphTaskOperation": 500, + "useVirtuosoForExpensiveSelects": true, + "errorGraph": "http://mu.semte.ch/graphs/jobs", + "loginPath": "/worship-posts/login", + "logOutgoingDelta": true + } +} From a7d6dcfb3226beb618da9c2ee3ccd09ae4e28e08 Mon Sep 17 00:00:00 2001 From: Riad Abdallah Date: Thu, 3 Oct 2024 16:39:14 +0200 Subject: [PATCH 02/14] Move streams to consolidated background job initiator --- .../background-job-initiator/config.json | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/config/delta-producer/background-job-initiator/config.json b/config/delta-producer/background-job-initiator/config.json index 379c1c2..a0c8411 100644 --- a/config/delta-producer/background-job-initiator/config.json +++ b/config/delta-producer/background-job-initiator/config.json @@ -1,12 +1,42 @@ [ - { - "name": "organizations", - "jobsGraph": "http://mu.semte.ch/graphs/jobs", - "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations", - "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations", - "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations", - "cronPatternDumpJob": "42 * * * *", - "cronPatternHealingJob": "0 0 4 * * *", - "startInitialSync": true - } -] \ No newline at end of file + { + "name": "organizations", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations", + "cronPatternDumpJob": "42 * * * *", + "cronPatternHealingJob": "0 0 4 * * *", + "startInitialSync": true + }, + { + "name": "public", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public", + "cronPatternDumpJob": "11 */4 * * *", + "cronPatternHealingJob": "0 0 4 * * *", + "startInitialSync": true + }, + { + "name": "public-info", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public-info", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public-info", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public-info", + "cronPatternDumpJob": "27 */4 * * *", + "cronPatternHealingJob": "0 0 4 * * *", + "startInitialSync": true + }, + { + "name": "worship-posts", + "jobsGraph": "http://mu.semte.ch/graphs/jobs", + "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts", + "cronPatternDumpJob": "17 */4 * * *", + "cronPatternHealingJob": "0 0 3 * * *", + "startInitialSync": true + } +] From d3b40b88ea0ea1759ab3e90034ce9b16392fd2fd Mon Sep 17 00:00:00 2001 From: Riad Abdallah Date: Thu, 3 Oct 2024 16:40:08 +0200 Subject: [PATCH 03/14] Remove individual streams from docker compose configs --- docker-compose.dev.yml | 14 +--- docker-compose.yml | 157 +---------------------------------------- 2 files changed, 4 insertions(+), 167 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3a59abd..e0218de 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -46,9 +46,9 @@ services: restart: "no" adressenregister: restart: "no" - frontend: - image: lblod/frontend-organization-portal:latest - restart: "no" + frontend: + image: lblod/frontend-organization-portal:latest + restart: "no" uri-info: restart: "no" delta-producer-background-jobs-initiator: @@ -75,14 +75,6 @@ services: restart: "no" construct-organization-relationships: restart: "no" - delta-producer-bg-jobs-initiator-worship-posts: - restart: "no" - delta-producer-pub-graph-maintainer-worship-posts: - restart: "no" - delta-producer-bg-jobs-initiator-public: - restart: "no" - delta-producer-pub-graph-maintainer-public: - restart: "no" mandatarissen-consumer: restart: "no" leidinggevenden-consumer: diff --git a/docker-compose.yml b/docker-compose.yml index ec24309..a185621 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -344,7 +344,7 @@ services: restart: always logging: *default-logging delta-producer-publication-graph-maintainer: - image: lblod/delta-producer-publication-graph-maintainer:1.1.1-rc.1 + image: lblod/delta-producer-publication-graph-maintainer:1.2.2 environment: MAX_BODY_SIZE: "50mb" PRETTY_PRINT_DIFF_JSON: "true" @@ -363,161 +363,6 @@ services: restart: always logging: *default-logging - ################################################################################ - # DELTA ORGANIZATIONS PUBLIC INFO: START - ################################################################################ - delta-producer-bg-jobs-initiator-organizations-public-info: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-organizations-public-info: - image: lblod/delta-producer-publication-graph-maintainer:0.15.5 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/organizations-public-info" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" - PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/organizations-public-info:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - labels: - - "logging=true" - logging: *default-logging - ################################################################################ - # DELTA ORGANIZATIONS PUBLIC INFO: END - ################################################################################ - - ################################################################################ - # DELTA PUBLIC INFORMATION: START - ################################################################################ - delta-producer-bg-jobs-initiator-public: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - CRON_PATTERN_DUMP_JOB: "11 */4 * * *" # At minute 11 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-public: - image: lblod/delta-producer-publication-graph-maintainer:0.15.5 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/public" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/public:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - labels: - - "logging=true" - logging: *default-logging - ################################################################################ - # DELTA PUBLIC INFORMATION: END - ################################################################################ - - ################################################################################ - # DELTA WORSHIP POSTS: START - ################################################################################ - delta-producer-bg-jobs-initiator-worship-posts: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - CRON_PATTERN_DUMP_JOB: "17 */4 * * *" # At minute 17 every 4 hours - CRON_PATTERN_HEALING_JOB: "0 0 3 * * *" # everyday at 3 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-worship-posts: - image: lblod/delta-producer-publication-graph-maintainer:0.15.5 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" - RELATIVE_FILE_PATH: "deltas/worship-posts" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - SERVE_DELTA_FILES: "true" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" - volumes: - - ./config/delta-producer/worship-posts:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - labels: - - "logging=true" - logging: *default-logging - ################################################################################ - # DELTA WORSHIP POSTS: END - ################################################################################ - ################################################################################ # DELTA CONSUMERS ################################################################################ From 162a6ff65454225a76ca83363d316c21aa2a1471 Mon Sep 17 00:00:00 2001 From: Riad Abdallah Date: Thu, 3 Oct 2024 16:42:00 +0200 Subject: [PATCH 04/14] Update dispatcher rules for the delta streams --- config/dispatcher/dispatcher.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index c8f984c..cbdfe24 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -366,15 +366,15 @@ defmodule Dispatcher do ################################################################# get "/sync/organizations/files/*path" do - Proxy.forward conn, path, "http://delta-producer-publication-graph-maintainer/files/" + Proxy.forward conn, path, "http://delta-producer-publication-graph-maintainer/organizations/files/" end ################################################################# # DELTA: organizations-public-info ################################################################# - get "/sync/organizations-public-info/files/*path" do - Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations-public-info/files/" + get "/sync/public-info/files/*path" do + Proxy.forward(conn, path, "http://delta-producer-publication-graph-maintainer/public-info/files/") end ################################################################# @@ -382,7 +382,7 @@ defmodule Dispatcher do ################################################################# get "/sync/public/files/*path" do - Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-public/files/" + Proxy.forward(conn, path, "http://delta-producer-publication-graph-maintainer/public/files/") end ################################################################# @@ -390,7 +390,7 @@ defmodule Dispatcher do ################################################################# get "/sync/worship-posts/files/*path" do - Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-worship-posts/files/" + Proxy.forward(conn, path, "http://delta-producer-publication-graph-maintainer/worship-posts/files/") end ################################################################# From a98f860d3c8816f3a83f3b72c684941d4bbf4922 Mon Sep 17 00:00:00 2001 From: Riad Abdallah Date: Thu, 3 Oct 2024 17:05:19 +0200 Subject: [PATCH 05/14] Remove individual delta rules --- config/delta/rules.js | 57 +------------------------------------------ 1 file changed, 1 insertion(+), 56 deletions(-) diff --git a/config/delta/rules.js b/config/delta/rules.js index 376ce20..966c2ba 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -33,7 +33,6 @@ export default [ ignoreFromSelf: true } }, - { match: { graph: { @@ -51,7 +50,6 @@ export default [ ignoreFromSelf: true } }, - { match: { graph: { @@ -69,7 +67,6 @@ export default [ ignoreFromSelf: true } }, - { match: { predicate: { @@ -92,59 +89,7 @@ export default [ match: { }, callback: { - url: 'http://delta-producer-publication-graph-maintainer/organizations/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] - } - }, - { - match: { - }, - callback: { - url: 'http://delta-producer-pub-graph-maintainer-organizations-public-info/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] - } - }, - { - match: { - }, - callback: { - url: 'http://delta-producer-pub-graph-maintainer-public/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] - } - }, - { - match: { - // anything - }, - callback: { - url: 'http://delta-producer-pub-graph-maintainer-worship-posts/delta', + url: 'http://delta-producer-publication-graph-maintainer/delta', method: 'POST' }, options: { From 797d3a8d445156e00fc4ab8b1aa89371d6235ccd Mon Sep 17 00:00:00 2001 From: Riad Abdallah Date: Tue, 8 Oct 2024 13:19:00 +0200 Subject: [PATCH 06/14] Fix public info stream name --- .../publication-graph-maintainer/config.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/delta-producer/publication-graph-maintainer/config.json b/config/delta-producer/publication-graph-maintainer/config.json index 3bfcd68..ec06806 100644 --- a/config/delta-producer/publication-graph-maintainer/config.json +++ b/config/delta-producer/publication-graph-maintainer/config.json @@ -46,16 +46,16 @@ }, "public-info": { "deltaInterval": 10000, - "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-public-info", + "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations-public-info", "exportConfigPath": "/config/organizations-public-info/export.json", - "filesPath": "/public-info/files", - "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public-info", - "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public-info", + "filesPath": "/organizations-public-info/files", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info", "jobsGraph": "http://mu.semte.ch/graphs/jobs", - "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/public-info", - "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/public-info", + "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info", "queuePollInterval": 3000, - "relativeFilePath": "/deltas/public-info", + "relativeFilePath": "/deltas/organizations-public-info", "serveDeltaFiles": true, "skipMuAuthInitialSync": true, "skipMuAuthHealing": true, From b6dd8b45193159d6a9799895f9e37f6ce7a3c94a Mon Sep 17 00:00:00 2001 From: Riad Abdallah Date: Tue, 8 Oct 2024 13:35:35 +0200 Subject: [PATCH 07/14] Fix public info stream name --- config/delta-producer/publication-graph-maintainer/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/delta-producer/publication-graph-maintainer/config.json b/config/delta-producer/publication-graph-maintainer/config.json index ec06806..a40806e 100644 --- a/config/delta-producer/publication-graph-maintainer/config.json +++ b/config/delta-producer/publication-graph-maintainer/config.json @@ -64,7 +64,7 @@ "healingPatchPublicationGraphTaskOperation": 500, "useVirtuosoForExpensiveSelects": true, "errorGraph": "http://mu.semte.ch/graphs/jobs", - "loginPath": "/public-info/login", + "loginPath": "/organizations-public-info/login", "logOutgoingDelta": true }, "worship-posts": { From 7e0a6685734f0b55d33380342d6078fd1298da5e Mon Sep 17 00:00:00 2001 From: felix Date: Mon, 14 Oct 2024 14:50:14 +0200 Subject: [PATCH 08/14] Restore the old public path, to not break clients --- config/dispatcher/dispatcher.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index cbdfe24..d3d2574 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -373,8 +373,8 @@ defmodule Dispatcher do # DELTA: organizations-public-info ################################################################# - get "/sync/public-info/files/*path" do - Proxy.forward(conn, path, "http://delta-producer-publication-graph-maintainer/public-info/files/") + get "/sync/organizations-public-info/files/*path" do + Proxy.forward(conn, path, "http://delta-producer-publication-graph-maintainer/organizations-public-info/files/") end ################################################################# From 6397ce292300ba7066f550feb9e7c53112706ef0 Mon Sep 17 00:00:00 2001 From: felix Date: Mon, 14 Oct 2024 14:50:54 +0200 Subject: [PATCH 09/14] Change the namse of the stream to be in line with previous --- config/delta-producer/background-job-initiator/config.json | 2 +- config/delta-producer/publication-graph-maintainer/config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/delta-producer/background-job-initiator/config.json b/config/delta-producer/background-job-initiator/config.json index a0c8411..f8b3625 100644 --- a/config/delta-producer/background-job-initiator/config.json +++ b/config/delta-producer/background-job-initiator/config.json @@ -20,7 +20,7 @@ "startInitialSync": true }, { - "name": "public-info", + "name": "organizations-public-info", "jobsGraph": "http://mu.semte.ch/graphs/jobs", "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public-info", "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public-info", diff --git a/config/delta-producer/publication-graph-maintainer/config.json b/config/delta-producer/publication-graph-maintainer/config.json index a40806e..ccd83c8 100644 --- a/config/delta-producer/publication-graph-maintainer/config.json +++ b/config/delta-producer/publication-graph-maintainer/config.json @@ -44,7 +44,7 @@ "loginPath": "/public/login", "logOutgoingDelta": true }, - "public-info": { + "organizations-public-info": { "deltaInterval": 10000, "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations-public-info", "exportConfigPath": "/config/organizations-public-info/export.json", From 3f460ce3294472a314598b3047ca6b05fd4aa2a7 Mon Sep 17 00:00:00 2001 From: felix Date: Mon, 14 Oct 2024 14:51:20 +0200 Subject: [PATCH 10/14] Some uri's of JobOperations were off. Fixing it here. --- config/delta-producer/background-job-initiator/config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/delta-producer/background-job-initiator/config.json b/config/delta-producer/background-job-initiator/config.json index f8b3625..fb0f696 100644 --- a/config/delta-producer/background-job-initiator/config.json +++ b/config/delta-producer/background-job-initiator/config.json @@ -22,9 +22,9 @@ { "name": "organizations-public-info", "jobsGraph": "http://mu.semte.ch/graphs/jobs", - "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public-info", - "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public-info", - "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public-info", + "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info", + "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info", + "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info", "cronPatternDumpJob": "27 */4 * * *", "cronPatternHealingJob": "0 0 4 * * *", "startInitialSync": true From cb0191f28b32f382384429848b05a3fd8c54998e Mon Sep 17 00:00:00 2001 From: felix Date: Mon, 14 Oct 2024 15:04:22 +0200 Subject: [PATCH 11/14] Fix syntax issue in docker-compose.dev.yml --- docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e0218de..145ad80 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -46,8 +46,8 @@ services: restart: "no" adressenregister: restart: "no" - frontend: image: lblod/frontend-organization-portal:latest + frontend: restart: "no" uri-info: restart: "no" From 77245adbaa4d74a5758b2a2e6ebf3a1fd3dbe7ae Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 11 Oct 2024 14:41:15 +0200 Subject: [PATCH 12/14] Link missing classifications to public producer concept scheme --- ...-to-public-producers-concept-scheme.sparql | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 config/migrations/2024/20241011105932-link-bestuursfunctie-to-public-producers-concept-scheme.sparql diff --git a/config/migrations/2024/20241011105932-link-bestuursfunctie-to-public-producers-concept-scheme.sparql b/config/migrations/2024/20241011105932-link-bestuursfunctie-to-public-producers-concept-scheme.sparql new file mode 100644 index 0000000..f10144c --- /dev/null +++ b/config/migrations/2024/20241011105932-link-bestuursfunctie-to-public-producers-concept-scheme.sparql @@ -0,0 +1,34 @@ +INSERT DATA { + GRAPH { + + . + + + . + + + . + + + . + + + . + + + . + + + . + } +} + +# # To find the relevant classifications: +# SELECT DISTINCT ?classification WHERE { +# ?bestuursfunctie a . +# +# ?orgaanInTime ?bestuursfunctie ; +# ?orgaan . +# +# ?orgaan ?classification . +# } From f08e2e3df0c217eaf429e06ced93ca827165627f Mon Sep 17 00:00:00 2001 From: felix Date: Mon, 14 Oct 2024 11:07:06 +0200 Subject: [PATCH 13/14] changed mock-login-service to tagged version in drc.dev.yml --- docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3a59abd..ffb8bc6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -30,7 +30,7 @@ services: deltanotifier: restart: "no" mocklogin: - image: lblod/mock-login-service:latest + image: lblod/mock-login-service:0.7.0 links: - db:database restart: "no" From 340937bc85de56bb6e88118c26f24be5fc8eee54 Mon Sep 17 00:00:00 2001 From: felix Date: Mon, 14 Oct 2024 11:06:36 +0200 Subject: [PATCH 14/14] Remove the :latest tag from frontend in docker-compose.dev.yml --- docker-compose.dev.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ffb8bc6..b793350 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -47,7 +47,6 @@ services: adressenregister: restart: "no" frontend: - image: lblod/frontend-organization-portal:latest restart: "no" uri-info: restart: "no"