diff --git a/CHANGELOG.md b/CHANGELOG.md index b4cef9f..4af96d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased +- Remove deprecated organizations-public-info producer [OP-3581] + +### Deploy instructions +``` +drc restart dispatcher delta-producer-dump-file-publisher delta-producer-background-jobs-initiator delta-producer-publication-graph-maintainer jobs-controller +drc up -d publication-triplestore-migrations +rm -r data/files/deltas/organizations-public-info +rm -r data/files/delta-producer-dumps/dump-organizations-public-info +``` + ## 1.37.0 (2025-10-24) - Added cleanup job to clean duplicated data of municipalities and provinces [OP-3676] diff --git a/config/delta-producer/background-job-initiator/config.json b/config/delta-producer/background-job-initiator/config.json index fb0f696..55078d2 100644 --- a/config/delta-producer/background-job-initiator/config.json +++ b/config/delta-producer/background-job-initiator/config.json @@ -19,16 +19,6 @@ "cronPatternHealingJob": "0 0 4 * * *", "startInitialSync": true }, - { - "name": "organizations-public-info", - "jobsGraph": "http://mu.semte.ch/graphs/jobs", - "dumpFileCreationJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info", - "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info", - "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info", - "cronPatternDumpJob": "27 */4 * * *", - "cronPatternHealingJob": "0 0 4 * * *", - "startInitialSync": true - }, { "name": "worship-posts", "jobsGraph": "http://mu.semte.ch/graphs/jobs", diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 59e312b..c488b67 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -17,24 +17,6 @@ "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", - "fileBaseName": "dump-organizations-public-info", - "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", - "fileBaseName": "dump-organizations-public-info", - "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", diff --git a/config/delta-producer/publication-graph-maintainer/config.json b/config/delta-producer/publication-graph-maintainer/config.json index ccd83c8..9fa4d5c 100644 --- a/config/delta-producer/publication-graph-maintainer/config.json +++ b/config/delta-producer/publication-graph-maintainer/config.json @@ -44,29 +44,6 @@ "loginPath": "/public/login", "logOutgoingDelta": true }, - "organizations-public-info": { - "deltaInterval": 10000, - "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-organizations-public-info", - "exportConfigPath": "/config/organizations-public-info/export.json", - "filesPath": "/organizations-public-info/files", - "healingJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info", - "initialPublicationGraphSyncJobOperation": "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info", - "jobsGraph": "http://mu.semte.ch/graphs/jobs", - "publicationGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", - "publisherUri": "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info", - "queuePollInterval": 3000, - "relativeFilePath": "/deltas/organizations-public-info", - "serveDeltaFiles": true, - "skipMuAuthInitialSync": true, - "skipMuAuthHealing": true, - "useFileDiff": true, - "healingInitialBatchSizeInsert": 1000, - "healingPatchPublicationGraphTaskOperation": 500, - "useVirtuosoForExpensiveSelects": true, - "errorGraph": "http://mu.semte.ch/graphs/jobs", - "loginPath": "/organizations-public-info/login", - "logOutgoingDelta": true - }, "worship-posts": { "deltaInterval": 10000, "errorCreatorUri": "http://lblod.data.gift/services/delta-producer-publication-graph-maintainer-worship-posts", diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 4a1591d..418a124 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -441,14 +441,6 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://delta-producer-publication-graph-maintainer/organizations/files/" end - ################################################################# - # DELTA: organizations-public-info - ################################################################# - - get "/sync/organizations-public-info/files/*path" do - Proxy.forward(conn, path, "http://delta-producer-publication-graph-maintainer/organizations-public-info/files/") - end - ################################################################# # DELTA: public ################################################################# diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 4115501..e22a768 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -63,38 +63,6 @@ } ] }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { - "tasksConfiguration": [ - { - "currentOperation": null, - "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", - "nextIndex": "0" - }, - { - "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", - "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", - "nextIndex": "1" - } - ] - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info": { - "tasksConfiguration": [ - { - "currentOperation": null, - "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", - "nextIndex": "0" - } - ] - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { - "tasksConfiguration": [ - { - "currentOperation": null, - "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", - "nextIndex": "0" - } - ] - }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { "tasksConfiguration": [ { diff --git a/config/publication-triplestore-migrations/2025/20251024152300-clear-graph-organization-public-producer.sparql b/config/publication-triplestore-migrations/2025/20251024152300-clear-graph-organization-public-producer.sparql new file mode 100644 index 0000000..6f223b7 --- /dev/null +++ b/config/publication-triplestore-migrations/2025/20251024152300-clear-graph-organization-public-producer.sparql @@ -0,0 +1 @@ +CLEAR GRAPH diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3231ca2..f15c77b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -19,6 +19,8 @@ services: restart: "no" migrations-triggering-indexing: restart: "no" + publication-triplestore-migrations: + restart: "no" resource: restart: "no" cache: diff --git a/docker-compose.yml b/docker-compose.yml index 5e5dc8e..8ce6769 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -78,6 +78,16 @@ services: - "logging=true" restart: always logging: *default-logging + publication-triplestore-migrations: + image: semtech/mu-migrations-service:0.9.0 + links: + - publication-triplestore:database + volumes: + - ./config/publication-triplestore-migrations:/data/migrations + labels: + - "logging=true" + restart: always + logging: *default-logging resource: image: semtech/mu-cl-resources:1.24.0 environment: