From f2d66ba03bd738e760af35684449146bfecf989d Mon Sep 17 00:00:00 2001 From: Damon Date: Fri, 21 Nov 2025 16:13:46 +0100 Subject: [PATCH 1/7] OP-3680: added vendor-management-consumer to ingest data from vendor management --- config/authorization/config.ex | 23 +++++++++++++++++++++++ config/resources/domain.json | 28 +++++++++++++++++++++++++++- docker-compose.yml | 21 +++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/config/authorization/config.ex b/config/authorization/config.ex index 2ef99a3..5268cc5 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -89,6 +89,11 @@ defmodule Acl.UserGroups.Config do "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject" ] + @vendor_type [ + "http://mu.semte.ch/vocabularies/ext/Vendor" + ] + + defp access_by_role(group_string) do %AccessByQuery{ # vars: ["session_group","session_role"], todo we might have to create multiple graph depending on role @@ -169,6 +174,12 @@ defmodule Acl.UserGroups.Config do constraint: %ResourceConstraint{ resource_types: @org_type ++ @worship_type } + }, + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/vendor-management-data", + constraint: %ResourceConstraint{ + resource_types: @vendor_type + } } ] }, @@ -182,6 +193,12 @@ defmodule Acl.UserGroups.Config do constraint: %ResourceConstraint{ resource_types: @org_type ++ @error_type ++ @worship_type } + }, + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/vendor-management-data", + constraint: %ResourceConstraint{ + resource_types: @vendor_type + } } ] }, @@ -195,6 +212,12 @@ defmodule Acl.UserGroups.Config do constraint: %ResourceConstraint{ resource_types: @org_type ++ @error_type ++ @worship_type } + }, + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/vendor-management-data", + constraint: %ResourceConstraint{ + resource_types: @vendor_type + } } ] }, diff --git a/config/resources/domain.json b/config/resources/domain.json index f335d76..5b32279 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -30,7 +30,8 @@ "ext": "http://mu.semte.ch/vocabularies/ext/", "geo": "http://www.opengis.net/ont/geosparql#", "nacebel": "http://data.gift/vocabularies/nace-bel/", - "time": "http://www.w3.org/2006/time#" + "time": "http://www.w3.org/2006/time#", + "muAccount": "http://mu.semte.ch/vocabularies/account/" }, "resources": { "agents": { @@ -553,6 +554,23 @@ }, "new-resource-base": "http://data.lblod.info/id/eredienstBestuurseenheden/" }, + "vendors": { + "class": "ext:Vendor", + "attributes": { + "name": { + "type": "string", + "predicate": "foaf:name" + } + }, + "relationships": { + "worship-service": { + "predicate": "muAccount:canActOnBehalfOf", + "target": "worship-services", + "cardinality": "many" + } + }, + "new-resource-base": "http://data.lblod.info/vendors/" + }, "worship-services": { "class": "ere:BestuurVanDeEredienst", "super": ["worship-administrative-units"], @@ -566,6 +584,14 @@ "predicate": "ere:grensoverschrijdend" } }, + "relationships": { + "vendor": { + "predicate": "muAccount:canActOnBehalfOf", + "target": "vendors", + "cardinality": "one", + "inverse": true + } + }, "new-resource-base": "http://data.lblod.info/id/besturenVanDeEredienst/" }, "recognized-worship-types": { diff --git a/docker-compose.yml b/docker-compose.yml index a758bf8..3af5632 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -540,6 +540,27 @@ services: - triplestore:virtuoso logging: *default-logging + vendor-management-consumer: + image: lblod/delta-consumer:0.0.24 + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info/" + DCR_SYNC_LOGIN_ENDPOINT: 'https://loket.lblod.info/sync/vendor-management/login' # Add DCR_SECRET_KEY in docker-compose.override.yml + DCR_SERVICE_NAME: "vendor-management-consumer" + DCR_SYNC_FILES_PATH: "/sync/vendor-management/files" + DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/VendorManagementCacheGraphDump" + DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/vendor-management" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/vendor-management-consumer" + DCR_DISABLE_INITIAL_SYNC: "true" + DCR_KEEP_DELTA_FILES: "true" + DCR_DELTA_FILE_FOLDER: "/consumer-files" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/VendorManagementFileSyncing" + INGEST_GRAPH: "http://mu.semte.ch/graphs/vendor-management-data" + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging ################################################################################ # DELTA CONSUMERS: END ################################################################################ From 0c4df815e936f913a9a76e2eb56c6316164fd9e5 Mon Sep 17 00:00:00 2001 From: Damon Date: Tue, 25 Nov 2025 10:59:05 +0100 Subject: [PATCH 2/7] OP-3680: vendor-management-consumer works with triple remapping to only get the needed data --- .../mapping/queries/dl2op/pass/vendor.sparql | 18 ++++++++++++++++ docker-compose.yml | 21 ++++++++++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql diff --git a/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql b/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql new file mode 100644 index 0000000..8bb79f8 --- /dev/null +++ b/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql @@ -0,0 +1,18 @@ +CONSTRUCT { + ?s ?p ?o. +} WHERE { + ?s a ; + ?p ?o. + + FILTER (?s IN ( + + )) + + FILTER (?p IN ( + , + , + + )) + + +} diff --git a/docker-compose.yml b/docker-compose.yml index 3af5632..d4e8f39 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -541,24 +541,35 @@ services: logging: *default-logging vendor-management-consumer: - image: lblod/delta-consumer:0.0.24 + image: lblod/delta-consumer:0.1.1 environment: DCR_SYNC_BASE_URL: "https://loket.lblod.info/" - DCR_SYNC_LOGIN_ENDPOINT: 'https://loket.lblod.info/sync/vendor-management/login' # Add DCR_SECRET_KEY in docker-compose.override.yml + DCR_SYNC_LOGIN_ENDPOINT: 'https://loket.lblod.info/sync/vendor-management/login' + DCR_SECRET_KEY: "" DCR_SERVICE_NAME: "vendor-management-consumer" - DCR_SYNC_FILES_PATH: "/sync/vendor-management/files" DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/VendorManagementCacheGraphDump" DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/vendor-management" + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/VendorManagementFileSyncing" DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/vendor-management-consumer" DCR_DISABLE_INITIAL_SYNC: "true" + DCR_SYNC_FILES_PATH: "/sync/vendor-management/files" DCR_KEEP_DELTA_FILES: "true" + DCR_DELTA_JOBS_RETENTION_PERIOD: "30" + DCR_ENABLE_TRIPLE_REMAPPING: "true" + DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/vendor-management-data" + DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/vendor-management-data" + BATCH_SIZE: "500" + SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" + SUDO_QUERY_RETRY: "true" DCR_DELTA_FILE_FOLDER: "/consumer-files" - DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/VendorManagementFileSyncing" - INGEST_GRAPH: "http://mu.semte.ch/graphs/vendor-management-data" + volumes: + - ./config/delta-consumers/vendor-management/mapping:/config/mapping + - ./data/files/consumer-files/vendor-management:/consumer-files/ labels: - "logging=true" links: - db:database + - triplestore:virtuoso restart: always logging: *default-logging ################################################################################ From 380dd9afaf2f3509ca9854e44223df4c79367007 Mon Sep 17 00:00:00 2001 From: Damon Date: Fri, 28 Nov 2025 09:54:30 +0100 Subject: [PATCH 3/7] OP-3680: added name to vendor-management --- .../vendor-management/mapping/queries/dl2op/pass/vendor.sparql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql b/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql index 8bb79f8..b387da5 100644 --- a/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql +++ b/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql @@ -11,7 +11,8 @@ CONSTRUCT { FILTER (?p IN ( , , - + , + )) From c27870a274bbeeccb0889883465d1d3616e2fc19 Mon Sep 17 00:00:00 2001 From: Damon Date: Mon, 1 Dec 2025 14:00:37 +0100 Subject: [PATCH 4/7] OP-3680: PR comments -> vendors for all organizations + multiple vendors --- config/authorization/config.ex | 18 ++++++++++++++++ .../mapping/queries/dl2op/pass/vendor.sparql | 19 ----------------- config/resources/domain.json | 20 ++++++++---------- docker-compose.yml | 21 ++++--------------- 4 files changed, 31 insertions(+), 47 deletions(-) delete mode 100644 config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql diff --git a/config/authorization/config.ex b/config/authorization/config.ex index 5268cc5..9d91fd6 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -135,6 +135,12 @@ defmodule Acl.UserGroups.Config do constraint: %ResourceConstraint{ resource_types: @org_type } + }, + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/vendor-management-data", + constraint: %ResourceConstraint{ + resource_types: @vendor_type + } } ] }, @@ -148,6 +154,12 @@ defmodule Acl.UserGroups.Config do constraint: %ResourceConstraint{ resource_types: @org_type ++ @error_type } + }, + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/vendor-management-data", + constraint: %ResourceConstraint{ + resource_types: @vendor_type + } } ] }, @@ -161,6 +173,12 @@ defmodule Acl.UserGroups.Config do constraint: %ResourceConstraint{ resource_types: @org_type ++ @error_type } + }, + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/vendor-management-data", + constraint: %ResourceConstraint{ + resource_types: @vendor_type + } } ] }, diff --git a/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql b/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql deleted file mode 100644 index b387da5..0000000 --- a/config/delta-consumers/vendor-management/mapping/queries/dl2op/pass/vendor.sparql +++ /dev/null @@ -1,19 +0,0 @@ -CONSTRUCT { - ?s ?p ?o. -} WHERE { - ?s a ; - ?p ?o. - - FILTER (?s IN ( - - )) - - FILTER (?p IN ( - , - , - , - - )) - - -} diff --git a/config/resources/domain.json b/config/resources/domain.json index 5b32279..139fe4f 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -468,7 +468,13 @@ "target": "kbo-organizations", "cardinality": "one", "inverse": true - } + }, + "vendors": { + "predicate": "muAccount:canActOnBehalfOf", + "target": "vendors", + "cardinality": "many", + "inverse": true + } }, "features": ["include-uri"], "new-resource-base": "http://data.lblod.info/id/organisaties/" @@ -563,9 +569,9 @@ } }, "relationships": { - "worship-service": { + "organizations": { "predicate": "muAccount:canActOnBehalfOf", - "target": "worship-services", + "target": "organizations", "cardinality": "many" } }, @@ -584,14 +590,6 @@ "predicate": "ere:grensoverschrijdend" } }, - "relationships": { - "vendor": { - "predicate": "muAccount:canActOnBehalfOf", - "target": "vendors", - "cardinality": "one", - "inverse": true - } - }, "new-resource-base": "http://data.lblod.info/id/besturenVanDeEredienst/" }, "recognized-worship-types": { diff --git a/docker-compose.yml b/docker-compose.yml index d4e8f39..a31923f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -544,32 +544,19 @@ services: image: lblod/delta-consumer:0.1.1 environment: DCR_SYNC_BASE_URL: "https://loket.lblod.info/" - DCR_SYNC_LOGIN_ENDPOINT: 'https://loket.lblod.info/sync/vendor-management/login' - DCR_SECRET_KEY: "" + DCR_SYNC_LOGIN_ENDPOINT: 'https://loket.lblod.info/sync/vendor-management/login' # Add DCR_SECRET_KEY in docker-compose.override.yml DCR_SERVICE_NAME: "vendor-management-consumer" + DCR_SYNC_FILES_PATH: "/sync/vendor-management/files" DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/VendorManagementCacheGraphDump" DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/vendor-management" - DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/VendorManagementFileSyncing" DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/vendor-management-consumer" DCR_DISABLE_INITIAL_SYNC: "true" - DCR_SYNC_FILES_PATH: "/sync/vendor-management/files" DCR_KEEP_DELTA_FILES: "true" - DCR_DELTA_JOBS_RETENTION_PERIOD: "30" - DCR_ENABLE_TRIPLE_REMAPPING: "true" - DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/vendor-management-data" - DCR_REMAPPING_GRAPH: "http://mu.semte.ch/graphs/vendor-management-data" - BATCH_SIZE: "500" - SUDO_QUERY_RETRY_FOR_HTTP_STATUS_CODES: "404,500,503" - SUDO_QUERY_RETRY: "true" DCR_DELTA_FILE_FOLDER: "/consumer-files" - volumes: - - ./config/delta-consumers/vendor-management/mapping:/config/mapping - - ./data/files/consumer-files/vendor-management:/consumer-files/ + DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/VendorManagementFileSyncing" + INGEST_GRAPH: "http://mu.semte.ch/graphs/vendor-management-data" labels: - "logging=true" - links: - - db:database - - triplestore:virtuoso restart: always logging: *default-logging ################################################################################ From cec0ccca77ca1c2cf8479fa06385987ad2db38f6 Mon Sep 17 00:00:00 2001 From: Damon Date: Mon, 1 Dec 2025 14:17:56 +0100 Subject: [PATCH 5/7] changelog edit --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9519308..62a759a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## unreleased +### Backend +- Indicate whether a worship service is a customer of ReligioPoint or Loket voor Lokale Besturen [OP-3680] + ## 1.37.3 (2025-11-14) ### Backend - Added cleanup job to clean duplicated data of municipalities and provinces [OP-3676] From 7b3d0d94d18703993b03028c0ae34a722ca8e204 Mon Sep 17 00:00:00 2001 From: Damon Date: Mon, 8 Dec 2025 12:58:10 +0100 Subject: [PATCH 6/7] OP-3680: vendor-management-consumer db link fix --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index a31923f..a15960a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -557,6 +557,8 @@ services: INGEST_GRAPH: "http://mu.semte.ch/graphs/vendor-management-data" labels: - "logging=true" + links: + - db:database restart: always logging: *default-logging ################################################################################ From a7d14d42af05d124a1b6733150020e1879c250e6 Mon Sep 17 00:00:00 2001 From: Damon Date: Mon, 8 Dec 2025 16:01:45 +0100 Subject: [PATCH 7/7] OP-3680: changelog + readme edits --- CHANGELOG.md | 2 ++ README.md | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fc78ee..f8224c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ ``` drc restart migrations drc restart delta-producer-publication-graph-maintainer resource +drc pull vendor-management-consumer; drc up -d vendor-management-consumer +drc restart db resource ``` ## 1.37.3 (2025-11-14) diff --git a/README.md b/README.md index c98fd33..8239e8c 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,31 @@ drc up -d sh scripts/reset-elastic.sh ``` +#### Setup vendors sync +Note: this is assuming you have never ran this before. +``` +drc down; +``` +Update `docker-compose.override.yml` to: +``` + vendor-management-consumer: + environment: + DCR_SYNC_BASE_URL: "https://dev.loket.lblod.info/" # or another endpoint + DCR_DISABLE_INITIAL_SYNC: "true" + BATCH_SIZE: "500" + DCR_SECRET_KEY: ">" # fill in key for selected endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://dev.loket.lblod.info/sync/vendor-management/login" # login for selected endpoint +``` +Then: +``` +drc up -d migrations +drc up -d db vendor-management-consumer +``` +Finally, put the stack back up. +``` +drc up -d +``` + ### Setting up the delta-producers related services To make sure the app can share data, producers need to be set up. There is an intial sync, that is potentially very expensive, and must be started manually