From bbb0f7a373ec22343286d118748e03b0cbbd7a53 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 17 Jan 2025 12:43:13 +0100 Subject: [PATCH] Update Changelog and Readme Inspired by @cecemel in https://github.com/lblod/app-organization-portal/commit/631ba827ee2246c25b4edcc7936fdd0e1c3c7988, thanks! --- CHANGELOG.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3ead38..adb97b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,64 @@ # Changelog +## Unreleased +### Backend + - Upgraded `worship-services-sensitive-consumer` [OP-3337] + +### Deploy notes +``` +drc down; +``` +Update `docker-compose.override.yml` to remove the config of `worship-services-sensitive-consumer` and replace it by: +``` + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_DISABLE_INITIAL_SYNC: "false" + DCR_DISABLE_DELTA_INGEST: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_REMAPPING_DATABASE: "virtuoso" # for the initial sync, we go directly to virtuoso + DCR_DISABLE_INITIAL_SYNC: "false" + DCR_DISABLE_DELTA_INGEST: "false" +``` +Then: +``` +drc up -d triplestore migrations +drc up -d db worship-services-main-info-consumer worship-services-private-info-consumer +# Wait until success of the previous step +``` +Then, update `docker-compose.override.yml` to: +``` + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "database" + DCR_REMAPPING_DATABASE: "database" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lblod.info" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lblod.info/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "database" + DCR_REMAPPING_DATABASE: "database" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +``` +drc up -d +sh scripts/reset-elastic.sh +``` ## 1.28.5 (TODO) ### Frontend - Bump to [v1.28.2](https://github.com/lblod/frontend-organization-portal/releases/tag/v1.28.2) diff --git a/README.md b/README.md index a59e0e3..8928eab 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,68 @@ Backend for the organization portal application, based on the mu.semte.ch micros You can shut down using `docker-compose stop` and remove everything using `docker-compose rm`. +#### Setup consumers +The application depends on external data. You'll have to perform extra steps if you want this data to be available. +These are multiple data sets, we'll try to be exhaustive on how to ingest these. Even though the procedures are similar. + +#### Setup worship mandatees sync +Note: this is assuming you have never ran this before. +``` +drc down; +``` +Update `docker-compose.override.yml` to: +``` + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lokaalbestuur.vlaanderen.be/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "virtuoso" + DCR_REMAPPING_DATABASE: "virtuoso" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lokaalbestuur.vlaanderen.be/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "virtuoso" + DCR_REMAPPING_DATABASE: "virtuoso" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +Then: +``` +drc up -d migrations +drc up -d db worship-services-main-info-consumer worship-services-private-info-consumer +# Wait until success of the previous step +``` +Then, update `docker-compose.override.yml` to: +``` + worship-services-main-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lokaalbestuur.vlaanderen.be/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "database" + DCR_REMAPPING_DATABASE: "database" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" + worship-services-private-info-consumer: + environment: + DCR_SYNC_BASE_URL: "https://loket.lokaalbestuur.vlaanderen.be" # or another endpoint + DCR_SYNC_LOGIN_ENDPOINT: "https://loket.lokaalbestuur.vlaanderen.be/sync/worship-services-sensitive/login" # or another endpoint + DCR_SECRET_KEY: "key-of-the-producer" + DCR_LANDING_ZONE_DATABASE: "database" + DCR_REMAPPING_DATABASE: "database" + DCR_DISABLE_DELTA_INGEST: "false" + DCR_DISABLE_INITIAL_SYNC: "false" +``` +Finally, put the stack back up and reindex elasticsearch +``` +drc up -d +sh scripts/reset-elastic.sh +``` ### Setting up the delta-producers related services To make sure the app can share data, producers need to be set up. There is an intial sync, that is potentially very expensive, and must be started manually