From 99644e7925916d2dd13def54fdf43819d9845bfc Mon Sep 17 00:00:00 2001 From: Sam Van Campenhout Date: Tue, 1 Feb 2022 15:23:01 +0100 Subject: [PATCH 01/24] Add a type attribute to the contact-points model (#95) --- config/resources/domain.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/resources/domain.json b/config/resources/domain.json index 596699c..2a90c1d 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -210,6 +210,10 @@ "website": { "type": "uri", "predicate": "foaf:page" + }, + "type": { + "type": "string", + "predicate": "schema:contactType" } }, "relationships": { From de2a143218af00488fd2191ce1d88d71b363a220 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Tue, 1 Feb 2022 18:04:19 +0100 Subject: [PATCH 02/24] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5c5e602..0468789 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -158,7 +158,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-contact-hub:1.5.2 + image: lblod/frontend-contact-hub:1.5.3 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 3bcc53b0bc8127376f1f52bb13c3652145f5471d Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Wed, 2 Feb 2022 14:59:02 +0100 Subject: [PATCH 03/24] add contact types (#88) * add contact types * move migration to 2022 directory Co-authored-by: Nordine Bittich --- .../20220202120000-primary-contacts.sparql | 22 +++++++++++++++++ .../20220203100000-secondary-contacts.sparql | 24 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 config/migrations/2022/20220202120000-primary-contacts.sparql create mode 100644 config/migrations/2022/20220203100000-secondary-contacts.sparql diff --git a/config/migrations/2022/20220202120000-primary-contacts.sparql b/config/migrations/2022/20220202120000-primary-contacts.sparql new file mode 100644 index 0000000..89c771d --- /dev/null +++ b/config/migrations/2022/20220202120000-primary-contacts.sparql @@ -0,0 +1,22 @@ +# cf 20220110100000-secondary-contacts.sparql for context. +# In case it's not a secondary contactpoint (a contactpoint without an address if multiple contact points exist for one entity/mandataris) +PREFIX locn: +PREFIX schema: +PREFIX rdf: +PREFIX rdfs: +INSERT { + GRAPH { + ?contactPoint + schema:contactType "Primary" + } +} +WHERE { + GRAPH { + ?contactPoint + a schema:ContactPoint. + FILTER NOT EXISTS { + ?contactPoint + schema:contactType "Secondary" + } + } +} \ No newline at end of file diff --git a/config/migrations/2022/20220203100000-secondary-contacts.sparql b/config/migrations/2022/20220203100000-secondary-contacts.sparql new file mode 100644 index 0000000..d6db186 --- /dev/null +++ b/config/migrations/2022/20220203100000-secondary-contacts.sparql @@ -0,0 +1,24 @@ +PREFIX locn: +PREFIX schema: +PREFIX rdf: +PREFIX rdfs: +INSERT { + GRAPH { + ?secondary + schema:contactType "Secondary" + } +} +WHERE { + GRAPH { + ?primary + a schema:ContactPoint; + locn:address ?a1; + schema:telephone ?t1. + ?mandataris schema:contactPoint ?primary, ?secondary. + ?secondary + schema:telephone ?t2. + FILTER NOT EXISTS { + ?secondary locn:address ?a2. + } + } +} \ No newline at end of file From 3e14d92c29e8e8b612ea511d186dff725c29102d Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 7 Feb 2022 07:50:37 +0100 Subject: [PATCH 04/24] mu search page person (#97) --- config/delta/rules.js | 17 ++++ config/dispatcher/dispatcher.ex | 8 ++ config/search/config.json | 133 ++++++++++++++++++++++++++++++++ docker-compose.yml | 19 +++++ 4 files changed, 177 insertions(+) create mode 100644 config/search/config.json diff --git a/config/delta/rules.js b/config/delta/rules.js index c476cfb..c1d5575 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -1,4 +1,21 @@ export default [ + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/organisatieportaal" + } + }, + callback: { + url: 'http://search/update', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 10000, + ignoreFromSelf: true + } + }, { match: { predicate: { diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index fb94483..71e6750 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -221,7 +221,15 @@ defmodule Dispatcher do match "/sessions/*path", %{ accept: [:any], layer: :api} do Proxy.forward conn, path, "http://login/sessions/" end + + ############################################################### + # SEARCH + ############################################################### + match "/search/*path", %{ accept: %{ json: true }, layer: :api_services} do + IO.puts("hey") + Proxy.forward conn, path, "http://search/" + end ############### # API SERVICES diff --git a/config/search/config.json b/config/search/config.json new file mode 100644 index 0000000..d1da3ac --- /dev/null +++ b/config/search/config.json @@ -0,0 +1,133 @@ +{ + "automatic_index_updates": true, + "persist_indexes": true, + "number_of_threads": 10, + "eager_indexing_groups": [ + [{ "variables": [], "name": "acmidm-lezer" }], + [{ "variables": [], "name": "acmidm-editeerder" }], + [{ "variables": [], "name": "acmidm-beheerder" }] + ], + "types": [ + { + "type": "person", + "on_path": "people", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", + "properties": { + "given_name": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://xmlns.com/foaf/0.1/givenName"], + "id": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://mu.semte.ch/vocabularies/core/uuid"], + "family_name": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://xmlns.com/foaf/0.1/familyName"], + "first_name_used": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam"], + "end_date": "http://data.vlaanderen.be/ns/mandaat#einde", + "organization_id": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "organization_name": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "organization_municipality": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ], + "organization_province": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "position_id": [ + "http://www.w3.org/ns/org#holds", + "http://www.w3.org/ns/org#role", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "position_name": [ + "http://www.w3.org/ns/org#holds", + "http://www.w3.org/ns/org#role", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "planned_end_date": "https://data.lblod.info/vocabularies/erediensten/geplandeEinddatumAanstelling" + }, + + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { "type": "text" }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { "type": "text" }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { "type": "date" }, + "planned_end_date": { "type": "date" } + } + } + } + + ] +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0468789..fa9fbc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -332,3 +332,22 @@ services: ################################################################################ # DELTA ORGANIZATIONS: END ################################################################################ + + +################################################################################ +# MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH +################################################################################ + search: + image: nbittich/mu-search:wip + links: + - db:database + volumes: + - ./config/search:/config + elasticsearch: + ports: + - "9200:9200" + image: semtech/mu-search-elastic-backend:latest + volumes: + - ./data/elasticsearch/:/usr/share/elasticsearch/data + environment: + - discovery.type=single-node \ No newline at end of file From e52d71682e717cc5554f2a8234f6cc0bab0a93cd Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:22:27 +0100 Subject: [PATCH 05/24] fix relationship (#99) --- config/resources/domain.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/resources/domain.json b/config/resources/domain.json index 2a90c1d..44a0268 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -46,9 +46,10 @@ }, "relationships": { "mandatories": { - "predicate": "mandaat:isAangesteldAls", + "predicate": "mandaat:isBestuurlijkeAliasVan", "target": "mandatories", - "cardinality": "many" + "cardinality": "many", + "inverse": true }, "agents-in-position": { "predicate": "org:heldBy", From 0ecbca52d296f0f2be866d63de6c3f1ddf46307c Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 8 Feb 2022 15:35:13 +0100 Subject: [PATCH 06/24] Bump adresenregister --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index fa9fbc4..db79c6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -152,7 +152,7 @@ services: restart: always logging: *default-logging adressenregister: - image: lblod/adressenregister-fuzzy-search-service:0.5.0 + image: lblod/adressenregister-fuzzy-search-service:0.6.0 labels: - "logging=true" restart: always From 1f48744b62068bae3243c27f527dd8f86fabf26d Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 10 Feb 2022 14:09:17 +0100 Subject: [PATCH 07/24] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index db79c6d..5ba8e09 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -158,7 +158,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-contact-hub:1.5.3 + image: lblod/frontend-contact-hub:1.5.4 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 2c6269d4876c66cfb82693399d2ac880069593a9 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 10 Feb 2022 16:01:33 +0100 Subject: [PATCH 08/24] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5ba8e09..825d5c3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -158,7 +158,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-contact-hub:1.5.4 + image: lblod/frontend-contact-hub:1.6.0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 2cb5576e5884fc26699926c982adddca58b64699 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 10 Feb 2022 19:38:26 +0100 Subject: [PATCH 09/24] remove port mapping --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 825d5c3..0f20a7a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -344,8 +344,6 @@ services: volumes: - ./config/search:/config elasticsearch: - ports: - - "9200:9200" image: semtech/mu-search-elastic-backend:latest volumes: - ./data/elasticsearch/:/usr/share/elasticsearch/data From f6b2c963f32574f3c593deee8c6032bb00077562 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 10 Feb 2022 09:23:52 +0100 Subject: [PATCH 10/24] improve perf bestuur --- .gitignore | 2 + config/search/config.json | 102 +++++++++++++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 39c29a4..c5d5d7c 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,5 @@ source.json out.json security.yml config/proxy-https/proxy_ssl.conf + +update-handler.store \ No newline at end of file diff --git a/config/search/config.json b/config/search/config.json index d1da3ac..99ff776 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -127,7 +127,107 @@ "planned_end_date": { "type": "date" } } } + }, + { + "type": "unit", + "on_path": "units", + "rdf_type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "sub_types": [ + "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", + "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan" + ], + "properties": { + "name": "http://www.w3.org/2004/02/skos/core#prefLabel", + "classification_id": [ + "http://www.w3.org/ns/org#classification", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "classification_name": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "status": [ + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "status_id": [ + "http://www.w3.org/ns/regorg#orgStatus", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "recognized_worship_type_id": [ + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "recognized_worship_type_name": [ + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "province": [ + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "municipality": [ + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ] + }, + "mappings": { + "properties": { + "classification_id": { "type": "text" }, + "recognized_worship_type_id": { "type": "text" }, + "name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "classification_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "recognized_worship_type_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + } + } + } } - ] } \ No newline at end of file From ddc679b2d7b34294da4a5e85026b4eb6a2647345 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Fri, 11 Feb 2022 11:34:42 +0100 Subject: [PATCH 11/24] bugfix --- config/search/config.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config/search/config.json b/config/search/config.json index 99ff776..7122002 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -131,11 +131,10 @@ { "type": "unit", "on_path": "units", - "rdf_type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid", "sub_types": [ "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", - "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", - "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan" + "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst" ], "properties": { "name": "http://www.w3.org/2004/02/skos/core#prefLabel", From 976c7b54781c08ff99d734b9aaacfbaaf21fd77b Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Mon, 14 Feb 2022 14:37:42 +0100 Subject: [PATCH 12/24] update docker restart policy --- config/search/update-handler.store | 3 +++ docker-compose.dev.yml | 16 ++++++++++++++++ docker-compose.yml | 5 ++++- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 config/search/update-handler.store diff --git a/config/search/update-handler.store b/config/search/update-handler.store new file mode 100644 index 0000000..90757d3 --- /dev/null +++ b/config/search/update-handler.store @@ -0,0 +1,3 @@ +--- +queue: [] +index: {} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 4c25a54..5666a47 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -44,3 +44,19 @@ services: restart: "no" uri-info: restart: "no" + delta-producer-bg-jobs-initiator-administrative-units: + restart: "no" + delta-producer-pub-graph-maintainer-administrative-units: + restart: "no" + delta-producer-json-diff-publisher-administrative-units: + restart: "no" + delta-producer-bg-jobs-initiator-organizations: + restart: "no" + delta-producer-pub-graph-maintainer-organizations: + restart: "no" + delta-producer-json-diff-publisher-organizations: + restart: "no" + search: + restart: "no" + elasticsearch: + restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index 0f20a7a..79a9974 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -341,6 +341,7 @@ services: image: nbittich/mu-search:wip links: - db:database + restart: always volumes: - ./config/search:/config elasticsearch: @@ -348,4 +349,6 @@ services: volumes: - ./data/elasticsearch/:/usr/share/elasticsearch/data environment: - - discovery.type=single-node \ No newline at end of file + - discovery.type=single-node + restart: always + \ No newline at end of file From 1941b09790983985daefab74565200c8e7044fe3 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Mon, 14 Feb 2022 14:40:06 +0100 Subject: [PATCH 13/24] gitignore update --- .gitignore | 1 + config/search/update-handler.store | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 config/search/update-handler.store diff --git a/.gitignore b/.gitignore index 39c29a4..ef8f148 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ source.json out.json security.yml config/proxy-https/proxy_ssl.conf +/config/update-handler.store \ No newline at end of file diff --git a/config/search/update-handler.store b/config/search/update-handler.store deleted file mode 100644 index 90757d3..0000000 --- a/config/search/update-handler.store +++ /dev/null @@ -1,3 +0,0 @@ ---- -queue: [] -index: {} From 11202a0b82499b0a5b75b42cd13752702601326c Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Mon, 14 Feb 2022 15:23:29 +0100 Subject: [PATCH 14/24] missing file & dump file --- docker-compose.dev.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5666a47..4b0e957 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -60,3 +60,7 @@ services: restart: "no" elasticsearch: restart: "no" + file: + restart: "no" + delta-producer-dump-file-publisher: + restart: "no" \ No newline at end of file From bfee6de5a95f5a9175717a2b68168fdfc34e862d Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 24 Feb 2022 18:25:33 +0100 Subject: [PATCH 15/24] Remove unused config and bump kallipe-api --- config/dispatcher/dispatcher.ex | 6 +----- docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 71e6750..9854ec6 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -221,7 +221,7 @@ defmodule Dispatcher do match "/sessions/*path", %{ accept: [:any], layer: :api} do Proxy.forward conn, path, "http://login/sessions/" end - + ############################################################### # SEARCH ############################################################### @@ -239,10 +239,6 @@ defmodule Dispatcher do forward conn, path, "http://kalliope-api/v3/api-docs/" end - match "/changes/*path", %{ layer: :api_services, accept: %{ json: true } } do - forward conn, path, "http://kalliope-api/changes/" - end - match "/consolidated/*path", %{ layer: :api_services, accept: %{ json: true } } do forward conn, path, "http://kalliope-api/consolidated/" end diff --git a/docker-compose.yml b/docker-compose.yml index 79a9974..2c9f1e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,7 +79,7 @@ services: restart: always logging: *default-logging kalliope-api: - image: lblod/jsonld-delta-service:0.5.4 + image: lblod/jsonld-delta-service:0.5.5-beta.1 volumes: - ./config/kalliope:/config - ./data/files:/share @@ -90,6 +90,7 @@ services: LOGGING_LEVEL: "INFO" SPARQL_ENDPOINT: "http://db:8890/sparql" LIMIT_SIZE: "10000" + MU_AUTH_SUDO: "false" labels: - "logging=true" restart: always @@ -351,4 +352,3 @@ services: environment: - discovery.type=single-node restart: always - \ No newline at end of file From 94e8d8cf06f44c1ce7de104b1c06ba3d8af54b86 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 25 Feb 2022 10:03:17 +0100 Subject: [PATCH 16/24] Add missing no restart --- docker-compose.dev.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 4b0e957..cc6d251 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -29,6 +29,7 @@ services: image: lblod/mock-login-service:latest links: - db:database + restart: "no" sink: restart: "no" login: From c552d76e1414eafe570f001c22ca973348cfdd88 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 25 Feb 2022 10:39:53 +0100 Subject: [PATCH 17/24] Add fixed image versions when possible --- docker-compose.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 79a9974..22dfa45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: restart: always logging: *default-logging dispatcher: - image: semtech/mu-dispatcher:latest + image: semtech/mu-dispatcher:2.1.0-beta.2 volumes: - ./config/dispatcher:/config labels: @@ -61,7 +61,7 @@ services: restart: always logging: *default-logging resource: - image: semtech/mu-cl-resources:feature-all-errors-as-json-api-errors + image: semtech/mu-cl-resources:feature-all-errors-as-json-api-errors # Bump to fixed version when merged and release links: - db:database volumes: @@ -127,7 +127,7 @@ services: links: - db:database accountdetail: - image: lblod/account-detail-service:latest + image: lblod/account-detail-service:0.1.0 labels: - "logging=true" environment: @@ -338,14 +338,14 @@ services: # MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH ################################################################################ search: - image: nbittich/mu-search:wip - links: - - db:database - restart: always - volumes: - - ./config/search:/config + image: nbittich/mu-search:wip # Bump to fixed version when merged and release to the original repo + links: + - db:database + restart: always + volumes: + - ./config/search:/config elasticsearch: - image: semtech/mu-search-elastic-backend:latest + image: semtech/mu-search-elastic-backend:1.0.1 volumes: - ./data/elasticsearch/:/usr/share/elasticsearch/data environment: From 61deab821295e9dc80f7c04bb64c23080d917700 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 28 Feb 2022 15:26:39 +0100 Subject: [PATCH 18/24] List all positions for a person (ministers + mandatories) & bug fix editing issue with delta (#105) --- config/search/config.json | 270 ++++++++++++++++++++++++++++- config/search/update-handler.store | 3 + docker-compose.yml | 2 +- 3 files changed, 268 insertions(+), 7 deletions(-) create mode 100644 config/search/update-handler.store diff --git a/config/search/config.json b/config/search/config.json index 7122002..f3d5ae4 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -9,12 +9,13 @@ ], "types": [ { - "type": "person", - "on_path": "people", + "type": "mandatory", + "on_path": "mandatories", "rdf_type": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", "properties": { "given_name": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://xmlns.com/foaf/0.1/givenName"], "id": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://mu.semte.ch/vocabularies/core/uuid"], + "uuid": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://mu.semte.ch/vocabularies/core/uuid"], "family_name": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://xmlns.com/foaf/0.1/familyName"], "first_name_used": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam"], "end_date": "http://data.vlaanderen.be/ns/mandaat#einde", @@ -59,8 +60,7 @@ "http://www.w3.org/ns/org#holds", "http://www.w3.org/ns/org#role", "http://www.w3.org/2004/02/skos/core#prefLabel" - ], - "planned_end_date": "https://data.lblod.info/vocabularies/erediensten/geplandeEinddatumAanstelling" + ] }, "mappings": { @@ -123,11 +123,269 @@ } } }, - "end_date": { "type": "date" }, - "planned_end_date": { "type": "date" } + "end_date": { "type": "date" } } } }, + { + "type": "minister", + "on_path": "ministers", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", + "sub_types": ["http://data.lblod.info/vocabularies/contacthub/AgentInPositie"], + "properties": { + "given_name": ["http://www.w3.org/ns/org#heldBy","http://xmlns.com/foaf/0.1/givenName"], + "id": ["http://www.w3.org/ns/org#heldBy","http://mu.semte.ch/vocabularies/core/uuid"], + "uuid": ["http://www.w3.org/ns/org#heldBy","http://mu.semte.ch/vocabularies/core/uuid"], + "family_name": ["http://www.w3.org/ns/org#heldBy","http://xmlns.com/foaf/0.1/familyName"], + "first_name_used": ["http://www.w3.org/ns/org#heldBy","https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam"], + "end_date": "http://data.lblod.info/vocabularies/contacthub/eindedatum", + "organization_id": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "organization_name": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "organization_municipality": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ], + "organization_province": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "position_id": [ + "http://www.w3.org/ns/org#holds", + "http://data.lblod.info/vocabularies/erediensten/functie", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "position_name": [ + "http://www.w3.org/ns/org#holds", + "http://data.lblod.info/vocabularies/erediensten/functie", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { "type": "text" }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { "type": "text" }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { "type": "date" } + } + } + }, + { + "type" : "person", + "composite_types" : ["minister", "mandatory"], + "on_path" : "people", + "properties" : [ + { + "name" : "given_name", + "mappings" : { + "minister" : "given_name", + "mandatory" : "given_name" + } + }, + { + "name" : "id", + "mappings" : { + "minister" : "id", + "mandatory" : "id" + } + }, + { + "name" : "family_name", + "mappings" : { + "minister" : "family_name", + "mandatory" : "family_name" + } + }, + { + "name" : "first_name_used", + "mappings" : { + "minister" : "first_name_used", + "mandatory" : "first_name_used" + } + }, + { + "name" : "end_date", + "mappings" : { + "minister" : "end_date", + "mandatory" : "end_date" + } + }, + { + "name" : "organization_id", + "mappings" : { + "minister" : "organization_id", + "mandatory" : "organization_id" + } + }, + { + "name" : "organization_name", + "mappings" : { + "minister" : "organization_name", + "mandatory" : "organization_name" + } + }, + { + "name" : "organization_municipality", + "mappings" : { + "minister" : "organization_municipality", + "mandatory" : "organization_municipality" + } + }, + { + "name" : "organization_province", + "mappings" : { + "minister" : "organization_province", + "mandatory" : "organization_province" + } + }, + { + "name" : "position_id", + "mappings" : { + "minister" : "position_id", + "mandatory" : "position_id" + } + }, + { + "name" : "position_name", + "mappings" : { + "minister" : "position_name", + "mandatory" : "position_name" + } + } + ], + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { "type": "text" }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { "type": "text" }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { "type": "date" } + } + } + }, { "type": "unit", "on_path": "units", diff --git a/config/search/update-handler.store b/config/search/update-handler.store new file mode 100644 index 0000000..90757d3 --- /dev/null +++ b/config/search/update-handler.store @@ -0,0 +1,3 @@ +--- +queue: [] +index: {} diff --git a/docker-compose.yml b/docker-compose.yml index 79a9974..ae221c6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -338,7 +338,7 @@ services: # MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH ################################################################################ search: - image: nbittich/mu-search:wip + image: nbittich/mu-search:wip-inheritance links: - db:database restart: always From cd185f427e9777e51732472667b91c1c073f9811 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 28 Feb 2022 17:24:42 +0100 Subject: [PATCH 19/24] Bump kalliope api to v0.5.5 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2c9f1e6..5dfbbb5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,7 +79,7 @@ services: restart: always logging: *default-logging kalliope-api: - image: lblod/jsonld-delta-service:0.5.5-beta.1 + image: lblod/jsonld-delta-service:0.5.5 volumes: - ./config/kalliope:/config - ./data/files:/share From df9a25210c0e945e1f841a73e650478ba5c5a490 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 1 Mar 2022 08:46:24 +0100 Subject: [PATCH 20/24] Update gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 611cdc4..8eaa314 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,4 @@ source.json out.json security.yml config/proxy-https/proxy_ssl.conf -/config/update-handler.store +/config/search/update-handler.store From b3c064b04a0305ac1cd3ef906069792541e8d2af Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Tue, 1 Mar 2022 10:35:28 +0100 Subject: [PATCH 21/24] use compressed dumps --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4fd0244..db9e39f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -200,7 +200,7 @@ services: # DELTA GENERAL ################################################################################ delta-producer-dump-file-publisher: - image: lblod/delta-producer-dump-file-publisher:0.4.0 + image: bdevloed/delta-producer-dump-file-publisher:0.5.0-beta volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share From e744642b19ff1ccd7075be5909713eb8f29f1809 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Wed, 2 Mar 2022 17:06:43 +0100 Subject: [PATCH 22/24] reschedule op dump creation --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index db9e39f..1e99619 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -282,7 +282,7 @@ services: DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations' INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations' HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations' - CRON_PATTERN_DUMP_JOB: '30 9 8-18/2 * * MON-FRI' # At second :30, at minute :09, every 2 hours between hours 08am and 18pm, every day between Monday and Friday, every month + CRON_PATTERN_DUMP_JOB: '42 * * * *' # At second :30, at minute :09, every 2 hours between hours 08am and 18pm, every day between Monday and Friday, every month CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. labels: From 5c7833e53f6499cb56f7ae005a878e3c980f4ae4 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Wed, 2 Mar 2022 17:15:29 +0100 Subject: [PATCH 23/24] update comment --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1e99619..848b6d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -282,7 +282,7 @@ services: DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations' INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations' HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations' - CRON_PATTERN_DUMP_JOB: '42 * * * *' # At second :30, at minute :09, every 2 hours between hours 08am and 18pm, every day between Monday and Friday, every month + CRON_PATTERN_DUMP_JOB: '42 * * * *' # At minute 42 every hour CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. labels: From dbf5fce41ea5260e01006337910942e26caacdc9 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 3 Mar 2022 09:21:40 +0100 Subject: [PATCH 24/24] bump frontend & reduce nb threads for mu search indexing --- config/search/config.json | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/search/config.json b/config/search/config.json index f3d5ae4..78fe1ac 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -1,7 +1,7 @@ { "automatic_index_updates": true, "persist_indexes": true, - "number_of_threads": 10, + "number_of_threads": 6, "eager_indexing_groups": [ [{ "variables": [], "name": "acmidm-lezer" }], [{ "variables": [], "name": "acmidm-editeerder" }], diff --git a/docker-compose.yml b/docker-compose.yml index 848b6d3..dd4c91e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-contact-hub:1.6.0 + image: lblod/frontend-contact-hub:1.6.3 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: