From 44c5cc4f67863b00f0ae9b42c89dddea383e8d3b Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Wed, 9 Mar 2022 16:45:26 +0100 Subject: [PATCH 01/52] fix index (#116) * fix index * add combination of editeerder / beheerder * add all possible combinations --- config/authorization/config.ex | 26 +- config/search/config.json | 386 ++++++++++++++++------ config/search/dev/config.json | 584 +++++++++++++++++++++++++++++++++ docker-compose.dev.yml | 2 + 4 files changed, 865 insertions(+), 133 deletions(-) create mode 100644 config/search/dev/config.json diff --git a/config/authorization/config.ex b/config/authorization/config.ex index d1784cb..e99ed52 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -68,7 +68,7 @@ defmodule Acl.UserGroups.Config do defp sparql_query_for_access_role( group_string ) do "PREFIX ext: PREFIX mu: - SELECT ?session_group ?session_role WHERE { + SELECT distinct ?session_group ?session_role WHERE { ext:sessionGroup/mu:uuid ?session_group; ext:sessionRole ?session_role. FILTER( ?session_role = \"#{group_string}\" ) @@ -116,30 +116,6 @@ defmodule Acl.UserGroups.Config do } ] }, - %GroupSpec{ - name: "ch-lezer", - useage: [:read], - access: access_by_role( "ABBOrganisatiePortaalGebruiker-lezer" ), - graphs: [ %GraphSpec{ - graph: "http://mu.semte.ch/graphs/organisatieportaal", - constraint: %ResourceConstraint{ - resource_types: @protected_resource_type } } ] }, - %GroupSpec{ - name: "ch-editeerder", - useage: [:read, :write, :read_for_write], - access: access_by_role( "ABBOrganisatiePortaalGebruiker-editeerder" ), - graphs: [ %GraphSpec{ - graph: "http://mu.semte.ch/graphs/organisatieportaal", - constraint: %ResourceConstraint{ - resource_types: @protected_resource_type } } ] }, - %GroupSpec{ - name: "ch-beheerder", - useage: [:read, :write, :read_for_write], - access: access_by_role( "ABBOrganisatiePortaalGebruiker-beheerder" ), - graphs: [ %GraphSpec{ - graph: "http://mu.semte.ch/graphs/organisatieportaal", - constraint: %ResourceConstraint{ - resource_types: @protected_resource_type } } ] }, %GroupSpec{ name: "public", useage: [:read], diff --git a/config/search/config.json b/config/search/config.json index 78fe1ac..4954cb8 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -3,9 +3,124 @@ "persist_indexes": true, "number_of_threads": 6, "eager_indexing_groups": [ - [{ "variables": [], "name": "acmidm-lezer" }], - [{ "variables": [], "name": "acmidm-editeerder" }], - [{ "variables": [], "name": "acmidm-beheerder" }] + [ + { + "variables": [], + "name": "acmidm-lezer" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ], + [ + { + "variables": [], + "name": "acmidm-editeerder" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ], + [ + { + "variables": [], + "name": "acmidm-beheerder" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ], + [ + { + "variables": [], + "name": "acmidm-beheerder" + }, + { + "variables": [], + "name": "acmidm-editeerder" + }, + { + "variables": [], + "name": "acmidm-lezer" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ], + [ + { + "variables": [], + "name": "acmidm-editeerder" + }, + { + "variables": [], + "name": "acmidm-lezer" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ], + [ + { + "variables": [], + "name": "acmidm-editeerder" + }, + { + "variables": [], + "name": "acmidm-beheerder" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ], + [ + { + "variables": [], + "name": "acmidm-lezer" + }, + { + "variables": [], + "name": "acmidm-beheerder" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ] ], "types": [ { @@ -13,11 +128,26 @@ "on_path": "mandatories", "rdf_type": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", "properties": { - "given_name": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://xmlns.com/foaf/0.1/givenName"], - "id": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://mu.semte.ch/vocabularies/core/uuid"], - "uuid": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://mu.semte.ch/vocabularies/core/uuid"], - "family_name": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://xmlns.com/foaf/0.1/familyName"], - "first_name_used": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam"], + "given_name": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "http://xmlns.com/foaf/0.1/givenName" + ], + "id": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "uuid": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "family_name": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "http://xmlns.com/foaf/0.1/familyName" + ], + "first_name_used": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam" + ], "end_date": "http://data.vlaanderen.be/ns/mandaat#einde", "organization_id": [ "http://www.w3.org/ns/org#holds", @@ -62,7 +192,6 @@ "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, - "mappings": { "properties": { "family_name": { @@ -89,7 +218,9 @@ } } }, - "organization_id": { "type": "text" }, + "organization_id": { + "type": "text" + }, "organization_province": { "type": "text", "fields": { @@ -114,7 +245,9 @@ } } }, - "position_id": { "type": "text" }, + "position_id": { + "type": "text" + }, "position_name": { "type": "text", "fields": { @@ -123,7 +256,9 @@ } } }, - "end_date": { "type": "date" } + "end_date": { + "type": "date" + } } } }, @@ -131,13 +266,30 @@ "type": "minister", "on_path": "ministers", "rdf_type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", - "sub_types": ["http://data.lblod.info/vocabularies/contacthub/AgentInPositie"], + "sub_types": [ + "http://data.lblod.info/vocabularies/contacthub/AgentInPositie" + ], "properties": { - "given_name": ["http://www.w3.org/ns/org#heldBy","http://xmlns.com/foaf/0.1/givenName"], - "id": ["http://www.w3.org/ns/org#heldBy","http://mu.semte.ch/vocabularies/core/uuid"], - "uuid": ["http://www.w3.org/ns/org#heldBy","http://mu.semte.ch/vocabularies/core/uuid"], - "family_name": ["http://www.w3.org/ns/org#heldBy","http://xmlns.com/foaf/0.1/familyName"], - "first_name_used": ["http://www.w3.org/ns/org#heldBy","https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam"], + "given_name": [ + "http://www.w3.org/ns/org#heldBy", + "http://xmlns.com/foaf/0.1/givenName" + ], + "id": [ + "http://www.w3.org/ns/org#heldBy", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "uuid": [ + "http://www.w3.org/ns/org#heldBy", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "family_name": [ + "http://www.w3.org/ns/org#heldBy", + "http://xmlns.com/foaf/0.1/familyName" + ], + "first_name_used": [ + "http://www.w3.org/ns/org#heldBy", + "https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam" + ], "end_date": "http://data.lblod.info/vocabularies/contacthub/eindedatum", "organization_id": [ "http://www.w3.org/ns/org#holds", @@ -174,7 +326,6 @@ "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, - "mappings": { "properties": { "family_name": { @@ -201,7 +352,9 @@ } } }, - "organization_id": { "type": "text" }, + "organization_id": { + "type": "text" + }, "organization_province": { "type": "text", "fields": { @@ -226,7 +379,9 @@ } } }, - "position_id": { "type": "text" }, + "position_id": { + "type": "text" + }, "position_name": { "type": "text", "fields": { @@ -235,92 +390,97 @@ } } }, - "end_date": { "type": "date" } + "end_date": { + "type": "date" + } } } }, { - "type" : "person", - "composite_types" : ["minister", "mandatory"], - "on_path" : "people", - "properties" : [ - { - "name" : "given_name", - "mappings" : { - "minister" : "given_name", - "mandatory" : "given_name" - } - }, - { - "name" : "id", - "mappings" : { - "minister" : "id", - "mandatory" : "id" - } - }, - { - "name" : "family_name", - "mappings" : { - "minister" : "family_name", - "mandatory" : "family_name" - } - }, - { - "name" : "first_name_used", - "mappings" : { - "minister" : "first_name_used", - "mandatory" : "first_name_used" - } - }, - { - "name" : "end_date", - "mappings" : { - "minister" : "end_date", - "mandatory" : "end_date" - } - }, - { - "name" : "organization_id", - "mappings" : { - "minister" : "organization_id", - "mandatory" : "organization_id" - } - }, - { - "name" : "organization_name", - "mappings" : { - "minister" : "organization_name", - "mandatory" : "organization_name" - } - }, - { - "name" : "organization_municipality", - "mappings" : { - "minister" : "organization_municipality", - "mandatory" : "organization_municipality" - } - }, - { - "name" : "organization_province", - "mappings" : { - "minister" : "organization_province", - "mandatory" : "organization_province" - } - }, - { - "name" : "position_id", - "mappings" : { - "minister" : "position_id", - "mandatory" : "position_id" - } - }, - { - "name" : "position_name", - "mappings" : { - "minister" : "position_name", - "mandatory" : "position_name" - } + "type": "person", + "composite_types": [ + "minister", + "mandatory" + ], + "on_path": "people", + "properties": [ + { + "name": "given_name", + "mappings": { + "minister": "given_name", + "mandatory": "given_name" } + }, + { + "name": "id", + "mappings": { + "minister": "id", + "mandatory": "id" + } + }, + { + "name": "family_name", + "mappings": { + "minister": "family_name", + "mandatory": "family_name" + } + }, + { + "name": "first_name_used", + "mappings": { + "minister": "first_name_used", + "mandatory": "first_name_used" + } + }, + { + "name": "end_date", + "mappings": { + "minister": "end_date", + "mandatory": "end_date" + } + }, + { + "name": "organization_id", + "mappings": { + "minister": "organization_id", + "mandatory": "organization_id" + } + }, + { + "name": "organization_name", + "mappings": { + "minister": "organization_name", + "mandatory": "organization_name" + } + }, + { + "name": "organization_municipality", + "mappings": { + "minister": "organization_municipality", + "mandatory": "organization_municipality" + } + }, + { + "name": "organization_province", + "mappings": { + "minister": "organization_province", + "mandatory": "organization_province" + } + }, + { + "name": "position_id", + "mappings": { + "minister": "position_id", + "mandatory": "position_id" + } + }, + { + "name": "position_name", + "mappings": { + "minister": "position_name", + "mandatory": "position_name" + } + } ], "mappings": { "properties": { @@ -348,7 +508,9 @@ } } }, - "organization_id": { "type": "text" }, + "organization_id": { + "type": "text" + }, "organization_province": { "type": "text", "fields": { @@ -373,7 +535,9 @@ } } }, - "position_id": { "type": "text" }, + "position_id": { + "type": "text" + }, "position_name": { "type": "text", "fields": { @@ -382,10 +546,12 @@ } } }, - "end_date": { "type": "date" } + "end_date": { + "type": "date" + } } } - }, + }, { "type": "unit", "on_path": "units", @@ -433,8 +599,12 @@ }, "mappings": { "properties": { - "classification_id": { "type": "text" }, - "recognized_worship_type_id": { "type": "text" }, + "classification_id": { + "type": "text" + }, + "recognized_worship_type_id": { + "type": "text" + }, "name": { "type": "text", "fields": { diff --git a/config/search/dev/config.json b/config/search/dev/config.json new file mode 100644 index 0000000..38571e2 --- /dev/null +++ b/config/search/dev/config.json @@ -0,0 +1,584 @@ +{ + "automatic_index_updates": true, + "persist_indexes": true, + "number_of_threads": 6, + "eager_indexing_groups": [ + [ + { + "variables": [], + "name": "acmidm-lezer" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ], + [ + { + "variables": [], + "name": "acmidm-editeerder" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ], + [ + { + "variables": [], + "name": "acmidm-beheerder" + }, + { + "variables": [], + "name": "clean" + }, + { + "variables": [], + "name": "public" + } + ] + ], + "types": [ + { + "type": "mandatory", + "on_path": "mandatories", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", + "properties": { + "given_name": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "http://xmlns.com/foaf/0.1/givenName" + ], + "id": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "uuid": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "family_name": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "http://xmlns.com/foaf/0.1/familyName" + ], + "first_name_used": [ + "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan", + "https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam" + ], + "end_date": "http://data.vlaanderen.be/ns/mandaat#einde", + "organization_id": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "organization_name": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "organization_municipality": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ], + "organization_province": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "position_id": [ + "http://www.w3.org/ns/org#holds", + "http://www.w3.org/ns/org#role", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "position_name": [ + "http://www.w3.org/ns/org#holds", + "http://www.w3.org/ns/org#role", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { + "type": "text" + }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { + "type": "text" + }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { + "type": "date" + } + } + } + }, + { + "type": "minister", + "on_path": "ministers", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", + "sub_types": [ + "http://data.lblod.info/vocabularies/contacthub/AgentInPositie" + ], + "properties": { + "given_name": [ + "http://www.w3.org/ns/org#heldBy", + "http://xmlns.com/foaf/0.1/givenName" + ], + "id": [ + "http://www.w3.org/ns/org#heldBy", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "uuid": [ + "http://www.w3.org/ns/org#heldBy", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "family_name": [ + "http://www.w3.org/ns/org#heldBy", + "http://xmlns.com/foaf/0.1/familyName" + ], + "first_name_used": [ + "http://www.w3.org/ns/org#heldBy", + "https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam" + ], + "end_date": "http://data.lblod.info/vocabularies/contacthub/eindedatum", + "organization_id": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "organization_name": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "organization_municipality": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ], + "organization_province": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "position_id": [ + "http://www.w3.org/ns/org#holds", + "http://data.lblod.info/vocabularies/erediensten/functie", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "position_name": [ + "http://www.w3.org/ns/org#holds", + "http://data.lblod.info/vocabularies/erediensten/functie", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { + "type": "text" + }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { + "type": "text" + }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { + "type": "date" + } + } + } + }, + { + "type": "person", + "composite_types": [ + "minister", + "mandatory" + ], + "on_path": "people", + "properties": [ + { + "name": "given_name", + "mappings": { + "minister": "given_name", + "mandatory": "given_name" + } + }, + { + "name": "id", + "mappings": { + "minister": "id", + "mandatory": "id" + } + }, + { + "name": "family_name", + "mappings": { + "minister": "family_name", + "mandatory": "family_name" + } + }, + { + "name": "first_name_used", + "mappings": { + "minister": "first_name_used", + "mandatory": "first_name_used" + } + }, + { + "name": "end_date", + "mappings": { + "minister": "end_date", + "mandatory": "end_date" + } + }, + { + "name": "organization_id", + "mappings": { + "minister": "organization_id", + "mandatory": "organization_id" + } + }, + { + "name": "organization_name", + "mappings": { + "minister": "organization_name", + "mandatory": "organization_name" + } + }, + { + "name": "organization_municipality", + "mappings": { + "minister": "organization_municipality", + "mandatory": "organization_municipality" + } + }, + { + "name": "organization_province", + "mappings": { + "minister": "organization_province", + "mandatory": "organization_province" + } + }, + { + "name": "position_id", + "mappings": { + "minister": "position_id", + "mandatory": "position_id" + } + }, + { + "name": "position_name", + "mappings": { + "minister": "position_name", + "mandatory": "position_name" + } + } + ], + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { + "type": "text" + }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { + "type": "text" + }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { + "type": "date" + } + } + } + }, + { + "type": "unit", + "on_path": "units", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid", + "sub_types": [ + "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst" + ], + "properties": { + "name": "http://www.w3.org/2004/02/skos/core#prefLabel", + "classification_id": [ + "http://www.w3.org/ns/org#classification", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "classification_name": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "status": [ + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "status_id": [ + "http://www.w3.org/ns/regorg#orgStatus", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "recognized_worship_type_id": [ + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "recognized_worship_type_name": [ + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "province": [ + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "municipality": [ + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ] + }, + "mappings": { + "properties": { + "classification_id": { + "type": "text" + }, + "recognized_worship_type_id": { + "type": "text" + }, + "name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "classification_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "recognized_worship_type_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 61f384e..26a56be 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -61,6 +61,8 @@ services: restart: "no" search: restart: "no" + volumes: + - ./config/search/dev:/config elasticsearch: restart: "no" file: From 4f54ef476ce04fbd33f7ee5abb2152d184b3dfb5 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 10 Mar 2022 19:05:22 +0100 Subject: [PATCH 02/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index dd4c91e..ec6a374 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-contact-hub:1.6.3 + image: lblod/frontend-contact-hub:1.6.4-0 # Pre patch (similar to "-beta" but using the convention of release-it) volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 58110ad8e022fa81354a8a84154200f5c0c2f749 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Mon, 14 Mar 2022 13:23:11 +0100 Subject: [PATCH 03/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ec6a374..5e24a46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-contact-hub:1.6.4-0 # Pre patch (similar to "-beta" but using the convention of release-it) + image: lblod/frontend-contact-hub:1.6.4-1 # Pre patch (similar to "-beta" but using the convention of release-it) volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 86afcb6ea5de1a80cfc64e40a54de8b40d9f9bc4 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 15 Mar 2022 09:33:58 +0100 Subject: [PATCH 04/52] Add new change event date --- config/authorization/config.ex | 1 + config/dispatcher/dispatcher.ex | 3 +++ config/resources/change-events.json | 25 +++++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/config/authorization/config.ex b/config/authorization/config.ex index e99ed52..fb10aaa 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -10,6 +10,7 @@ defmodule Acl.UserGroups.Config do @protected_resource_type [ "http://www.w3.org/ns/org#Organization", "http://data.vlaanderen.be/ns/besluit#Besluit", + "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit", "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index b155a18..88d8395 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -195,6 +195,9 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://cache/decisions/" end + match "/decision-activities/*path", %{ accept: [:json], layer: :api} do + Proxy.forward conn, path, "http://cache/decision-activities/" + end ############### # LOGIN diff --git a/config/resources/change-events.json b/config/resources/change-events.json index 2ac99e5..1ed1021 100644 --- a/config/resources/change-events.json +++ b/config/resources/change-events.json @@ -98,7 +98,32 @@ "predicate": "foaf:page" } }, + "relationships": { + "has-decision-activity": { + "predicate": "http://www.w3.org/ns/prov#generated", + "target": "decision-activities", + "cardinality": "one", + "inverse": true + } + }, "new-resource-base": "http://lblod.data.info/id/besluiten/" + }, + "decision-activities": { + "class": "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit", + "attributes": { + "end-date": { + "type": "date", + "predicate": "https://data.vlaanderen.be/ns/dossier#Activiteit.einddatum" + } + }, + "relationships": { + "gives-cause-to": { + "predicate": "http://www.w3.org/ns/prov#generated", + "target": "decisions", + "cardinality": "many" + } + }, + "new-resource-base": "http://lblod.data.info/id/beslissingsactiviteit/" } } } From 01968b304415f55a56563c1b89e5fbcbe3eaf53d Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 17 Mar 2022 11:32:42 +0100 Subject: [PATCH 05/52] Delete bestuurseenheden without status (op-1075) --- ...000-remove-bestuurseenheden-op-1075.sparql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 config/migrations/2022/20220317100000-remove-bestuurseenheden-op-1075.sparql diff --git a/config/migrations/2022/20220317100000-remove-bestuurseenheden-op-1075.sparql b/config/migrations/2022/20220317100000-remove-bestuurseenheden-op-1075.sparql new file mode 100644 index 0000000..9d75fbc --- /dev/null +++ b/config/migrations/2022/20220317100000-remove-bestuurseenheden-op-1075.sparql @@ -0,0 +1,19 @@ +# Deletes 2 worships services and their attributes (id, address, ...) + +DELETE { + GRAPH ?g { + ?bestuur ?p ?o. + ?o ?p2 ?o2. + ?o2 ?p3 ?o3. + } +} WHERE { + GRAPH ?g { + VALUES ?bestuur { + + + } + ?bestuur ?p ?o. + ?o ?p2 ?o2. + ?o2 ?p3 ?o3. + } +} From e1209a5473e10a27b3ba4fb0da68ba667ee00cc5 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 17 Mar 2022 17:29:19 +0100 Subject: [PATCH 06/52] query rewrite/fix --- ...000-remove-bestuurseenheden-op-1075.sparql | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/config/migrations/2022/20220317100000-remove-bestuurseenheden-op-1075.sparql b/config/migrations/2022/20220317100000-remove-bestuurseenheden-op-1075.sparql index 9d75fbc..ca963b3 100644 --- a/config/migrations/2022/20220317100000-remove-bestuurseenheden-op-1075.sparql +++ b/config/migrations/2022/20220317100000-remove-bestuurseenheden-op-1075.sparql @@ -1,10 +1,12 @@ -# Deletes 2 worships services and their attributes (id, address, ...) - DELETE { GRAPH ?g { - ?bestuur ?p ?o. - ?o ?p2 ?o2. - ?o2 ?p3 ?o3. + ?representatieveOrgaan ?bestuur . + ?bestuur ?pbestuur ?obestuur . + ?primarySite ?pprimarySite ?oprimarySite . + ?address ?paddress ?oaddress . + ?contactPoint ?pcontactPoint ?ocontactPoint . + ?identifier ?pidentifier ?oidentifier . + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . } } WHERE { GRAPH ?g { @@ -12,8 +14,23 @@ DELETE { } - ?bestuur ?p ?o. - ?o ?p2 ?o2. - ?o2 ?p3 ?o3. + + ?bestuur ?primarySite ; + ?identifier ; + ?pbestuur ?obestuur . + + ?primarySite ?address ; + ?contactPoint ; + ?pprimarySite ?oprimarySite . + + ?address ?paddress ?oaddress . + + ?contactPoint ?pcontactPoint ?ocontactPoint . + + ?identifier ?structuredIdentifier ; + ?pidentifier ?oidentifier . + + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . } } + From db7fe5ed4f9921ee12e86f51236a74ee5f5be3ca Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Fri, 18 Mar 2022 09:12:10 +0100 Subject: [PATCH 07/52] sensitive data (#118) --- config/dispatcher/dispatcher.ex | 8 ++++++++ docker-compose.yml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 88d8395..0adabb7 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -254,6 +254,14 @@ defmodule Dispatcher do forward conn, path, "http://privacy/person-information-requests" end + match "/person-information-ask/*path", %{ layer: :api_services, accept: %{ json: true } } do + forward conn, path, "http://privacy/person-information-ask/" + end + + match "/person-information-validate-ssn/*path", %{ layer: :api_services, accept: %{ json: true } } do + forward conn, path, "http://privacy/person-information-validate-ssn/" + end + get "/uri-info/*path", %{ layer: :api_services, accept: %{ json: true } } do forward conn, path, "http://uri-info/" end diff --git a/docker-compose.yml b/docker-compose.yml index 5e24a46..ed16ead 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,7 +138,7 @@ services: restart: always logging: *default-logging privacy: - image: lblod/privacy-centric-service:0.3.0 + image: lblod/privacy-centric-service:0.4.0 labels: - "logging=true" environment: From 5ee1ba43f42e177c07477066779ba85635583775 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 21 Mar 2022 15:10:49 +0100 Subject: [PATCH 08/52] Rename the project from contact-hub to organization-portal --- README.md | 4 ++-- docker-compose.dev.yml | 2 +- docker-compose.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4741e2..a59e0e3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Contact Hub +# Organization portal -Backend for the Contact Hub application based on the mu.semte.ch microservices stack. +Backend for the organization portal application, based on the mu.semte.ch microservices stack. ## How to diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 26a56be..b705c26 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -43,7 +43,7 @@ services: adressenregister: restart: "no" frontend: - image: lblod/frontend-contact-hub:latest + image: lblod/frontend-organization-portal:latest restart: "no" uri-info: restart: "no" diff --git a/docker-compose.yml b/docker-compose.yml index 5e24a46..ae95189 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-contact-hub:1.6.4-1 # Pre patch (similar to "-beta" but using the convention of release-it) + image: lblod/frontend-organization-portal:1.6.4-2 # Pre patch (similar to "-beta" but using the convention of release-it) volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From cd66d02e4fea4a31c6aae71a6c30e9d741750a7d Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Mon, 21 Mar 2022 16:22:34 +0100 Subject: [PATCH 09/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ae95189..8d2c7d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.6.4-2 # Pre patch (similar to "-beta" but using the convention of release-it) + image: lblod/frontend-organization-portal:1.6.4-3 # Pre patch (similar to "-beta" but using the convention of release-it) volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 7532f0fd026d236041c3e69fc0e3421583eab8e6 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 15 Mar 2022 09:33:58 +0100 Subject: [PATCH 10/52] Add new change event date --- config/authorization/config.ex | 1 + config/dispatcher/dispatcher.ex | 3 +++ config/resources/change-events.json | 25 +++++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/config/authorization/config.ex b/config/authorization/config.ex index e99ed52..fb10aaa 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -10,6 +10,7 @@ defmodule Acl.UserGroups.Config do @protected_resource_type [ "http://www.w3.org/ns/org#Organization", "http://data.vlaanderen.be/ns/besluit#Besluit", + "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit", "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index b155a18..88d8395 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -195,6 +195,9 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://cache/decisions/" end + match "/decision-activities/*path", %{ accept: [:json], layer: :api} do + Proxy.forward conn, path, "http://cache/decision-activities/" + end ############### # LOGIN diff --git a/config/resources/change-events.json b/config/resources/change-events.json index 2ac99e5..1ed1021 100644 --- a/config/resources/change-events.json +++ b/config/resources/change-events.json @@ -98,7 +98,32 @@ "predicate": "foaf:page" } }, + "relationships": { + "has-decision-activity": { + "predicate": "http://www.w3.org/ns/prov#generated", + "target": "decision-activities", + "cardinality": "one", + "inverse": true + } + }, "new-resource-base": "http://lblod.data.info/id/besluiten/" + }, + "decision-activities": { + "class": "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit", + "attributes": { + "end-date": { + "type": "date", + "predicate": "https://data.vlaanderen.be/ns/dossier#Activiteit.einddatum" + } + }, + "relationships": { + "gives-cause-to": { + "predicate": "http://www.w3.org/ns/prov#generated", + "target": "decisions", + "cardinality": "many" + } + }, + "new-resource-base": "http://lblod.data.info/id/beslissingsactiviteit/" } } } From e799bdac565169118e7cdc37bbcb459df34ab3ea Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 21 Mar 2022 17:00:41 +0100 Subject: [PATCH 11/52] Bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8d2c7d8..b8b6eca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.6.4-3 # Pre patch (similar to "-beta" but using the convention of release-it) + image: lblod/frontend-organization-portal:1.6.4-4 # Pre patch (similar to "-beta" but using the convention of release-it) volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From d0950575cca9eabab556d5b2f032cce3fef0c7ed Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Tue, 22 Mar 2022 09:32:38 +0100 Subject: [PATCH 12/52] bump privacy service --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c83fa44..4452f95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,7 +138,7 @@ services: restart: always logging: *default-logging privacy: - image: lblod/privacy-centric-service:0.4.0 + image: lblod/privacy-centric-service:0.4.1 labels: - "logging=true" environment: From 956faa2d6f04c17289c292518e696fe8fb90f20b Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Tue, 22 Mar 2022 14:59:49 +0100 Subject: [PATCH 13/52] fix relationship ws (#121) --- config/resources/domain.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/resources/domain.json b/config/resources/domain.json index f1bd579..6f5142c 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -392,6 +392,11 @@ "predicate": "ere:typeEredienst", "target": "recognized-worship-types", "cardinality": "one" + }, + "minister-positions": { + "predicate": "ere:wordtBediendDoor", + "target": "minister-positions", + "cardinality": "many" } }, "new-resource-base": "http://data.lblod.info/id/eredienstBestuurseenheden/" @@ -416,11 +421,6 @@ "target": "local-involvements", "cardinality": "many", "inverse": true - }, - "minister-positions": { - "predicate": "ere:wordtBediendDoor", - "target": "minister-positions", - "cardinality": "many" } }, "new-resource-base": "http://data.lblod.info/id/besturenVanDeEredienst/" @@ -756,7 +756,7 @@ }, "worship-service": { "predicate": "ere:wordtBediendDoor", - "target": "worship-services", + "target": "worship-administrative-units", "cardinality": "one", "inverse": true }, From b1f5dd09d6df609e6077578f7900c3465dc92b81 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Wed, 23 Mar 2022 09:56:35 +0100 Subject: [PATCH 14/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b8b6eca..672eb1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.6.4-4 # Pre patch (similar to "-beta" but using the convention of release-it) + image: lblod/frontend-organization-portal:1.6.4-5 # Pre patch (similar to "-beta" but using the convention of release-it) volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 6bd631fa23beb594cc1da3b02f020205145db81a Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 24 Mar 2022 19:14:07 +0100 Subject: [PATCH 15/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 672eb1e..768dd1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.6.4-5 # Pre patch (similar to "-beta" but using the convention of release-it) + image: lblod/frontend-organization-portal:1.6.4 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From a82cb06cc7f02e7dcd3949185d1068dd6db50bcb Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Fri, 25 Mar 2022 12:18:18 +0100 Subject: [PATCH 16/52] bump frontend prerelase 1.7.0-0 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5103ad9..abc0703 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.6.4 + image: lblod/frontend-organization-portal:1.7.0-0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From ebb29eb07536d0654068284fadf67b07c72df064 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Fri, 25 Mar 2022 17:00:29 +0100 Subject: [PATCH 17/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index abc0703..931d7ed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.7.0-0 + image: lblod/frontend-organization-portal:1.7.0-1 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 064eda052618f3e0e1c21de4e4f9a31d0f669cff Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 28 Mar 2022 17:11:25 +0200 Subject: [PATCH 18/52] Organize migrations --- config/migrations/{ => 2021}/20210701120000-meta.ttl | 0 .../20210729182256-bestuurseenheid-classificatie-code-uuid.ttl | 0 .../20210729182256-bestuurseenheid-classificatie-code.ttl | 0 config/migrations/{ => 2021}/20210729182256-geslacht-code.ttl | 0 .../{ => 2021}/20210729182256-mandataris-status-code-uuid.ttl | 0 .../{ => 2021}/20210729182256-mandataris-status-code.ttl | 0 config/migrations/{ => 2021}/20210729182257-reason-code.ttl | 0 .../20210830174500-delete-label-from-bestuurseenheid.sparql | 0 .../migrations/{ => 2021}/20210831193441-codelist-nationality.ttl | 0 .../{ => 2021}/20210902170100-rename-location-fusion.sparql | 0 .../{ => 2021}/20210906150000-fix-classification-property.sparql | 0 .../{ => 2021}/20210909152600-fix-percentage-sort-part-1.sparql | 0 .../{ => 2021}/20210909152630-fix-percentage-sort-part-2.sparql | 0 .../migrations/{ => 2021}/20211119151700-move-reason-codes.sparql | 0 .../migrations/{ => 2021}/20211120120000-create-mock-group.sparql | 0 .../migrations/{ => 2021}/20211120130000-create-mock-user.sparql | 0 config/migrations/{ => 2021}/20211120140000-auth.sparql | 0 .../20211120150000-remove-acmidm-account-from-public.sparql | 0 config/migrations/{ => 2021}/20211120160000-source-crab.sparql | 0 config/migrations/{ => 2021}/20211123120000-private-data.sparql | 0 config/migrations/{ => 2021}/20211203120000-bestuur-admidm.sparql | 0 .../migrations/{ => 2021}/20211203130000-clear-mock-data.sparql | 0 .../migrations/{ => 2021}/20211206120000-mock-user-roles.sparql | 0 config/migrations/{ => 2021}/20211206130000-move-graph.sparql | 0 .../{ => 2022}/20220110120000-delete-old-change-events.sparql | 0 .../{ => 2022}/20220120100418-remove-old-reasons.sparql | 0 .../{ => 2022}/20220120101318-insert-new-reasons.sparql | 0 27 files changed, 0 insertions(+), 0 deletions(-) rename config/migrations/{ => 2021}/20210701120000-meta.ttl (100%) rename config/migrations/{ => 2021}/20210729182256-bestuurseenheid-classificatie-code-uuid.ttl (100%) rename config/migrations/{ => 2021}/20210729182256-bestuurseenheid-classificatie-code.ttl (100%) rename config/migrations/{ => 2021}/20210729182256-geslacht-code.ttl (100%) rename config/migrations/{ => 2021}/20210729182256-mandataris-status-code-uuid.ttl (100%) rename config/migrations/{ => 2021}/20210729182256-mandataris-status-code.ttl (100%) rename config/migrations/{ => 2021}/20210729182257-reason-code.ttl (100%) rename config/migrations/{ => 2021}/20210830174500-delete-label-from-bestuurseenheid.sparql (100%) rename config/migrations/{ => 2021}/20210831193441-codelist-nationality.ttl (100%) rename config/migrations/{ => 2021}/20210902170100-rename-location-fusion.sparql (100%) rename config/migrations/{ => 2021}/20210906150000-fix-classification-property.sparql (100%) rename config/migrations/{ => 2021}/20210909152600-fix-percentage-sort-part-1.sparql (100%) rename config/migrations/{ => 2021}/20210909152630-fix-percentage-sort-part-2.sparql (100%) rename config/migrations/{ => 2021}/20211119151700-move-reason-codes.sparql (100%) rename config/migrations/{ => 2021}/20211120120000-create-mock-group.sparql (100%) rename config/migrations/{ => 2021}/20211120130000-create-mock-user.sparql (100%) rename config/migrations/{ => 2021}/20211120140000-auth.sparql (100%) rename config/migrations/{ => 2021}/20211120150000-remove-acmidm-account-from-public.sparql (100%) rename config/migrations/{ => 2021}/20211120160000-source-crab.sparql (100%) rename config/migrations/{ => 2021}/20211123120000-private-data.sparql (100%) rename config/migrations/{ => 2021}/20211203120000-bestuur-admidm.sparql (100%) rename config/migrations/{ => 2021}/20211203130000-clear-mock-data.sparql (100%) rename config/migrations/{ => 2021}/20211206120000-mock-user-roles.sparql (100%) rename config/migrations/{ => 2021}/20211206130000-move-graph.sparql (100%) rename config/migrations/{ => 2022}/20220110120000-delete-old-change-events.sparql (100%) rename config/migrations/{ => 2022}/20220120100418-remove-old-reasons.sparql (100%) rename config/migrations/{ => 2022}/20220120101318-insert-new-reasons.sparql (100%) diff --git a/config/migrations/20210701120000-meta.ttl b/config/migrations/2021/20210701120000-meta.ttl similarity index 100% rename from config/migrations/20210701120000-meta.ttl rename to config/migrations/2021/20210701120000-meta.ttl diff --git a/config/migrations/20210729182256-bestuurseenheid-classificatie-code-uuid.ttl b/config/migrations/2021/20210729182256-bestuurseenheid-classificatie-code-uuid.ttl similarity index 100% rename from config/migrations/20210729182256-bestuurseenheid-classificatie-code-uuid.ttl rename to config/migrations/2021/20210729182256-bestuurseenheid-classificatie-code-uuid.ttl diff --git a/config/migrations/20210729182256-bestuurseenheid-classificatie-code.ttl b/config/migrations/2021/20210729182256-bestuurseenheid-classificatie-code.ttl similarity index 100% rename from config/migrations/20210729182256-bestuurseenheid-classificatie-code.ttl rename to config/migrations/2021/20210729182256-bestuurseenheid-classificatie-code.ttl diff --git a/config/migrations/20210729182256-geslacht-code.ttl b/config/migrations/2021/20210729182256-geslacht-code.ttl similarity index 100% rename from config/migrations/20210729182256-geslacht-code.ttl rename to config/migrations/2021/20210729182256-geslacht-code.ttl diff --git a/config/migrations/20210729182256-mandataris-status-code-uuid.ttl b/config/migrations/2021/20210729182256-mandataris-status-code-uuid.ttl similarity index 100% rename from config/migrations/20210729182256-mandataris-status-code-uuid.ttl rename to config/migrations/2021/20210729182256-mandataris-status-code-uuid.ttl diff --git a/config/migrations/20210729182256-mandataris-status-code.ttl b/config/migrations/2021/20210729182256-mandataris-status-code.ttl similarity index 100% rename from config/migrations/20210729182256-mandataris-status-code.ttl rename to config/migrations/2021/20210729182256-mandataris-status-code.ttl diff --git a/config/migrations/20210729182257-reason-code.ttl b/config/migrations/2021/20210729182257-reason-code.ttl similarity index 100% rename from config/migrations/20210729182257-reason-code.ttl rename to config/migrations/2021/20210729182257-reason-code.ttl diff --git a/config/migrations/20210830174500-delete-label-from-bestuurseenheid.sparql b/config/migrations/2021/20210830174500-delete-label-from-bestuurseenheid.sparql similarity index 100% rename from config/migrations/20210830174500-delete-label-from-bestuurseenheid.sparql rename to config/migrations/2021/20210830174500-delete-label-from-bestuurseenheid.sparql diff --git a/config/migrations/20210831193441-codelist-nationality.ttl b/config/migrations/2021/20210831193441-codelist-nationality.ttl similarity index 100% rename from config/migrations/20210831193441-codelist-nationality.ttl rename to config/migrations/2021/20210831193441-codelist-nationality.ttl diff --git a/config/migrations/20210902170100-rename-location-fusion.sparql b/config/migrations/2021/20210902170100-rename-location-fusion.sparql similarity index 100% rename from config/migrations/20210902170100-rename-location-fusion.sparql rename to config/migrations/2021/20210902170100-rename-location-fusion.sparql diff --git a/config/migrations/20210906150000-fix-classification-property.sparql b/config/migrations/2021/20210906150000-fix-classification-property.sparql similarity index 100% rename from config/migrations/20210906150000-fix-classification-property.sparql rename to config/migrations/2021/20210906150000-fix-classification-property.sparql diff --git a/config/migrations/20210909152600-fix-percentage-sort-part-1.sparql b/config/migrations/2021/20210909152600-fix-percentage-sort-part-1.sparql similarity index 100% rename from config/migrations/20210909152600-fix-percentage-sort-part-1.sparql rename to config/migrations/2021/20210909152600-fix-percentage-sort-part-1.sparql diff --git a/config/migrations/20210909152630-fix-percentage-sort-part-2.sparql b/config/migrations/2021/20210909152630-fix-percentage-sort-part-2.sparql similarity index 100% rename from config/migrations/20210909152630-fix-percentage-sort-part-2.sparql rename to config/migrations/2021/20210909152630-fix-percentage-sort-part-2.sparql diff --git a/config/migrations/20211119151700-move-reason-codes.sparql b/config/migrations/2021/20211119151700-move-reason-codes.sparql similarity index 100% rename from config/migrations/20211119151700-move-reason-codes.sparql rename to config/migrations/2021/20211119151700-move-reason-codes.sparql diff --git a/config/migrations/20211120120000-create-mock-group.sparql b/config/migrations/2021/20211120120000-create-mock-group.sparql similarity index 100% rename from config/migrations/20211120120000-create-mock-group.sparql rename to config/migrations/2021/20211120120000-create-mock-group.sparql diff --git a/config/migrations/20211120130000-create-mock-user.sparql b/config/migrations/2021/20211120130000-create-mock-user.sparql similarity index 100% rename from config/migrations/20211120130000-create-mock-user.sparql rename to config/migrations/2021/20211120130000-create-mock-user.sparql diff --git a/config/migrations/20211120140000-auth.sparql b/config/migrations/2021/20211120140000-auth.sparql similarity index 100% rename from config/migrations/20211120140000-auth.sparql rename to config/migrations/2021/20211120140000-auth.sparql diff --git a/config/migrations/20211120150000-remove-acmidm-account-from-public.sparql b/config/migrations/2021/20211120150000-remove-acmidm-account-from-public.sparql similarity index 100% rename from config/migrations/20211120150000-remove-acmidm-account-from-public.sparql rename to config/migrations/2021/20211120150000-remove-acmidm-account-from-public.sparql diff --git a/config/migrations/20211120160000-source-crab.sparql b/config/migrations/2021/20211120160000-source-crab.sparql similarity index 100% rename from config/migrations/20211120160000-source-crab.sparql rename to config/migrations/2021/20211120160000-source-crab.sparql diff --git a/config/migrations/20211123120000-private-data.sparql b/config/migrations/2021/20211123120000-private-data.sparql similarity index 100% rename from config/migrations/20211123120000-private-data.sparql rename to config/migrations/2021/20211123120000-private-data.sparql diff --git a/config/migrations/20211203120000-bestuur-admidm.sparql b/config/migrations/2021/20211203120000-bestuur-admidm.sparql similarity index 100% rename from config/migrations/20211203120000-bestuur-admidm.sparql rename to config/migrations/2021/20211203120000-bestuur-admidm.sparql diff --git a/config/migrations/20211203130000-clear-mock-data.sparql b/config/migrations/2021/20211203130000-clear-mock-data.sparql similarity index 100% rename from config/migrations/20211203130000-clear-mock-data.sparql rename to config/migrations/2021/20211203130000-clear-mock-data.sparql diff --git a/config/migrations/20211206120000-mock-user-roles.sparql b/config/migrations/2021/20211206120000-mock-user-roles.sparql similarity index 100% rename from config/migrations/20211206120000-mock-user-roles.sparql rename to config/migrations/2021/20211206120000-mock-user-roles.sparql diff --git a/config/migrations/20211206130000-move-graph.sparql b/config/migrations/2021/20211206130000-move-graph.sparql similarity index 100% rename from config/migrations/20211206130000-move-graph.sparql rename to config/migrations/2021/20211206130000-move-graph.sparql diff --git a/config/migrations/20220110120000-delete-old-change-events.sparql b/config/migrations/2022/20220110120000-delete-old-change-events.sparql similarity index 100% rename from config/migrations/20220110120000-delete-old-change-events.sparql rename to config/migrations/2022/20220110120000-delete-old-change-events.sparql diff --git a/config/migrations/20220120100418-remove-old-reasons.sparql b/config/migrations/2022/20220120100418-remove-old-reasons.sparql similarity index 100% rename from config/migrations/20220120100418-remove-old-reasons.sparql rename to config/migrations/2022/20220120100418-remove-old-reasons.sparql diff --git a/config/migrations/20220120101318-insert-new-reasons.sparql b/config/migrations/2022/20220120101318-insert-new-reasons.sparql similarity index 100% rename from config/migrations/20220120101318-insert-new-reasons.sparql rename to config/migrations/2022/20220120101318-insert-new-reasons.sparql From ef692cf6a8f565622bf73110cac786f6e99ab0b6 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Tue, 29 Mar 2022 10:10:56 +0200 Subject: [PATCH 19/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 931d7ed..c72c538 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.7.0-1 + image: lblod/frontend-organization-portal:1.7.0-2 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 947b4666581e89d3abed8153a67b4008cfadd459 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Tue, 29 Mar 2022 15:07:23 +0200 Subject: [PATCH 20/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c72c538..3a0b0d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.7.0-2 + image: lblod/frontend-organization-portal:1.7.0-3 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From a1655337c27e0ea67bb61e08c8357831a259136c Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 29 Mar 2022 15:55:43 +0200 Subject: [PATCH 21/52] Adds migrations to fix wrong data --- ...1923-update-date-change-event-Eeklo.sparql | 15 ++++++++ ...3-update-date-change-event-Glabbeek.sparql | 15 ++++++++ ...-de-Anglicaanse-Eredienst-in-Belgie.sparql | 19 ++++++++++ ...ions-Protestantse-Kerk-van-Kortrijk.sparql | 38 +++++++++++++++++++ ...9154322-delete-positions-ckb-staden.sparql | 38 +++++++++++++++++++ 5 files changed, 125 insertions(+) create mode 100644 config/migrations/2022/20220328171923-update-date-change-event-Eeklo.sparql create mode 100644 config/migrations/2022/20220328172743-update-date-change-event-Glabbeek.sparql create mode 100644 config/migrations/2022/20220329114750-remove-kbo-of-Centraal-Comite-van-de-Anglicaanse-Eredienst-in-Belgie.sparql create mode 100644 config/migrations/2022/20220329150729-delete-positions-Protestantse-Kerk-van-Kortrijk.sparql create mode 100644 config/migrations/2022/20220329154322-delete-positions-ckb-staden.sparql diff --git a/config/migrations/2022/20220328171923-update-date-change-event-Eeklo.sparql b/config/migrations/2022/20220328171923-update-date-change-event-Eeklo.sparql new file mode 100644 index 0000000..9739adf --- /dev/null +++ b/config/migrations/2022/20220328171923-update-date-change-event-Eeklo.sparql @@ -0,0 +1,15 @@ +PREFIX dct: + +DELETE { + GRAPH ?g { + dct:date "2018-06-09"^^xsd:date . + } +} INSERT { + GRAPH ?g { + dct:date "2018-06-15"^^xsd:date . + } +} WHERE { + GRAPH ?g { + dct:date "2018-06-09"^^xsd:date . + } +} diff --git a/config/migrations/2022/20220328172743-update-date-change-event-Glabbeek.sparql b/config/migrations/2022/20220328172743-update-date-change-event-Glabbeek.sparql new file mode 100644 index 0000000..314a72c --- /dev/null +++ b/config/migrations/2022/20220328172743-update-date-change-event-Glabbeek.sparql @@ -0,0 +1,15 @@ +PREFIX dct: + +DELETE { + GRAPH ?g { + dct:date "2021-02-04"^^xsd:date . + } +} INSERT { + GRAPH ?g { + dct:date "2022-02-04"^^xsd:date . + } +} WHERE { + GRAPH ?g { + dct:date "2021-02-04"^^xsd:date . + } +} diff --git a/config/migrations/2022/20220329114750-remove-kbo-of-Centraal-Comite-van-de-Anglicaanse-Eredienst-in-Belgie.sparql b/config/migrations/2022/20220329114750-remove-kbo-of-Centraal-Comite-van-de-Anglicaanse-Eredienst-in-Belgie.sparql new file mode 100644 index 0000000..a888347 --- /dev/null +++ b/config/migrations/2022/20220329114750-remove-kbo-of-Centraal-Comite-van-de-Anglicaanse-Eredienst-in-Belgie.sparql @@ -0,0 +1,19 @@ +DELETE { + GRAPH ?g { + + ?identifier . + ?identifier + ?structuredIdentifier ; + ?pidentifier ?oidentifier . + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} WHERE { + GRAPH ?g { + + ?identifier . + ?identifier + ?structuredIdentifier ; + ?pidentifier ?oidentifier . + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} diff --git a/config/migrations/2022/20220329150729-delete-positions-Protestantse-Kerk-van-Kortrijk.sparql b/config/migrations/2022/20220329150729-delete-positions-Protestantse-Kerk-van-Kortrijk.sparql new file mode 100644 index 0000000..3cd75ec --- /dev/null +++ b/config/migrations/2022/20220329150729-delete-positions-Protestantse-Kerk-van-Kortrijk.sparql @@ -0,0 +1,38 @@ +DELETE { + GRAPH ?g { + + ?post . + + ?post ?ppost ?opost . + + ?role ?post ; + ?contactPoint ; + ?prole ?orole . + + ?contactPoint ?address ; + ?pcontactPoint ?ocontactPoint . + + ?address ?paddress ?oaddress . + } +} WHERE { + GRAPH ?g { + + ?post . + + ?post ?ppost ?opost . + + ?role ?post ; + ?prole ?orole . + + OPTIONAL { + ?role ?contactPoint . + + ?contactPoint ?pcontactPoint ?ocontactPoint . + + OPTIONAL { + ?contactPoint ?address . + ?address ?paddress ?oaddress . + } + } + } +} diff --git a/config/migrations/2022/20220329154322-delete-positions-ckb-staden.sparql b/config/migrations/2022/20220329154322-delete-positions-ckb-staden.sparql new file mode 100644 index 0000000..32cb92e --- /dev/null +++ b/config/migrations/2022/20220329154322-delete-positions-ckb-staden.sparql @@ -0,0 +1,38 @@ +DELETE { + GRAPH ?g { + + ?post . + + ?post ?ppost ?opost . + + ?role ?post ; + ?contactPoint ; + ?prole ?orole . + + ?contactPoint ?address ; + ?pcontactPoint ?ocontactPoint . + + ?address ?paddress ?oaddress . + } +} WHERE { + GRAPH ?g { + + ?post . + + ?post ?ppost ?opost . + + ?role ?post ; + ?prole ?orole . + + OPTIONAL { + ?role ?contactPoint . + + ?contactPoint ?pcontactPoint ?ocontactPoint . + + OPTIONAL { + ?contactPoint ?address . + ?address ?paddress ?oaddress . + } + } + } +} From 7866e89d4e890c29d4cccf437cd1b2bed712ca6e Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Wed, 30 Mar 2022 13:38:10 +0200 Subject: [PATCH 22/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3a0b0d7..20c87d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.7.0-3 + image: lblod/frontend-organization-portal:1.7.0-4 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From d2d60e13396290ef68393c916214f917dca2ef34 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 31 Mar 2022 14:48:45 +0200 Subject: [PATCH 23/52] Bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 20c87d9..74a17a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.7.0-4 + image: lblod/frontend-organization-portal:1.7.0-5 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 2a1e75628d51bd7dd303d2c47f666f337ec0f513 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Thu, 31 Mar 2022 14:59:32 +0200 Subject: [PATCH 24/52] bump java deps (#124) --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 74a17a6..fab6c76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,7 +79,7 @@ services: restart: always logging: *default-logging kalliope-api: - image: lblod/jsonld-delta-service:0.5.5 + image: lblod/jsonld-delta-service:0.5.6 volumes: - ./config/kalliope:/config - ./data/files:/share @@ -138,7 +138,7 @@ services: restart: always logging: *default-logging privacy: - image: lblod/privacy-centric-service:0.4.1 + image: lblod/privacy-centric-service:0.4.2 labels: - "logging=true" environment: From 1411dd872a7edf127f9a97ddb773db837c370c1d Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 31 Mar 2022 19:52:28 +0200 Subject: [PATCH 25/52] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index fab6c76..ca28252 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.7.0-5 + image: lblod/frontend-organization-portal:1.7.0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 5f96fb22e1bfbc2498f5654a28dde427520d30c1 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 1 Apr 2022 16:38:28 +0200 Subject: [PATCH 26/52] Add migration to reformat weird kbos --- ...401162146-fix-wrongly-formatted-kbo.sparql | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 config/migrations/2022/20220401162146-fix-wrongly-formatted-kbo.sparql diff --git a/config/migrations/2022/20220401162146-fix-wrongly-formatted-kbo.sparql b/config/migrations/2022/20220401162146-fix-wrongly-formatted-kbo.sparql new file mode 100644 index 0000000..41a491a --- /dev/null +++ b/config/migrations/2022/20220401162146-fix-wrongly-formatted-kbo.sparql @@ -0,0 +1,51 @@ +PREFIX generiek: + +DELETE { + GRAPH ?g { + generiek:lokaleIdentificator ?kbo1 . + generiek:lokaleIdentificator ?kbo2 . + generiek:lokaleIdentificator ?kbo3 . + generiek:lokaleIdentificator ?kbo4 . + generiek:lokaleIdentificator ?kbo5 . + generiek:lokaleIdentificator ?kbo6 . + generiek:lokaleIdentificator ?kbo7 . + generiek:lokaleIdentificator ?kbo8 . + generiek:lokaleIdentificator ?kbo9 . + generiek:lokaleIdentificator ?kbo10 . + generiek:lokaleIdentificator ?kbo11 . + generiek:lokaleIdentificator ?kbo12 . + generiek:lokaleIdentificator ?kbo13 . + } +} INSERT { + GRAPH ?g { + generiek:lokaleIdentificator "0777683642" . + generiek:lokaleIdentificator "0780386972" . + generiek:lokaleIdentificator "0408140861" . + generiek:lokaleIdentificator "0408472839" . + generiek:lokaleIdentificator "0409677223" . + generiek:lokaleIdentificator "0409771253" . + generiek:lokaleIdentificator "0410195380" . + generiek:lokaleIdentificator "0412914944" . + generiek:lokaleIdentificator "0413234648" . + generiek:lokaleIdentificator "0464533790" . + generiek:lokaleIdentificator "0687815914" . + generiek:lokaleIdentificator "0864330673" . + generiek:lokaleIdentificator "0898987189" . + } +} WHERE { + GRAPH ?g { + generiek:lokaleIdentificator ?kbo1 . + generiek:lokaleIdentificator ?kbo2 . + generiek:lokaleIdentificator ?kbo3 . + generiek:lokaleIdentificator ?kbo4 . + generiek:lokaleIdentificator ?kbo5 . + generiek:lokaleIdentificator ?kbo6 . + generiek:lokaleIdentificator ?kbo7 . + generiek:lokaleIdentificator ?kbo8 . + generiek:lokaleIdentificator ?kbo9 . + generiek:lokaleIdentificator ?kbo10 . + generiek:lokaleIdentificator ?kbo11 . + generiek:lokaleIdentificator ?kbo12 . + generiek:lokaleIdentificator ?kbo13 . + } +} From dcb0ad63ba0b8a5291858cdb61230499f1e56b8f Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 4 Apr 2022 15:15:22 +0200 Subject: [PATCH 27/52] fix sorting case insensitive (#123) --- .gitignore | 1 + config/search/config.json | 87 +++++++++++++++++++++++--------- config/search/dev/config.json | 93 +++++++++++++++++++++++++---------- 3 files changed, 131 insertions(+), 50 deletions(-) diff --git a/.gitignore b/.gitignore index 8eaa314..cc42a73 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ out.json security.yml config/proxy-https/proxy_ssl.conf /config/search/update-handler.store +/config/search/dev/update-handler.store diff --git a/config/search/config.json b/config/search/config.json index 4954cb8..07ba765 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -122,6 +122,21 @@ } ] ], + "default_settings" : { + "analysis": { + "normalizer": { + "custom_sort_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase", + "trim", + "asciifolding" + ] + } + } + } + }, "types": [ { "type": "mandatory", @@ -198,7 +213,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -214,7 +230,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -225,7 +242,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -233,7 +251,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -241,7 +260,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -252,7 +272,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -332,7 +353,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -348,7 +370,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -359,7 +382,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -367,7 +391,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -375,7 +400,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -386,7 +412,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -488,7 +515,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -504,7 +532,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -515,7 +544,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -523,7 +553,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -531,7 +562,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -542,7 +574,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -609,7 +642,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -617,7 +651,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -625,7 +660,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -633,7 +669,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -641,7 +678,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -649,7 +687,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } } diff --git a/config/search/dev/config.json b/config/search/dev/config.json index 38571e2..269b7a6 100644 --- a/config/search/dev/config.json +++ b/config/search/dev/config.json @@ -1,7 +1,7 @@ { "automatic_index_updates": true, "persist_indexes": true, - "number_of_threads": 6, + "number_of_threads": 11, "eager_indexing_groups": [ [ { @@ -46,6 +46,21 @@ } ] ], + "default_settings" : { + "analysis": { + "normalizer": { + "custom_sort_normalizer": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase", + "trim", + "asciifolding" + ] + } + } + } + }, "types": [ { "type": "mandatory", @@ -122,7 +137,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -138,7 +154,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -149,7 +166,9 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" + } } }, @@ -157,7 +176,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -165,7 +185,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -176,7 +197,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -256,7 +278,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -272,7 +295,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -283,7 +307,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -291,7 +316,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -299,7 +325,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -310,7 +337,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -412,7 +440,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -420,7 +449,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -428,7 +458,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -439,7 +470,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -447,7 +479,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -455,7 +488,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -466,7 +500,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -533,7 +568,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -541,7 +577,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -549,7 +586,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -557,7 +595,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -565,7 +604,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } }, @@ -573,7 +613,8 @@ "type": "text", "fields": { "field": { - "type": "keyword" + "type": "keyword", + "normalizer": "custom_sort_normalizer" } } } From 068f9cd8a754ec2a7ab47538045cf16fd8b0c194 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 5 Apr 2022 08:44:49 +0200 Subject: [PATCH 28/52] Add migrations to remove dangling triples --- ...lete-triples-pointing-towards-werkingsgebieden.sparql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 config/migrations/2022/20220405084015-delete-triples-pointing-towards-werkingsgebieden.sparql diff --git a/config/migrations/2022/20220405084015-delete-triples-pointing-towards-werkingsgebieden.sparql b/config/migrations/2022/20220405084015-delete-triples-pointing-towards-werkingsgebieden.sparql new file mode 100644 index 0000000..7e20f97 --- /dev/null +++ b/config/migrations/2022/20220405084015-delete-triples-pointing-towards-werkingsgebieden.sparql @@ -0,0 +1,9 @@ +DELETE { + GRAPH ?g { + ?s ?o . + } +} WHERE { + GRAPH ?g { + ?s ?o . + } +} From 5714ea3f14fa9d999c24379964293b06bf0b2d61 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 6 Apr 2022 11:52:13 +0200 Subject: [PATCH 29/52] Add migration to add link gemeente <-> province --- ...6100547-add-link-gemeente-provincie.sparql | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 config/migrations/2022/20220406100547-add-link-gemeente-provincie.sparql diff --git a/config/migrations/2022/20220406100547-add-link-gemeente-provincie.sparql b/config/migrations/2022/20220406100547-add-link-gemeente-provincie.sparql new file mode 100644 index 0000000..0c58b23 --- /dev/null +++ b/config/migrations/2022/20220406100547-add-link-gemeente-provincie.sparql @@ -0,0 +1,319 @@ +INSERT DATA { + GRAPH { + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + . + } +} From 6d492bb1a0c5462906ebc997ed842b7e9a57a7d2 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 6 Apr 2022 14:13:42 +0200 Subject: [PATCH 30/52] Fix status ckb melle --- ...20406134511-change-status-ckb-melle.sparql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 config/migrations/2022/20220406134511-change-status-ckb-melle.sparql diff --git a/config/migrations/2022/20220406134511-change-status-ckb-melle.sparql b/config/migrations/2022/20220406134511-change-status-ckb-melle.sparql new file mode 100644 index 0000000..ea1a9e0 --- /dev/null +++ b/config/migrations/2022/20220406134511-change-status-ckb-melle.sparql @@ -0,0 +1,19 @@ +PREFIX regorg: + +DELETE { + GRAPH ?g { + + regorg:orgStatus . # Actief + } +} INSERT { + GRAPH ?g { + + regorg:orgStatus . # Niet Actief + + } +} WHERE { + GRAPH ?g { + + regorg:orgStatus . + } +} From 40bd74a3c58b80df8fd7d588df33ce8f10fecd56 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:21:54 +0200 Subject: [PATCH 31/52] fix regression mu search (#128) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ca28252..7abc5cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -339,7 +339,7 @@ services: # MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH ################################################################################ search: - image: nbittich/mu-search:wip-inheritance # Bump to fixed version when merged and release to the original repo + image: nbittich/mu-search:wip-inheritance-alpha2 # Bump to fixed version when merged and release to the original repo links: - db:database restart: always From 1b152e688b5b9861decbb36536bca7fbbf16966f Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:21:54 +0200 Subject: [PATCH 32/52] fix regression mu search (#128) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ca28252..7abc5cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -339,7 +339,7 @@ services: # MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH ################################################################################ search: - image: nbittich/mu-search:wip-inheritance # Bump to fixed version when merged and release to the original repo + image: nbittich/mu-search:wip-inheritance-alpha2 # Bump to fixed version when merged and release to the original repo links: - db:database restart: always From 54212c864db3cb86ed04a75f60734bb63f0a2c3b Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 12 Apr 2022 15:14:09 +0200 Subject: [PATCH 33/52] Add migration to update gender codelist --- ...220412134535-update-gender-codelist.sparql | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 config/migrations/2022/20220412134535-update-gender-codelist.sparql diff --git a/config/migrations/2022/20220412134535-update-gender-codelist.sparql b/config/migrations/2022/20220412134535-update-gender-codelist.sparql new file mode 100644 index 0000000..13f4c53 --- /dev/null +++ b/config/migrations/2022/20220412134535-update-gender-codelist.sparql @@ -0,0 +1,42 @@ +DELETE { + GRAPH ?g { + ?s ?p ?o . + } +} WHERE { + GRAPH ?g { + ?s ?p ?o . + VALUES ?s { + + + } + } +} + +; + +INSERT DATA { + GRAPH { + a , + ; + "c11b47dd-cacd-4664-9a72-14730e00eaed"; + "Ander"; + ; + . + } +} + +; + +DELETE { + GRAPH ?g { + skos:prefLabel ?label . + } +} INSERT { + GRAPH ?g { + skos:prefLabel "Niet gekend" . + } +} WHERE { + GRAPH ?g { + skos:prefLabel ?label . + } +} \ No newline at end of file From 3f2ff1f1c0f6eae973ffbb36a22a94072d3b5cf4 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 12 Apr 2022 15:14:09 +0200 Subject: [PATCH 34/52] Add migration to update gender codelist --- ...220412134535-update-gender-codelist.sparql | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 config/migrations/2022/20220412134535-update-gender-codelist.sparql diff --git a/config/migrations/2022/20220412134535-update-gender-codelist.sparql b/config/migrations/2022/20220412134535-update-gender-codelist.sparql new file mode 100644 index 0000000..13f4c53 --- /dev/null +++ b/config/migrations/2022/20220412134535-update-gender-codelist.sparql @@ -0,0 +1,42 @@ +DELETE { + GRAPH ?g { + ?s ?p ?o . + } +} WHERE { + GRAPH ?g { + ?s ?p ?o . + VALUES ?s { + + + } + } +} + +; + +INSERT DATA { + GRAPH { + a , + ; + "c11b47dd-cacd-4664-9a72-14730e00eaed"; + "Ander"; + ; + . + } +} + +; + +DELETE { + GRAPH ?g { + skos:prefLabel ?label . + } +} INSERT { + GRAPH ?g { + skos:prefLabel "Niet gekend" . + } +} WHERE { + GRAPH ?g { + skos:prefLabel ?label . + } +} \ No newline at end of file From 5aaa45c186aa8a40fdab11edb6683f75ce3433db Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 14 Apr 2022 10:56:11 +0200 Subject: [PATCH 35/52] Add migration to remove some bedinaren --- .../20220414092643-remove-bedinaren.sparql | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 config/migrations/2022/20220414092643-remove-bedinaren.sparql diff --git a/config/migrations/2022/20220414092643-remove-bedinaren.sparql b/config/migrations/2022/20220414092643-remove-bedinaren.sparql new file mode 100644 index 0000000..8b55a4e --- /dev/null +++ b/config/migrations/2022/20220414092643-remove-bedinaren.sparql @@ -0,0 +1,35 @@ +DELETE { + GRAPH ?g { + + ?position . + + ?position ?pposition ?oposition . + + ?role ?prole ?orole . + + ?contactPoint ?pcontactPoint ?ocontactPoint . + + ?address ?paddress ?oaddress . + } +} WHERE { + GRAPH ?g { + + ?position . + + ?position ?pposition ?oposition . + + ?role ?position ; + ?prole ?orole . + + OPTIONAL { + ?role ?contactPoint . + + ?contactPoint ?pcontactPoint ?ocontactPoint . + + OPTIONAL { + ?contactPoint ?address . + ?address ?paddress ?oaddress . + } + } + } +} From 148a3f3ca7bc78a20a0d8f173916cfd08a446c14 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 14 Apr 2022 13:41:33 +0200 Subject: [PATCH 36/52] Fix change event resulting org and statuses --- ...fix-wrong-resulting-org-after-merge.sparql | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 config/migrations/2022/20220414132656-fix-wrong-resulting-org-after-merge.sparql diff --git a/config/migrations/2022/20220414132656-fix-wrong-resulting-org-after-merge.sparql b/config/migrations/2022/20220414132656-fix-wrong-resulting-org-after-merge.sparql new file mode 100644 index 0000000..86dd408 --- /dev/null +++ b/config/migrations/2022/20220414132656-fix-wrong-resulting-org-after-merge.sparql @@ -0,0 +1,42 @@ +# Update status of the bestuurseenheden as well as the resulting status of the change event + +PREFIX regorg: +PREFIX lblodOrg: + +DELETE { + GRAPH ?g { + + regorg:orgStatus . # Actief + + lblodOrg:resulterendeStatus . # Actief + + + regorg:orgStatus . # Niet Actief + + lblodOrg:resulterendeStatus . # Niet Actief + } +} INSERT { + GRAPH ?g { + + regorg:orgStatus . # Niet Actief + + lblodOrg:resulterendeStatus . # Niet Actief + + + regorg:orgStatus . # Actief + + lblodOrg:resulterendeStatus . # Actief + } +} WHERE { + GRAPH ?g { + + regorg:orgStatus . # Actief + + lblodOrg:resulterendeStatus . # Actief + + + regorg:orgStatus . # Niet Actief + + lblodOrg:resulterendeStatus . # Niet Actief + } +} From 7b160e40de209cab57d36832aa1f121e3e5ff56a Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 14 Apr 2022 15:37:02 +0200 Subject: [PATCH 37/52] Add migration to fix resulting organization --- ...431-upate-resulting-org-change-event.sparql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 config/migrations/2022/20220414152431-upate-resulting-org-change-event.sparql diff --git a/config/migrations/2022/20220414152431-upate-resulting-org-change-event.sparql b/config/migrations/2022/20220414152431-upate-resulting-org-change-event.sparql new file mode 100644 index 0000000..0e8a9ff --- /dev/null +++ b/config/migrations/2022/20220414152431-upate-resulting-org-change-event.sparql @@ -0,0 +1,18 @@ +PREFIX org: + +DELETE { + GRAPH ?g { + + org:resultingOrganization . + } +} INSERT { + GRAPH ?g { + + org:resultingOrganization . + } +} WHERE { + GRAPH ?g { + + org:resultingOrganization . + } +} From 289eb375a52be6d575ef6ca8f47dd514efcfb6cb Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 14 Apr 2022 16:23:12 +0200 Subject: [PATCH 38/52] Fix missing original organization --- ...161826-add-missing-original-org-to-change-event.sparql | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 config/migrations/2022/20220414161826-add-missing-original-org-to-change-event.sparql diff --git a/config/migrations/2022/20220414161826-add-missing-original-org-to-change-event.sparql b/config/migrations/2022/20220414161826-add-missing-original-org-to-change-event.sparql new file mode 100644 index 0000000..28c254f --- /dev/null +++ b/config/migrations/2022/20220414161826-add-missing-original-org-to-change-event.sparql @@ -0,0 +1,8 @@ +PREFIX org: + +INSERT DATA { + GRAPH { + + org:originalOrganization . + } +} From 16fc4b3a5b257f6e7591a7e92524a6f4b468c406 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 15 Apr 2022 09:17:54 +0200 Subject: [PATCH 39/52] Fix change event --- .../20220415090554-fix-change-event.sparql | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 config/migrations/2022/20220415090554-fix-change-event.sparql diff --git a/config/migrations/2022/20220415090554-fix-change-event.sparql b/config/migrations/2022/20220415090554-fix-change-event.sparql new file mode 100644 index 0000000..b150023 --- /dev/null +++ b/config/migrations/2022/20220415090554-fix-change-event.sparql @@ -0,0 +1,36 @@ +# Update status of the bestuurseenheden as well as the resulting status of the change event + +PREFIX regorg: +PREFIX lblodOrg: + +DELETE { + GRAPH ?g { + + lblodOrg:resulterendeStatus . # Actief + + + regorg:orgStatus . # Niet Actief + + lblodOrg:resulterendeStatus . # Niet Actief + } +} INSERT { + GRAPH ?g { + + lblodOrg:resulterendeStatus . # Niet Actief + + + regorg:orgStatus . # Actief + + lblodOrg:resulterendeStatus . # Actief + } +} WHERE { + GRAPH ?g { + + lblodOrg:resulterendeStatus . # Actief + + + regorg:orgStatus . # Niet Actief + + lblodOrg:resulterendeStatus . # Niet Actief + } +} From ae0565b2cd9403a91be6bf2f960383978e1165e8 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Tue, 19 Apr 2022 09:35:57 +0200 Subject: [PATCH 40/52] publish public info --- .../dump-file-publisher/config.json | 10 + config/delta-producer/public/export.json | 266 ++++++++++++++++++ config/delta/rules.js | 35 +++ ...407110000-add-export-concept-scheme.sparql | 43 +++ 4 files changed, 354 insertions(+) create mode 100644 config/delta-producer/public/export.json create mode 100644 config/migrations/2022/20220407110000-add-export-concept-scheme.sparql diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 66391ca..8d33759 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -18,5 +18,15 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations" + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" : { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "fileBaseName": "dump-public" + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" : { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "fileBaseName": "dump-public" } } diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json new file mode 100644 index 0000000..5cb4fa9 --- /dev/null +++ b/config/delta-producer/public/export.json @@ -0,0 +1,266 @@ +{ + "conceptScheme": "http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b", + "export": [ + { + "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", + "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau", + "http://data.lblod.info/vocabularies/erediensten/denominatie", + "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", + "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://www.w3.org/ns/org#Site", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + + ], + "properties": [ + "http://www.w3.org/ns/org#siteAddress", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://data.lblod.info/vocabularies/erediensten/vestigingstype" + ] + }, + { + "type": "http://www.w3.org/ns/locn#Address", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "^http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + + ], + "properties": [ + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer", + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer", + "http://www.w3.org/ns/locn#thoroughfare", + "http://www.w3.org/ns/locn#postCode", + "https://data.vlaanderen.be/ns/adres#gemeentenaam", + "http://www.w3.org/ns/locn#adminUnitL2", + "https://data.vlaanderen.be/ns/adres#land", + "http://www.w3.org/ns/locn#fullAddress", + "https://data.vlaanderen.be/ns/adres#verwijstNaar", + "http://purl.org/dc/terms/source" + ] + }, + { + "type": "http://schema.org/ContactPoint", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^http://www.w3.org/ns/org#siteAddress", + "^http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/locn#address", + "http://xmlns.com/foaf/0.1/page", + "http://schema.org/faxNumber", + "http://schema.org/contactType", + "http://xmlns.com/foaf/0.1/page", + "http://schema.org/email", + "http://schema.org/telephone" + + ] + }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ] + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator", + "^http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://data.vlaanderen.be/ns/mandaat#bindingEinde", + "http://data.vlaanderen.be/ns/mandaat#bindingStart", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#classification", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://www.w3.org/ns/org#ChangeEvent", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://data.lblod.info/vocabularies/contacthub/typeWijziging", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://purl.org/dc/terms/date", + "http://purl.org/dc/terms/description", + "http://data.lblod.info/vocabularies/contacthub/typeWijziging", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#changedBy" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + } + ] +} diff --git a/config/delta/rules.js b/config/delta/rules.js index c1d5575..75e80b8 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -104,6 +104,41 @@ export default [ optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] } }, + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-public/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://redpencil.data.gift/id/deltas/producer/public' + } + }, + callback: { + url: 'http://delta-producer-json-diff-publisher-public/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, { match: { predicate: { diff --git a/config/migrations/2022/20220407110000-add-export-concept-scheme.sparql b/config/migrations/2022/20220407110000-add-export-concept-scheme.sparql new file mode 100644 index 0000000..3f76325 --- /dev/null +++ b/config/migrations/2022/20220407110000-add-export-concept-scheme.sparql @@ -0,0 +1,43 @@ +PREFIX mu: +PREFIX skos: + +INSERT { + GRAPH { + + a + skos:ConceptScheme ; + mu:uuid + "10caf4a4-1ff5-4c05-917e-56489910d68b" ; + skos:prefLabel + "Public OP Export" . + ?code + skos:inScheme + . + } +} WHERE { + VALUES ?type { + + + + + + } + GRAPH { + ?code + a + ?type . + } +} + +INSERT { + GRAPH { + + skos:inScheme + . + + + skos:inScheme + . + } +} + From 4cc6640590c3216fb05c597b31f0fa7ef48c8175 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Tue, 19 Apr 2022 10:07:30 +0200 Subject: [PATCH 41/52] docker compose --- docker-compose.yml | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 7abc5cc..94f3a0a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -335,6 +335,69 @@ services: ################################################################################ +################################################################################ +# DELTA PUBLIC INFORMATION: START +################################################################################ + delta-producer-bg-jobs-initiator-public: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public' + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public' + HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public' + CRON_PATTERN_DUMP_JOB: '42 * * * *' # At minute 42 every hour + CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM + START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-public: + image: lblod/delta-producer-publication-graph-maintainer:0.15.1 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + RELATIVE_FILE_PATH: "deltas/public" + PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/public' + HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public' + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public' + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: 'true' + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + volumes: + - ./config/delta-producer/public:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + logging: *default-logging + delta-producer-json-diff-publisher-public: + image: lblod/delta-producer-json-diff-file-publisher:0.2.3 + environment: + MAX_BODY_SIZE: "50mb" + ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + RELATIVE_FILE_PATH: "deltas/public" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" + PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/public' + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + volumes: + - ./data/files:/share + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging +################################################################################ +# DELTA PUBLIC INFORMATION: END +################################################################################ + + ################################################################################ # MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH ################################################################################ From c1a361b3388f17e2c634fd4fe6e610d34b6ddcff Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Wed, 20 Apr 2022 15:13:37 +0200 Subject: [PATCH 42/52] fix caching issues (#138) --- config/delta/rules.js | 17 +++++++++++++++++ docker-compose.yml | 2 ++ 2 files changed, 19 insertions(+) diff --git a/config/delta/rules.js b/config/delta/rules.js index c1d5575..7f3265b 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -16,6 +16,23 @@ export default [ ignoreFromSelf: true } }, + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/organisatieportaal" + } + }, + callback: { + url: 'http://resource/.mu/delta', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 250, + ignoreFromSelf: true + } + }, { match: { predicate: { diff --git a/docker-compose.yml b/docker-compose.yml index 7abc5cc..59b5e1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,8 @@ services: logging: *default-logging resource: image: semtech/mu-cl-resources:feature-all-errors-as-json-api-errors # Bump to fixed version when merged and release + environment: + CACHE_CLEAR_PATH: "http://cache/.mu/clear-keys" links: - db:database volumes: From 5cab3665ea0bdeb478999e283d24d306cfb65466 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 21 Apr 2022 14:53:26 +0200 Subject: [PATCH 43/52] publish public op data --- .../dump-file-publisher/config.json | 18 +++++++---- config/jobs-controller/config.json | 32 +++++++++++++++++++ config/resources/dcat.json | 4 +++ docker-compose.yml | 4 +-- 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 8d33759..5f0fd27 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -2,31 +2,37 @@ "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", - "fileBaseName": "dump-administrative-units" + "fileBaseName": "dump-administrative-units", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", - "fileBaseName": "dump-administrative-units" + "fileBaseName": "dump-administrative-units", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", - "fileBaseName": "dump-organizations" + "fileBaseName": "dump-organizations", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", - "fileBaseName": "dump-organizations" + "fileBaseName": "dump-organizations", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", - "fileBaseName": "dump-public" + "fileBaseName": "dump-public", + "compression": false }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", - "fileBaseName": "dump-public" + "fileBaseName": "dump-public", + "compression": false } } diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 317c490..928128b 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -62,5 +62,37 @@ "nextIndex": "0" } ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextIndex": "0" + }, + { + "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "1" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "0" + } + ] } } diff --git a/config/resources/dcat.json b/config/resources/dcat.json index eba75ea..4945a66 100644 --- a/config/resources/dcat.json +++ b/config/resources/dcat.json @@ -133,6 +133,10 @@ "byte-size": { "type": "number", "predicate": "dcat:byteSize" + }, + "compression": { + "type": "url", + "predicate": "dcat:compressFormat" } }, "relationships": { diff --git a/docker-compose.yml b/docker-compose.yml index 94f3a0a..8f3a5af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -200,7 +200,7 @@ services: # DELTA GENERAL ################################################################################ delta-producer-dump-file-publisher: - image: bdevloed/delta-producer-dump-file-publisher:0.5.0-beta + image: bdevloed/delta-producer-dump-file-publisher:0.8.0-beta volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share @@ -345,7 +345,7 @@ services: DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public' INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public' HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public' - CRON_PATTERN_DUMP_JOB: '42 * * * *' # At minute 42 every hour + CRON_PATTERN_DUMP_JOB: '11 */4 * * *' # At minute 11 every 4 hours CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. labels: From 8afaf03854513852ce8d1260e400cde044ae1c2a Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 21 Apr 2022 16:56:30 +0200 Subject: [PATCH 44/52] add missing change event properties --- config/delta-producer/public/export.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index 5cb4fa9..49d2931 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -223,7 +223,11 @@ "http://purl.org/dc/terms/description", "http://data.lblod.info/vocabularies/contacthub/typeWijziging", "http://www.w3.org/ns/org#resultedFrom", - "http://www.w3.org/ns/org#changedBy" + "http://www.w3.org/ns/org#changedBy", + "http://lblod.data.gift/vocabularies/organisatie/veranderingsgebeurtenisResultaat", + "http://www.w3.org/ns/org#originalOrganization", + "http://www.w3.org/ns/org#resultingOrganization", + "http://data.europa.eu/m8g/hasFormalFramework" ] }, { From 1fb7dd694fe31187ff5f7510c1dbc09eb5eaea48 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 25 Apr 2022 09:04:00 +0200 Subject: [PATCH 45/52] bump mu-cl resource for cache clearing fix (#140) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c80080b..7af05d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,7 +61,7 @@ services: restart: always logging: *default-logging resource: - image: semtech/mu-cl-resources:feature-all-errors-as-json-api-errors # Bump to fixed version when merged and release + image: semtech/mu-cl-resources:1.21.1 environment: CACHE_CLEAR_PATH: "http://cache/.mu/clear-keys" links: From 95853bcfdf73123f8d8307b6aca1ae95f20eec5c Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Thu, 28 Apr 2022 11:39:35 +0200 Subject: [PATCH 46/52] bump privacy (#141) --- docker-compose.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7af05d8..c7c3406 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -140,20 +140,17 @@ services: restart: always logging: *default-logging privacy: - image: lblod/privacy-centric-service:0.4.2 + image: lblod/privacy-centric-service:0.5.0 labels: - "logging=true" environment: - SERVER_PORT: "80" - SPARQL_ENDPOINT: "http://db:8890/sparql" - LOGGING_LEVEL: "INFO" DEFAULT_GRAPH: "http://mu.semte.ch/graphs/privacy-centric-graph" SESSION_GRAPH: "http://mu.semte.ch/graphs/sessions" APP_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" - labels: - - "logging=true" restart: always logging: *default-logging + links: + - db:database adressenregister: image: lblod/adressenregister-fuzzy-search-service:0.6.0 labels: From 48fd0d02555b392c63098d38119ba5672e5f0c51 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 29 Apr 2022 14:26:54 +0200 Subject: [PATCH 47/52] Add migration to remove person and position --- ...1-remove-person-and-related-positions.sparql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 config/migrations/2022/20220429135441-remove-person-and-related-positions.sparql diff --git a/config/migrations/2022/20220429135441-remove-person-and-related-positions.sparql b/config/migrations/2022/20220429135441-remove-person-and-related-positions.sparql new file mode 100644 index 0000000..a063a30 --- /dev/null +++ b/config/migrations/2022/20220429135441-remove-person-and-related-positions.sparql @@ -0,0 +1,17 @@ +# A person has been wrongly added (it's a duplicate) and someone attached to him a wrong position + +DELETE { + GRAPH ?g { + ?pperson ?operson . + ?pmandataris ?omandataris . + ?pcontact ?ocontact . + ?paddress ?oaddress . + } +} WHERE { + GRAPH ?g { + ?pperson ?operson . + ?pmandataris ?omandataris . + ?pcontact ?ocontact . + ?paddress ?oaddress . + } +} From 724635d9aae616fe86ee2782249c4473439eea0f Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 3 May 2022 19:22:33 +0200 Subject: [PATCH 48/52] Add missing person-position link --- ...921-fix-broken-link-person-position.sparql | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 config/migrations/2022/20220503141921-fix-broken-link-person-position.sparql diff --git a/config/migrations/2022/20220503141921-fix-broken-link-person-position.sparql b/config/migrations/2022/20220503141921-fix-broken-link-person-position.sparql new file mode 100644 index 0000000..4adec4d --- /dev/null +++ b/config/migrations/2022/20220503141921-fix-broken-link-person-position.sparql @@ -0,0 +1,21 @@ +INSERT DATA { + GRAPH { + . + } +} + +# The associated call to trigger a reindex is : +# curl --location --request POST '/update' \ +# --header 'Content-Type: application/vnd.api+json' \ +# --data-raw ' [ +# { +# "deletes": [], +# "inserts": [ +# { +# "subject": { "type": "uri", "value": "http://data.lblod.info/id/mandatarissen/626BCD9AFEE49A5CF118F54B" }, +# "predicate": { "type": "uri", "value": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" }, +# "object": { "type": "uri", "value": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris" } +# } +# ] +# } +# ]' From 24f13fa6deb02f8d4c2f76de2d952b855d65da0a Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Wed, 4 May 2022 10:28:20 +0200 Subject: [PATCH 49/52] missing governing body (#143) --- ...0504081921-fix-missing-governing-body.sparql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 config/migrations/2022/20220504081921-fix-missing-governing-body.sparql diff --git a/config/migrations/2022/20220504081921-fix-missing-governing-body.sparql b/config/migrations/2022/20220504081921-fix-missing-governing-body.sparql new file mode 100644 index 0000000..f12b2c7 --- /dev/null +++ b/config/migrations/2022/20220504081921-fix-missing-governing-body.sparql @@ -0,0 +1,17 @@ +PREFIX mu: +PREFIX org: +PREFIX besluit: +PREFIX generiek: + +INSERT DATA { + GRAPH { + + a ; + mu:uuid "3703cf09-f9c0-451b-b8b1-e373e1367a85"; + besluit:bestuurt ; + org:classification . + a ; + mu:uuid "bdf61666-05fe-4a70-8e13-7a98e8e72707"; + generiek:isTijdspecialisatieVan . + } +} From 2ae11f707c6d0e541691a0182d1d217a2efbce33 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 6 May 2022 12:20:21 +0200 Subject: [PATCH 50/52] Bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7abc5cc..d66bf7a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.7.0 + image: lblod/frontend-organization-portal:1.7.1 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 1e05382a6da28a92d998813b49654b993b5e4f46 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Mon, 9 May 2022 08:52:31 +0200 Subject: [PATCH 51/52] fix wrong classification --- ...09081921-fix-gb-classification-code.sparql | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 config/migrations/2022/20220509081921-fix-gb-classification-code.sparql diff --git a/config/migrations/2022/20220509081921-fix-gb-classification-code.sparql b/config/migrations/2022/20220509081921-fix-gb-classification-code.sparql new file mode 100644 index 0000000..f3d7cf4 --- /dev/null +++ b/config/migrations/2022/20220509081921-fix-gb-classification-code.sparql @@ -0,0 +1,22 @@ +PREFIX mu: +PREFIX org: +PREFIX besluit: +PREFIX generiek: + + +DELETE { + graph ?g { + org:classification . + } +} +INSERT { + graph ?g { + org:classification . + } + +}WHERE { + graph ?g { + org:classification . + } + +} From 693763c3b4246e9310e58f172354874b1ac68e57 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 9 May 2022 09:12:21 +0200 Subject: [PATCH 52/52] Bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d66bf7a..2554d1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -159,7 +159,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.7.1 + image: lblod/frontend-organization-portal:1.7.2 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: