diff --git a/.gitignore b/.gitignore index 39c29a4..8eaa314 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ source.json out.json security.yml config/proxy-https/proxy_ssl.conf +/config/search/update-handler.store diff --git a/config/delta/rules.js b/config/delta/rules.js index c476cfb..c1d5575 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -1,4 +1,21 @@ export default [ + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/organisatieportaal" + } + }, + callback: { + url: 'http://search/update', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 10000, + ignoreFromSelf: true + } + }, { match: { predicate: { diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index fb94483..9854ec6 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -222,6 +222,14 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://login/sessions/" end + ############################################################### + # SEARCH + ############################################################### + + match "/search/*path", %{ accept: %{ json: true }, layer: :api_services} do + IO.puts("hey") + Proxy.forward conn, path, "http://search/" + end ############### # API SERVICES @@ -231,10 +239,6 @@ defmodule Dispatcher do forward conn, path, "http://kalliope-api/v3/api-docs/" end - match "/changes/*path", %{ layer: :api_services, accept: %{ json: true } } do - forward conn, path, "http://kalliope-api/changes/" - end - match "/consolidated/*path", %{ layer: :api_services, accept: %{ json: true } } do forward conn, path, "http://kalliope-api/consolidated/" end diff --git a/config/resources/domain.json b/config/resources/domain.json index 2a90c1d..44a0268 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -46,9 +46,10 @@ }, "relationships": { "mandatories": { - "predicate": "mandaat:isAangesteldAls", + "predicate": "mandaat:isBestuurlijkeAliasVan", "target": "mandatories", - "cardinality": "many" + "cardinality": "many", + "inverse": true }, "agents-in-position": { "predicate": "org:heldBy", diff --git a/config/search/config.json b/config/search/config.json new file mode 100644 index 0000000..78fe1ac --- /dev/null +++ b/config/search/config.json @@ -0,0 +1,490 @@ +{ + "automatic_index_updates": true, + "persist_indexes": true, + "number_of_threads": 6, + "eager_indexing_groups": [ + [{ "variables": [], "name": "acmidm-lezer" }], + [{ "variables": [], "name": "acmidm-editeerder" }], + [{ "variables": [], "name": "acmidm-beheerder" }] + ], + "types": [ + { + "type": "mandatory", + "on_path": "mandatories", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", + "properties": { + "given_name": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://xmlns.com/foaf/0.1/givenName"], + "id": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://mu.semte.ch/vocabularies/core/uuid"], + "uuid": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://mu.semte.ch/vocabularies/core/uuid"], + "family_name": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","http://xmlns.com/foaf/0.1/familyName"], + "first_name_used": ["http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan","https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam"], + "end_date": "http://data.vlaanderen.be/ns/mandaat#einde", + "organization_id": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "organization_name": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "organization_municipality": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ], + "organization_province": [ + "http://www.w3.org/ns/org#holds", + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "position_id": [ + "http://www.w3.org/ns/org#holds", + "http://www.w3.org/ns/org#role", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "position_name": [ + "http://www.w3.org/ns/org#holds", + "http://www.w3.org/ns/org#role", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { "type": "text" }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { "type": "text" }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { "type": "date" } + } + } + }, + { + "type": "minister", + "on_path": "ministers", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", + "sub_types": ["http://data.lblod.info/vocabularies/contacthub/AgentInPositie"], + "properties": { + "given_name": ["http://www.w3.org/ns/org#heldBy","http://xmlns.com/foaf/0.1/givenName"], + "id": ["http://www.w3.org/ns/org#heldBy","http://mu.semte.ch/vocabularies/core/uuid"], + "uuid": ["http://www.w3.org/ns/org#heldBy","http://mu.semte.ch/vocabularies/core/uuid"], + "family_name": ["http://www.w3.org/ns/org#heldBy","http://xmlns.com/foaf/0.1/familyName"], + "first_name_used": ["http://www.w3.org/ns/org#heldBy","https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam"], + "end_date": "http://data.lblod.info/vocabularies/contacthub/eindedatum", + "organization_id": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "organization_name": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "organization_municipality": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ], + "organization_province": [ + "http://www.w3.org/ns/org#holds", + "^http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "position_id": [ + "http://www.w3.org/ns/org#holds", + "http://data.lblod.info/vocabularies/erediensten/functie", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "position_name": [ + "http://www.w3.org/ns/org#holds", + "http://data.lblod.info/vocabularies/erediensten/functie", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { "type": "text" }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { "type": "text" }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { "type": "date" } + } + } + }, + { + "type" : "person", + "composite_types" : ["minister", "mandatory"], + "on_path" : "people", + "properties" : [ + { + "name" : "given_name", + "mappings" : { + "minister" : "given_name", + "mandatory" : "given_name" + } + }, + { + "name" : "id", + "mappings" : { + "minister" : "id", + "mandatory" : "id" + } + }, + { + "name" : "family_name", + "mappings" : { + "minister" : "family_name", + "mandatory" : "family_name" + } + }, + { + "name" : "first_name_used", + "mappings" : { + "minister" : "first_name_used", + "mandatory" : "first_name_used" + } + }, + { + "name" : "end_date", + "mappings" : { + "minister" : "end_date", + "mandatory" : "end_date" + } + }, + { + "name" : "organization_id", + "mappings" : { + "minister" : "organization_id", + "mandatory" : "organization_id" + } + }, + { + "name" : "organization_name", + "mappings" : { + "minister" : "organization_name", + "mandatory" : "organization_name" + } + }, + { + "name" : "organization_municipality", + "mappings" : { + "minister" : "organization_municipality", + "mandatory" : "organization_municipality" + } + }, + { + "name" : "organization_province", + "mappings" : { + "minister" : "organization_province", + "mandatory" : "organization_province" + } + }, + { + "name" : "position_id", + "mappings" : { + "minister" : "position_id", + "mandatory" : "position_id" + } + }, + { + "name" : "position_name", + "mappings" : { + "minister" : "position_name", + "mandatory" : "position_name" + } + } + ], + "mappings": { + "properties": { + "family_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "id": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "given_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_id": { "type": "text" }, + "organization_province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "organization_municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "position_id": { "type": "text" }, + "position_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "end_date": { "type": "date" } + } + } + }, + { + "type": "unit", + "on_path": "units", + "rdf_type": "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid", + "sub_types": [ + "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst" + ], + "properties": { + "name": "http://www.w3.org/2004/02/skos/core#prefLabel", + "classification_id": [ + "http://www.w3.org/ns/org#classification", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "classification_name": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "status": [ + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "status_id": [ + "http://www.w3.org/ns/regorg#orgStatus", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "recognized_worship_type_id": [ + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://mu.semte.ch/vocabularies/core/uuid" + ], + "recognized_worship_type_name": [ + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ], + "province": [ + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://www.w3.org/ns/locn#adminUnitL2" + ], + "municipality": [ + "http://www.w3.org/ns/org#hasPrimarySite", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "https://data.vlaanderen.be/ns/adres#gemeentenaam" + ] + }, + "mappings": { + "properties": { + "classification_id": { "type": "text" }, + "recognized_worship_type_id": { "type": "text" }, + "name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "classification_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "recognized_worship_type_name": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "province": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + }, + "municipality": { + "type": "text", + "fields": { + "field": { + "type": "keyword" + } + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 4c25a54..cc6d251 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -29,6 +29,7 @@ services: image: lblod/mock-login-service:latest links: - db:database + restart: "no" sink: restart: "no" login: @@ -44,3 +45,23 @@ services: restart: "no" uri-info: restart: "no" + delta-producer-bg-jobs-initiator-administrative-units: + restart: "no" + delta-producer-pub-graph-maintainer-administrative-units: + restart: "no" + delta-producer-json-diff-publisher-administrative-units: + restart: "no" + delta-producer-bg-jobs-initiator-organizations: + restart: "no" + delta-producer-pub-graph-maintainer-organizations: + restart: "no" + delta-producer-json-diff-publisher-organizations: + restart: "no" + search: + restart: "no" + elasticsearch: + restart: "no" + file: + restart: "no" + delta-producer-dump-file-publisher: + restart: "no" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index fa457ae..dd4c91e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: restart: always logging: *default-logging dispatcher: - image: semtech/mu-dispatcher:latest + image: semtech/mu-dispatcher:2.1.0-beta.2 volumes: - ./config/dispatcher:/config labels: @@ -61,7 +61,7 @@ services: restart: always logging: *default-logging resource: - image: semtech/mu-cl-resources:feature-all-errors-as-json-api-errors + image: semtech/mu-cl-resources:feature-all-errors-as-json-api-errors # Bump to fixed version when merged and release links: - db:database volumes: @@ -79,7 +79,7 @@ services: restart: always logging: *default-logging kalliope-api: - image: lblod/jsonld-delta-service:0.5.4 + image: lblod/jsonld-delta-service:0.5.5 volumes: - ./config/kalliope:/config - ./data/files:/share @@ -90,6 +90,7 @@ services: LOGGING_LEVEL: "INFO" SPARQL_ENDPOINT: "http://db:8890/sparql" LIMIT_SIZE: "10000" + MU_AUTH_SUDO: "false" labels: - "logging=true" restart: always @@ -127,7 +128,7 @@ services: links: - db:database accountdetail: - image: lblod/account-detail-service:latest + image: lblod/account-detail-service:0.1.0 labels: - "logging=true" environment: @@ -152,13 +153,13 @@ services: restart: always logging: *default-logging adressenregister: - image: lblod/adressenregister-fuzzy-search-service:0.5.0 + image: lblod/adressenregister-fuzzy-search-service:0.6.0 labels: - "logging=true" restart: always logging: *default-logging frontend: - image: lblod/frontend-contact-hub:1.6.0 + image: lblod/frontend-contact-hub:1.6.3 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: @@ -199,7 +200,7 @@ services: # DELTA GENERAL ################################################################################ delta-producer-dump-file-publisher: - image: lblod/delta-producer-dump-file-publisher:0.4.0 + image: bdevloed/delta-producer-dump-file-publisher:0.5.0-beta volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share @@ -281,7 +282,7 @@ services: DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations' INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations' HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations' - CRON_PATTERN_DUMP_JOB: '30 9 8-18/2 * * MON-FRI' # At second :30, at minute :09, every 2 hours between hours 08am and 18pm, every day between Monday and Friday, every month + CRON_PATTERN_DUMP_JOB: '42 * * * *' # At minute 42 every hour CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. labels: @@ -332,3 +333,22 @@ services: ################################################################################ # DELTA ORGANIZATIONS: END ################################################################################ + + +################################################################################ +# MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH +################################################################################ + search: + image: nbittich/mu-search:wip-inheritance # Bump to fixed version when merged and release to the original repo + links: + - db:database + restart: always + volumes: + - ./config/search:/config + elasticsearch: + image: semtech/mu-search-elastic-backend:1.0.1 + volumes: + - ./data/elasticsearch/:/usr/share/elasticsearch/data + environment: + - discovery.type=single-node + restart: always