From c5e1436ea8fed2e4ff609d22a41cc0ab9e86104e Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 19 Jan 2023 17:50:21 +0100 Subject: [PATCH 01/58] Enable local involvements for central worship services I had to deactivate parts of the cache, causing the frontend to have very weird behaviours when the data was correct --- config/authorization/config.ex | 1 + config/dispatcher/dispatcher.ex | 4 +- ...le-betrokken-lokale-besturen-for-cb.sparql | 19 + ...55023-add-missing-betrokken-besturen.graph | 1 + ...9155023-add-missing-betrokken-besturen.ttl | 3070 +++++++++++++++++ config/resources/domain.json | 18 +- 6 files changed, 3101 insertions(+), 12 deletions(-) create mode 100644 config/migrations/2023/20230119153244-delete-double-betrokken-lokale-besturen-for-cb.sparql create mode 100644 config/migrations/2023/20230119155023-add-missing-betrokken-besturen.graph create mode 100644 config/migrations/2023/20230119155023-add-missing-betrokken-besturen.ttl diff --git a/config/authorization/config.ex b/config/authorization/config.ex index f7a758d..e52cb73 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -18,6 +18,7 @@ defmodule Acl.UserGroups.Config do "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit", "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", + "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid", "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", "http://data.lblod.info/vocabularies/erediensten/VoorwaardenBedienaar", diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index a86c47a..1a8eb1b 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -97,7 +97,7 @@ defmodule Dispatcher do end match "/central-worship-services/*path", %{ accept: [:json], layer: :api} do - Proxy.forward conn, path, "http://cache/central-worship-services/" + Proxy.forward conn, path, "http://resource/central-worship-services/" end match "/representative-bodies/*path", %{ accept: [:json], layer: :api} do @@ -113,7 +113,7 @@ defmodule Dispatcher do end match "/local-involvements/*path", %{ accept: [:json], layer: :api} do - Proxy.forward conn, path, "http://cache/local-involvements/" + Proxy.forward conn, path, "http://resource/local-involvements/" end match "/identifiers/*path", %{ accept: [:json], layer: :api} do diff --git a/config/migrations/2023/20230119153244-delete-double-betrokken-lokale-besturen-for-cb.sparql b/config/migrations/2023/20230119153244-delete-double-betrokken-lokale-besturen-for-cb.sparql new file mode 100644 index 0000000..7b1873d --- /dev/null +++ b/config/migrations/2023/20230119153244-delete-double-betrokken-lokale-besturen-for-cb.sparql @@ -0,0 +1,19 @@ +# We already had betrokken lokale besturen entered for some CB. +# Too keep data aligned with the other applications (Loket, CVP etc), we first flush them +# before re-adding only the necessary ones + +DELETE { + GRAPH ?g { + ?localInvolvement ?plocalInvolvement ?olocalInvolvement . + ?bestuur ?localInvolvement . + } +} WHERE { + GRAPH ?g { + ?centraalBestuur a . + + ?localInvolvement ?centraalBestuur ; + ?plocalInvolvement ?olocalInvolvement . + + ?bestuur ?localInvolvement . + } +} diff --git a/config/migrations/2023/20230119155023-add-missing-betrokken-besturen.graph b/config/migrations/2023/20230119155023-add-missing-betrokken-besturen.graph new file mode 100644 index 0000000..7489cd5 --- /dev/null +++ b/config/migrations/2023/20230119155023-add-missing-betrokken-besturen.graph @@ -0,0 +1 @@ +http://mu.semte.ch/graphs/organisatieportaal \ No newline at end of file diff --git a/config/migrations/2023/20230119155023-add-missing-betrokken-besturen.ttl b/config/migrations/2023/20230119155023-add-missing-betrokken-besturen.ttl new file mode 100644 index 0000000..a886300 --- /dev/null +++ b/config/migrations/2023/20230119155023-add-missing-betrokken-besturen.ttl @@ -0,0 +1,3070 @@ + + + a ; + 100; + ; + "39db6dfe-579c-4c1d-9851-f18547643a33"; + . + + + a ; + 100; + ; + "14d5d68e-ac34-4793-8fde-e43569f0c37d"; + . + + + a ; + 100; + ; + "54281766-9fce-42be-9c83-b8d1ee171e74"; + . + + + a ; + 100; + ; + "c0bcd3a6-517a-4669-a376-c31e3ae6cc17"; + . + + + a ; + 100; + ; + "cd8d283b-7e5d-427a-be95-6ff9b87db757"; + . + + + a ; + 100; + ; + "86b1eac5-aa95-475c-93df-b9586eb077de"; + . + + + a ; + 100; + ; + "15acbee6-a360-43ff-b5cc-fe5dc3bc03ce"; + . + + + a ; + 100; + ; + "296eff94-b3cc-473b-abef-5abf68590916"; + . + + + a ; + 100; + ; + "bba8e9b3-547d-4f52-88cf-36ba90b0cfd8"; + . + + + a ; + 100; + ; + "e2dfa734-ed22-4109-bae1-979575030fbc"; + . + + + a ; + 100; + ; + "291d611f-e5e8-4e65-b0d9-4ddd39026ce0"; + . + + + a ; + 100; + ; + "34c15ea2-7347-4ad7-a3fe-4c0540e6a00b"; + . + + + a ; + 100; + ; + "fcb1b005-226d-4b9a-94ee-25bab1950a52"; + . + + + a ; + 100; + ; + "134dec04-7788-49af-b539-1b2a3b7b2f9b"; + . + + + a ; + 100; + ; + "8f8d28c9-5776-4bb7-9da5-ca8498d36130"; + . + + + a ; + 100; + ; + "91d5cddd-2127-42c4-a14b-9ac09410d35f"; + . + + + a ; + 100; + ; + "8609c2af-ab32-4b1d-8166-5a8aa9549145"; + . + + + a ; + 100; + ; + "60535a06-07a0-4071-8a1b-f776f7d4d68d"; + . + + + a ; + 100; + ; + "a44ea2b7-65df-41ee-a747-7065a7f8df1a"; + . + + + a ; + 100; + ; + "339eb0e3-fbb1-4a6d-9284-e4b47c7afb82"; + . + + + a ; + 100; + ; + "206dfcd4-a6b2-4b04-b83a-3ef2b7a0d630"; + . + + + a ; + 100; + ; + "744e85a0-5217-4f1e-a863-5845f7b19b5a"; + . + + + a ; + 100; + ; + "306b56d0-5294-4ee6-bf22-b62cfdda8d32"; + . + + + a ; + 100; + ; + "7a4c7098-a524-4c1c-abdd-d674a31df7b1"; + . + + + a ; + 100; + ; + "127859a0-41f8-4593-9e65-a0ecdc354bdc"; + . + + + a ; + 100; + ; + "ce1026f8-7474-47f2-86bc-80851d14b15b"; + . + + + a ; + 100; + ; + "77e4d005-5acc-45f7-ab52-c37cdbd27840"; + . + + + a ; + 100; + ; + "0e305e28-e4c8-485a-9d78-4c8a4932108d"; + . + + + a ; + 100; + ; + "c86dfd33-5286-4981-911a-b14182685c0e"; + . + + + a ; + 100; + ; + "f857ddda-4349-4918-9eb0-7fa95c82f903"; + . + + + a ; + 100; + ; + "d8d2a773-c569-4ac5-b0f4-f67560705124"; + . + + + a ; + 100; + ; + "90a72485-877d-428e-9b7a-41a5e282acf3"; + . + + + a ; + 100; + ; + "a3dba296-e6a9-481c-b708-6c57b04c30e0"; + . + + + a ; + 100; + ; + "2fd9d299-0826-47ab-8a4d-e56247fe703a"; + . + + + a ; + 100; + ; + "04a73cb0-8e64-4609-804d-8d189f3e8ceb"; + . + + + a ; + 100; + ; + "8fe7a0f7-767f-4fd2-94f0-8e4c0c3b529e"; + . + + + a ; + 100; + ; + "7c989a6c-ef15-490c-a972-13b380892fb4"; + . + + + a ; + 100; + ; + "18ad5664-1873-4ef1-8be0-7235f903d4c2"; + . + + + a ; + 100; + ; + "8ebc757c-c4d8-486f-a01c-93e25c72708e"; + . + + + a ; + 100; + ; + "7316fb2f-388b-40f8-b6fa-49a91255d6c4"; + . + + + a ; + 100; + ; + "6018fdd0-5588-4652-935f-72fb9f48a03b"; + . + + + a ; + 100; + ; + "5bbe9ad0-2066-4ae4-b893-301ae1a3c8b8"; + . + + + a ; + 100; + ; + "d2a63b79-a907-4e67-a283-dde537b38eca"; + . + + + a ; + 100; + ; + "8c7ad881-4f96-4beb-b69a-0e4a9ac3048a"; + . + + + a ; + 100; + ; + "ad165bd6-0fb9-43df-8e74-28370f306aed"; + . + + + a ; + 100; + ; + "2b5d3333-d0c0-4d56-9435-fa56b32bba7a"; + . + + + a ; + 100; + ; + "c8a4618d-e6f8-4ed4-8dc0-0a1ebc19edf3"; + . + + + a ; + 100; + ; + "992017f5-247b-4e02-acfd-e9882a336cef"; + . + + + a ; + 100; + ; + "94ee9cd9-350e-4cf4-8037-d84e989b13cd"; + . + + + a ; + 100; + ; + "239249a4-1121-48fa-a4d2-99f57a1a6ffa"; + . + + + a ; + 100; + ; + "565404af-88c0-4005-a976-1fcd2f786e48"; + . + + + a ; + 100; + ; + "787879fb-d398-4cdf-9d3e-31bd3fcde711"; + . + + + a ; + 100; + ; + "13737cc0-59e5-439c-95b0-fddc4352a4dc"; + . + + + a ; + 100; + ; + "ca1146a6-786d-4e67-897a-6f9be3a63f8d"; + . + + + a ; + 100; + ; + "9dbc6857-0647-4e8c-85e9-83042bdc0e5c"; + . + + + a ; + 100; + ; + "f83ee6da-c5c6-48cf-b47a-bd7cde8ef7d1"; + . + + + a ; + 100; + ; + "4219f672-aa0c-4d1e-97b0-ed779d14e390"; + . + + + a ; + 100; + ; + "6798d9cd-f314-4bca-8488-2f376439abdd"; + . + + + a ; + 100; + ; + "428f7bb2-ec40-4ec8-be93-cc1f9cf118c2"; + . + + + a ; + 100; + ; + "64d7773a-810b-4eee-9172-bffaa2946ee1"; + . + + + a ; + 100; + ; + "e088c7f4-b5b4-4f55-9815-43c1be94bd2f"; + . + + + a ; + 100; + ; + "83cff3a2-7ab0-4cb7-b9b1-01f88265817e"; + . + + + a ; + 100; + ; + "d1917ea6-f1c5-41cf-b2af-447f1d370d01"; + . + + + a ; + 100; + ; + "64ae6d76-aa1c-4df7-99d3-4efd9a4320df"; + . + + + a ; + 100; + ; + "570ac453-b6b7-47ea-869e-2047b0f7906e"; + . + + + a ; + 100; + ; + "741ddc80-fdff-432f-895d-484ee3fb035c"; + . + + + a ; + 100; + ; + "f0c82c4c-e50d-4885-a769-3ecd1edd8ee7"; + . + + + a ; + 100; + ; + "dc963735-8bb7-4faa-9c57-c2bcdbc9782a"; + . + + + a ; + 100; + ; + "7a25c32f-e161-43da-af61-133179ff6fdf"; + . + + + a ; + 100; + ; + "25c11629-4084-42f8-94ac-918f3ef6cfd4"; + . + + + a ; + 100; + ; + "61e30f3a-54b2-41fb-b2ff-f96207a98d8c"; + . + + + a ; + 100; + ; + "278ccdf8-5170-4501-9daa-d4a853de40cd"; + . + + + a ; + 100; + ; + "cc97c7ae-8b3f-464d-89c9-30e74abaea79"; + . + + + a ; + 100; + ; + "9a3a5452-e626-4886-9499-d4826e70206c"; + . + + + a ; + 100; + ; + "b7695bf9-8f3e-4a90-b029-230e069fc38e"; + . + + + a ; + 100; + ; + "4ebb7193-400b-4415-9508-4a3af27a029a"; + . + + + a ; + 100; + ; + "31658bc3-3296-4f2f-9e91-0badefc5c790"; + . + + + a ; + 100; + ; + "28246920-b91c-4321-8e10-65c1edc3c7ab"; + . + + + a ; + 100; + ; + "250a4a81-2394-4828-8aa8-3ec6e2625908"; + . + + + a ; + 100; + ; + "868c83a4-bd00-436f-93a8-3fae5f32ccea"; + . + + + a ; + 100; + ; + "baa67736-47f5-4838-a5f0-3b2e50af4ae9"; + . + + + a ; + 100; + ; + "8e0e36f0-99fe-4a4f-af12-b0408163ce21"; + . + + + a ; + 100; + ; + "f5413fe1-5117-473a-8f7f-d9da25e2f5a6"; + . + + + a ; + 100; + ; + "e4455570-25ed-478e-8fa4-be7ecac66c11"; + . + + + a ; + 100; + ; + "50a9a3b9-a9ed-4d94-891d-4d70b8cc1a2a"; + . + + + a ; + 100; + ; + "62bd8d74-c266-4a42-8a43-bf8fb8887bf5"; + . + + + a ; + 100; + ; + "214bc598-39ea-496e-906b-60ec04543b08"; + . + + + a ; + 100; + ; + "49751f48-7076-40ac-8002-b0793aa4aa5d"; + . + + + a ; + 100; + ; + "68c31d29-4373-4215-a9dd-e26b90be567c"; + . + + + a ; + 100; + ; + "d243b200-81a3-43d7-8dbf-df8f89fae3a3"; + . + + + a ; + 100; + ; + "7bcd745d-6fa6-4a57-94ec-5aae6afd05a1"; + . + + + a ; + 100; + ; + "62786f76-4e5d-46df-958f-b45c95e6aa03"; + . + + + a ; + 100; + ; + "3c6efedb-efb6-4aaf-877d-7cbbe3bc9aa3"; + . + + + a ; + 100; + ; + "ed3bebca-9177-4e8c-8e65-27616d334a18"; + . + + + a ; + 100; + ; + "e9c7a18b-197c-401e-8239-14f990c950fa"; + . + + + a ; + 100; + ; + "568e868f-7005-40b9-957c-9e02054b1a80"; + . + + + a ; + 100; + ; + "162a3bc7-ec69-4dd2-b352-6848290f4f7a"; + . + + + a ; + 100; + ; + "12a63c62-dc35-40fa-a9c4-33890fb75b9e"; + . + + + a ; + 100; + ; + "fe93089a-d4c1-4dc7-a774-b3d01decb8c3"; + . + + + a ; + 100; + ; + "35c5df16-5ef4-43bf-97f0-681654a1549e"; + . + + + a ; + 100; + ; + "4ca67124-51c6-4c39-b2c2-76f332d74a90"; + . + + + a ; + 100; + ; + "ba45706a-e24e-4adc-96a1-f6475d9e72a0"; + . + + + a ; + 100; + ; + "a1ffa407-3e09-4c59-8666-41e4d06c93c1"; + . + + + a ; + 100; + ; + "e894ea54-e3f5-497b-b2db-99244d7b8492"; + . + + + a ; + 100; + ; + "b6e810a9-565a-4157-b3bb-dcc88e8e5338"; + . + + + a ; + 100; + ; + "e6b76587-cb8c-4689-9f21-5cfb9f108f9d"; + . + + + a ; + 100; + ; + "9f0c28c3-6859-46b9-ac36-b53efaa333d7"; + . + + + a ; + 100; + ; + "4fb4a21b-74f0-4e98-afb3-62e32c03f793"; + . + + + a ; + 100; + ; + "a11ea4bb-6f42-40b5-948f-2d4645e96b10"; + . + + + a ; + 100; + ; + "aa9f88e5-132b-4bfa-9c5f-b226c6ef44f1"; + . + + + a ; + 100; + ; + "d22dc214-5d35-46a8-85b6-8f0b2793b7eb"; + . + + + a ; + 100; + ; + "79c1a286-69a7-4c6d-bfd9-eb82749fd759"; + . + + + a ; + 100; + ; + "697412fc-de35-41ee-9786-81e62ef25f56"; + . + + + a ; + 100; + ; + "207be3f8-e1fc-46f0-acfb-3e9f9660b344"; + . + + + a ; + 100; + ; + "2ac22503-9450-42c3-82a5-96cab6bd72d4"; + . + + + a ; + 100; + ; + "6572b77d-be77-4c5f-b1fb-473f34ace8ac"; + . + + + a ; + 100; + ; + "dd510104-5352-4a44-962f-0424c960c251"; + . + + + a ; + 100; + ; + "d02b415a-c66c-42b6-a15b-0e5101edf9cc"; + . + + + a ; + 100; + ; + "0632f407-2829-4a17-afb6-5725fbcf80db"; + . + + + a ; + 100; + ; + "f1dcd064-35aa-4d47-a8a9-5f3532e4787f"; + . + + + a ; + 100; + ; + "e7c618b7-f47b-46cd-b081-59839d20398f"; + . + + + a ; + 100; + ; + "ff579e0c-fce9-4855-8a06-ede8de9e9754"; + . + + + a ; + 100; + ; + "eb38cf59-77d6-4fb1-8912-183d90a6affb"; + . + + + a ; + 100; + ; + "87d179f6-7413-4137-bdf6-63ac9d4c16d7"; + . + + + a ; + 100; + ; + "466e8074-bd15-46d7-95e1-02a03527948e"; + . + + + a ; + 100; + ; + "8579babe-cfa9-4dc5-92ed-15044ef5a098"; + . + + + a ; + 100; + ; + "71680486-630e-46a4-8325-37b3026b1385"; + . + + + a ; + 100; + ; + "6417268c-ec01-4873-89d1-7bb777ad1583"; + . + + + a ; + 100; + ; + "64128756-b6c8-4e0b-8fce-df7bbf327c0c"; + . + + + a ; + 100; + ; + "9905d0ad-9ca5-4077-b5b5-452f207b45f7"; + . + + + a ; + 100; + ; + "77e1f4cb-1055-46d1-b052-852bdd79fe68"; + . + + + a ; + 100; + ; + "d19a92da-1ddc-4410-9188-9c3941f41173"; + . + + + a ; + 100; + ; + "e0c8b859-c442-4e0f-8735-160a95ba8ac3"; + . + + + a ; + 100; + ; + "09dede7e-1e6e-4c0a-b0f6-2b3c52291d5b"; + . + + + a ; + 100; + ; + "937533ac-0cb8-4e78-91f7-c23dfee89bb2"; + . + + + a ; + 100; + ; + "f80a6a4c-9b90-42ae-ab5c-8ab9bbfe2724"; + . + + + a ; + 100; + ; + "089cb5a2-8b1c-4c5b-a49e-5a176a0d145a"; + . + + + a ; + 100; + ; + "706b2012-3aba-477e-9c9c-ce47f29e3840"; + . + + + a ; + 100; + ; + "b88e6c3d-21ac-4252-8526-a8fad885002e"; + . + + + a ; + 100; + ; + "3611f01a-6439-4631-9ae1-da38a5ba83f0"; + . + + + a ; + 100; + ; + "f74bebb6-1322-4d3d-b96c-acce0320ae36"; + . + + + a ; + 100; + ; + "d915a89f-703d-49c0-9dc2-7eaf5c419a99"; + . + + + a ; + 100; + ; + "ce3cce03-8a26-4eb2-8b22-887c99dd27b9"; + . + + + a ; + 100; + ; + "213e15a7-e334-4aa3-ad36-8cded8a27157"; + . + + + a ; + 100; + ; + "589e11e6-367b-4649-a15a-ac14ba1c4d8e"; + . + + + a ; + 100; + ; + "9246c603-62b1-4984-a90b-4f1608b600e4"; + . + + + a ; + 100; + ; + "35274e45-b5b6-46b2-afe1-98e0aaff5a58"; + . + + + a ; + 100; + ; + "5fb9cd00-c6b5-49b5-8eb5-381bb8167446"; + . + + + a ; + 100; + ; + "c34307ad-7555-4140-827d-34f49191622a"; + . + + + a ; + 100; + ; + "27a6cb6a-e17b-4f57-96e4-b3ff6f783e5a"; + . + + + a ; + 100; + ; + "128b0d77-a84f-404b-87c7-8a979cbfac44"; + . + + + a ; + 100; + ; + "70e8bbcc-3fec-45fb-855a-f10029468a19"; + . + + + a ; + 100; + ; + "9cba7852-98c8-4e71-b702-4d4057b8f2c5"; + . + + + a ; + 100; + ; + "b0a6cc03-f5d0-4232-89ed-9c9d4a82f211"; + . + + + a ; + 100; + ; + "7ad2d2ad-c003-472d-a9d3-cc0217e8244c"; + . + + + a ; + 100; + ; + "4a062eeb-8a0c-4fee-9d65-95ee2201f2a6"; + . + + + a ; + 100; + ; + "01a80a07-f933-4eb3-ad55-abbce96232f1"; + . + + + a ; + 100; + ; + "b7bbabee-aaea-4380-a6ea-ef80450a5419"; + . + + + a ; + 100; + ; + "50ae7530-3c5c-47c5-a4c3-e867f7b8d913"; + . + + + a ; + 100; + ; + "b91371e9-96fd-40e7-94ac-2dff4b25bf94"; + . + + + a ; + 100; + ; + "8a01e75c-a469-484c-b16a-925066ce84f6"; + . + + + a ; + 100; + ; + "fa7e9864-e350-4183-adb1-c448689fdc10"; + . + + + a ; + 100; + ; + "77bf3b5f-3f68-429a-885e-d88dbfefb3d2"; + . + + + a ; + 100; + ; + "c53d5dba-3877-43fd-9d41-d73b92549174"; + . + + + a ; + 100; + ; + "58808ab5-24a8-413b-80fb-68972c666bdd"; + . + + + a ; + 100; + ; + "3a91c63a-e100-49f8-9385-89c08c87675a"; + . + + + a ; + 100; + ; + "cabaeee7-baa5-453c-85c5-64b797f74734"; + . + + + a ; + 100; + ; + "2a51b7c0-2708-4c9b-9ce8-8eb2c1c16b11"; + . + + + a ; + 100; + ; + "0b24ff63-9e02-4149-b0da-5795de03ae6f"; + . + + + a ; + 100; + ; + "5464eefb-2fb0-4737-9304-466669a5ddc3"; + . + + + a ; + 100; + ; + "047418fa-8aa8-4524-a4a3-65fdde2bd492"; + . + + + a ; + 100; + ; + "b09b6515-5781-48cc-ba0b-2d22e765b432"; + . + + + a ; + 100; + ; + "fadbd6f2-0118-43a8-9eb7-e8d4a91bc8ed"; + . + + + a ; + 100; + ; + "80f91b42-8387-40e3-a57d-682f35b5954d"; + . + + + a ; + 100; + ; + "df27db01-f88c-4c60-988b-74fec112c8b4"; + . + + + a ; + 100; + ; + "81da9158-07fa-44bb-b598-31ad4ac20dc6"; + . + + + a ; + 100; + ; + "bfabc47a-37f0-4ed5-b4ba-70052e83d135"; + . + + + a ; + 100; + ; + "a259314c-5931-4122-bac1-e7f9813cc10c"; + . + + + a ; + 100; + ; + "16243d63-2cfd-42a4-834a-3ccb9a01a2f6"; + . + + + a ; + 100; + ; + "6a929c1b-82e4-4d93-a3fe-d91a75217f46"; + . + + + a ; + 100; + ; + "7ff5d6d2-d991-486a-ad03-2beba2a88f68"; + . + + + a ; + 100; + ; + "28b914fa-0e7d-47e2-99fa-9fa3a409be54"; + . + + + a ; + 100; + ; + "8b4a085c-99a1-4789-b566-b6ad837bcd44"; + . + + + a ; + 100; + ; + "ad535a13-cff6-422b-a083-c827cd77f232"; + . + + + a ; + 100; + ; + "709b701c-573c-42b3-ae08-4db84215308b"; + . + + + a ; + 100; + ; + "cb8b230f-f1e4-4992-981d-e4d0ecf889a3"; + . + + + a ; + 100; + ; + "c345fbfc-ab9b-4211-a032-e41753da5288"; + . + + + a ; + 100; + ; + "9a7efbb9-d5ff-418d-b02e-f9c09d8489f5"; + . + + + a ; + 100; + ; + "d7565813-18f0-4dad-b6b8-41b91947f105"; + . + + + a ; + 100; + ; + "f7054ad2-b4e6-4117-9f8a-d40e03da4ce5"; + . + + + a ; + 100; + ; + "5ec8ca42-4386-4888-a875-f2175106a813"; + . + + + a ; + 100; + ; + "4baf611b-0850-4c92-9dd2-8872c5c4f629"; + . + + + a ; + 100; + ; + "93db383d-b5f2-4bde-bf0e-2d36c5d2efa0"; + . + + + a ; + 100; + ; + "d2398702-63bf-498d-870d-6060cb716bc4"; + . + + + a ; + 100; + ; + "1047d898-f0a3-4f45-ae81-ea8e44e1599d"; + . + + + a ; + 100; + ; + "88a03a37-2ff7-47e5-a79b-9dc6e5b99a50"; + . + + + a ; + 100; + ; + "0206d727-0cf5-4cb7-bc80-9af44cb205e2"; + . + + + a ; + 100; + ; + "9985cafe-8eeb-4ca0-bdea-6f644f1466a1"; + . + + + a ; + 100; + ; + "54958763-5035-4429-a978-2f390c5e2b6f"; + . + + + a ; + 100; + ; + "d09c807b-a88a-452e-8522-ab1192e4c14f"; + . + + + a ; + 100; + ; + "2feff2ed-ea4d-451e-b128-de14e7c2327e"; + . + + + a ; + 100; + ; + "2ed9ac9d-5131-40aa-9881-c23860283973"; + . + + + a ; + 100; + ; + "dfa69fc9-3739-4e65-9d18-1c8fd7721e34"; + . + + + a ; + 100; + ; + "fe97d33f-d6f9-45c4-9526-caac68f2e880"; + . + + + a ; + 100; + ; + "7d22ca2f-1203-4333-bc93-f8e7826dfbf9"; + . + + + a ; + 100; + ; + "935cc3be-c4e1-4b3f-9aaf-c12c2b294b3d"; + . + + + a ; + 100; + ; + "b37752e5-6010-4bde-9528-d75267568b46"; + . + + + a ; + 100; + ; + "fd55d258-10c1-41fc-8158-ebe18ef9a3cf"; + . + + + a ; + 100; + ; + "b02539bd-7de9-4b53-914d-b77a0b70b346"; + . + + + a ; + 100; + ; + "c4ea099a-26b3-4a6c-8d81-6b7985579499"; + . + + + a ; + 100; + ; + "3440173c-27e5-4f0d-8f02-be3b91f605be"; + . + + + a ; + 100; + ; + "c4c82ddf-4cd4-4e82-8976-45987e24c0fe"; + . + + + a ; + 100; + ; + "7f467c15-7ac5-4fc5-92af-83aa20bc920b"; + . + + + a ; + 100; + ; + "ba677230-e9bd-4902-b763-7672e5a3a979"; + . + + + a ; + 100; + ; + "d8dfe222-6a06-4b8c-a925-84bbc1d1cace"; + . + + + a ; + 100; + ; + "efea1be4-8f83-41d3-a3b1-44aa09d1e258"; + . + + + a ; + 100; + ; + "509a0e42-bc30-442f-a872-d39fe8388f33"; + . + + + a ; + 100; + ; + "74abb738-8f89-413b-81b6-bc7f461b278a"; + . + + + a ; + 100; + ; + "bf6ecbe4-7ce3-4711-918f-04f498367d47"; + . + + + a ; + 100; + ; + "78aa54ff-a9ef-47d9-81d0-9266cb8ef940"; + . + + + a ; + 100; + ; + "e2620be3-8f6c-4dee-a0ef-ed02009aa7dc"; + . + + + a ; + 100; + ; + "5ed9eb05-627f-4b28-8ed5-813e4b3dbf38"; + . + + + a ; + 100; + ; + "cb597309-1b3e-41e2-8798-1670b0b3c2ad"; + . + + + a ; + 100; + ; + "a215fa25-a829-48b2-b40c-6edea1494795"; + . + + + a ; + 100; + ; + "a6309fc7-47e5-48a8-882b-2995f54463ad"; + . + + + a ; + 100; + ; + "961861e5-a0a8-4591-b50a-815de0c98136"; + . + + + a ; + 100; + ; + "bf59dce8-dc6d-4cd6-a47a-aebc02e50e5b"; + . + + + a ; + 100; + ; + "0ccd39a1-6007-49d8-b4a3-57b5ec6cfc3b"; + . + + + a ; + 100; + ; + "65771337-68d9-4e12-821f-21e8bb2f3cdf"; + . + + + a ; + 100; + ; + "5890a48a-bff5-4dce-869a-64f386827dc1"; + . + + + a ; + 100; + ; + "4c2744fb-08d7-4be9-8392-d1fdf4664095"; + . + + + a ; + 100; + ; + "5572181b-d91c-4258-828f-372ae64f10e9"; + . + + + a ; + 100; + ; + "1a01900a-a9ce-4d10-87aa-df79e0d529ad"; + . + + + a ; + 100; + ; + "cb39f5f3-ca4e-4d17-b9f2-9cd0b9a5cfdc"; + . + + + a ; + 100; + ; + "e356d500-f70b-4f96-b2aa-030331b3368c"; + . + + + a ; + 100; + ; + "f62426cf-e089-49f3-8222-7e5af777ad0a"; + . + + + a ; + 100; + ; + "af47b89b-a457-43c8-aad3-d0c2f03b097b"; + . + + + a ; + 100; + ; + "e19033b0-8f1e-4f67-b2b4-a293a59f4aab"; + . + + + a ; + 100; + ; + "af5ff968-30f0-48c6-80bd-17c56501ed60"; + . + + + a ; + 100; + ; + "84b30cfb-94c9-419a-8bf7-041a1ab94ac4"; + . + + + a ; + 100; + ; + "b96042b6-bfbb-4187-803d-6661675958c3"; + . + + + a ; + 100; + ; + "0ac22758-7380-46be-b881-cb962acbcc4f"; + . + + + a ; + 100; + ; + "d119dc08-213d-4030-ab98-fdbe796d4667"; + . + + + a ; + 100; + ; + "9fffe76f-9cfb-4e91-8408-d70c4feb75bc"; + . + + + a ; + 100; + ; + "64c9f77a-5078-449e-966d-10de580ae689"; + . + + + a ; + 100; + ; + "be57a498-43d8-4c3a-8a5e-a1dc5f9d1fc3"; + . + + + a ; + 100; + ; + "06d5bcbe-7917-48f8-9e55-3a24a632b627"; + . + + + a ; + 100; + ; + "a242d7d2-ab33-4cb6-8490-22584778764c"; + . + + + a ; + 100; + ; + "714a2f24-e9ba-4219-89ab-49129fba7986"; + . + + + a ; + 100; + ; + "8daf621b-766d-45d0-b071-9ca71bcb4ff5"; + . + + + a ; + 100; + ; + "a8f2b132-4257-422e-9bd0-0f8081311fef"; + . + + + a ; + 100; + ; + "65163706-f40e-4944-9f18-ac0bdc51a368"; + . + + + a ; + 100; + ; + "273f3fe0-fd88-4c20-9674-de1ac7a598f8"; + . + + + a ; + 100; + ; + "f1819db5-b9b5-45a2-9897-3fd3e551e0c3"; + . + + + a ; + 100; + ; + "281edce4-7008-4464-8ad5-18e74b878124"; + . + + + a ; + 100; + ; + "b663f2e9-22c5-4308-82da-668b33efbb2d"; + . + + + a ; + 100; + ; + "fba041d9-1d8b-4c86-8454-6ba72da0db82"; + . + + + a ; + 100; + ; + "c8dc25bf-8622-4f53-a61e-f2a852b8f441"; + . + + + a ; + 100; + ; + "7ec382e1-fb93-4fb8-8a33-35491e963ad1"; + . + + + a ; + 100; + ; + "14c3966b-ab20-4c0d-b61d-04c386072052"; + . + + + a ; + 100; + ; + "3a2fff57-3aca-4278-a7cc-c44ce6504d99"; + . + + + a ; + 100; + ; + "ae1fb5b6-b210-4b1a-81e7-429cfa4c6321"; + . + + + a ; + 100; + ; + "a3b5ba49-7826-4221-afbe-58ea09f7e9ae"; + . + + + a ; + 100; + ; + "a96b574f-354e-45b5-b5c7-323c3686b20f"; + . + + + a ; + 100; + ; + "d7cd7da2-d97c-4496-9ed3-c423a0db4292"; + . + + + a ; + 100; + ; + "c9fd8612-c459-40e7-bff9-de204a9e4733"; + . + + + a ; + 100; + ; + "6462d6a9-604d-4860-a480-06de8ab88488"; + . + + + a ; + 100; + ; + "5d2b4250-0927-4c24-aab7-08ba65a4f56a"; + . + + + a ; + 100; + ; + "cd84600f-5d7a-43c8-b971-8285b67ae173"; + . + + + a ; + 100; + ; + "d7672d1f-893b-496d-ae39-55fa40ab99b6"; + . + + + a ; + 100; + ; + "7a1bc677-2030-48fb-85c3-7633a5c6a0a6"; + . + + + a ; + 100; + ; + "4dd4ef9f-36cd-4a40-a16d-1cd65bbaf3ec"; + . + + + a ; + 100; + ; + "4bdbb2b6-3ce4-41c7-b05a-10867a142668"; + . + + + a ; + 100; + ; + "ec55bfbc-67cd-41d9-8823-3aca1d4c959b"; + . + + + a ; + 100; + ; + "8d13312e-69c3-4284-b371-737b58be2361"; + . + + + a ; + 100; + ; + "3a1038f8-6e42-49b2-9f25-3bab894414f1"; + . + + + a ; + 100; + ; + "4c392c17-bff7-4f24-b288-99e3df00b589"; + . + + + a ; + 100; + ; + "213df356-1b68-45c0-a1db-8feab78aec81"; + . + + + a ; + 100; + ; + "692921a6-2465-47eb-8e04-2c29b49f50de"; + . + + + a ; + 100; + ; + "dcb29e8e-263d-490a-97ef-b312252822d8"; + . + + + a ; + 100; + ; + "4dc96030-88e8-4748-adbf-1aebdc2ea2b3"; + . + + + a ; + 100; + ; + "ad539bb3-f41a-44a9-947f-0dc6ecb29fea"; + . + + + a ; + 100; + ; + "03ddc34f-7990-4bb2-888b-37bce5ce044e"; + . + + + a ; + 100; + ; + "9773c47e-4206-44cd-81ac-a437e080b9d0"; + . + + + a ; + 100; + ; + "637812c3-34c4-4339-8618-bcc2675d83d4"; + . + + + a ; + 100; + ; + "94f23563-6df6-4808-8324-5924c86b4f25"; + . + + + a ; + 100; + ; + "dd898f3b-87e5-432e-9e00-5d0d3fa54b05"; + . + + + a ; + 100; + ; + "849f49cf-bd09-4f88-96ea-e1bd702a94ca"; + . + + + a ; + 100; + ; + "5e40cc59-194b-4b42-b64a-6a53a8cb1152"; + . + + + a ; + 100; + ; + "37b2f71b-c14f-4fa5-ba7d-8a765c87859e"; + . + + + a ; + 100; + ; + "4ba2e1d8-fa5d-4002-9d1b-c168a2a84045"; + . + + + a ; + 100; + ; + "34837fc6-c5f9-424e-8d9e-b9fef8751a8a"; + . + + + a ; + 100; + ; + "aeeed194-f748-4ec1-a2c8-c54e8fd60545"; + . + + + a ; + 100; + ; + "b8d72a74-90c1-4241-8b3f-9df3abb6299b"; + . + + + a ; + 100; + ; + "d5f854b4-0751-4a3a-8f4b-227e3dd8252e"; + . + + + a ; + 100; + ; + "e395419c-b06e-484e-8302-cd2d1011601f"; + . + + + a ; + 100; + ; + "2924cc14-f51c-4692-ae4b-84f886d4daf9"; + . + + + a ; + 100; + ; + "c6e14db2-95da-4189-9d60-051164b5e0b5"; + . + + + a ; + 100; + ; + "9ae6d05c-a9db-429b-8c7c-65367ff00fcb"; + . + + + a ; + 100; + ; + "49d534b3-50ca-41ed-9fea-80cd49f67454"; + . + + + a ; + 100; + ; + "ef6637fa-f831-4c6a-92c7-2f8a1c29e383"; + . + + + a ; + 100; + ; + "7d08085a-3e2a-46ba-9347-8681b8173137"; + . + + + a ; + 100; + ; + "ade9acde-d2e6-4ae3-b733-733d8465e83b"; + . + + + a ; + 100; + ; + "44d35426-24a8-4516-b9bb-6c8b56417e85"; + . + + + a ; + 100; + ; + "2e60a1c4-358c-409f-94ab-cbf817d55569"; + . + + + a ; + 100; + ; + "626974f2-ad28-4a67-a59c-a60cd0c0ab77"; + . + + + a ; + 100; + ; + "a03b1945-3232-4561-b719-78b3e77b366e"; + . + + + a ; + 100; + ; + "40e1c80d-17ab-47ce-ba7c-bfcb39e7cd57"; + . + + + a ; + 100; + ; + "f09f7a20-5c11-4277-902c-851c1f98e031"; + . + + + a ; + 100; + ; + "f28864b3-d734-4c52-bf0c-e6335be2f714"; + . + + + a ; + 100; + ; + "12451bc0-ed31-4c6b-9d1e-0c68ec2d23b5"; + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + , + , + , + , + , + , + , + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + . + + + , + . + + + . + + + . + + + . \ No newline at end of file diff --git a/config/resources/domain.json b/config/resources/domain.json index c524da8..6a29794 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -492,6 +492,12 @@ "predicate": "ere:wordtBediendDoor", "target": "minister-positions", "cardinality": "many" + }, + "involvements": { + "predicate": "org:organization", + "target": "local-involvements", + "cardinality": "many", + "inverse": true } }, "new-resource-base": "http://data.lblod.info/id/eredienstBestuurseenheden/" @@ -510,14 +516,6 @@ "predicate": "ere:grensoverschrijdend" } }, - "relationships": { - "involvements": { - "predicate": "org:organization", - "target": "local-involvements", - "cardinality": "many", - "inverse": true - } - }, "new-resource-base": "http://data.lblod.info/id/besturenVanDeEredienst/" }, "recognized-worship-types": { @@ -631,9 +629,9 @@ "target": "involvement-types", "cardinality": "one" }, - "worship-service": { + "worship-administrative-unit": { "predicate": "org:organization", - "target": "worship-services", + "target": "worship-administrative-units", "cardinality": "one" }, "administrative-unit": { From 03e5c42dcc23bcdcc2c2239b57e3ddfc9bc6ebbc Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 26 Jan 2023 10:01:44 +0000 Subject: [PATCH 02/58] removed comma --- config/resources/domain.json | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/config/resources/domain.json b/config/resources/domain.json index 6a29794..164d134 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -23,7 +23,7 @@ "skos": "http://www.w3.org/2004/02/skos/core#", "euvoc": "http://publications.europa.eu/ontology/euvoc#", "ere": "http://data.lblod.info/vocabularies/erediensten/", - "ch": "http://data.lblod.info/vocabularies/contacthub/", + "ch": "http://data.lblod.info/vocabularies/contacthub/", "code": "http://lblod.data.gift/vocabularies/organisatie/", "ext": "http://mu.semte.ch/vocabularies/ext/", "geo": "http://www.opengis.net/ont/geosparql#" @@ -63,7 +63,7 @@ "target": "agents", "cardinality": "many", "inverse": true - }, + } }, "new-resource-base": "http://data.lblod.info/id/personen/" }, @@ -223,7 +223,7 @@ "type": "string", "predicate": "schema:contactType" } - }, + }, "relationships": { "contact-address": { "predicate": "locn:address", @@ -269,7 +269,7 @@ "type": "datetime", "predicate": "mandaat:einde" } - }, + }, "relationships": { "board-position": { "predicate": "org:holds", @@ -358,7 +358,7 @@ "predicate": "skos:prefLabel" }, "alternative-name": { - "type": "string", + "type": "string", "predicate": "skos:altLabel" } }, @@ -501,7 +501,6 @@ } }, "new-resource-base": "http://data.lblod.info/id/eredienstBestuurseenheden/" - }, "worship-services": { "class": "ere:BestuurVanDeEredienst", @@ -709,7 +708,6 @@ "type": "url", "predicate": "adres:verwijstNaar" } - }, "relationships": { "source": { @@ -836,9 +834,9 @@ "predicate": "ere:voldoetAan", "target": "minister-conditions", "cardinality": "many" - } + } }, - "new-resource-base": "http://data.lblod.info/id/rollenBedienaar/" + "new-resource-base": "http://data.lblod.info/id/rollenBedienaar/" }, "minister-conditions": { "class": "ere:VoorwaardenBedienaar", @@ -966,4 +964,4 @@ "new-resource-base": "http://lblod.data.gift/concepts/" } } -} \ No newline at end of file +} From eb78e3a6aa3d00a626fb565eac44cb3a9a6baaec Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 26 Jan 2023 11:01:58 +0000 Subject: [PATCH 03/58] typo in target --- config/resources/domain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/resources/domain.json b/config/resources/domain.json index 164d134..a763d2a 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -108,7 +108,7 @@ }, "organization": { "predicate": "org:postIn", - "target": "organization", + "target": "organizations", "cardinality": "one" }, "agents-in-position": { From 9b4fe1c76ed89c256251301a334af67f9519e09f Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 14 Feb 2023 16:44:24 +0100 Subject: [PATCH 04/58] Remove unused admin unit producer --- .../administrative-units/export.json | 54 ---------------- .../dump-file-publisher/config.json | 16 ----- docker-compose.dev.yml | 6 -- docker-compose.yml | 61 ------------------- 4 files changed, 137 deletions(-) delete mode 100644 config/delta-producer/administrative-units/export.json diff --git a/config/delta-producer/administrative-units/export.json b/config/delta-producer/administrative-units/export.json deleted file mode 100644 index d38b7b0..0000000 --- a/config/delta-producer/administrative-units/export.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "export": [ - { - "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", - "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" - ], - "properties": [ - "http://mu.semte.ch/vocabularies/core/uuid", - "http://www.w3.org/2004/02/skos/core#prefLabel", - "http://www.w3.org/2004/02/skos/core#scopeNote" - ] - }, - { - "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", - "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" - ], - "properties": [ - "http://mu.semte.ch/vocabularies/core/uuid", - "http://www.w3.org/2004/02/skos/core#prefLabel", - "http://www.w3.org/2004/02/skos/core#altLabel", - "http://data.vlaanderen.be/ns/besluit#werkingsgebied", - "http://www.w3.org/ns/org#classification" - ] - }, - { - "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", - "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" - ], - "properties": [ - "http://mu.semte.ch/vocabularies/core/uuid", - "http://www.w3.org/2004/02/skos/core#prefLabel", - "http://data.vlaanderen.be/ns/besluit#bestuurt", - "http://www.w3.org/ns/org#classification", - "http://data.vlaanderen.be/ns/mandaat#bindingEinde", - "http://data.vlaanderen.be/ns/mandaat#bindingStart", - "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan" - ] - }, - { - "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", - "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" - ], - "properties": [ - "http://mu.semte.ch/vocabularies/core/uuid", - "http://www.w3.org/2004/02/skos/core#prefLabel", - "http://www.w3.org/2004/02/skos/core#scopeNote" - ] - } - ] -} diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index d8c54f3..a73ccad 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -1,20 +1,4 @@ { - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", - "fileBaseName": "dump-administrative-units", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units": { - "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", - "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", - "fileBaseName": "dump-administrative-units", - "targetDcatGraph": "http://mu.semte.ch/graphs/public", - "targetFilesGraph": "http://mu.semte.ch/graphs/public", - "cleanupOldDumps": true - }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5d68e6d..c793b9d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -49,12 +49,6 @@ services: restart: "no" uri-info: restart: "no" - delta-producer-bg-jobs-initiator-administrative-units: - restart: "no" - delta-producer-pub-graph-maintainer-administrative-units: - restart: "no" - delta-producer-json-diff-publisher-administrative-units: - restart: "no" delta-producer-bg-jobs-initiator-organizations: restart: "no" delta-producer-pub-graph-maintainer-organizations: diff --git a/docker-compose.yml b/docker-compose.yml index 6d1f4cb..2f35a1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -227,67 +227,6 @@ services: restart: always logging: *default-logging ################################################################################ - # DELTA ADMINISTRATIVE-UNITS: START - ################################################################################ - delta-producer-bg-jobs-initiator-administrative-units: - image: lblod/delta-producer-background-jobs-initiator:0.3.1 - environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" - DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units" - CRON_PATTERN_DUMP_JOB: "0 0 4 * * 6" # weekly on saturday - CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM - START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - delta-producer-pub-graph-maintainer-administrative-units: - image: lblod/delta-producer-publication-graph-maintainer:0.7.0 - environment: - MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" - RELATIVE_FILE_PATH: "deltas/administrative-units" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/administrative-units" - HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/administrative-units" - INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" - QUEUE_POLL_INTERVAL: 3000 - USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive - SKIP_MU_AUTH_INITIAL_SYNC: "true" - HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - volumes: - - ./config/delta-producer/administrative-units:/config - - ./data/files/:/share - links: - - db:database - - triplestore:virtuoso - restart: always - logging: *default-logging - delta-producer-json-diff-publisher-administrative-units: - image: lblod/delta-producer-json-diff-file-publisher:0.2.3 - environment: - MAX_BODY_SIZE: "50mb" - ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" - RELATIVE_FILE_PATH: "deltas/administrative-units" - PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/administrative-units" - PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/administrative-units" - DELTA_INTERVAL_MS: 10000 - PRETTY_PRINT_DIFF_JSON: "true" - volumes: - - ./data/files:/share - labels: - - "logging=true" - links: - - db:database - restart: always - logging: *default-logging - ################################################################################ - # DELTA ADMINISTRATIVE-UNITS: END - ################################################################################ - ################################################################################ # DELTA ORGANIZATIONS: START ################################################################################ delta-producer-bg-jobs-initiator-organizations: From f4b24b10d8e52c25aba8c57310246c139c42b2e3 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Wed, 15 Feb 2023 10:46:45 +0100 Subject: [PATCH 05/58] add new codelist (#236) --- .../20230213113324-add-new-codelist.sparql | 18 + .../20230215094055-add-missing-position.graph | 1 + .../20230215094055-add-missing-position.ttl | 613 ++++++++++++++++++ 3 files changed, 632 insertions(+) create mode 100644 config/migrations/2023/20230213113324-add-new-codelist.sparql create mode 100644 config/migrations/2023/20230215094055-add-missing-position.graph create mode 100644 config/migrations/2023/20230215094055-add-missing-position.ttl diff --git a/config/migrations/2023/20230213113324-add-new-codelist.sparql b/config/migrations/2023/20230213113324-add-new-codelist.sparql new file mode 100644 index 0000000..ca4a6d3 --- /dev/null +++ b/config/migrations/2023/20230213113324-add-new-codelist.sparql @@ -0,0 +1,18 @@ + +PREFIX org: +PREFIX ere: +PREFIX skos: +PREFIX mu: + + +INSERT DATA { + graph { + a ; + mu:uuid "7d8e0c15-7726-4196-9416-6dc40ed858bb"; + skos:prefLabel "Waarnemende Imam"; + ; + . + } +} + + diff --git a/config/migrations/2023/20230215094055-add-missing-position.graph b/config/migrations/2023/20230215094055-add-missing-position.graph new file mode 100644 index 0000000..00b1762 --- /dev/null +++ b/config/migrations/2023/20230215094055-add-missing-position.graph @@ -0,0 +1 @@ +http://mu.semte.ch/graphs/organisatieportaal diff --git a/config/migrations/2023/20230215094055-add-missing-position.ttl b/config/migrations/2023/20230215094055-add-missing-position.ttl new file mode 100644 index 0000000..f385e01 --- /dev/null +++ b/config/migrations/2023/20230215094055-add-missing-position.ttl @@ -0,0 +1,613 @@ +@prefix ns0: . +@prefix ns1: . +@prefix ns2: . +@prefix rdf: . +@prefix ns4: . +@prefix ns5: . + + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:e06b3e3d005d25e4a23459dd5773c00e . + + ns0:wordtBediendDoor ns1:a3d7cda7f4acad34682b8086d13ea397 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:a6735a9f75e9f30f8d31198e18e39192 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:e39958a6e2d38e23b5669a3b9ae1e719 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:c1efedae090d2c4be97d529c73b5ae87 . + + ns0:wordtBediendDoor ns1:ef9ebb3d9429b24c87a6576bfe302d85 . + + ns0:wordtBediendDoor . +ns2:ef77bb3094fdc4a7ed7e09551500acfe + ns0:wordtBediendDoor . +ns2:a8faa182c59167da7551f40979dc5022 + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:c924a44f9163ec6c946505f5f687708d . +ns2:ca0daee18a29e187017bf236e26f1ec2 + ns0:wordtBediendDoor ns1:cb4bf613bd82f99785df7aedb65b3367 . +ns2:f95b761af55091b491fd62bd6bed11dc + ns0:wordtBediendDoor ns1:d1dd10ca84303e547546f7e7a502dc6c . + + ns0:wordtBediendDoor . +ns2:c1c334a2cc56ac501badce5e879aec65 + ns0:wordtBediendDoor ns1:b74b9e77b49bbe8f5be5766be677c9bb . + + ns0:wordtBediendDoor ns1:bfe1868d6a41e073f114dd2efdcf51a5 . + + ns0:wordtBediendDoor . +ns2:a6e0531c6c4b5885552604163cfabfbc + ns0:wordtBediendDoor ns1:cdb151adeceb026fc5e81ea80d729391 . + + ns0:wordtBediendDoor ns1:c94d252366917f598a19df33680bbb29 . +ns2:cd7d4208e8e6291d75b33d89e3a0a439 + ns0:wordtBediendDoor . +ns2:a8d9f1a45c11c6761306bdf680349593 + ns0:wordtBediendDoor ns1:fad9626cd2ab433ce884ef75d06e840b . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:d5ef4ca7efebfdafc4d28d2d8ab019b1 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . +ns2:a8880f3d8b482788c5f3adbf58f01b4f + ns0:wordtBediendDoor . +ns2:adeaab8a02815bf2c5816165a45ad4c5 + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . +ns2:c7d6827eb88cd977146e72583461071b + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . +ns2:b81199eddd02f291bd4c1a0e29e5b9c8 + ns0:wordtBediendDoor ns1:cf320c8fb2e9db3a488b0ac016440f33 . +ns2:db46915e77fbf8f083e9861c802b2b13 + ns0:wordtBediendDoor ns1:b6c9aeea4bf9daa1520156b9fe5a49e5 . +ns2:fd4ae252fffca29852354b12ef37b086 + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . +ns2:b540096ac8be2c9c72dafd59e61080c5 + ns0:wordtBediendDoor ns1:f9a134e17b7ac4a42a66b7b66d86d00b . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:d4505963c15a4dd1cc77b34977b727c0 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:f90bc6678d032b1203c3e003b05a903e . + + ns0:wordtBediendDoor ns1:a1a4364ce6767907e600d8025c2f8cb9 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:baa288870012628b450bd134175cb91a . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:f1ac3ad7a3cf88917b6ff453bcdae693 . + + ns0:wordtBediendDoor ns1:e3c51e910902a9b4087f9d2dfe8629a2 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:ad8e06f595ccd82c799c81e7838db45e . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:e32bf54030b9a31d84249386a289287a . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:cf701aabf06d091083135506f7d8b785 . + + ns0:wordtBediendDoor ns1:f484ae309f3f981489724334f7a9e90b . + + ns0:wordtBediendDoor ns1:ebfd92d4fa2fb375fb407070e5275bc1 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:d82546b18b55081cc9c8e46aff579c75 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "72c14871c7a0f28cd52ebeb73d6d0fee" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "203b6f9c7e6eec3f62860b378e8371b0" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "79a1d65018cd0f86fa5757f9594ed52b" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "6436abc14a9615cfaa26f4dc7609bef8" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "116c6e00577975a5f41d4789166549d4" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "4a0c024fae6dcfb732006eeacf926c1c" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "0cde683248e90e543936deea015ec7f8" ; + ns0:functie . +ns1:e06b3e3d005d25e4a23459dd5773c00e + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "e06b3e3d005d25e4a23459dd5773c00e" ; + ns0:functie . +ns1:a3d7cda7f4acad34682b8086d13ea397 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "a3d7cda7f4acad34682b8086d13ea397" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "16c09cdc3608605b3ee88bf9ab9c7dac" ; + ns0:functie . +ns1:a6735a9f75e9f30f8d31198e18e39192 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "a6735a9f75e9f30f8d31198e18e39192" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "0351db31503375a8fb31b7089c65ec83" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "85d28128b4cc9476fef4eeeffa2530a2" ; + ns0:functie . +ns1:e39958a6e2d38e23b5669a3b9ae1e719 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "e39958a6e2d38e23b5669a3b9ae1e719" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "594841e8ec952d28ee1f09bbe1d3890a" ; + ns0:functie . +ns1:c1efedae090d2c4be97d529c73b5ae87 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "c1efedae090d2c4be97d529c73b5ae87" ; + ns0:functie . +ns1:ef9ebb3d9429b24c87a6576bfe302d85 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "ef9ebb3d9429b24c87a6576bfe302d85" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "569b7a4720c5d44269ab96663f3f5021" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "453521f31bee624be96caa2f1a2e7048" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "87db8210da3911674adf40d01076fd81" ; + ns0:functie . +ns1:c924a44f9163ec6c946505f5f687708d + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "c924a44f9163ec6c946505f5f687708d" ; + ns0:functie . +ns1:cb4bf613bd82f99785df7aedb65b3367 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "cb4bf613bd82f99785df7aedb65b3367" ; + ns0:functie . +ns1:d1dd10ca84303e547546f7e7a502dc6c + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "d1dd10ca84303e547546f7e7a502dc6c" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "95222f20d3ff233e0623dc8c811375e4" ; + ns0:functie . +ns1:b74b9e77b49bbe8f5be5766be677c9bb + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "b74b9e77b49bbe8f5be5766be677c9bb" ; + ns0:functie . +ns1:bfe1868d6a41e073f114dd2efdcf51a5 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "bfe1868d6a41e073f114dd2efdcf51a5" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "59b02bc348ce9c8b26fd2eb6b6a9bd86" ; + ns0:functie . +ns1:cdb151adeceb026fc5e81ea80d729391 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "cdb151adeceb026fc5e81ea80d729391" ; + ns0:functie . +ns1:c94d252366917f598a19df33680bbb29 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "c94d252366917f598a19df33680bbb29" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "5a87bfd76cc6d2f582ef1272116c863a" ; + ns0:functie . +ns1:fad9626cd2ab433ce884ef75d06e840b + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "fad9626cd2ab433ce884ef75d06e840b" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "9cea42c28a90ce1ac0cdf3bdb01199b8" ; + ns0:functie . +ns1:d5ef4ca7efebfdafc4d28d2d8ab019b1 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "d5ef4ca7efebfdafc4d28d2d8ab019b1" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "8ef8c30c7e682a8c56af185c91c898ad" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "1903dde5ac84bece6238f20175ba583e" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "1bb5f658bd511ed2752ae4ad57ef8528" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "57bce94d3031b52f361a3c043b1678d9" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "776bc597d74bba0ad0c60c81b4c22529" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "259b382ef9f0a8eaaf84954bc09e3f38" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "09ee3daf174b1069b4ae66e31f5b8728" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "77e3e5bcb167367015c391dfa91cb01f" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "8b98c482cb67f55c2fb7174a00cdf09d" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "5963146dc30cba248bb24e29b5002b78" ; + ns0:functie . +ns1:cf320c8fb2e9db3a488b0ac016440f33 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "cf320c8fb2e9db3a488b0ac016440f33" ; + ns0:functie . +ns1:b6c9aeea4bf9daa1520156b9fe5a49e5 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "b6c9aeea4bf9daa1520156b9fe5a49e5" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "8eab113defa6ac21fd87c0828292e926" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "0aaf20e1103fec92356373dd7de19887" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "46cac4be99a8036f729ddd2df0cbc3c4" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "8c2ab509b1e9cfa3dba90f9a4cb3037d" ; + ns0:functie . +ns1:f9a134e17b7ac4a42a66b7b66d86d00b + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "f9a134e17b7ac4a42a66b7b66d86d00b" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "0a46fb4249e6a218b943a8f6fe1ac090" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "19838c3b991ad9f816ef491d96a721d3" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "76419a6001ca12e313e13bde8840e405" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "60ae1e4b56e80af0b81a20b96d4187b1" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "1c1d1fd0c40ffe62076203d7ea2060a6" ; + ns0:functie . +ns1:d4505963c15a4dd1cc77b34977b727c0 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "d4505963c15a4dd1cc77b34977b727c0" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "0a69b193cce1416d4d6366fff46add3f" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "41ae97e88af106e5c9ee3588adc9e2ba" ; + ns0:functie . +ns1:f90bc6678d032b1203c3e003b05a903e + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "f90bc6678d032b1203c3e003b05a903e" ; + ns0:functie . +ns1:a1a4364ce6767907e600d8025c2f8cb9 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "a1a4364ce6767907e600d8025c2f8cb9" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "5687522854e85ab6f4c324b273467ce4" ; + ns0:functie . +ns1:baa288870012628b450bd134175cb91a + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "baa288870012628b450bd134175cb91a" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "031e9b8845e50e0f2484ed7eac94324e" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "233a795834c4f862342250b0c8647d15" ; + ns0:functie . +ns1:f1ac3ad7a3cf88917b6ff453bcdae693 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "f1ac3ad7a3cf88917b6ff453bcdae693" ; + ns0:functie . +ns1:e3c51e910902a9b4087f9d2dfe8629a2 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "e3c51e910902a9b4087f9d2dfe8629a2" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "02c677db7c013653e475984bebf59b58" ; + ns0:functie . +ns1:ad8e06f595ccd82c799c81e7838db45e + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "ad8e06f595ccd82c799c81e7838db45e" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "57fdb71d4feb243e727005e16d31d98a" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "2a425d7fa4022573cb4824e91e2f5264" ; + ns0:functie . +ns1:e32bf54030b9a31d84249386a289287a + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "e32bf54030b9a31d84249386a289287a" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "0315bd04919b2884af8f3c0a8b820ba0" ; + ns0:functie . +ns1:cf701aabf06d091083135506f7d8b785 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "cf701aabf06d091083135506f7d8b785" ; + ns0:functie . +ns1:f484ae309f3f981489724334f7a9e90b + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "f484ae309f3f981489724334f7a9e90b" ; + ns0:functie . +ns1:ebfd92d4fa2fb375fb407070e5275bc1 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "ebfd92d4fa2fb375fb407070e5275bc1" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "638c32d8700a7c8891e7ae2ed1f86dfc" ; + ns0:functie . +ns1:d82546b18b55081cc9c8e46aff579c75 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "d82546b18b55081cc9c8e46aff579c75" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "87e2496b12c3a58672970e4db4a013ed" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "6c177d4df98c1ae795146245e1b8ded3" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "41ee849be721d50f1012987ec14cf4d5" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "36357858aff9e011120ce12a4c9a735d" ; + ns0:functie . +ns1:c654c9e1b47c7acd6c994f9213c02671 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "c654c9e1b47c7acd6c994f9213c02671" ; + ns0:functie . +ns1:face7445fb50e36d72b142e862381b05 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "face7445fb50e36d72b142e862381b05" ; + ns0:functie . +ns1:f24f1b4a152c70b2dbe51ad0c3171de3 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "f24f1b4a152c70b2dbe51ad0c3171de3" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "09bbb9454457c17975696ac680349a0a" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "809e08adf06dad4286658ba80dcb3ec0" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "014c834f16b6e3780d6e0613e1b469e1" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "2715eda6c886e5931ae16e7bc877db6c" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "06c83ff34ec4231674e90095e24fbb12" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "9e59891d2fff60c9a325c0760263ac53" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "8eccf8c0577135ac23cfc2514d4e41ce" ; + ns0:functie . +ns1:dd14c36fdcb4c93c80d8b8b6a9d7c400 + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "dd14c36fdcb4c93c80d8b8b6a9d7c400" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "7708bcfebdd905bd82c3dddd7861e7a1" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "49641427d736deffcdcbe2d086a6238f" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "5f0961a539d8dc48fcfef822f71d735c" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "5b18d66f489d69900a22782a1fd9f273" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "6f10f26c5cfbe3398b24a5d0cb360ee2" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "87fb9d24420b7def7d58e9eb5d014145" ; + ns0:functie . +ns1:d45e2fdff3725615b024f7b71fadef7e + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "d45e2fdff3725615b024f7b71fadef7e" ; + ns0:functie . +ns1:c275c2b71b28e21ccaf2e16582f59e0d + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "c275c2b71b28e21ccaf2e16582f59e0d" ; + ns0:functie . + + rdf:type ns4:Post , ns0:PositieBedienaar ; + ns5:uuid "25a8dd65997f9ba30e7416bbf774594f" ; + ns0:functie . + + ns0:wordtBediendDoor ns1:c654c9e1b47c7acd6c994f9213c02671 . + + ns0:wordtBediendDoor ns1:face7445fb50e36d72b142e862381b05 . + + ns0:wordtBediendDoor ns1:f24f1b4a152c70b2dbe51ad0c3171de3 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:dd14c36fdcb4c93c80d8b8b6a9d7c400 . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor . + + ns0:wordtBediendDoor ns1:d45e2fdff3725615b024f7b71fadef7e . + + ns0:wordtBediendDoor ns1:c275c2b71b28e21ccaf2e16582f59e0d . + + ns0:wordtBediendDoor . From d756113dcf2969ebb2f64031690b21626189a762 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 16 Feb 2023 09:19:22 +0100 Subject: [PATCH 06/58] Remove admin unit producer's data --- ...3952-clear-admin-unit-producer-data.sparql | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 config/migrations/2023/20230214163952-clear-admin-unit-producer-data.sparql diff --git a/config/migrations/2023/20230214163952-clear-admin-unit-producer-data.sparql b/config/migrations/2023/20230214163952-clear-admin-unit-producer-data.sparql new file mode 100644 index 0000000..e5938f6 --- /dev/null +++ b/config/migrations/2023/20230214163952-clear-admin-unit-producer-data.sparql @@ -0,0 +1,61 @@ +DELETE { + GRAPH ?g { + ?s ?p ?o . + } +} WHERE { + GRAPH ?g { + ?s a ; + ; + ?p ?o . + } +} +; +DELETE { + GRAPH ?g { + ?s ?p ?o . + } +} WHERE { + GRAPH ?g { + ?s a ; + ; + ?p ?o . + } +} +; +DELETE { + GRAPH ?g { + ?s ?p ?o . + } +} WHERE { + GRAPH ?g { + ?s a ; + ; + ?p ?o . + } +} +; +DELETE { + GRAPH ?g { + ?s ?p ?o . + } +} WHERE { + GRAPH ?g { + ?s a ; + ; + ?p ?o . + } +} +; +DELETE { + GRAPH ?g { + ?s ?p ?o . + } +} WHERE { + GRAPH ?g { + ?s a ; + ; + ?p ?o . + } +} +; +CLEAR GRAPH From fcf46273836c43e862324df18f68ef58898a2513 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 16 Feb 2023 09:20:35 +0100 Subject: [PATCH 07/58] Extend worship posts producer's export config --- .../delta-producer/worship-posts/export.json | 185 +++++++++++++++++- 1 file changed, 179 insertions(+), 6 deletions(-) diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index 9c4b9ad..bbf8a55 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -12,25 +12,40 @@ "additionalFilter": "?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" }, { - "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", - "strictTypeExport": "true", + "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "graphsFilter": [ "http://mu.semte.ch/graphs/organisatieportaal" ], "properties": [ - "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor" - ] + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/regorg#orgStatus", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://data.vlaanderen.be/ns/besluit#werkingsgebied", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite" + ], + "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" }, { "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", - "strictTypeExport": "true", "graphsFilter": [ "http://mu.semte.ch/graphs/organisatieportaal" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#classification", + "http://data.vlaanderen.be/ns/mandaat#bindingEinde", + "http://data.vlaanderen.be/ns/mandaat#bindingStart", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", "http://www.w3.org/ns/org#hasPost" ], - "additionalFilter": "?subject / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" + "additionalFilter": "?subject (//)|(/) ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" }, { "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", @@ -42,6 +57,164 @@ "http://www.w3.org/ns/org#role", "http://data.lblod.info/vocabularies/erediensten/functie" ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://data.lblod.info/vocabularies/erediensten/typebetrokkenheid", + "http://www.w3.org/ns/org#organization" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#linkedTo", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://www.w3.org/ns/org#Site", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://data.lblod.info/vocabularies/erediensten/vestigingstype" + ] + }, + { + "type": "http://www.w3.org/ns/locn#Address", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer", + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer", + "http://www.w3.org/ns/locn#thoroughfare", + "http://www.w3.org/ns/locn#postCode", + "https://data.vlaanderen.be/ns/adres#gemeentenaam", + "http://www.w3.org/ns/locn#adminUnitL2", + "http://www.w3.org/ns/locn#fullAddress" + ], + "additionalFilter": "?subject ^ ?site. ?site a ." + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#scopeNote" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#scopeNote" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeBetrokkenheid", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursfunctieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/EredienstBeroepen", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://mu.semte.ch/vocabularies/ext/GeslachtCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://publications.europa.eu/ontology/euvoc#Country", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2000/01/rdf-schema#label", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] } ] } From f110f1c7b1903332279847a0ad6e758f35ff5d03 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 17 Feb 2023 16:11:23 +0100 Subject: [PATCH 08/58] Add missing property in worship posts export --- config/delta-producer/worship-posts/export.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index bbf8a55..38a08ae 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -26,7 +26,8 @@ "http://www.w3.org/ns/org#classification", "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", "http://www.w3.org/ns/org#hasPrimarySite", - "http://www.w3.org/ns/org#hasSite" + "http://www.w3.org/ns/org#hasSite", + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur" ], "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" }, From eb99aab77bf47cb1a93105cd48976421cf502060 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 20 Feb 2023 17:31:43 +0100 Subject: [PATCH 09/58] Dispatch files to resource directly --- config/dispatcher/dispatcher.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 1a8eb1b..7f5dbb7 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -220,7 +220,7 @@ defmodule Dispatcher do ############### match "/accounts", %{ accept: [:json], layer: :api} do - Proxy.forward conn, [], "http://resource/accounts/" + Proxy.forward conn, [], "http://resource/accounts/" end match "/accounts/*path", %{ accept: [:json], layer: :api} do Proxy.forward conn, path, "http://accountdetail/accounts/" @@ -242,7 +242,7 @@ defmodule Dispatcher do ############################################################### match "/search/*path", %{ accept: %{ json: true }, layer: :api_services} do - IO.puts("hey") + IO.puts("hey") Proxy.forward conn, path, "http://search/" end @@ -303,7 +303,7 @@ defmodule Dispatcher do end get "/files/*path" , %{ layer: :api_services, accept: %{ json: true } } do - Proxy.forward conn, path, "http://cache/files/" + Proxy.forward conn, path, "http://resource/files/" end ################################################################# From 157fe4f3745d78e836b23e3275f83c4e0e6f3bd1 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Feb 2023 20:30:52 +0100 Subject: [PATCH 10/58] Fix producer not exporting all worship admin units Apparently the producer doesn't take into account inheritance when constructing the queries, so we have to explicitely define the necessary types --- .../delta-producer/worship-posts/export.json | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index 38a08ae..83a173d 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -12,7 +12,7 @@ "additionalFilter": "?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" }, { - "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "graphsFilter": [ "http://mu.semte.ch/graphs/organisatieportaal" ], @@ -26,10 +26,43 @@ "http://www.w3.org/ns/org#classification", "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/regorg#orgStatus", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://data.vlaanderen.be/ns/besluit#werkingsgebied", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/regorg#orgStatus", + "http://data.vlaanderen.be/ns/besluit#werkingsgebied", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/ns/org#hasPrimarySite", "http://www.w3.org/ns/org#hasSite", "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur" ], - "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" + "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" }, { "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", From acaca124c279c6a07506cb5b2049efe304b5da7e Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 27 Feb 2023 15:28:18 +0100 Subject: [PATCH 11/58] hard split (#223) --- .gitignore | 1 + config/authorization/config.ex | 182 +++++++---- .../dump-file-publisher/config.json | 4 +- .../delta-producer/organizations/export.json | 101 +++--- config/delta-producer/public/export.json | 47 +-- .../delta-producer/worship-posts/export.json | 51 +-- config/delta/rules.js | 75 +++-- config/dispatcher/dispatcher.ex | 27 +- .../leidinggevenden-dispatching/config.js | 4 +- .../delta-sync-dispatching.js | 68 ++-- .../initial-sync-dispatching.js | 36 +-- .../leidinggevenden-dispatching/util.js | 2 +- .../mandate-dispatching/config.js | 4 +- .../delta-sync-dispatching.js | 68 ++-- .../initial-sync-dispatching.js | 36 +-- .../mandate-dispatching/util.js | 2 +- .../positions-dispatching/config.js | 43 ++- .../delta-sync-dispatching.js | 6 +- .../initial-sync-dispatching.js | 65 ++-- .../positions-dispatching/util.js | 2 +- ...20230401132500-delete-elastic-index.sparql | 1 + ...230401132600-move-all-worship-types.sparql | 44 +++ ...01-move-all-worsip-governing-bodies.sparql | 90 ++++++ ...-move-all-worsip-governing-bodies-2.sparql | 73 +++++ ...-move-all-worsip-governing-bodies-3.sparql | 93 ++++++ .../20230401132604-move-bestrokken.sparql | 36 +++ ...32605-move-all-worsip-change-events.sparql | 87 ++++++ .../20230401132606-duplicate-person.sparql | 76 +++++ .../20230401132607-move-shared-types.sparql | 290 ++++++++++++++++++ .../20230401132608-move-public-types.sparql | 40 +++ ...1132609-move-to-administrative-unit.sparql | 3 + .../20230401132610-add-mock-user-1.sparql | 44 +++ .../20230401132611-add-mock-user-2.sparql | 42 +++ .../20230401132612-add-mock-user-3.sparql | 30 ++ .../20230401132613-add-mock-user-4.sparql | 53 ++++ .../20230401132614-move-privacy-data.sparql | 128 ++++++++ ...2105324-move-jobs-to-specific-graph.sparql | 21 ++ .../20230403132612-add-mock-user-5.sparql | 23 ++ .../20230403132613-add-mock-user-6.sparql | 22 ++ config/resources/domain.json | 60 +++- config/search/config.json | 58 +--- config/search/dev/config.json | 18 +- config/tlsproxy-mock/Caddyfile | 4 + data/db/virtuoso.ini | 0 docker-compose.dev.yml | 16 +- docker-compose.yml | 54 ++-- 46 files changed, 1799 insertions(+), 431 deletions(-) create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132500-delete-elastic-index.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132600-move-all-worship-types.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132601-move-all-worsip-governing-bodies.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132602-move-all-worsip-governing-bodies-2.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132603-move-all-worsip-governing-bodies-3.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132604-move-bestrokken.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132605-move-all-worsip-change-events.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132606-duplicate-person.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132607-move-shared-types.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132608-move-public-types.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132609-move-to-administrative-unit.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132610-add-mock-user-1.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132611-add-mock-user-2.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132612-add-mock-user-3.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132613-add-mock-user-4.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230401132614-move-privacy-data.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230402105324-move-jobs-to-specific-graph.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230403132612-add-mock-user-5.sparql create mode 100644 config/migrations/2023/20230401132400-hard-split/20230403132613-add-mock-user-6.sparql create mode 100644 config/tlsproxy-mock/Caddyfile delete mode 100755 data/db/virtuoso.ini diff --git a/.gitignore b/.gitignore index d6b6a17..3f398c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ # Virtuoso files /data/db +/data/db.bck/** /data/db/* !/data/db/virtuoso.ini !/data/db/toLoad diff --git a/config/authorization/config.ex b/config/authorization/config.ex index e52cb73..30364c4 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -8,70 +8,86 @@ alias Acl.GroupSpec.GraphCleanup, as: GraphCleanup defmodule Acl.UserGroups.Config do - @error_type [ - "https://docs.oasis-open-projects.org/oslc-op/core/v3.0/os/core-vocab.html#Error" - ] + @error_type [ + "https://docs.oasis-open-projects.org/oslc-op/core/v3.0/os/core-vocab.html#Error" + ] - @protected_resource_type [ - "http://www.w3.org/ns/org#Organization", - "http://data.vlaanderen.be/ns/besluit#Besluit", - "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit", - "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + @worship_type [ + "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid", - "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", "http://data.lblod.info/vocabularies/erediensten/VoorwaardenBedienaar", - "https://data.vlaanderen.be/ns/persoon#Geboorte", - "http://mu.semte.ch/vocabularies/ext/GeslachtCode", "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", - "http://www.w3.org/ns/adms#Identifier", - "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", - "http://www.w3.org/ns/org#ChangeEvent", "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen", "http://data.lblod.info/vocabularies/erediensten/VerbondenJuridischeStructuren", - "http://www.w3.org/ns/prov#Location", - "http://www.w3.org/ns/org#Site", - "http://www.w3.org/ns/locn#Address", - "http://www.w3.org/ns/person#Person", - "http://data.vlaanderen.be/ns/mandaat#Mandaat", - "http://data.vlaanderen.be/ns/mandaat#Mandataris", - "http://data.lblod.info/vocabularies/leidinggevenden/FunctionarisStatusCode", - "http://data.lblod.info/vocabularies/leidinggevenden/Functionaris", - "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", - "http://schema.org/ContactPoint", - "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie", "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan", + ] + + @public_type [ "http://www.w3.org/ns/org#Role", - "http://www.w3.org/ns/org#Post", - "http://data.lblod.info/vocabularies/contacthub/AgentInPositie", + "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "http://xmlns.com/foaf/0.1/Person", + "http://xmlns.com/foaf/0.1/OnlineAccount", + "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject", + "http://www.w3.org/ns/dcat#Dataset", + "http://www.w3.org/ns/dcat#Distribution", + "http://www.w3.org/ns/dcat#Catalog", + "http://mu.semte.ch/vocabularies/ext/GeslachtCode", + "http://data.lblod.info/vocabularies/leidinggevenden/FunctionarisStatusCode", "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", "http://lblod.data.gift/vocabularies/organisatie/BestuursfunctieCode", "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", - "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", - "http://lblod.data.gift/vocabularies/organisatie/HelftVerkiezing", - "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", - "http://lblod.data.gift/vocabularies/organisatie/VeranderingsgebeurtenisResultaat", "http://lblod.data.gift/vocabularies/organisatie/MandatarisStatusCode", "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", + "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", + "http://www.w3.org/2004/02/skos/core#ConceptScheme", + "http://lblod.data.gift/vocabularies/organisatie/BedienaarFinanceringCode", + "http://www.w3.org/2004/02/skos/core#Concept", + "http://publications.europa.eu/ontology/euvoc#Country", + "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", + "http://lblod.data.gift/vocabularies/organisatie/HelftVerkiezing", + "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", "http://lblod.data.gift/vocabularies/organisatie/TypeBetrokkenheid", "http://lblod.data.gift/vocabularies/organisatie/VoorwaardenBedienaarCriterium", "http://lblod.data.gift/vocabularies/organisatie/BedienaarCriteriumBewijsstuk", "http://lblod.data.gift/vocabularies/organisatie/EredienstBeroepen", - "http://lblod.data.gift/vocabularies/organisatie/BedienaarFinanceringCode", - "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", - "http://lblod.data.gift/vocabularies/organisatie/Rechtsvormtype", - "http://www.w3.org/2004/02/skos/core#ConceptScheme", - "http://www.w3.org/2004/02/skos/core#Concept", + "http://lblod.data.gift/vocabularies/organisatie/Rechtsvormtype" + ] + + @org_type [ + "http://www.w3.org/ns/org#Organization", + "http://data.vlaanderen.be/ns/besluit#Besluit", + "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit", + "http://www.w3.org/ns/org#Site", + "http://www.w3.org/ns/locn#Address", + "http://www.w3.org/ns/person#Person", + "http://schema.org/ContactPoint", + "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", + "http://www.w3.org/ns/adms#Identifier", + "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "http://data.vlaanderen.be/ns/mandaat#Mandaat", + "http://data.vlaanderen.be/ns/mandaat#Mandataris", + "http://data.lblod.info/vocabularies/leidinggevenden/Functionaris", + "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie", + "http://lblod.data.gift/vocabularies/organisatie/VeranderingsgebeurtenisResultaat", + "http://www.w3.org/ns/org#Post", + "http://www.w3.org/ns/org#ChangeEvent", + "http://data.lblod.info/vocabularies/contacthub/AgentInPositie" + ] + + @shared_protected_type [ + "http://www.w3.org/ns/prov#Location", "http://xmlns.com/foaf/0.1/Image", - "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject", - "http://publications.europa.eu/ontology/euvoc#Country" - ] + "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject" + ] + defp access_by_role( group_string ) do %AccessByQuery{ # vars: ["session_group","session_role"], todo we might have to create multiple graph depending on role - vars: [], + vars: [], query: sparql_query_for_access_role( group_string ) } end @@ -80,48 +96,88 @@ defmodule Acl.UserGroups.Config do PREFIX mu: SELECT distinct ?session_group ?session_role WHERE { ext:sessionGroup/mu:uuid ?session_group; - ext:sessionRole ?session_role. + ext:activeSessionRole ?session_role. FILTER( ?session_role = \"#{group_string}\" ) }" end def user_groups do [ - %GroupSpec{ + + %GroupSpec{ name: "acmidm-lezer", useage: [:read], access: access_by_role( "ABBOrganisatiePortaalGebruiker-lezer" ), graphs: [ %GraphSpec{ - graph: "http://mu.semte.ch/graphs/organisatieportaal", + graph: "http://mu.semte.ch/graphs/administrative-unit", constraint: %ResourceConstraint{ - resource_types: @protected_resource_type + resource_types: @org_type } } ] }, - %GroupSpec{ + %GroupSpec{ name: "acmidm-editeerder", useage: [:read, :write, :read_for_write], access: access_by_role( "ABBOrganisatiePortaalGebruiker-editeerder" ), graphs: [ %GraphSpec{ - graph: "http://mu.semte.ch/graphs/organisatieportaal", + graph: "http://mu.semte.ch/graphs/administrative-unit", constraint: %ResourceConstraint{ - resource_types: @protected_resource_type ++ @error_type + resource_types: @org_type ++ @error_type } } ] }, - %GroupSpec{ + %GroupSpec{ name: "acmidm-beheerder", useage: [:read, :write, :read_for_write], access: access_by_role( "ABBOrganisatiePortaalGebruiker-beheerder" ), graphs: [ %GraphSpec{ - graph: "http://mu.semte.ch/graphs/organisatieportaal", + graph: "http://mu.semte.ch/graphs/administrative-unit", constraint: %ResourceConstraint{ - resource_types: @protected_resource_type ++ @error_type + resource_types: @org_type ++ @error_type + } + } + ] + }, + %GroupSpec{ + name: "acmidm-worship-lezer", + useage: [:read], + access: access_by_role( "ABBOrganisatiePortaalErediensten-lezer" ), + graphs: [ + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/worship-service", + constraint: %ResourceConstraint{ + resource_types: @org_type ++ @worship_type + } + } + ] + }, + %GroupSpec{ + name: "acmidm-worship-editeerder", + useage: [:read, :write, :read_for_write], + access: access_by_role( "ABBOrganisatiePortaalErediensten-editeerder" ), + graphs: [ + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/worship-service", + constraint: %ResourceConstraint{ + resource_types: @org_type ++ @error_type ++ @worship_type + } + } + ] + }, + %GroupSpec{ + name: "acmidm-worship-beheerder", + useage: [:read, :write, :read_for_write], + access: access_by_role( "ABBOrganisatiePortaalErediensten-beheerder" ), + graphs: [ + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/worship-service", + constraint: %ResourceConstraint{ + resource_types: @org_type ++ @error_type ++ @worship_type } } ] @@ -130,19 +186,21 @@ defmodule Acl.UserGroups.Config do name: "public", useage: [:read], access: %AlwaysAccessible{}, - graphs: [ %GraphSpec{ - graph: "http://mu.semte.ch/graphs/public", # mock login only + graphs: [ + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/public", constraint: %ResourceConstraint{ - resource_types: [ - "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", - "http://xmlns.com/foaf/0.1/Person", - "http://xmlns.com/foaf/0.1/OnlineAccount", - "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject", - "http://www.w3.org/ns/dcat#Dataset", - "http://www.w3.org/ns/dcat#Distribution", - "http://www.w3.org/ns/dcat#Catalog" - ] - } }]}, + resource_types: @public_type + } + }, + %GraphSpec{ + graph: "http://mu.semte.ch/graphs/shared", + constraint: %ResourceConstraint{ + resource_types: @shared_protected_type ++ @org_type + } + } + + ]}, # // CLEANUP # diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index a73ccad..42c183f 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -31,7 +31,7 @@ "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", "fileBaseName": "dump-worship-posts", @@ -39,7 +39,7 @@ "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true }, - "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" : { + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", "fileBaseName": "dump-worship-posts", diff --git a/config/delta-producer/organizations/export.json b/config/delta-producer/organizations/export.json index bb86cfb..510c48b 100644 --- a/config/delta-producer/organizations/export.json +++ b/config/delta-producer/organizations/export.json @@ -3,9 +3,11 @@ { "type": "http://www.w3.org/ns/person#Person", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" ], - "properties": [ + "properties": [ "http://xmlns.com/foaf/0.1/givenName", "http://xmlns.com/foaf/0.1/familyName", "https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam", @@ -16,7 +18,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/MandatarisStatusCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -26,7 +28,8 @@ "type": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared" ], "properties": [ "http://data.lblod.info/vocabularies/contacthub/startdatum", @@ -46,7 +49,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/HelftVerkiezing", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -55,7 +58,8 @@ { "type": "http://schema.org/ContactPoint", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "http://schema.org/email", @@ -67,7 +71,9 @@ { "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "http://www.w3.org/ns/org#role", @@ -78,7 +84,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursfunctieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -87,7 +93,7 @@ { "type": "http://www.w3.org/ns/org#Role", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -96,7 +102,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -105,7 +111,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -115,7 +121,8 @@ "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel", @@ -142,7 +149,8 @@ "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel", @@ -166,7 +174,8 @@ "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel", @@ -187,7 +196,9 @@ "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "http://data.vlaanderen.be/ns/mandaat#bindingEinde", @@ -201,7 +212,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -210,7 +221,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://data.lblod.info/vocabularies/erediensten/financieringspercentage", @@ -221,7 +233,9 @@ { "type": "http://www.w3.org/ns/adms#Identifier", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/administrative-unit", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://www.w3.org/2004/02/skos/core#notation", @@ -231,7 +245,9 @@ { "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/administrative-unit", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" @@ -240,7 +256,9 @@ { "type": "http://www.w3.org/ns/locn#Address", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer", @@ -258,7 +276,8 @@ { "type": "http://www.w3.org/ns/org#Site", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "http://www.w3.org/ns/org#siteAddress", @@ -269,7 +288,8 @@ { "type": "http://www.w3.org/ns/org#ChangeEvent", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "http://purl.org/dc/terms/date", @@ -282,7 +302,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -291,7 +311,7 @@ { "type": "http://www.w3.org/2004/02/skos/core#Concept", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -300,7 +320,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -309,16 +329,17 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeBetrokkenheid", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, { - "type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", + "type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://data.lblod.info/vocabularies/contacthub/startdatum", @@ -333,26 +354,30 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/VoorwaardenBedienaar", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://data.lblod.info/vocabularies/erediensten/voldaan", "http://data.lblod.info/vocabularies/erediensten/criterium", "http://data.lblod.info/vocabularies/erediensten/criteriumbewijsstuktype" ] - },{ + }, + { "type": "http://publications.europa.eu/ontology/euvoc#Country", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel", "http://www.w3.org/2000/01/rdf-schema#label" ] - },{ + }, + { "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://www.w3.org/ns/org#role", @@ -364,7 +389,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/EredienstBeroepen", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -373,7 +398,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BedienaarFinanceringCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -382,7 +407,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/VoorwaardenBedienaarCriterium", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -391,7 +416,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BedienaarCriteriumBewijsstuk", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" @@ -400,7 +425,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index 49d2931..2cbfa05 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -5,7 +5,8 @@ "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "pathToConceptScheme": [ "http://www.w3.org/ns/org#classification", @@ -36,7 +37,8 @@ "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "pathToConceptScheme": [ "http://www.w3.org/ns/org#classification", @@ -63,13 +65,13 @@ { "type": "http://www.w3.org/ns/org#Site", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^http://www.w3.org/ns/org#hasPrimarySite", "http://www.w3.org/ns/org#classification", "http://www.w3.org/2004/02/skos/core#inScheme" - ], "properties": [ "http://www.w3.org/ns/org#siteAddress", @@ -80,14 +82,14 @@ { "type": "http://www.w3.org/ns/locn#Address", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^https://data.vlaanderen.be/ns/organisatie#bestaatUit", "^http://www.w3.org/ns/org#hasPrimarySite", "http://www.w3.org/ns/org#classification", "http://www.w3.org/2004/02/skos/core#inScheme" - ], "properties": [ "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer", @@ -105,7 +107,9 @@ { "type": "http://schema.org/ContactPoint", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^http://www.w3.org/ns/org#siteAddress", @@ -122,13 +126,14 @@ "http://xmlns.com/foaf/0.1/page", "http://schema.org/email", "http://schema.org/telephone" - ] }, { "type": "http://www.w3.org/ns/adms#Identifier", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^http://www.w3.org/ns/adms#identifier", @@ -143,7 +148,9 @@ { "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator", @@ -158,7 +165,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -171,7 +178,9 @@ "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "http://www.w3.org/ns/org#classification", @@ -188,7 +197,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -200,7 +209,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -212,7 +221,9 @@ { "type": "http://www.w3.org/ns/org#ChangeEvent", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "http://data.lblod.info/vocabularies/contacthub/typeWijziging", @@ -233,7 +244,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -245,7 +256,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -257,7 +268,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index 83a173d..ef2656a 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -3,7 +3,8 @@ { "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -14,7 +15,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -32,7 +34,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -49,7 +52,9 @@ { "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -67,7 +72,8 @@ { "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -84,7 +90,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -95,7 +102,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -106,7 +114,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -122,7 +131,8 @@ { "type": "http://www.w3.org/ns/org#Site", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -133,7 +143,8 @@ { "type": "http://www.w3.org/ns/locn#Address", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -150,7 +161,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -161,7 +172,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -172,7 +183,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -182,7 +193,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -192,7 +203,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeBetrokkenheid", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -202,7 +213,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -212,7 +223,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursfunctieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -222,7 +233,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/EredienstBeroepen", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -232,7 +243,7 @@ { "type": "http://mu.semte.ch/vocabularies/ext/GeslachtCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -242,7 +253,7 @@ { "type": "http://publications.europa.eu/ontology/euvoc#Country", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", diff --git a/config/delta/rules.js b/config/delta/rules.js index 2b2899f..6d9ffc6 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -3,7 +3,7 @@ export default [ match: { graph: { type: "uri", - value: "http://mu.semte.ch/graphs/organisatieportaal" + value: "http://mu.semte.ch/graphs/administrative-unit" } }, callback: { @@ -20,7 +20,25 @@ export default [ match: { graph: { type: "uri", - value: "http://mu.semte.ch/graphs/organisatieportaal" + value: "http://mu.semte.ch/graphs/worship-service" + } + }, + callback: { + url: 'http://search/update', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 10000, + ignoreFromSelf: true + } + }, + + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/administrative-unit" } }, callback: { @@ -33,6 +51,25 @@ export default [ ignoreFromSelf: true } }, + + { + match: { + graph: { + type: "uri", + value: "http://mu.semte.ch/graphs/worship-service" + } + }, + callback: { + url: 'http://resource/.mu/delta', + method: 'POST' + }, + options: { + resourceFormat: "v0.0.1", + gracePeriod: 250, + ignoreFromSelf: true + } + }, + { match: { predicate: { @@ -48,7 +85,7 @@ export default [ resourceFormat: 'v0.0.1', gracePeriod: 1000, ignoreFromSelf: true, - optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } }, { @@ -63,9 +100,9 @@ export default [ gracePeriod: 1000, ignoreFromSelf: true, optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] } }, { @@ -83,7 +120,7 @@ export default [ resourceFormat: 'v0.0.1', gracePeriod: 1000, ignoreFromSelf: true, - optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } }, { @@ -98,9 +135,9 @@ export default [ gracePeriod: 1000, ignoreFromSelf: true, optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] } }, { @@ -118,7 +155,7 @@ export default [ resourceFormat: 'v0.0.1', gracePeriod: 1000, ignoreFromSelf: true, - optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } }, { @@ -133,9 +170,9 @@ export default [ gracePeriod: 1000, ignoreFromSelf: true, optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] } }, { @@ -153,7 +190,7 @@ export default [ resourceFormat: 'v0.0.1', gracePeriod: 1000, ignoreFromSelf: true, - optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } }, { @@ -169,9 +206,9 @@ export default [ gracePeriod: 1000, ignoreFromSelf: true, optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] } }, { @@ -193,7 +230,7 @@ export default [ resourceFormat: 'v0.0.1', gracePeriod: 1000, ignoreFromSelf: true, - optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } } ] diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 7f5dbb7..e72e468 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -225,6 +225,9 @@ defmodule Dispatcher do match "/accounts/*path", %{ accept: [:json], layer: :api} do Proxy.forward conn, path, "http://accountdetail/accounts/" end + match "/active-role/*path", %{ accept: [:json], layer: :api} do + Proxy.forward conn, path, "http://roles/" + end match "/groups/*path", %{ accept: [:json], layer: :api} do Proxy.forward conn, path, "http://resource/groups/" end @@ -262,20 +265,36 @@ defmodule Dispatcher do forward conn, path, "http://adressenregister/" end + match "/worship/person-information-updates/*path", %{ layer: :api_services, accept: %{ json: true } } do + forward conn, path, "http://privacy-worship/person-information-updates" + end + + match "/worship/person-information-requests/*path", %{ layer: :api_services, accept: %{ json: true } } do + forward conn, path, "http://privacy-worship/person-information-requests" + end + + match "/worship/person-information-ask/*path", %{ layer: :api_services, accept: %{ json: true } } do + forward conn, path, "http://privacy-worship/person-information-ask/" + end + + match "/worship/person-information-validate-ssn/*path", %{ layer: :api_services, accept: %{ json: true } } do + forward conn, path, "http://privacy-worship/person-information-validate-ssn/" + end + match "/person-information-updates/*path", %{ layer: :api_services, accept: %{ json: true } } do - forward conn, path, "http://privacy/person-information-updates" + forward conn, path, "http://privacy-unit/person-information-updates" end match "/person-information-requests/*path", %{ layer: :api_services, accept: %{ json: true } } do - forward conn, path, "http://privacy/person-information-requests" + forward conn, path, "http://privacy-unit/person-information-requests" end match "/person-information-ask/*path", %{ layer: :api_services, accept: %{ json: true } } do - forward conn, path, "http://privacy/person-information-ask/" + forward conn, path, "http://privacy-unit/person-information-ask/" end match "/person-information-validate-ssn/*path", %{ layer: :api_services, accept: %{ json: true } } do - forward conn, path, "http://privacy/person-information-validate-ssn/" + forward conn, path, "http://privacy-unit/person-information-validate-ssn/" end get "/uri-info/*path", %{ layer: :api_services, accept: %{ json: true } } do diff --git a/config/loket-consumer/leidinggevenden-dispatching/config.js b/config/loket-consumer/leidinggevenden-dispatching/config.js index e280733..2afb525 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/config.js +++ b/config/loket-consumer/leidinggevenden-dispatching/config.js @@ -7,7 +7,7 @@ const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_AT const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); -const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`; +const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`; if (!process.env.FILE_SYNC_GRAPH) @@ -26,4 +26,4 @@ module.exports = { SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, INGEST_GRAPH, FILE_SYNC_GRAPH -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js index c8d8ba1..669895b 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js +++ b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js @@ -1,13 +1,10 @@ -const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); -const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, +const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util'); +const { BATCH_SIZE, MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH, - FILE_SYNC_GRAPH + INGEST_GRAPH } = require('./config'); /** @@ -34,43 +31,40 @@ async function dispatch(lib, data) { const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); if (deleteStatements.length) { - await transformStatements(fetch, deleteStatements).then( - transformedStatements => { - deleteFromAllGraphs( - muAuthSudo.updateSudo, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - ); - } - ) + const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements); + if (transformedStatementsToDelete.length) { + await deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatementsToDelete, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } } - if (insertStatements.length) { - await transformStatements(fetch, insertStatements).then( - transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "INSERT" - ); - } - ) + const transformedStatementsToInsert = await transformStatements(fetch, insertStatements); + if (transformedStatementsToInsert.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatementsToInsert, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + "INSERT" + ); + } } } } module.exports = { dispatch -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js index 88017c1..7967073 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js +++ b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js @@ -1,4 +1,4 @@ -const { transformStatements, batchedDbUpdate, partition } = require('./util'); +const { transformStatements, batchedDbUpdate } = require('./util'); const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, DIRECT_DATABASE_ENDPOINT, MU_CALL_SCOPE_ID_INITIAL_SYNC, @@ -6,8 +6,7 @@ const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH, - FILE_SYNC_GRAPH + INGEST_GRAPH } = require('./config'); const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; @@ -36,24 +35,23 @@ async function dispatch(lib, data) { if (termObjects.length) { originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - await transformStatements(fetch, originalInsertTriples).then( - transformedInsertTriples => { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ) - } - ) + const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples); + if (transformedStatementsToInsert.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatementsToInsert, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } } } module.exports = { dispatch -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/util.js b/config/loket-consumer/leidinggevenden-dispatching/util.js index ff3932f..bdf477b 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/util.js +++ b/config/loket-consumer/leidinggevenden-dispatching/util.js @@ -150,4 +150,4 @@ module.exports = { deleteFromAllGraphs, partition, transformStatements -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/mandate-dispatching/config.js b/config/loket-consumer/mandate-dispatching/config.js index e280733..2afb525 100644 --- a/config/loket-consumer/mandate-dispatching/config.js +++ b/config/loket-consumer/mandate-dispatching/config.js @@ -7,7 +7,7 @@ const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_AT const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); -const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`; +const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`; if (!process.env.FILE_SYNC_GRAPH) @@ -26,4 +26,4 @@ module.exports = { SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, INGEST_GRAPH, FILE_SYNC_GRAPH -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js index c8d8ba1..669895b 100644 --- a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js +++ b/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js @@ -1,13 +1,10 @@ -const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); -const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, +const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util'); +const { BATCH_SIZE, MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH, - FILE_SYNC_GRAPH + INGEST_GRAPH } = require('./config'); /** @@ -34,43 +31,40 @@ async function dispatch(lib, data) { const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); if (deleteStatements.length) { - await transformStatements(fetch, deleteStatements).then( - transformedStatements => { - deleteFromAllGraphs( - muAuthSudo.updateSudo, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - ); - } - ) + const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements); + if (transformedStatementsToDelete.length) { + await deleteFromAllGraphs( + muAuthSudo.updateSudo, + transformedStatementsToDelete, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + ); + } } - if (insertStatements.length) { - await transformStatements(fetch, insertStatements).then( - transformedStatements => { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatements, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "INSERT" - ); - } - ) + const transformedStatementsToInsert = await transformStatements(fetch, insertStatements); + if (transformedStatementsToInsert.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatementsToInsert, + { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, + process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + "INSERT" + ); + } } } } module.exports = { dispatch -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js index 88017c1..152abda 100644 --- a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js +++ b/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js @@ -1,4 +1,4 @@ -const { transformStatements, batchedDbUpdate, partition } = require('./util'); +const { transformStatements, batchedDbUpdate } = require('./util'); const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, DIRECT_DATABASE_ENDPOINT, MU_CALL_SCOPE_ID_INITIAL_SYNC, @@ -6,8 +6,7 @@ const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH, - FILE_SYNC_GRAPH + INGEST_GRAPH } = require('./config'); const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; @@ -36,24 +35,23 @@ async function dispatch(lib, data) { if (termObjects.length) { originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - await transformStatements(fetch, originalInsertTriples).then( - transformedInsertTriples => { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ) - } - ) + const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples); + if (transformedStatementsToInsert.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatementsToInsert, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ) + } } } module.exports = { dispatch -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/mandate-dispatching/util.js b/config/loket-consumer/mandate-dispatching/util.js index ff3932f..bdf477b 100644 --- a/config/loket-consumer/mandate-dispatching/util.js +++ b/config/loket-consumer/mandate-dispatching/util.js @@ -150,4 +150,4 @@ module.exports = { deleteFromAllGraphs, partition, transformStatements -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/positions-dispatching/config.js b/config/loket-consumer/positions-dispatching/config.js index 911277b..ab34eca 100644 --- a/config/loket-consumer/positions-dispatching/config.js +++ b/config/loket-consumer/positions-dispatching/config.js @@ -1,18 +1,32 @@ const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; -const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; -const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; -const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = + process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || + "http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync"; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = + process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == "true" ? true : false; +const DIRECT_DATABASE_ENDPOINT = + process.env.DIRECT_DATABASE_ENDPOINT || "http://virtuoso:8890/sparql"; const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); -const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); -const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); -const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); -const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); -const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`; -const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/privacy-centric-graph`; -const UNFILTERED_MAPPING = 'main'; -const MAIN_INFO_MAPPING = 'main-info'; -const PRIVATE_INFO_MAPPING = 'private-info'; - +const MAX_REASONING_RETRY_ATTEMPTS = parseInt( + process.env.MAX_REASONING_RETRY_ATTEMPTS || 5 +); +const SLEEP_BETWEEN_BATCHES = parseInt( + process.env.SLEEP_BETWEEN_BATCHES || 1000 +); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt( + process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000 +); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt( + process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000 +); +const INGEST_GRAPH = + process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/worship-service`; +const PRIVACY_SENSITIVE_GRAPH = + process.env.PRIVACY_SENSITIVE_GRAPH || + `http://mu.semte.ch/graphs/worship-privacy-centric-graph`; +const UNFILTERED_MAPPING = "main"; +const MAIN_INFO_MAPPING = "main-info"; +const PRIVATE_INFO_MAPPING = "private-info"; if (!process.env.FILE_SYNC_GRAPH) throw `Expected 'FILE_SYNC_GRAPH' to be provided.`; @@ -33,4 +47,5 @@ module.exports = { UNFILTERED_MAPPING, MAIN_INFO_MAPPING, PRIVATE_INFO_MAPPING, -}; \ No newline at end of file + FILE_SYNC_GRAPH, +}; diff --git a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js index d826b2e..5384005 100644 --- a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js +++ b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js @@ -1,4 +1,4 @@ -const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util'); +const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util'); const { BATCH_SIZE, MAX_DB_RETRY_ATTEMPTS, @@ -72,7 +72,7 @@ async function dispatch(lib, data) { PRIVACY_SENSITIVE_GRAPH, transformedStatementsToInsertPrivate, { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth + process.env.MU_SPARQL_ENDPOINT, // Note: this is the default endpoint through auth BATCH_SIZE, MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, @@ -85,4 +85,4 @@ async function dispatch(lib, data) { module.exports = { dispatch -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js index f5fedc8..86f34b7 100644 --- a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js +++ b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js @@ -38,43 +38,38 @@ async function dispatch(lib, data) { if (termObjects.length) { originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - await transformStatements(fetch, originalInsertTriples, MAIN_INFO_MAPPING).then( - transformedInsertTriples => { - if (transformedInsertTriples.length) { - batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ) - } - } - ); - await transformStatements(fetch, originalInsertTriples, PRIVATE_INFO_MAPPING).then( - transformedInsertTriples => { - if (transformedInsertTriples.length) { - batchedDbUpdate( - muAuthSudo.updateSudo, - PRIVACY_SENSITIVE_GRAPH, - transformedInsertTriples, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ) - } - } - ); + const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples, MAIN_INFO_MAPPING); + if (transformedStatementsToInsert.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + INGEST_GRAPH, + transformedStatementsToInsert, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } + + const transformedStatementsToInsertPrivate = await transformStatements(fetch, originalInsertTriples, PRIVATE_INFO_MAPPING); + if (transformedStatementsToInsertPrivate.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + transformedStatementsToInsertPrivate, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } } } module.exports = { dispatch -}; \ No newline at end of file +}; diff --git a/config/loket-consumer/positions-dispatching/util.js b/config/loket-consumer/positions-dispatching/util.js index ce38369..f6411a2 100644 --- a/config/loket-consumer/positions-dispatching/util.js +++ b/config/loket-consumer/positions-dispatching/util.js @@ -151,4 +151,4 @@ module.exports = { deleteFromAllGraphs, partition, transformStatements -}; \ No newline at end of file +}; diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132500-delete-elastic-index.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132500-delete-elastic-index.sparql new file mode 100644 index 0000000..8d7e1b4 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132500-delete-elastic-index.sparql @@ -0,0 +1 @@ +delete where { graph ?g{ ?s a ; ?p ?o } } diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132600-move-all-worship-types.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132600-move-all-worship-types.sparql new file mode 100644 index 0000000..7a791af --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132600-move-all-worship-types.sparql @@ -0,0 +1,44 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: + + +DELETE { + GRAPH { + ?s ?p ?o + } +} +INSERT { + GRAPH { + ?s ?p ?o + } +} + +WHERE { + GRAPH { + ?s a ?type; ?p ?o. + + filter (?type in ( + ere:EredienstMandataris, + ere:BestuurVanDeEredienst, + ere:PositieBedienaar, + ere:BetrokkenLokaleBesturen, + ere:VerbondenJuridischeStructuren, + ere:RolBedienaar, + ere:EredienstBestuurseenheid, + ere:VoorwaardenBedienaar, + ere:CentraalBestuurVanDeEredienst, + ere:RepresentatiefOrgaan + )) + + } +} diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132601-move-all-worsip-governing-bodies.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132601-move-all-worsip-governing-bodies.sparql new file mode 100644 index 0000000..ef16265 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132601-move-all-worsip-governing-bodies.sparql @@ -0,0 +1,90 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: + + +DELETE { + GRAPH { + ?governingBody ?p ?o. + } +} +INSERT { + GRAPH { + ?governingBody ?p ?o. + } +} + +WHERE { + GRAPH { + ?governingBody ?post; ?p ?o. + + filter exists { + GRAPH { + ?mandatories ?post. + + } + } + } + +} + +; + +DELETE { + GRAPH { + ?specialization ?specializationP ?specializationO. + } +} +INSERT { + GRAPH { + ?specialization ?specializationP ?specializationO. + } +} + +WHERE { + GRAPH { + ?specialization ?specializationP ?specializationO. + + filter exists { + GRAPH { + ?governingBody generiek:isTijdspecialisatieVan ?specialization. + } + } + } + +} + +; + +DELETE { + GRAPH { + ?bestuur ?bestuurP ?bestuurO. + } +} +INSERT { + GRAPH { + ?bestuur ?bestuurP ?bestuurO. + } +} + +WHERE { + GRAPH { + ?bestuur ?bestuurP ?bestuurO. + filter exists { + GRAPH { + + ?specialization besluit:bestuurt ?bestuur. + } + } + } + +} diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132602-move-all-worsip-governing-bodies-2.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132602-move-all-worsip-governing-bodies-2.sparql new file mode 100644 index 0000000..173d4dc --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132602-move-all-worsip-governing-bodies-2.sparql @@ -0,0 +1,73 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: + + +DELETE { + GRAPH { + ?governingBody ?p ?o. + } +} +INSERT { + GRAPH { + ?governingBody ?p ?o. + } +} + +WHERE { + GRAPH { + ?governingBody + ?post; ?p ?o. + + filter exists { + GRAPH { + ?mandatories ?post. + + } + } + } + +} +; + + +DELETE { + GRAPH { + ?post a ?type; + mu:uuid ?uuid; + ?role. + } +} +INSERT { + GRAPH { + ?post a ?type; mu:uuid ?uuid; + ?role. + } +} + +WHERE { + GRAPH { + ?post a ?type; + mu:uuid ?uuid; + + ?role. + + filter exists { + GRAPH { + ?mandatories ?post. + + } + } + } + +} + diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132603-move-all-worsip-governing-bodies-3.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132603-move-all-worsip-governing-bodies-3.sparql new file mode 100644 index 0000000..e4ece33 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132603-move-all-worsip-governing-bodies-3.sparql @@ -0,0 +1,93 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: + + +DELETE { + GRAPH { + ?governingBody ?p ?o. + } +} +INSERT { + GRAPH { + ?governingBody ?p ?o. + } +} + +WHERE { + GRAPH { + ?governingBody ?post; ?p ?o. + + filter exists { + GRAPH { + ?mandatories ?post. + + } + } + } + +} + +; + +DELETE { + GRAPH { + ?specialization ?specializationP ?specializationO. + } +} +INSERT { + GRAPH { + ?specialization ?specializationP ?specializationO. + } +} + +WHERE { + GRAPH { + ?specialization ?specializationP ?specializationO. + + filter exists { + GRAPH { + ?governingBody generiek:isTijdspecialisatieVan ?specialization. + } + } + } + +} + +; + + +PREFIX besluit: + +DELETE { + GRAPH { + ?bestuur ?bestuurP ?bestuurO. + } +} +INSERT { + GRAPH { + ?bestuur ?bestuurP ?bestuurO. + } +} + +WHERE { + GRAPH { + ?bestuur besluit:bestuurt ?s;?bestuurP ?bestuurO. + filter exists { + GRAPH { + + ?s ?p ?o. + } + } + } + +} diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132604-move-bestrokken.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132604-move-bestrokken.sparql new file mode 100644 index 0000000..62a10af --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132604-move-bestrokken.sparql @@ -0,0 +1,36 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: + + +DELETE { + GRAPH { + ?s ?bestrokke. + } +} +INSERT { + GRAPH { + ?s ?bestrokke. + } +} + +WHERE { + GRAPH { + + ?s ?bestrokke. + + } + +} + + + diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132605-move-all-worsip-change-events.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132605-move-all-worsip-change-events.sparql new file mode 100644 index 0000000..b07ee2f --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132605-move-all-worsip-change-events.sparql @@ -0,0 +1,87 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: + + +DELETE { + GRAPH { + ?s ?p ?o. + } +} +INSERT { + GRAPH { + ?s ?p ?o. + } +} + +WHERE { + GRAPH { + ?resultingOrganization a ?type. + + } + GRAPH { + + ?s ext:resultingOrganization ?resultingOrganization. + ?s ?p ?o. + } + +} +; + +DELETE { + GRAPH { + ?s ?p ?o. + } +} +INSERT { + GRAPH { + ?s ?p ?o. + } +} + +WHERE { + GRAPH { + ?resultingOrganization a ?type. + + } + GRAPH { + + ?s ?resultingOrganization. + ?s ?p ?o. + } + +} +; +DELETE { + GRAPH { + ?s ?p ?o. + } +} +INSERT { + GRAPH { + ?s ?p ?o. + } +} + +WHERE { + GRAPH { + ?resultingOrganization a ?type. + + } + GRAPH { + + ?s org:resultingOrganization ?resultingOrganization. + ?s ?p ?o. + } + +} + diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132606-duplicate-person.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132606-duplicate-person.sparql new file mode 100644 index 0000000..6ccb0ab --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132606-duplicate-person.sparql @@ -0,0 +1,76 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: + +INSERT { + graph { + ?person a person:Person; + mu:uuid ?uuid; + foaf:givenName ?givenName; + foaf:familyName ?familyName; + ?isAangesteldAls; + persoon:gebruikteVoornaam ?firstNameUsed. + } + +} +WHERE { + graph { + ?person a person:Person; + mu:uuid ?uuid; + foaf:givenName ?givenName; + foaf:familyName ?familyName. + optional {?person persoon:gebruikteVoornaam ?firstNameUsed. } + optional {?person ?isAangesteldAls.} + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} +; + +# delete orphans +DELETE { + graph { + ?person a person:Person; + foaf:givenName ?givenName; + foaf:familyName ?familyName; + ?isAangesteldAls; + persoon:gebruikteVoornaam ?firstNameUsed. + } + +} +WHERE { + graph { + ?person a person:Person; + foaf:givenName ?givenName; + foaf:familyName ?familyName. + optional{?person persoon:gebruikteVoornaam ?firstNameUsed.} + optional {?person ?isAangesteldAls.} + + FILTER((NOT EXISTS { ?mandatories ?person. }) + && (NOT EXISTS { ?ministers ?person. })) + + } +} + + diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132607-move-shared-types.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132607-move-shared-types.sparql new file mode 100644 index 0000000..1376169 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132607-move-shared-types.sparql @@ -0,0 +1,290 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: +PREFIX skos: +PREFIX schema: +PREFIX organisatie: +PREFIX locn: +# move geboorte to private graph + +DELETE { + GRAPH { + ?s ?p ?o + } +} + INSERT { + GRAPH { + ?s ?p ?o + } +} +WHERE { + + GRAPH { + ?s a ; ?p ?o. + } +} + +; + +DELETE { + graph { + ?contactPoint ?p ?o. + ?addr ?x ?y. + } +} +insert { + graph { + ?contactPoint ?p ?o. + ?addr ?x ?y. + + } +} +where { + graph { + + ?contactPoint ?p ?o. + optional { + ?contactPoint ?addr. + ?addr ?x ?y. + } + filter (exists { + graph { + ?s ?contactPoint. + } + }) + } +} +; + + +DELETE { + GRAPH { + ?primarySite ?siteAddr; ?p ?o. + ?siteAddr ?x ?y. + ?addr ?addrP ?addrO. + ?locAddr ?locP ?locO + } +} +INSERT { + GRAPH { + ?primarySite ?siteAddr; ?p ?o. + ?siteAddr ?x ?y. + ?addr ?addrP ?addrO. + ?locAddr ?locP ?locO + } +} +Where { + Graph { + ?primarySite ?siteAddr; + ?p ?o. + ?siteAddr ?x ?y. + optional { + ?siteAddr locn:address ?locAddr . + ?locAddr ?locP ?locO. + } + optional { + ?primarySite organisatie:bestaatUit ?addr. + ?addr ?addrP ?addrO. + } + + + filter (exists { + graph { + ?s ?primarySite. + } + } + ) + + + } +} +; + +DELETE { + GRAPH { + ?s a ?type; ?p ?o. + } +} +INSERT { + GRAPH { + ?s a ?type; ?p ?o. + } +} +Where { + Graph { + ?s a ?type; ?p ?o. + filter (?type in(, + , + + ) + && exists { + graph { + ?x ?y ?z. + filter(?z=?s) + } + } + ) + + + } +} + +; + +DELETE { + GRAPH { + + ?primarySite ?bestaatUit; ?p ?o. + ?bestaatUit ?bestaatUitP ?bestaatUitO. + + } +} +INSERT { + GRAPH { + + ?primarySite ?bestaatUit; ?p ?o. + ?bestaatUit ?bestaatUitP ?bestaatUitO. +} +} +Where { + Graph { + ?primarySite ?bestaatUit; ?p ?o. + ?bestaatUit ?bestaatUitP ?bestaatUitO. + + + filter (exists { + graph { + ?unit ?primarySite. + } + } + ) + + + } +} +; + +DELETE { + GRAPH { + ?s a ?type; ?p ?o. + } +} +INSERT { + GRAPH { + ?s a ?type; ?p ?o. + } +} + +Where { + Graph { + ?s a ?type; ?p ?o. + filter (?type in(, + , + , + + ) + ) + + + } +} + + + +; + +DELETE { + GRAPH { + ?s a ; + mu:uuid ?uuid; + ?classification; + org:hasSubOrganization ?subOrg; + skos:prefLabel ?name. + + + } +} +INSERT { + GRAPH { + ?s a ; + mu:uuid ?uuid; + ?classification; + org:hasSubOrganization ?subOrg; + skos:prefLabel ?name. + + } +} + +WHERE { + GRAPH { + ?s a ; + ?classification; + mu:uuid ?uuid; + org:hasSubOrganization ?subOrg; + skos:prefLabel ?name. + + filter(?classification in ( + , + + )) + + + } +} + +; + +DELETE { + GRAPH { + ?s a ; + mu:uuid ?uuid; + ?classification; + skos:prefLabel ?name. + + + } +} +INSERT { + GRAPH { + ?s a ; + mu:uuid ?uuid; + ?classification; + + skos:prefLabel ?name. + + } +} + +WHERE { + GRAPH { + ?s a ; + ?classification; + mu:uuid ?uuid; + skos:prefLabel ?name. + + + filter(?classification in ( + , + + )&& exists { + + graph { + + ?subOrg org:hasSubOrganization ?s . +} + +}) + + + } +} + diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132608-move-public-types.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132608-move-public-types.sparql new file mode 100644 index 0000000..9b929d1 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132608-move-public-types.sparql @@ -0,0 +1,40 @@ +delete { + graph { + ?s ?p ?o + } +} +insert { + graph { + ?s ?p ?o + } +} +where { + graph { + ?s a ?type; ?p ?o. + filter (?type in ( + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + + + + )) + } +} diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132609-move-to-administrative-unit.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132609-move-to-administrative-unit.sparql new file mode 100644 index 0000000..bab2563 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132609-move-to-administrative-unit.sparql @@ -0,0 +1,3 @@ +# must be the latest to run + +MOVE TO diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132610-add-mock-user-1.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132610-add-mock-user-1.sparql new file mode 100644 index 0000000..192c7b5 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132610-add-mock-user-1.sparql @@ -0,0 +1,44 @@ +PREFIX ext: + PREFIX besluit: + PREFIX mu: + PREFIX skos: + PREFIX dcterms: + PREFIX foaf: + + INSERT DATA { + GRAPH { + besluit:Bestuurseenheid . + mu:uuid "141d9d6b-54af-4d17-b313-8d1c30bc3f5b" . + skos:prefLabel "Agentschap Binnenlands Bestuur" . + dcterms:identifier "OVO001835" . + foaf:Person . + mu:uuid "d39fe09e-d339-4e43-8680-dbbcfb3d3470" . + foaf:firstName "Josephine" . + foaf:familyName "Lezer" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "3a91ff60-07c1-4136-ac5e-55cf401e0989" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalGebruiker-lezer" . + foaf:Person . + mu:uuid "d39fe09e-d339-4e43-8680-dbbcfb3d3471" . + foaf:firstName "Josephine" . + foaf:familyName "Editeerder" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "3a91ff60-07c1-4136-ac5e-55cf401e0957" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalGebruiker-editeerder" . + foaf:Person . + mu:uuid "d39fe09e-d339-4e43-8680-dbbcfb3d3472" . + foaf:firstName "Josephine" . + foaf:familyName "Beheerder" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "3a91ff60-07c1-4136-ac5e-55cf401e0958" . + foaf:accountServiceHomepage . + } + } diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132611-add-mock-user-2.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132611-add-mock-user-2.sparql new file mode 100644 index 0000000..361ea9e --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132611-add-mock-user-2.sparql @@ -0,0 +1,42 @@ +PREFIX ext: + PREFIX besluit: + PREFIX mu: + PREFIX skos: + PREFIX dcterms: + PREFIX foaf: + + INSERT DATA { + GRAPH { + ext:sessionRole "ABBOrganisatiePortaalGebruiker-beheerder" . + foaf:Person . + mu:uuid "2fe9885b-d058-4059-b036-69b54444273d" . + foaf:firstName "Josephine" . + foaf:familyName "WorshipLezer" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "8d411fea-5bf0-4925-ba97-14b2223f65ea" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalErediensten-lezer" . + foaf:Person . + mu:uuid "4925580c-a1a9-43d4-b3f6-10bd7ff57e25" . + foaf:firstName "Josephine" . + foaf:familyName "WorshipEditeerder" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "fe45b266-715c-481e-9a97-a70f6121166c" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalErediensten-editeerder" . + foaf:Person . + mu:uuid "f3610493-f6e1-4490-9723-628fe0213491" . + foaf:firstName "Josephine" . + foaf:familyName "WorshipBeheerder" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "2a88a4b4-4672-45eb-b072-c930bf3b40ae" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalErediensten-beheerder" . + } + } diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132612-add-mock-user-3.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132612-add-mock-user-3.sparql new file mode 100644 index 0000000..33e912c --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132612-add-mock-user-3.sparql @@ -0,0 +1,30 @@ +PREFIX ext: + PREFIX besluit: + PREFIX mu: + PREFIX skos: + PREFIX dcterms: + PREFIX foaf: + + INSERT DATA { + GRAPH { + foaf:Person . + mu:uuid "d39fe09e-d339-4e43-8680-dbbcfb3d3470" . + foaf:firstName "Josephine" . + foaf:familyName "Lezer" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "3a91ff60-07c1-4136-ac5e-55cf401e0989" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalGebruiker-lezer" . + foaf:Person . + mu:uuid "d39fe09e-d339-4e43-8680-dbbcfb3d3471" . + foaf:firstName "Josephine" . + foaf:familyName "Editeerder" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "3a91ff60-07c1-4136-ac5e-55cf401e0957" . + foaf:accountServiceHomepage . + } + } diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132613-add-mock-user-4.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132613-add-mock-user-4.sparql new file mode 100644 index 0000000..68534f9 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132613-add-mock-user-4.sparql @@ -0,0 +1,53 @@ +PREFIX ext: + PREFIX besluit: + PREFIX mu: + PREFIX skos: + PREFIX dcterms: + PREFIX foaf: + + INSERT DATA { + GRAPH { + ext:sessionRole "ABBOrganisatiePortaalGebruiker-editeerder" . + foaf:Person . + mu:uuid "d39fe09e-d339-4e43-8680-dbbcfb3d3472" . + foaf:firstName "Josephine" . + foaf:familyName "Beheerder" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "3a91ff60-07c1-4136-ac5e-55cf401e0958" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalGebruiker-beheerder" . + foaf:Person . + mu:uuid "2fe9885b-d058-4059-b036-69b54444273d" . + foaf:firstName "Josephine" . + foaf:familyName "WorshipLezer" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "8d411fea-5bf0-4925-ba97-14b2223f65ea" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalErediensten-lezer" . + foaf:Person . + mu:uuid "4925580c-a1a9-43d4-b3f6-10bd7ff57e25" . + foaf:firstName "Josephine" . + foaf:familyName "WorshipEditeerder" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "fe45b266-715c-481e-9a97-a70f6121166c" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalErediensten-editeerder" . + foaf:Person . + mu:uuid "f3610493-f6e1-4490-9723-628fe0213491" . + foaf:firstName "Josephine" . + foaf:familyName "WorshipBeheerder" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "2a88a4b4-4672-45eb-b072-c930bf3b40ae" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalErediensten-beheerder" . + +} + } diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132614-move-privacy-data.sparql b/config/migrations/2023/20230401132400-hard-split/20230401132614-move-privacy-data.sparql new file mode 100644 index 0000000..ea64b74 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230401132614-move-privacy-data.sparql @@ -0,0 +1,128 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: + + + + +insert { + +GRAPH { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + ?person persoon:geslacht ?geslacht. + ?person persoon:heeftNationaliteit ?nationaliteit. + + +} + +} + where { + GRAPH { + optional { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + } + optional { + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + } + optional { + ?person persoon:geslacht ?geslacht. + } + optional { + ?person persoon:heeftNationaliteit ?nationaliteit. + } + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + + + } + + + } + + ; + + + + + +insert { + +GRAPH { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + ?person persoon:geslacht ?geslacht. + ?person persoon:heeftNationaliteit ?nationaliteit. + + +} + +} + where { + GRAPH { + optional { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + } + optional { + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + } + optional { + ?person persoon:geslacht ?geslacht. + } + optional { + ?person persoon:heeftNationaliteit ?nationaliteit. + } + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + + + } + + + } + diff --git a/config/migrations/2023/20230401132400-hard-split/20230402105324-move-jobs-to-specific-graph.sparql b/config/migrations/2023/20230401132400-hard-split/20230402105324-move-jobs-to-specific-graph.sparql new file mode 100644 index 0000000..9cf2b5b --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230402105324-move-jobs-to-specific-graph.sparql @@ -0,0 +1,21 @@ +DELETE { + GRAPH ?g { + ?s ?p ?o . + } +} INSERT { + GRAPH { + ?s ?p ?o . + } +} WHERE { + GRAPH ?g { + VALUES ?type { + + + + + } + + ?s a ?type ; + ?p ?o . + } +} diff --git a/config/migrations/2023/20230401132400-hard-split/20230403132612-add-mock-user-5.sparql b/config/migrations/2023/20230401132400-hard-split/20230403132612-add-mock-user-5.sparql new file mode 100644 index 0000000..c87920f --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230403132612-add-mock-user-5.sparql @@ -0,0 +1,23 @@ +PREFIX ext: + PREFIX besluit: + PREFIX mu: + PREFIX skos: + PREFIX dcterms: + PREFIX foaf: + + INSERT DATA { + GRAPH { + foaf:Person . + mu:uuid "6928a160-f3fc-4b46-bae8-fb72a0ae833e" . + foaf:firstName "Josephine" . + foaf:familyName "BothRoles" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "cd52ed34-7bcd-41df-a7d4-c805c1d1a197" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalErediensten-beheerder", + "ABBOrganisatiePortaalGebruiker-beheerder" . + + } + } diff --git a/config/migrations/2023/20230401132400-hard-split/20230403132613-add-mock-user-6.sparql b/config/migrations/2023/20230401132400-hard-split/20230403132613-add-mock-user-6.sparql new file mode 100644 index 0000000..1f3e984 --- /dev/null +++ b/config/migrations/2023/20230401132400-hard-split/20230403132613-add-mock-user-6.sparql @@ -0,0 +1,22 @@ +PREFIX ext: + PREFIX besluit: + PREFIX mu: + PREFIX skos: + PREFIX dcterms: + PREFIX foaf: + + INSERT DATA { + GRAPH { + foaf:Person . + mu:uuid "6928a160-f3fc-4b46-bae8-fb72a0ae833e" . + foaf:firstName "Josephine" . + foaf:familyName "BothRoles" . + foaf:member . + foaf:account . + foaf:OnlineAccount . + mu:uuid "cd52ed34-7bcd-41df-a7d4-c805c1d1a197" . + foaf:accountServiceHomepage . + ext:sessionRole "ABBOrganisatiePortaalErediensten-beheerder", "ABBOrganisatiePortaalGebruiker-beheerder" . + + } + } diff --git a/config/resources/domain.json b/config/resources/domain.json index a763d2a..4eba5e7 100644 --- a/config/resources/domain.json +++ b/config/resources/domain.json @@ -122,7 +122,9 @@ }, "mandatories": { "class": "mandaat:Mandataris", - "super": ["agents-in-position"], + "super": [ + "agents-in-position" + ], "attributes": { "start-date": { "type": "date", @@ -165,12 +167,16 @@ "predicate": "skos:prefLabel" } }, - "features": ["include-uri"], + "features": [ + "include-uri" + ], "new-resource-base": "http://data.vlaanderen.be/id/concept/MandatarisStatusCode/" }, "worship-mandatories": { "class": "ere:EredienstMandataris", - "super": ["mandatories"], + "super": [ + "mandatories" + ], "attributes": { "expected-end-date": { "type": "date", @@ -235,7 +241,9 @@ }, "mandates": { "class": "mandaat:Mandaat", - "super": ["posts"], + "super": [ + "posts" + ], "relationships": { "role-board": { "predicate": "org:role", @@ -259,7 +267,9 @@ }, "agents": { "class": "lblodlg:Functionaris", - "super": ["agents-in-position"], + "super": [ + "agents-in-position" + ], "attributes": { "start-date": { "type": "datetime", @@ -297,12 +307,16 @@ "predicate": "skos:prefLabel" } }, - "features": ["include-uri"], + "features": [ + "include-uri" + ], "new-resource-base": "http://data.vlaanderen.be/id/concept/functionarisStatusCode/" }, "board-positions": { "class": "lblodlg:Bestuursfunctie", - "super": ["posts"], + "super": [ + "posts" + ], "relationships": { "role-board": { "predicate": "org:role", @@ -433,7 +447,9 @@ }, "administrative-units": { "class": "besluit:Bestuurseenheid", - "super": ["organizations"], + "super": [ + "organizations" + ], "relationships": { "classification": { "predicate": "org:classification", @@ -481,7 +497,9 @@ }, "worship-administrative-units": { "class": "ere:EredienstBestuurseenheid", - "super": ["administrative-units"], + "super": [ + "administrative-units" + ], "relationships": { "recognized-worship-type": { "predicate": "ere:typeEredienst", @@ -504,7 +522,9 @@ }, "worship-services": { "class": "ere:BestuurVanDeEredienst", - "super": ["worship-administrative-units"], + "super": [ + "worship-administrative-units" + ], "attributes": { "denomination": { "type": "string", @@ -529,12 +549,16 @@ }, "central-worship-services": { "class": "ere:CentraalBestuurVanDeEredienst", - "super": ["worship-administrative-units"], + "super": [ + "worship-administrative-units" + ], "new-resource-base": "http://data.lblod.info/id/centraleBesturenVanDeEredienst/" }, "representative-bodies": { "class": "ere:RepresentatiefOrgaan", - "super": ["organizations"], + "super": [ + "organizations" + ], "relationships": { "recognized-worship-type": { "predicate": "ere:typeEredienst", @@ -765,7 +789,9 @@ "predicate": "skos:prefLabel" } }, - "features": ["include-uri"], + "features": [ + "include-uri" + ], "new-resource-base": "http://lblod.data.gift/concepts/" }, "locations": { @@ -818,7 +844,9 @@ }, "ministers": { "class": "ere:RolBedienaar", - "super": ["agents-in-position"], + "super": [ + "agents-in-position" + ], "relationships": { "minister-position": { "predicate": "org:holds", @@ -886,7 +914,9 @@ }, "minister-positions": { "class": "ere:PositieBedienaar", - "super": ["posts"], + "super": [ + "posts" + ], "relationships": { "function": { "predicate": "ere:functie", diff --git a/config/search/config.json b/config/search/config.json index 6649ae8..e62d951 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -48,15 +48,7 @@ [ { "variables": [], - "name": "acmidm-beheerder" - }, - { - "variables": [], - "name": "acmidm-editeerder" - }, - { - "variables": [], - "name": "acmidm-lezer" + "name": "acmidm-worship-beheerder" }, { "variables": [], @@ -70,11 +62,7 @@ [ { "variables": [], - "name": "acmidm-editeerder" - }, - { - "variables": [], - "name": "acmidm-lezer" + "name": "acmidm-worship-lezer" }, { "variables": [], @@ -88,29 +76,7 @@ [ { "variables": [], - "name": "acmidm-editeerder" - }, - { - "variables": [], - "name": "acmidm-beheerder" - }, - { - "variables": [], - "name": "clean" - }, - { - "variables": [], - "name": "public" - } - ], - [ - { - "variables": [], - "name": "acmidm-lezer" - }, - { - "variables": [], - "name": "acmidm-beheerder" + "name": "acmidm-worship-editeerder" }, { "variables": [], @@ -122,19 +88,19 @@ } ] ], - "default_settings" : { + "default_settings": { "analysis": { "normalizer": { "custom_sort_normalizer": { - "type": "custom", - "char_filter": [], - "filter": [ - "lowercase", - "trim", - "asciifolding" - ] + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase", + "trim", + "asciifolding" + ] } - } + } } }, "types": [ diff --git a/config/search/dev/config.json b/config/search/dev/config.json index 153ec14..6dd6b9e 100644 --- a/config/search/dev/config.json +++ b/config/search/dev/config.json @@ -1,26 +1,12 @@ { "automatic_index_updates": true, "persist_indexes": true, - "number_of_threads": 8, + "number_of_threads": 6, "eager_indexing_groups": [ [ { "variables": [], - "name": "acmidm-lezer" - }, - { - "variables": [], - "name": "clean" - }, - { - "variables": [], - "name": "public" - } - ], - [ - { - "variables": [], - "name": "acmidm-editeerder" + "name": "acmidm-worship-beheerder" }, { "variables": [], diff --git a/config/tlsproxy-mock/Caddyfile b/config/tlsproxy-mock/Caddyfile new file mode 100644 index 0000000..b041c33 --- /dev/null +++ b/config/tlsproxy-mock/Caddyfile @@ -0,0 +1,4 @@ +https://{$OP_ADDRESS} { + reverse_proxy identifier:80 + tls internal +} diff --git a/data/db/virtuoso.ini b/data/db/virtuoso.ini deleted file mode 100755 index e69de29..0000000 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index c793b9d..52c53e8 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,4 +1,4 @@ -version: '3.4' +version: "3.4" services: identifier: @@ -40,7 +40,9 @@ services: restart: "no" accountdetail: restart: "no" - privacy: + privacy-unit: + restart: "no" + privacy-worship: restart: "no" adressenregister: restart: "no" @@ -71,3 +73,13 @@ services: restart: "no" delta-producer-pub-graph-maintainer-worship-posts: restart: "no" + # use this to test acmidm + # tlsproxy: + # image: caddy:2 + # restart: "no" + # ports: + # - "443:443" + # environment: + # OP_ADDRESS: "organisaties.abb.lblod.info" + # volumes: + # - ./config/tlsproxy-mock/Caddyfile:/etc/caddy/Caddyfile diff --git a/docker-compose.yml b/docker-compose.yml index 2f35a1f..2e4f940 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -148,14 +148,26 @@ services: - db:database restart: always logging: *default-logging - privacy: + privacy-worship: image: lblod/privacy-centric-service:0.6.0 labels: - "logging=true" environment: - DEFAULT_GRAPH: "http://mu.semte.ch/graphs/privacy-centric-graph" + DEFAULT_GRAPH: "http://mu.semte.ch/graphs/worship-privacy-centric-graph" SESSION_GRAPH: "http://mu.semte.ch/graphs/sessions" - APP_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + APP_GRAPH: "http://mu.semte.ch/graphs/public" + restart: always + logging: *default-logging + links: + - db:database + privacy-unit: + image: lblod/privacy-centric-service:0.6.0 + labels: + - "logging=true" + environment: + DEFAULT_GRAPH: "http://mu.semte.ch/graphs/administrative-unit-privacy-centric-graph" + SESSION_GRAPH: "http://mu.semte.ch/graphs/sessions" + APP_GRAPH: "http://mu.semte.ch/graphs/public" restart: always logging: *default-logging links: @@ -232,11 +244,11 @@ services: delta-producer-bg-jobs-initiator-organizations: image: lblod/delta-producer-background-jobs-initiator:0.3.1 environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" - CRON_PATTERN_DUMP_JOB: "40 * * * *" # At minute 40 every hour + CRON_PATTERN_DUMP_JOB: "42 * * * *" # At minute 42 every hour CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM START_INITIAL_SYNC: "false" # prefer to disable it now. We want to have feeling with how the DB acts. labels: @@ -249,7 +261,7 @@ services: image: lblod/delta-producer-publication-graph-maintainer:0.7.0 environment: MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" RELATIVE_FILE_PATH: "deltas/organizations" PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations" @@ -258,8 +270,6 @@ services: USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive SKIP_MU_AUTH_INITIAL_SYNC: "true" HEALING_PATCH_GRAPH_BATCH_SIZE: 500 - FILES_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-deltas" - ACCOUNT: "http://data.lblod.info/foaf/account/id/8df6f1e4-869f-4223-9834-961b01a7bd48" volumes: - ./config/delta-producer/organizations:/config - ./data/files/:/share @@ -272,7 +282,7 @@ services: image: lblod/delta-producer-json-diff-file-publisher:0.2.3 environment: MAX_BODY_SIZE: "50mb" - ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" RELATIVE_FILE_PATH: "deltas/organizations" PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations" PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations" @@ -296,7 +306,7 @@ services: delta-producer-bg-jobs-initiator-public: image: lblod/delta-producer-background-jobs-initiator:0.3.1 environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" @@ -313,7 +323,7 @@ services: image: lblod/delta-producer-publication-graph-maintainer:0.15.1 environment: MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" RELATIVE_FILE_PATH: "deltas/public" PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public" @@ -334,7 +344,7 @@ services: image: lblod/delta-producer-json-diff-file-publisher:0.2.3 environment: MAX_BODY_SIZE: "50mb" - ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" RELATIVE_FILE_PATH: "deltas/public" PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/public" @@ -358,7 +368,7 @@ services: delta-producer-bg-jobs-initiator-worship-posts: image: lblod/delta-producer-background-jobs-initiator:0.3.1 environment: - JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" @@ -375,7 +385,7 @@ services: image: lblod/delta-producer-publication-graph-maintainer:0.15.1 environment: MAX_BODY_SIZE: "50mb" - JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" RELATIVE_FILE_PATH: "deltas/worship-posts" PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/worship-posts" HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/worship-posts" @@ -388,7 +398,7 @@ services: PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/worship-posts" DELTA_INTERVAL_MS: 10000 PRETTY_PRINT_DIFF_JSON: "true" - ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" volumes: - ./config/delta-producer/worship-posts:/config - ./data/files/:/share @@ -516,10 +526,18 @@ services: links: - db:database + roles: + image: lblod/select-role-service + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging -################################################################################### -# MU SCRIPTS -# ################################################################################# + ################################################################################### + # MU SCRIPTS + # ################################################################################# project-scripts: image: semtech/mu-scripts:1.0.0 volumes: From c842e3a286c3d96d14e5e07d1fe25fc472fc56a6 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 27 Feb 2023 16:00:16 +0100 Subject: [PATCH 12/58] Delete a "veranderingsgebeurtenis" in worship service Tevhid (#242) --- ...15900-delete-duplicate-change-event.sparql | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 config/migrations/2023/20230227115900-delete-duplicate-change-event.sparql diff --git a/config/migrations/2023/20230227115900-delete-duplicate-change-event.sparql b/config/migrations/2023/20230227115900-delete-duplicate-change-event.sparql new file mode 100644 index 0000000..8b6bf15 --- /dev/null +++ b/config/migrations/2023/20230227115900-delete-duplicate-change-event.sparql @@ -0,0 +1,37 @@ +PREFIX rdf: +PREFIX rdfs: +PREFIX skos: +PREFIX org: +PREFIX mu: +PREFIX xsd: +PREFIX dct: +PREFIX skos: +PREFIX concepts: +PREFIX bestuur: +PREFIX org: +PREFIX rdf: +PREFIX contacthub: +PREFIX xsd: +PREFIX ce: +PREFIX cer: +PREFIX m8g: +PREFIX besluit: +PREFIX eli: +PREFIX foaf: + +DELETE WHERE { + graph ?g { + ?changeEvent a ; + ?changeEventResult; + mu:uuid ?uuidChangeEvent; + ?p ?o. + + ?changeEventResult a ; + mu:uuid ?uuidChangeEventResult; + ?cP ?cO. + + filter(?uuidChangeEvent ="63E4B56D1C2DE4F40F38C089") + + } +} + From 160c355d1c83b872af2feedb2d5657a61260234f Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 27 Feb 2023 16:13:22 +0100 Subject: [PATCH 13/58] =?UTF-8?q?Kerkfabriek=20St.-Antonius=20van=20Padua?= =?UTF-8?q?=20van=20Gent=20(Heirnis)=E2=80=9D=20with=20status=20=E2=80=9CN?= =?UTF-8?q?iet=20actief=E2=80=9D=20from=20OP=20(#241)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...0227114600-remove-worship-not-actif.sparql | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 config/migrations-triggering-indexing/2023/20230227114600-remove-worship-not-actif.sparql diff --git a/config/migrations-triggering-indexing/2023/20230227114600-remove-worship-not-actif.sparql b/config/migrations-triggering-indexing/2023/20230227114600-remove-worship-not-actif.sparql new file mode 100644 index 0000000..10d16d7 --- /dev/null +++ b/config/migrations-triggering-indexing/2023/20230227114600-remove-worship-not-actif.sparql @@ -0,0 +1,50 @@ +PREFIX org: +PREFIX adms: +PREFIX generiek: +PREFIX organisatie: + +DELETE { + GRAPH { + ?bestuur ?pbestuur ?obestuur . + ?primarySite ?pprimarySite ?oprimarySite . + ?address ?paddress ?oaddress . + ?contactPoint ?pcontactPoint ?ocontactPoint . + ?identifier ?pidentifier ?oidentifier . + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } +} WHERE { + GRAPH { + BIND( as ?bestuur) + + ?bestuur ?pbestuur ?obestuur . + + OPTIONAL { + + ?bestuur org:hasPrimarySite ?primarySite . + + OPTIONAL { + ?primarySite organisatie:bestaatUit ?address . + ?address ?paddress ?oaddress . + } + OPTIONAL { + ?primarySite org:siteAddress ?contactPoint . + ?contactPoint ?pcontactPoint ?ocontactPoint . + } + + ?primarySite ?pprimarySite ?oprimarySite . + + } + OPTIONAL { + + ?bestuur adms:identifier ?identifier . + + OPTIONAL { + ?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier . + ?structuredIdentifier ?pstructuredIdentifier ?ostructuredIdentifier . + } + + ?identifier ?pidentifier ?oidentifier . + + } + } +} From b5fbf09c086cb500d3db6c2cca02a2a5bc990ba9 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Tue, 28 Feb 2023 15:07:27 +0000 Subject: [PATCH 14/58] extend public producer --- config/delta-producer/public/export.json | 90 ++++++++++++++++++- ...lish-all-bestuurseenheden-in-public.sparql | 20 +++++ 2 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 config/migrations/2023/20230228155500-publish-all-bestuurseenheden-in-public.sparql diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index 49d2931..04ffd0a 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -25,7 +25,6 @@ "http://www.w3.org/ns/org#hasSubOrganization", "http://www.w3.org/ns/org#classification", "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", - "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau", "http://data.lblod.info/vocabularies/erediensten/denominatie", "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", @@ -56,10 +55,42 @@ "http://www.w3.org/ns/org#hasSubOrganization", "http://www.w3.org/ns/org#classification", "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", - "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau", "http://data.lblod.info/vocabularies/erediensten/typeEredienst" ] }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", + "http://data.lblod.info/vocabularies/erediensten/denominatie", + "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://data.vlaanderen.be/ns/besluit#werkingsgebied", + "http://www.opengis.net/ont/geosparql#sfWithin", + "http://www.w3.org/2002/07/owl#sameAs", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/regorg#orgStatus" + ] + }, { "type": "http://www.w3.org/ns/org#Site", "graphsFilter": [ @@ -182,7 +213,28 @@ "http://data.vlaanderen.be/ns/mandaat#bindingStart", "http://data.vlaanderen.be/ns/besluit#bestuurt", "http://www.w3.org/ns/org#classification", - "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan" + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://www.w3.org/ns/org#hasPost" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://data.vlaanderen.be/ns/mandaat#bindingEinde", + "http://data.vlaanderen.be/ns/mandaat#bindingStart", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#classification", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://www.w3.org/ns/org#hasPost" ] }, { @@ -265,6 +317,38 @@ "properties": [ "http://www.w3.org/2004/02/skos/core#prefLabel" ] + }, + { + "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", + "graphsFilter": [ + "^http://www.w3.org/ns/org#hasPost", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#role" + ], + "properties": [ + "http://www.w3.org/ns/org#role" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie", + "graphsFilter": [ + "^http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#role" + ], + "properties": [ + "http://www.w3.org/ns/org#role", + "http://www.w3.org/2004/02/skos/core#prefLabel" + + ] } ] } diff --git a/config/migrations/2023/20230228155500-publish-all-bestuurseenheden-in-public.sparql b/config/migrations/2023/20230228155500-publish-all-bestuurseenheden-in-public.sparql new file mode 100644 index 0000000..4fe6e36 --- /dev/null +++ b/config/migrations/2023/20230228155500-publish-all-bestuurseenheden-in-public.sparql @@ -0,0 +1,20 @@ +INSERT DATA { + GRAPH { + #Centraal bestuur van de eredienst" + #Gemeente" + #Provincie" + #Bestuur van de eredienst" + #Autonoom gemeentebedrijf" + #OCMW vereniging" + #Hulpverleningszone" + #Politiezone" + #Projectvereniging" + #Opdrachthoudende vereniging" + #Dienstverlenende vereniging" + #District" + #OCMW" + #Autonoom provinciebedrijf" + } +} + + From f7f654df2f70967a32c278e6de5fb0ee5dd28f21 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Tue, 28 Feb 2023 17:19:08 +0100 Subject: [PATCH 15/58] Feat/remove beheerder (#243) --- ...105324-move-jobs-to-specific-graph.sparql} | 0 ...0230228132500-delete-elastic-index.sparql} | 0 ...30228132600-move-all-worship-types.sparql} | 0 ...1-move-all-worsip-governing-bodies.sparql} | 0 ...move-all-worsip-governing-bodies-2.sparql} | 0 ...move-all-worsip-governing-bodies-3.sparql} | 0 .../20230228132604-move-bestrokken.sparql} | 0 ...2605-move-all-worsip-change-events.sparql} | 0 .../20230228132606-duplicate-person.sparql} | 0 .../20230228132607-move-shared-types.sparql} | 0 .../20230228132608-move-public-types.sparql} | 0 ...132609-move-to-administrative-unit.sparql} | 0 .../20230228132610-add-mock-user-1.sparql} | 0 .../20230228132611-add-mock-user-2.sparql} | 0 .../20230228132612-add-mock-user-3.sparql} | 0 .../20230228132613-add-mock-user-4.sparql} | 0 .../20230228132614-move-privacy-data.sparql} | 0 .../20230228142612-add-mock-user-5.sparql} | 0 .../20230228142712-add-mock-user-6.sparql} | 0 .../20230228152117-remove-beheerder.sparql | 58 +++++++++++++++++++ config/search/config.json | 28 --------- config/search/dev/config.json | 4 +- 22 files changed, 60 insertions(+), 30 deletions(-) rename config/migrations/2023/{20230401132400-hard-split/20230402105324-move-jobs-to-specific-graph.sparql => 20230228112800-hard-split/20230228105324-move-jobs-to-specific-graph.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132500-delete-elastic-index.sparql => 20230228112800-hard-split/20230228132500-delete-elastic-index.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132600-move-all-worship-types.sparql => 20230228112800-hard-split/20230228132600-move-all-worship-types.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132601-move-all-worsip-governing-bodies.sparql => 20230228112800-hard-split/20230228132601-move-all-worsip-governing-bodies.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132602-move-all-worsip-governing-bodies-2.sparql => 20230228112800-hard-split/20230228132602-move-all-worsip-governing-bodies-2.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132603-move-all-worsip-governing-bodies-3.sparql => 20230228112800-hard-split/20230228132603-move-all-worsip-governing-bodies-3.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132604-move-bestrokken.sparql => 20230228112800-hard-split/20230228132604-move-bestrokken.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132605-move-all-worsip-change-events.sparql => 20230228112800-hard-split/20230228132605-move-all-worsip-change-events.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132606-duplicate-person.sparql => 20230228112800-hard-split/20230228132606-duplicate-person.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132607-move-shared-types.sparql => 20230228112800-hard-split/20230228132607-move-shared-types.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132608-move-public-types.sparql => 20230228112800-hard-split/20230228132608-move-public-types.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132609-move-to-administrative-unit.sparql => 20230228112800-hard-split/20230228132609-move-to-administrative-unit.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132610-add-mock-user-1.sparql => 20230228112800-hard-split/20230228132610-add-mock-user-1.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132611-add-mock-user-2.sparql => 20230228112800-hard-split/20230228132611-add-mock-user-2.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132612-add-mock-user-3.sparql => 20230228112800-hard-split/20230228132612-add-mock-user-3.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132613-add-mock-user-4.sparql => 20230228112800-hard-split/20230228132613-add-mock-user-4.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230401132614-move-privacy-data.sparql => 20230228112800-hard-split/20230228132614-move-privacy-data.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230403132612-add-mock-user-5.sparql => 20230228112800-hard-split/20230228142612-add-mock-user-5.sparql} (100%) rename config/migrations/2023/{20230401132400-hard-split/20230403132613-add-mock-user-6.sparql => 20230228112800-hard-split/20230228142712-add-mock-user-6.sparql} (100%) create mode 100644 config/migrations/2023/20230228152117-remove-beheerder.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230402105324-move-jobs-to-specific-graph.sparql b/config/migrations/2023/20230228112800-hard-split/20230228105324-move-jobs-to-specific-graph.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230402105324-move-jobs-to-specific-graph.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228105324-move-jobs-to-specific-graph.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132500-delete-elastic-index.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132500-delete-elastic-index.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132500-delete-elastic-index.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132500-delete-elastic-index.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132600-move-all-worship-types.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132600-move-all-worship-types.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132600-move-all-worship-types.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132600-move-all-worship-types.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132601-move-all-worsip-governing-bodies.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132601-move-all-worsip-governing-bodies.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132601-move-all-worsip-governing-bodies.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132601-move-all-worsip-governing-bodies.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132602-move-all-worsip-governing-bodies-2.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132602-move-all-worsip-governing-bodies-2.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132602-move-all-worsip-governing-bodies-2.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132602-move-all-worsip-governing-bodies-2.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132603-move-all-worsip-governing-bodies-3.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132603-move-all-worsip-governing-bodies-3.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132603-move-all-worsip-governing-bodies-3.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132603-move-all-worsip-governing-bodies-3.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132604-move-bestrokken.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132604-move-bestrokken.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132604-move-bestrokken.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132604-move-bestrokken.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132605-move-all-worsip-change-events.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132605-move-all-worsip-change-events.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132605-move-all-worsip-change-events.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132605-move-all-worsip-change-events.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132606-duplicate-person.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132606-duplicate-person.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132606-duplicate-person.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132606-duplicate-person.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132607-move-shared-types.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132607-move-shared-types.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132607-move-shared-types.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132607-move-shared-types.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132608-move-public-types.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132608-move-public-types.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132608-move-public-types.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132608-move-public-types.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132609-move-to-administrative-unit.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132609-move-to-administrative-unit.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132609-move-to-administrative-unit.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132609-move-to-administrative-unit.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132610-add-mock-user-1.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132610-add-mock-user-1.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132610-add-mock-user-1.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132610-add-mock-user-1.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132611-add-mock-user-2.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132611-add-mock-user-2.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132611-add-mock-user-2.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132611-add-mock-user-2.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132612-add-mock-user-3.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132612-add-mock-user-3.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132612-add-mock-user-3.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132612-add-mock-user-3.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132613-add-mock-user-4.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132613-add-mock-user-4.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132613-add-mock-user-4.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132613-add-mock-user-4.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230401132614-move-privacy-data.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132614-move-privacy-data.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230401132614-move-privacy-data.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228132614-move-privacy-data.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230403132612-add-mock-user-5.sparql b/config/migrations/2023/20230228112800-hard-split/20230228142612-add-mock-user-5.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230403132612-add-mock-user-5.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228142612-add-mock-user-5.sparql diff --git a/config/migrations/2023/20230401132400-hard-split/20230403132613-add-mock-user-6.sparql b/config/migrations/2023/20230228112800-hard-split/20230228142712-add-mock-user-6.sparql similarity index 100% rename from config/migrations/2023/20230401132400-hard-split/20230403132613-add-mock-user-6.sparql rename to config/migrations/2023/20230228112800-hard-split/20230228142712-add-mock-user-6.sparql diff --git a/config/migrations/2023/20230228152117-remove-beheerder.sparql b/config/migrations/2023/20230228152117-remove-beheerder.sparql new file mode 100644 index 0000000..40f987c --- /dev/null +++ b/config/migrations/2023/20230228152117-remove-beheerder.sparql @@ -0,0 +1,58 @@ +PREFIX ext: +DELETE where { + graph { + ?p ?o. + ?aP ?aO. + ?wp ?wo. + ?wpap ?wpao. + } +} +; +DELETE where { + graph { + ?p ?o. + ?aP ?aO. + ?wp ?wo. + ?wpap ?wpao. + } +} +; + +DELETE WHERE { + graph { + ext:sessionRole "ABBOrganisatiePortaalErediensten-beheerder", + "ABBOrganisatiePortaalGebruiker-beheerder" . + + } +} + +; + +; + +DELETE WHERE { + graph { + ext:sessionRole "ABBOrganisatiePortaalErediensten-beheerder", + "ABBOrganisatiePortaalGebruiker-beheerder" . + + + } +} + + +INSERT DATA { +GRAPH { + ext:sessionRole "ABBOrganisatiePortaalErediensten-editeerder", + "ABBOrganisatiePortaalGebruiker-editeerder" . + +} +} +; + +INSERT DATA { +GRAPH { + ext:sessionRole "ABBOrganisatiePortaalErediensten-editeerder", + "ABBOrganisatiePortaalGebruiker-editeerder" . + +} +} diff --git a/config/search/config.json b/config/search/config.json index e62d951..4b95f52 100644 --- a/config/search/config.json +++ b/config/search/config.json @@ -31,34 +31,6 @@ "name": "public" } ], - [ - { - "variables": [], - "name": "acmidm-beheerder" - }, - { - "variables": [], - "name": "clean" - }, - { - "variables": [], - "name": "public" - } - ], - [ - { - "variables": [], - "name": "acmidm-worship-beheerder" - }, - { - "variables": [], - "name": "clean" - }, - { - "variables": [], - "name": "public" - } - ], [ { "variables": [], diff --git a/config/search/dev/config.json b/config/search/dev/config.json index 6dd6b9e..22b117e 100644 --- a/config/search/dev/config.json +++ b/config/search/dev/config.json @@ -6,7 +6,7 @@ [ { "variables": [], - "name": "acmidm-worship-beheerder" + "name": "acmidm-worship-editeerder" }, { "variables": [], @@ -20,7 +20,7 @@ [ { "variables": [], - "name": "acmidm-beheerder" + "name": "acmidm-editeerder" }, { "variables": [], From c5dc809d50edf9ed737589b310514a183b66c627 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Wed, 1 Mar 2023 10:45:42 +0100 Subject: [PATCH 16/58] cache not triggered for related organization (#245) --- config/dispatcher/dispatcher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index e72e468..885120a 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -77,7 +77,7 @@ defmodule Dispatcher do end match "/administrative-units/*path", %{ accept: [:json], layer: :api} do - Proxy.forward conn, path, "http://cache/administrative-units/" + Proxy.forward conn, path, "http://resource/administrative-units/" end match "/administrative-unit-classification-codes/*path", %{ accept: [:json], layer: :api} do From 5274f2e66cbb39197741c6d2313514f75cc953ef Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Thu, 2 Mar 2023 09:40:48 +0100 Subject: [PATCH 17/58] bump construct unit service --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2e4f940..52f0904 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -217,7 +217,7 @@ services: volumes: - ./config/jobs-controller/:/config/ construct-administrative-unit-relationships: - image: lblod/construct-administrative-unit-relationships-service:0.1.0 + image: lblod/construct-administrative-unit-relationships-service:0.1.1 links: - db:database restart: always From 50a3ff2929b46164dbcb4514d494d5c86e166dd6 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Thu, 2 Mar 2023 10:17:41 +0000 Subject: [PATCH 18/58] publish delta files --- config/dispatcher/dispatcher.ex | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 1a8eb1b..a426f37 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -335,6 +335,14 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-organizations/files/" end + ################################################################# + # DELTA: public + ################################################################# + + get "/sync/public/files/*path" do + Proxy.forward conn, path, "http://delta-producer-json-diff-publisher-public/files/" + end + ################################################################# # DELTA: posts ################################################################# From 8d1e434e5d98e73e1f31245d5c747377bb9ef927 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 3 Mar 2023 13:23:43 +0100 Subject: [PATCH 19/58] Fixing producer not dumping betrokkenBestuur A Bestuurseenheid can now have triples in multiple graphs in OP : worship graph, admin unit graph, shared graph. But when there is an additional filter on top, it expects all the triples to be in the same graph, causing triples in the worship graph not to be exported. Moving them to a separate block allows them to be safely exported :) --- config/delta-producer/worship-posts/export.json | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index ef2656a..384e44c 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -53,8 +53,7 @@ "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "graphsFilter": [ "http://mu.semte.ch/graphs/shared", - "http://mu.semte.ch/graphs/administrative-unit", - "http://mu.semte.ch/graphs/worship-service" + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -64,11 +63,19 @@ "http://data.vlaanderen.be/ns/besluit#werkingsgebied", "http://www.w3.org/ns/org#classification", "http://www.w3.org/ns/org#hasPrimarySite", - "http://www.w3.org/ns/org#hasSite", - "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur" + "http://www.w3.org/ns/org#hasSite" ], "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur" + ] + }, { "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "graphsFilter": [ From b38e2ce8167dccfdb9beb01ffd29b6fc8890d4d2 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 6 Mar 2023 09:08:27 +0100 Subject: [PATCH 20/58] disable cache because of a bug with local involvements (#246) --- config/dispatcher/dispatcher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index 885120a..cb7773e 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -85,7 +85,7 @@ defmodule Dispatcher do end match "/worship-administrative-units/*path", %{ accept: [:json], layer: :api} do - Proxy.forward conn, path, "http://cache/worship-administrative-units/" + Proxy.forward conn, path, "http://resource/worship-administrative-units/" end match "/worship-services/*path", %{ accept: [:json], layer: :api} do From 2ae1c9228ea9a9c15d1def2308620812b8c1832f Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 6 Mar 2023 12:39:46 +0100 Subject: [PATCH 21/58] Add missing appliesWithin link for worship services --- ...0306122751-add-missing-applies-within-link.sparql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 config/migrations/2023/20230306122751-add-missing-applies-within-link.sparql diff --git a/config/migrations/2023/20230306122751-add-missing-applies-within-link.sparql b/config/migrations/2023/20230306122751-add-missing-applies-within-link.sparql new file mode 100644 index 0000000..4f1478a --- /dev/null +++ b/config/migrations/2023/20230306122751-add-missing-applies-within-link.sparql @@ -0,0 +1,12 @@ +PREFIX mu: +PREFIX ext: + +INSERT DATA { + GRAPH { + ext:appliesWithin . # Centraal bestuur + ext:appliesWithin . # Comité + ext:appliesWithin . # Bestuursraad + ext:appliesWithin . # Kerkfabriekraad + ext:appliesWithin . # Kathedrale kerkraad + } +} From eff38a71de9ab698a0a0e373fd9bcb1987eaba19 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:22:31 +0100 Subject: [PATCH 22/58] fix orgaan wrong graph (#249) --- ...230306142222-move-orgaan-to-worship.sparql | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 config/migrations/2023/20230306142222-move-orgaan-to-worship.sparql diff --git a/config/migrations/2023/20230306142222-move-orgaan-to-worship.sparql b/config/migrations/2023/20230306142222-move-orgaan-to-worship.sparql new file mode 100644 index 0000000..7bef175 --- /dev/null +++ b/config/migrations/2023/20230306142222-move-orgaan-to-worship.sparql @@ -0,0 +1,107 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: +PREFIX ere: +PREFIX mandaat: +PREFIX org: +PREFIX generiek: + +DELETE { + GRAPH { + ?s ?p ?o. + } +} +INSERT { + GRAPH { + ?s ?p ?o. + } +} + +WHERE { + GRAPH { + ?s a ; + ?classification; + ?p ?o. + +filter exists { + graph { + ?classification ?appliesWithin. + ?appliesWithin mu:uuid ?classificationUuid. + + filter(?classificationUuid in ("66ec74fd-8cfc-4e16-99c6-350b35012e86","f9cac08a-13c1-49da-9bcb-f650b0604054")) + } +} + + + + +} + +} +; + +DELETE { + GRAPH { + ?specialization ?specializationP ?specializationO. + } +} +INSERT { + GRAPH { + ?specialization ?specializationP ?specializationO. + } +} + +WHERE { + GRAPH { + ?specialization a ; generiek:isTijdspecialisatieVan ?specialization2; ?specializationP ?specializationO. + +filter exists{ + +graph { + + + ?specialization2 a . +} +} +} + + +} +; + + + + +; + + + +DELETE { + GRAPH { + ?mandaat ?p ?o. + } +} +INSERT { + GRAPH { + ?mandaat ?p ?o. + } +} + +WHERE { + GRAPH { + ?mandaat ?p ?o. + filter exists { + GRAPH { + ?orgaan ?mandaat. + + } + } +} + + +} From 8c52365b8119b77559a5957d4e488ecd20c03169 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 7 Mar 2023 09:38:57 +0100 Subject: [PATCH 23/58] Remove config of previously deleted producer --- config/delta/rules.js | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/config/delta/rules.js b/config/delta/rules.js index 6d9ffc6..b502977 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -88,41 +88,6 @@ export default [ optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] } }, - { - match: { - }, - callback: { - url: 'http://delta-producer-pub-graph-maintainer-administrative-units/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: [ - "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", - "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" - ] - } - }, - { - match: { - graph: { - type: 'uri', - value: 'http://redpencil.data.gift/id/deltas/producer/administrative-units' - } - }, - callback: { - url: 'http://delta-producer-json-diff-publisher-administrative-units/delta', - method: 'POST' - }, - options: { - resourceFormat: 'v0.0.1', - gracePeriod: 1000, - ignoreFromSelf: true, - optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"] - } - }, { match: { }, From 0c9024b7c6d51dd3e8d13251960eba5c954fbff6 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Tue, 7 Mar 2023 10:58:05 +0100 Subject: [PATCH 24/58] Incident/move change event (#251) --- ...0227114600-remove-worship-not-actif.sparql | 0 ...fix-wrong-resulting-org-after-merge.sparql | 47 ++++++++++++++ .../20230307100012-change-event-move.json | 64 +++++++++++++++++++ 3 files changed, 111 insertions(+) rename config/{migrations-triggering-indexing => migrations}/2023/20230227114600-remove-worship-not-actif.sparql (100%) create mode 100644 config/migrations/2023/20230307091456-fix-wrong-resulting-org-after-merge.sparql create mode 100644 config/search-migrations/2023/20230307100012-change-event-move.json diff --git a/config/migrations-triggering-indexing/2023/20230227114600-remove-worship-not-actif.sparql b/config/migrations/2023/20230227114600-remove-worship-not-actif.sparql similarity index 100% rename from config/migrations-triggering-indexing/2023/20230227114600-remove-worship-not-actif.sparql rename to config/migrations/2023/20230227114600-remove-worship-not-actif.sparql diff --git a/config/migrations/2023/20230307091456-fix-wrong-resulting-org-after-merge.sparql b/config/migrations/2023/20230307091456-fix-wrong-resulting-org-after-merge.sparql new file mode 100644 index 0000000..dfb815b --- /dev/null +++ b/config/migrations/2023/20230307091456-fix-wrong-resulting-org-after-merge.sparql @@ -0,0 +1,47 @@ +# Update status of the bestuurseenheden as well as the resulting status of the change event + +PREFIX regorg: +PREFIX lblodOrg: + +DELETE { + GRAPH ?g { + + regorg:orgStatus . # Actief + + lblodOrg:resulterendeStatus . # Actief + + + regorg:orgStatus . # Niet Actief + + lblodOrg:resulterendeStatus . # Niet Actief + . + } +} INSERT { + GRAPH ?g { + + regorg:orgStatus . # Niet Actief + + lblodOrg:resulterendeStatus . # Niet Actief + + + regorg:orgStatus . # Actief + + lblodOrg:resulterendeStatus . # Actief + . + + } +} WHERE { + GRAPH ?g { + + regorg:orgStatus . # Actief + + lblodOrg:resulterendeStatus . # Actief + + + regorg:orgStatus . # Niet Actief + + lblodOrg:resulterendeStatus . # Niet Actief + . + + } +} diff --git a/config/search-migrations/2023/20230307100012-change-event-move.json b/config/search-migrations/2023/20230307100012-change-event-move.json new file mode 100644 index 0000000..fb10102 --- /dev/null +++ b/config/search-migrations/2023/20230307100012-change-event-move.json @@ -0,0 +1,64 @@ +[ + { + "deletes": [ + { + "subject": { + "type": "uri", + "value": "http://data.lblod.info/id/besturenVanDeEredienst/e158c6378d3751b88df4a8c0c2b2f99a" + }, + "predicate": { + "type": "uri", + "value": "http://www.w3.org/ns/regorg#orgStatus" + }, + "object": { + "type": "uri", + "value": "http://lblod.data.gift/concepts/63cc561de9188d64ba5840a42ae8f0d6" + } + }, + { + "subject": { + "type": "uri", + "value": "http://data.lblod.info/id/besturenVanDeEredienst/8a90a8de8b1f6bb0a6ab29cf81428754" + }, + "predicate": { + "type": "uri", + "value": "http://www.w3.org/ns/regorg#orgStatus" + }, + "object": { + "type": "uri", + "value": "http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311" + } + } + ], + "inserts": [ + { + "subject": { + "type": "uri", + "value": "http://data.lblod.info/id/besturenVanDeEredienst/e158c6378d3751b88df4a8c0c2b2f99a" + }, + "predicate": { + "type": "uri", + "value": "http://www.w3.org/ns/regorg#orgStatus" + }, + "object": { + "type": "uri", + "value": "http://lblod.data.gift/concepts/d02c4e12bf88d2fdf5123b07f29c9311" + } + }, + { + "subject": { + "type": "uri", + "value": "http://data.lblod.info/id/besturenVanDeEredienst/8a90a8de8b1f6bb0a6ab29cf81428754" + }, + "predicate": { + "type": "uri", + "value": "http://www.w3.org/ns/regorg#orgStatus" + }, + "object": { + "type": "uri", + "value": "http://lblod.data.gift/concepts/63cc561de9188d64ba5840a42ae8f0d6" + } + } + ] + } +] From fc443fa19be0465d77cc1830ca9d6f4fa1513fba Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Tue, 7 Mar 2023 13:28:56 +0100 Subject: [PATCH 25/58] move kbo to worships (#252) --- .../20230307125956-move-identifiers.sparql | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 config/migrations/2023/20230307125956-move-identifiers.sparql diff --git a/config/migrations/2023/20230307125956-move-identifiers.sparql b/config/migrations/2023/20230307125956-move-identifiers.sparql new file mode 100644 index 0000000..120c565 --- /dev/null +++ b/config/migrations/2023/20230307125956-move-identifiers.sparql @@ -0,0 +1,37 @@ +PREFIX regorg: +PREFIX lblodOrg: +PREFIX adms: +PREFIX mu: +PREFIX generiek: + +DELETE { + GRAPH { + + ?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier; + ?p ?o. + ?structuredIdentifier ?sp ?so. + + } +} INSERT { + GRAPH { + + ?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier; + ?p ?o. + ?structuredIdentifier ?sp ?so. + + } +} WHERE { + GRAPH { + + ?identifier generiek:gestructureerdeIdentificator ?structuredIdentifier; + ?p ?o. + ?structuredIdentifier ?sp ?so. + filter exists { + GRAPH { + ?org adms:identifier ?identifier. + + } + } + + } +} From 046ff4f9fb639c3ab02ff4ced874835c2d792954 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 7 Mar 2023 16:21:23 +0100 Subject: [PATCH 26/58] Fix posts producer --- .../delta-producer/worship-posts/export.json | 66 +++++++++++++++---- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index 384e44c..a43e54b 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -10,7 +10,7 @@ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/ns/org#role" ], - "additionalFilter": "?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" + "additionalFilter": "} GRAPH ?h { ?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" }, { "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", @@ -24,7 +24,6 @@ "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/regorg#orgStatus", "http://data.lblod.info/vocabularies/erediensten/typeEredienst", - "http://data.vlaanderen.be/ns/besluit#werkingsgebied", "http://www.w3.org/ns/org#classification", "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", "http://www.w3.org/ns/org#hasPrimarySite", @@ -43,7 +42,6 @@ "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/regorg#orgStatus", "http://data.lblod.info/vocabularies/erediensten/typeEredienst", - "http://data.vlaanderen.be/ns/besluit#werkingsgebied", "http://www.w3.org/ns/org#classification", "http://www.w3.org/ns/org#hasPrimarySite", "http://www.w3.org/ns/org#hasSite" @@ -59,13 +57,10 @@ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", "http://www.w3.org/2004/02/skos/core#altLabel", - "http://www.w3.org/ns/regorg#orgStatus", - "http://data.vlaanderen.be/ns/besluit#werkingsgebied", "http://www.w3.org/ns/org#classification", - "http://www.w3.org/ns/org#hasPrimarySite", - "http://www.w3.org/ns/org#hasSite" + "http://www.w3.org/ns/adms#identifier" ], - "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" + "additionalFilter": "} GRAPH ?h { ?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" }, { "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", @@ -92,7 +87,7 @@ "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", "http://www.w3.org/ns/org#hasPost" ], - "additionalFilter": "?subject (//)|(/) ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" + "additionalFilter": "} GRAPH ?h { ?subject (//)|(/) ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" }, { "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", @@ -132,9 +127,58 @@ "http://www.w3.org/ns/org#hasPrimarySite", "http://www.w3.org/ns/org#hasSite", "http://www.w3.org/ns/org#linkedTo", - "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/ns/adms#identifier" ] }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/administrative-unit", + "http://mu.semte.ch/graphs/shared" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ], + "additionalFilter": "} GRAPH ?h { ?bestuur ?subject ; ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/administrative-unit", + "http://mu.semte.ch/graphs/shared" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ], + "additionalFilter": "} GRAPH ?h { ?bestuur ?id ; ?bestuurClassification . FILTER (?bestuurClassification IN (, )) . ?id ?subject ." + }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ], + "additionalFilter": "} GRAPH ?h { ?bestuur ?subject . ?bestuur a ." + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ], + "additionalFilter": "} GRAPH ?h { ?bestuur ?id . ?bestuur a . ?id ?subject ." + }, { "type": "http://www.w3.org/ns/org#Site", "graphsFilter": [ @@ -163,7 +207,7 @@ "http://www.w3.org/ns/locn#adminUnitL2", "http://www.w3.org/ns/locn#fullAddress" ], - "additionalFilter": "?subject ^ ?site. ?site a ." + "additionalFilter": "} GRAPH ?h { ?subject ^ ?site. ?site a ." }, { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", From 8f311f7b9890db75db5f88f0ff74018dd1c55a56 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 8 Mar 2023 09:42:49 +0100 Subject: [PATCH 27/58] Bump posts producer --- config/delta-producer/worship-posts/export.json | 16 ++++++++-------- docker-compose.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index a43e54b..50b8ad4 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -10,7 +10,7 @@ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/ns/org#role" ], - "additionalFilter": "} GRAPH ?h { ?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" + "additionalFilter": "?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" }, { "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", @@ -60,7 +60,7 @@ "http://www.w3.org/ns/org#classification", "http://www.w3.org/ns/adms#identifier" ], - "additionalFilter": "} GRAPH ?h { ?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" + "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" }, { "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", @@ -87,7 +87,7 @@ "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", "http://www.w3.org/ns/org#hasPost" ], - "additionalFilter": "} GRAPH ?h { ?subject (//)|(/) ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" + "additionalFilter": "?subject (//)|(/) ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" }, { "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", @@ -142,7 +142,7 @@ "http://www.w3.org/2004/02/skos/core#notation", "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" ], - "additionalFilter": "} GRAPH ?h { ?bestuur ?subject ; ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" + "additionalFilter": "?bestuur ?subject ; ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" }, { "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", @@ -154,7 +154,7 @@ "http://mu.semte.ch/vocabularies/core/uuid", "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" ], - "additionalFilter": "} GRAPH ?h { ?bestuur ?id ; ?bestuurClassification . FILTER (?bestuurClassification IN (, )) . ?id ?subject ." + "additionalFilter": "?bestuur ?id ; ?bestuurClassification . FILTER (?bestuurClassification IN (, )) . ?id ?subject ." }, { "type": "http://www.w3.org/ns/adms#Identifier", @@ -166,7 +166,7 @@ "http://www.w3.org/2004/02/skos/core#notation", "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" ], - "additionalFilter": "} GRAPH ?h { ?bestuur ?subject . ?bestuur a ." + "additionalFilter": "?bestuur ?subject . ?bestuur a ." }, { "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", @@ -177,7 +177,7 @@ "http://mu.semte.ch/vocabularies/core/uuid", "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" ], - "additionalFilter": "} GRAPH ?h { ?bestuur ?id . ?bestuur a . ?id ?subject ." + "additionalFilter": "?bestuur ?id . ?bestuur a . ?id ?subject ." }, { "type": "http://www.w3.org/ns/org#Site", @@ -207,7 +207,7 @@ "http://www.w3.org/ns/locn#adminUnitL2", "http://www.w3.org/ns/locn#fullAddress" ], - "additionalFilter": "} GRAPH ?h { ?subject ^ ?site. ?site a ." + "additionalFilter": "?subject ^ ?site. ?site a ." }, { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", diff --git a/docker-compose.yml b/docker-compose.yml index 52f0904..a9d2eb8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -382,7 +382,7 @@ services: restart: always logging: *default-logging delta-producer-pub-graph-maintainer-worship-posts: - image: lblod/delta-producer-publication-graph-maintainer:0.15.1 + image: lblod/delta-producer-publication-graph-maintainer:0.15.3 environment: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" From a61d6bed356159fb6f233594960ca609c9def2a3 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Wed, 8 Mar 2023 14:10:43 +0000 Subject: [PATCH 28/58] fix query adding all bestuurseenheden --- ...ish-all-bestuurseenheden-in-public.sparql} | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) rename config/migrations/2023/{20230228155500-publish-all-bestuurseenheden-in-public.sparql => 20230228155501-publish-all-bestuurseenheden-in-public.sparql} (75%) diff --git a/config/migrations/2023/20230228155500-publish-all-bestuurseenheden-in-public.sparql b/config/migrations/2023/20230228155501-publish-all-bestuurseenheden-in-public.sparql similarity index 75% rename from config/migrations/2023/20230228155500-publish-all-bestuurseenheden-in-public.sparql rename to config/migrations/2023/20230228155501-publish-all-bestuurseenheden-in-public.sparql index 4fe6e36..6e9dc1c 100644 --- a/config/migrations/2023/20230228155500-publish-all-bestuurseenheden-in-public.sparql +++ b/config/migrations/2023/20230228155501-publish-all-bestuurseenheden-in-public.sparql @@ -1,19 +1,19 @@ INSERT DATA { GRAPH { - #Centraal bestuur van de eredienst" - #Gemeente" - #Provincie" - #Bestuur van de eredienst" - #Autonoom gemeentebedrijf" - #OCMW vereniging" - #Hulpverleningszone" - #Politiezone" - #Projectvereniging" - #Opdrachthoudende vereniging" - #Dienstverlenende vereniging" - #District" - #OCMW" - #Autonoom provinciebedrijf" + . #Centraal bestuur van de eredienst" + . #Gemeente" + . #Provincie" + . #Bestuur van de eredienst" + . #Autonoom gemeentebedrijf" + . #OCMW vereniging" + . #Hulpverleningszone" + . #Politiezone" + . #Projectvereniging" + . #Opdrachthoudende vereniging" + . #Dienstverlenende vereniging" + . #District" + . #OCMW" + . #Autonoom provinciebedrijf" } } From 928106c53c37d9e2044a1701b84f192614082375 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 8 Mar 2023 17:40:51 +0100 Subject: [PATCH 29/58] Fix posts producer --- .../delta-producer/worship-posts/export.json | 90 +++++++------------ 1 file changed, 34 insertions(+), 56 deletions(-) diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json index 50b8ad4..0f540bd 100644 --- a/config/delta-producer/worship-posts/export.json +++ b/config/delta-producer/worship-posts/export.json @@ -12,10 +12,31 @@ ], "additionalFilter": "?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))" }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/regorg#orgStatus", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/adms#identifier", + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur" + ], + "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))" + }, { "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "graphsFilter": [ - "http://mu.semte.ch/graphs/shared", "http://mu.semte.ch/graphs/worship-service" ], "properties": [ @@ -27,13 +48,14 @@ "http://www.w3.org/ns/org#classification", "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", "http://www.w3.org/ns/org#hasPrimarySite", - "http://www.w3.org/ns/org#hasSite" + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/adms#identifier" ] }, { "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/shared", "http://mu.semte.ch/graphs/worship-service" ], "properties": [ @@ -44,31 +66,8 @@ "http://data.lblod.info/vocabularies/erediensten/typeEredienst", "http://www.w3.org/ns/org#classification", "http://www.w3.org/ns/org#hasPrimarySite", - "http://www.w3.org/ns/org#hasSite" - ] - }, - { - "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", - "graphsFilter": [ - "http://mu.semte.ch/graphs/shared", - "http://mu.semte.ch/graphs/administrative-unit" - ], - "properties": [ - "http://mu.semte.ch/vocabularies/core/uuid", - "http://www.w3.org/2004/02/skos/core#prefLabel", - "http://www.w3.org/2004/02/skos/core#altLabel", - "http://www.w3.org/ns/org#classification", + "http://www.w3.org/ns/org#hasSite", "http://www.w3.org/ns/adms#identifier" - ], - "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" - }, - { - "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", - "graphsFilter": [ - "http://mu.semte.ch/graphs/worship-service" - ], - "properties": [ - "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur" ] }, { @@ -134,50 +133,29 @@ { "type": "http://www.w3.org/ns/adms#Identifier", "graphsFilter": [ - "http://mu.semte.ch/graphs/administrative-unit", - "http://mu.semte.ch/graphs/shared" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#notation", "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" ], - "additionalFilter": "?bestuur ?subject ; ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" + "additionalFilter": "?subject \"KBO nummer\" ." }, { "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", "graphsFilter": [ - "http://mu.semte.ch/graphs/administrative-unit", - "http://mu.semte.ch/graphs/shared" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" ], - "additionalFilter": "?bestuur ?id ; ?bestuurClassification . FILTER (?bestuurClassification IN (, )) . ?id ?subject ." - }, - { - "type": "http://www.w3.org/ns/adms#Identifier", - "graphsFilter": [ - "http://mu.semte.ch/graphs/worship-service" - ], - "properties": [ - "http://mu.semte.ch/vocabularies/core/uuid", - "http://www.w3.org/2004/02/skos/core#notation", - "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" - ], - "additionalFilter": "?bestuur ?subject . ?bestuur a ." - }, - { - "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", - "graphsFilter": [ - "http://mu.semte.ch/graphs/worship-service" - ], - "properties": [ - "http://mu.semte.ch/vocabularies/core/uuid", - "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" - ], - "additionalFilter": "?bestuur ?id . ?bestuur a . ?id ?subject ." + "additionalFilter": "?subject ^/ \"KBO nummer\" ." }, { "type": "http://www.w3.org/ns/org#Site", From 2a518169c77efe04c118bea475546ab8e8b93dc2 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 8 Mar 2023 18:02:27 +0100 Subject: [PATCH 30/58] Bump posts producer --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a9d2eb8..9e5760d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -382,7 +382,7 @@ services: restart: always logging: *default-logging delta-producer-pub-graph-maintainer-worship-posts: - image: lblod/delta-producer-publication-graph-maintainer:0.15.3 + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 environment: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" From 38e2c76fd8e5d10a7ccdedc9502dc83d7c1bee1f Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 9 Mar 2023 14:39:32 +0100 Subject: [PATCH 31/58] Remove cache from dcat resources --- config/dispatcher/dispatcher.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index cb7773e..6c5f881 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -330,11 +330,11 @@ defmodule Dispatcher do ################################################################# get "/datasets/*path", %{ layer: :api_services, accept: %{ json: true } } do - Proxy.forward conn, path, "http://cache/datasets/" + Proxy.forward conn, path, "http://resource/datasets/" end get "/distributions/*path", %{ layer: :api_services, accept: %{ json: true } } do - Proxy.forward conn, path, "http://cache/distributions/" + Proxy.forward conn, path, "http://resource/distributions/" end From 1624bf84fc5cbd55cc29ab7eeca5d22bb7c77093 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Fri, 10 Mar 2023 09:15:08 +0100 Subject: [PATCH 32/58] fix privacy graph (#253) --- .../20230228132614-move-privacy-data.sparql | 128 -------- .../20230308112614-move-privacy-data.sparql | 293 ++++++++++++++++++ .../20230308112615-move-privacy-data-2.sparql | 77 +++++ 3 files changed, 370 insertions(+), 128 deletions(-) delete mode 100644 config/migrations/2023/20230228112800-hard-split/20230228132614-move-privacy-data.sparql create mode 100644 config/migrations/2023/20230308112614-move-privacy-data.sparql create mode 100644 config/migrations/2023/20230308112615-move-privacy-data-2.sparql diff --git a/config/migrations/2023/20230228112800-hard-split/20230228132614-move-privacy-data.sparql b/config/migrations/2023/20230228112800-hard-split/20230228132614-move-privacy-data.sparql deleted file mode 100644 index ea64b74..0000000 --- a/config/migrations/2023/20230228112800-hard-split/20230228132614-move-privacy-data.sparql +++ /dev/null @@ -1,128 +0,0 @@ -PREFIX xsd: -PREFIX mu: -PREFIX persoon: -PREFIX ext: -PREFIX person: -PREFIX session: -PREFIX foaf: -PREFIX besluit: - - - - -insert { - -GRAPH { - ?person persoon:heeftGeboorte ?heeftGeboorte. - ?heeftGeboorte ?p ?o. - ?person persoon:registratie ?registration. - ?registration ?identificator. - ?identificator ?x ?y. - ?registration ?t ?j. - ?person persoon:geslacht ?geslacht. - ?person persoon:heeftNationaliteit ?nationaliteit. - - -} - -} - where { - GRAPH { - optional { - ?person persoon:heeftGeboorte ?heeftGeboorte. - ?heeftGeboorte ?p ?o. - } - optional { - ?person persoon:registratie ?registration. - ?registration ?identificator. - ?identificator ?x ?y. - ?registration ?t ?j. - } - optional { - ?person persoon:geslacht ?geslacht. - } - optional { - ?person persoon:heeftNationaliteit ?nationaliteit. - } - - filter (exists - { - graph { - ?mandatories ?person. - } - } - - || exists { - graph { - ?ministers ?person. - } - }) - - - - } - - - } - - ; - - - - - -insert { - -GRAPH { - ?person persoon:heeftGeboorte ?heeftGeboorte. - ?heeftGeboorte ?p ?o. - ?person persoon:registratie ?registration. - ?registration ?identificator. - ?identificator ?x ?y. - ?registration ?t ?j. - ?person persoon:geslacht ?geslacht. - ?person persoon:heeftNationaliteit ?nationaliteit. - - -} - -} - where { - GRAPH { - optional { - ?person persoon:heeftGeboorte ?heeftGeboorte. - ?heeftGeboorte ?p ?o. - } - optional { - ?person persoon:registratie ?registration. - ?registration ?identificator. - ?identificator ?x ?y. - ?registration ?t ?j. - } - optional { - ?person persoon:geslacht ?geslacht. - } - optional { - ?person persoon:heeftNationaliteit ?nationaliteit. - } - - filter (exists - { - graph { - ?mandatories ?person. - } - } - - || exists { - graph { - ?ministers ?person. - } - }) - - - - } - - - } - diff --git a/config/migrations/2023/20230308112614-move-privacy-data.sparql b/config/migrations/2023/20230308112614-move-privacy-data.sparql new file mode 100644 index 0000000..33dfbd4 --- /dev/null +++ b/config/migrations/2023/20230308112614-move-privacy-data.sparql @@ -0,0 +1,293 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: + +DELETE { + GRAPH { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + } +} +INSERT { +GRAPH { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + + } +} +WHERE { + GRAPH { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} +; + +DELETE { + GRAPH { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + } +} +INSERT { +GRAPH { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + + } +} +WHERE { + GRAPH { + ?person persoon:heeftGeboorte ?heeftGeboorte. + ?heeftGeboorte ?p ?o. + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} +; + +# geslacht + +DELETE { + GRAPH { + ?person persoon:geslacht ?geslacht. + } +} +INSERT { +GRAPH { + ?person persoon:geslacht ?geslacht. + + + } +} +WHERE { + GRAPH { + ?person persoon:geslacht ?geslacht. + + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} +; +DELETE { + GRAPH { + ?person persoon:geslacht ?geslacht. + } +} +INSERT { +GRAPH { + ?person persoon:geslacht ?geslacht. + + } +} +WHERE { + GRAPH { + + ?person persoon:geslacht ?geslacht. + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} +; + +# nationaliteit +DELETE { + GRAPH { + ?person persoon:heeftNationaliteit ?nationaliteit. + } +} +INSERT { +GRAPH { + ?person persoon:heeftNationaliteit ?nationaliteit. + } +} +WHERE { + GRAPH { + ?person persoon:heeftNationaliteit ?nationaliteit. + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} +; +DELETE { + GRAPH { + ?person persoon:heeftNationaliteit ?nationaliteit. + } +} +INSERT { +GRAPH { + ?person persoon:heeftNationaliteit ?nationaliteit. + } +} +WHERE { + GRAPH { + + ?person persoon:heeftNationaliteit ?nationaliteit. + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} +; + + +# registration +DELETE { + GRAPH { + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + + } +} +INSERT { +GRAPH { + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + } +} +WHERE { + GRAPH { + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + optional { ?registration ?t ?j. } + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} +; +DELETE { + GRAPH { + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + + } +} +INSERT { +GRAPH { + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + ?registration ?t ?j. + } +} +WHERE { + GRAPH { + ?person persoon:registratie ?registration. + ?registration ?identificator. + ?identificator ?x ?y. + optional {?registration ?t ?j.} + + filter (exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} + + diff --git a/config/migrations/2023/20230308112615-move-privacy-data-2.sparql b/config/migrations/2023/20230308112615-move-privacy-data-2.sparql new file mode 100644 index 0000000..fcf234f --- /dev/null +++ b/config/migrations/2023/20230308112615-move-privacy-data-2.sparql @@ -0,0 +1,77 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: + +DELETE { + GRAPH { + ?s ?p ?o + } +} +INSERT { +GRAPH { + ?s ?p ?o + } +} +WHERE { + GRAPH { + ?s a ?type; ?person; ?p ?o. + filter ( +(?type = || ?type = ) +&& + + exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} + +; + +DELETE { + GRAPH { + ?s ?p ?o + } +} +INSERT { +GRAPH { + ?s ?p ?o + } +} +WHERE { + GRAPH { + ?s a ?type; ?person; ?p ?o. + filter ( +(?type = || ?type = ) +&& + + exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} + From dcbc032fa897a2721bc3f0f6e460443842022699 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Fri, 10 Mar 2023 09:34:17 +0100 Subject: [PATCH 33/58] split query even more --- .../20230308112615-move-privacy-data-2.sparql | 34 --------------- .../20230308112616-move-privacy-data-3.sparql | 43 +++++++++++++++++++ 2 files changed, 43 insertions(+), 34 deletions(-) create mode 100644 config/migrations/2023/20230308112616-move-privacy-data-3.sparql diff --git a/config/migrations/2023/20230308112615-move-privacy-data-2.sparql b/config/migrations/2023/20230308112615-move-privacy-data-2.sparql index fcf234f..5fb62fb 100644 --- a/config/migrations/2023/20230308112615-move-privacy-data-2.sparql +++ b/config/migrations/2023/20230308112615-move-privacy-data-2.sparql @@ -40,38 +40,4 @@ WHERE { } } -; - -DELETE { - GRAPH { - ?s ?p ?o - } -} -INSERT { -GRAPH { - ?s ?p ?o - } -} -WHERE { - GRAPH { - ?s a ?type; ?person; ?p ?o. - filter ( -(?type = || ?type = ) -&& - - exists - { - graph { - ?mandatories ?person. - } - } - - || exists { - graph { - ?ministers ?person. - } - }) - - } -} diff --git a/config/migrations/2023/20230308112616-move-privacy-data-3.sparql b/config/migrations/2023/20230308112616-move-privacy-data-3.sparql new file mode 100644 index 0000000..727170c --- /dev/null +++ b/config/migrations/2023/20230308112616-move-privacy-data-3.sparql @@ -0,0 +1,43 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: + + +DELETE { + GRAPH { + ?s ?p ?o + } +} +INSERT { +GRAPH { + ?s ?p ?o + } +} +WHERE { + GRAPH { + ?s a ?type; ?person; ?p ?o. + filter ( +(?type = || ?type = ) +&& + + exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} + From 4bf0f6274ecf04b8e3b1be80cf19593b4f870ace Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Fri, 10 Mar 2023 09:45:55 +0100 Subject: [PATCH 34/58] optimize for timeouts --- ...0230308112615-move-privacy-data-2a.sparql} | 5 +-- ...20230308112616-move-privacy-data-2b.sparql | 40 +++++++++++++++++++ ...20230308112617-move-privacy-data-3.sparql} | 0 3 files changed, 41 insertions(+), 4 deletions(-) rename config/migrations/2023/{20230308112615-move-privacy-data-2.sparql => 20230308112615-move-privacy-data-2a.sparql} (82%) create mode 100644 config/migrations/2023/20230308112616-move-privacy-data-2b.sparql rename config/migrations/2023/{20230308112616-move-privacy-data-3.sparql => 20230308112617-move-privacy-data-3.sparql} (100%) diff --git a/config/migrations/2023/20230308112615-move-privacy-data-2.sparql b/config/migrations/2023/20230308112615-move-privacy-data-2a.sparql similarity index 82% rename from config/migrations/2023/20230308112615-move-privacy-data-2.sparql rename to config/migrations/2023/20230308112615-move-privacy-data-2a.sparql index 5fb62fb..2ad7d48 100644 --- a/config/migrations/2023/20230308112615-move-privacy-data-2.sparql +++ b/config/migrations/2023/20230308112615-move-privacy-data-2a.sparql @@ -19,11 +19,8 @@ GRAPH { } WHERE { GRAPH { - ?s a ?type; ?person; ?p ?o. + ?s a ; ?person; ?p ?o. filter ( -(?type = || ?type = ) -&& - exists { graph { diff --git a/config/migrations/2023/20230308112616-move-privacy-data-2b.sparql b/config/migrations/2023/20230308112616-move-privacy-data-2b.sparql new file mode 100644 index 0000000..b296ee7 --- /dev/null +++ b/config/migrations/2023/20230308112616-move-privacy-data-2b.sparql @@ -0,0 +1,40 @@ +PREFIX xsd: +PREFIX mu: +PREFIX persoon: +PREFIX ext: +PREFIX person: +PREFIX session: +PREFIX foaf: +PREFIX besluit: + +DELETE { + GRAPH { + ?s ?p ?o + } +} +INSERT { +GRAPH { + ?s ?p ?o + } +} +WHERE { + GRAPH { + ?s a ; ?person; ?p ?o. + filter ( + exists + { + graph { + ?mandatories ?person. + } + } + + || exists { + graph { + ?ministers ?person. + } + }) + + } +} + + diff --git a/config/migrations/2023/20230308112616-move-privacy-data-3.sparql b/config/migrations/2023/20230308112617-move-privacy-data-3.sparql similarity index 100% rename from config/migrations/2023/20230308112616-move-privacy-data-3.sparql rename to config/migrations/2023/20230308112617-move-privacy-data-3.sparql From 49ea6b3743b28a98735fa9176e7c786ca919d8b7 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Mon, 13 Mar 2023 10:14:53 +0100 Subject: [PATCH 35/58] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9e5760d..b36f816 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.15.0 + image: lblod/frontend-organization-portal:1.16.0-0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 7b2b577cc14c55e395afcc9d6a4d317dcd752ca1 Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Wed, 15 Mar 2023 16:27:00 +0100 Subject: [PATCH 36/58] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b36f816..aba21ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.16.0-0 + image: lblod/frontend-organization-portal:1.16.0-1 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 0e52e2bcf6dba75b08c26276668d0b0294e41f7a Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Mon, 20 Mar 2023 17:29:53 +0100 Subject: [PATCH 37/58] Add uuids and ROs to public producer --- config/delta-producer/public/export.json | 32 +++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index 04ffd0a..f929330 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -12,6 +12,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/adms#identifier", @@ -42,6 +43,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel", "http://www.w3.org/2004/02/skos/core#altLabel", "http://www.w3.org/ns/adms#identifier", @@ -69,6 +71,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", "http://data.lblod.info/vocabularies/erediensten/denominatie", "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", @@ -91,6 +94,19 @@ "http://www.w3.org/ns/regorg#orgStatus" ] }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/org#linkedTo", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/ns/adms#identifier" + ] + }, { "type": "http://www.w3.org/ns/org#Site", "graphsFilter": [ @@ -103,6 +119,7 @@ ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/ns/org#siteAddress", "https://data.vlaanderen.be/ns/organisatie#bestaatUit", "http://data.lblod.info/vocabularies/erediensten/vestigingstype" @@ -121,6 +138,7 @@ ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer", "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer", "http://www.w3.org/ns/locn#thoroughfare", @@ -167,6 +185,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#notation", "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" ] @@ -183,6 +202,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" ] }, @@ -195,6 +215,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, @@ -209,6 +230,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://data.vlaanderen.be/ns/mandaat#bindingEinde", "http://data.vlaanderen.be/ns/mandaat#bindingStart", "http://data.vlaanderen.be/ns/besluit#bestuurt", @@ -229,6 +251,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://data.vlaanderen.be/ns/mandaat#bindingEinde", "http://data.vlaanderen.be/ns/mandaat#bindingStart", "http://data.vlaanderen.be/ns/besluit#bestuurt", @@ -246,6 +269,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, @@ -258,6 +282,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, @@ -271,6 +296,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://purl.org/dc/terms/date", "http://purl.org/dc/terms/description", "http://data.lblod.info/vocabularies/contacthub/typeWijziging", @@ -291,6 +317,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, @@ -303,6 +330,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, @@ -315,6 +343,7 @@ "http://www.w3.org/2004/02/skos/core#inScheme" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/2004/02/skos/core#prefLabel" ] }, @@ -330,6 +359,7 @@ "http://www.w3.org/ns/org#role" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/ns/org#role" ] }, @@ -345,9 +375,9 @@ "http://www.w3.org/ns/org#role" ], "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/ns/org#role", "http://www.w3.org/2004/02/skos/core#prefLabel" - ] } ] From ba39434853abb4429ba51ec6e51dee2cd8b96bec Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 21 Mar 2023 08:39:46 +0100 Subject: [PATCH 38/58] Add betrokken and positie bedienaar to public export --- config/delta-producer/public/export.json | 25 ++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index f929330..587b3e2 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -25,7 +25,6 @@ "http://www.w3.org/ns/org#linkedTo", "http://www.w3.org/ns/org#hasSubOrganization", "http://www.w3.org/ns/org#classification", - "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", "http://data.lblod.info/vocabularies/erediensten/denominatie", "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", @@ -56,7 +55,6 @@ "http://www.w3.org/ns/org#linkedTo", "http://www.w3.org/ns/org#hasSubOrganization", "http://www.w3.org/ns/org#classification", - "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", "http://data.lblod.info/vocabularies/erediensten/typeEredienst" ] }, @@ -107,6 +105,29 @@ "http://www.w3.org/ns/adms#identifier" ] }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://data.lblod.info/vocabularies/erediensten/typebetrokkenheid", + "http://www.w3.org/ns/org#organization", + "http://data.lblod.info/vocabularies/erediensten/financieringspercentage" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role", + "http://data.lblod.info/vocabularies/erediensten/functie" + ] + }, { "type": "http://www.w3.org/ns/org#Site", "graphsFilter": [ From 7b37d91012c8d0fda17a6cda4783732c676e8e78 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 21 Mar 2023 11:10:05 +0100 Subject: [PATCH 39/58] Fix public producer config --- config/delta-producer/public/export.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index 587b3e2..384faad 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -371,14 +371,14 @@ { "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ "^http://www.w3.org/ns/org#hasPost", "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", "http://www.w3.org/ns/org#classification", "http://www.w3.org/2004/02/skos/core#inScheme" ], - "pathToConceptScheme": [ - "http://www.w3.org/ns/org#role" - ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/ns/org#role" @@ -387,14 +387,14 @@ { "type": "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie", "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ "^http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie", "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", "http://www.w3.org/ns/org#classification", "http://www.w3.org/2004/02/skos/core#inScheme" ], - "pathToConceptScheme": [ - "http://www.w3.org/ns/org#role" - ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", "http://www.w3.org/ns/org#role", From be12f3587ab042ac8cb3ccc29f8a34f41d00af6c Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Tue, 21 Mar 2023 11:30:03 +0100 Subject: [PATCH 40/58] Adapt producer to hardsplit graphs --- config/delta-producer/public/export.json | 71 ++++++++++++++++-------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index 384faad..8a726f9 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -5,7 +5,7 @@ "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service" ], "pathToConceptScheme": [ "http://www.w3.org/ns/org#classification", @@ -35,7 +35,7 @@ "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service" ], "pathToConceptScheme": [ "http://www.w3.org/ns/org#classification", @@ -62,7 +62,9 @@ "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "http://www.w3.org/ns/org#classification", @@ -95,7 +97,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -108,7 +111,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -120,7 +124,8 @@ { "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", @@ -131,7 +136,9 @@ { "type": "http://www.w3.org/ns/org#Site", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^http://www.w3.org/ns/org#hasPrimarySite", @@ -149,7 +156,9 @@ { "type": "http://www.w3.org/ns/locn#Address", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^https://data.vlaanderen.be/ns/organisatie#bestaatUit", @@ -175,7 +184,9 @@ { "type": "http://schema.org/ContactPoint", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^http://www.w3.org/ns/org#siteAddress", @@ -198,7 +209,9 @@ { "type": "http://www.w3.org/ns/adms#Identifier", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^http://www.w3.org/ns/adms#identifier", @@ -214,7 +227,9 @@ { "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator", @@ -230,7 +245,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -244,7 +259,9 @@ "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "http://www.w3.org/ns/org#classification", @@ -264,7 +281,9 @@ "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "strictTypeExport": "true", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", @@ -284,7 +303,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -297,7 +316,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -310,7 +329,9 @@ { "type": "http://www.w3.org/ns/org#ChangeEvent", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "http://data.lblod.info/vocabularies/contacthub/typeWijziging", @@ -332,7 +353,9 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -345,7 +368,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -358,7 +381,7 @@ { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/public" ], "pathToConceptScheme": [ "http://www.w3.org/2004/02/skos/core#inScheme" @@ -371,7 +394,9 @@ { "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^http://www.w3.org/ns/org#hasPost", @@ -387,7 +412,9 @@ { "type": "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie", "graphsFilter": [ - "http://mu.semte.ch/graphs/organisatieportaal" + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" ], "pathToConceptScheme": [ "^http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie", From 8d5c4958eafdae48448db60522846b64e45dd7c0 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Mar 2023 08:20:01 +0100 Subject: [PATCH 41/58] Fix auth config --- config/authorization/config.ex | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/config/authorization/config.ex b/config/authorization/config.ex index 82b2d30..1dd43ab 100644 --- a/config/authorization/config.ex +++ b/config/authorization/config.ex @@ -16,7 +16,6 @@ defmodule Acl.UserGroups.Config do "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris", "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", "http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid", - "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "http://data.lblod.info/vocabularies/erediensten/RolBedienaar", "http://data.lblod.info/vocabularies/erediensten/VoorwaardenBedienaar", "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", @@ -187,12 +186,12 @@ defmodule Acl.UserGroups.Config do name: "public", useage: [:read], access: %AlwaysAccessible{}, - graphs: [ + graphs: [ %GraphSpec{ - graph: "http://mu.semte.ch/graphs/public", + graph: "http://mu.semte.ch/graphs/public", constraint: %ResourceConstraint{ resource_types: @public_type - } + } }, %GraphSpec{ graph: "http://mu.semte.ch/graphs/shared", From 211708754cf8b7d25ec8f98ecc8c6766e189f9c3 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Mar 2023 10:35:48 +0100 Subject: [PATCH 42/58] Adapting query for QA and DEV deploy On QA and DEV, the hardsplit has already been deployed, so we need to adjust the migration. On prod, the migration has already run, so there is no risk of missing data. --- ...20230228155501-publish-all-bestuurseenheden-in-public.sparql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/migrations/2023/20230228155501-publish-all-bestuurseenheden-in-public.sparql b/config/migrations/2023/20230228155501-publish-all-bestuurseenheden-in-public.sparql index 6e9dc1c..80314a3 100644 --- a/config/migrations/2023/20230228155501-publish-all-bestuurseenheden-in-public.sparql +++ b/config/migrations/2023/20230228155501-publish-all-bestuurseenheden-in-public.sparql @@ -1,5 +1,5 @@ INSERT DATA { - GRAPH { + GRAPH { . #Centraal bestuur van de eredienst" . #Gemeente" . #Provincie" From b05f69a61504422888f4ed7b0dcf11a0334749ae Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Fri, 24 Mar 2023 13:47:24 +0100 Subject: [PATCH 43/58] bump frontend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index aba21ad..ba2164b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,7 @@ services: restart: always logging: *default-logging frontend: - image: lblod/frontend-organization-portal:1.16.0-1 + image: lblod/frontend-organization-portal:1.16.0 volumes: - ./config/frontend/add-x-frame-options-header.conf:/config/add-x-frame-options.conf environment: From 99fe30d2efd86025e4c6d82ee22f428e485c9677 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 22 Mar 2023 16:35:06 +0100 Subject: [PATCH 44/58] Bump dump file publisher --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ba2164b..4d02d8a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -230,7 +230,7 @@ services: # DELTA GENERAL ################################################################################ delta-producer-dump-file-publisher: - image: lblod/delta-producer-dump-file-publisher:0.9.0 + image: lblod/delta-producer-dump-file-publisher:0.9.1 volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share From 42dc3af86a125d6bdc5b31f197c964b91036b075 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 24 Mar 2023 15:34:35 +0100 Subject: [PATCH 45/58] Fix status and add missing orgaan I also had to bump mu-search, I was getting HTTP 500 errors otherwise (NoMethodError - undefined method `each' for nil:NilClass) About the mandates creation: - LidVanRechtswege : shared mandate for all the organen in time - Bestuursleden big half : shared with 2023-2026 because 2020 was the start of a big half - Bestuursleden small half : new mandate (should have been shared with 2017-2020, but we don't have this time period) - Voorzitter : new mandate - Secretaris : new mandate - Penningmeester : new mandate --- ...-of-st-egidius-van-sint-gillis-waas.sparql | 46 +++++++++++++++++++ docker-compose.yml | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 config/migrations-triggering-indexing/2023/20230324150743-update-status-and-orgaan-of-st-egidius-van-sint-gillis-waas.sparql diff --git a/config/migrations-triggering-indexing/2023/20230324150743-update-status-and-orgaan-of-st-egidius-van-sint-gillis-waas.sparql b/config/migrations-triggering-indexing/2023/20230324150743-update-status-and-orgaan-of-st-egidius-van-sint-gillis-waas.sparql new file mode 100644 index 0000000..060b4be --- /dev/null +++ b/config/migrations-triggering-indexing/2023/20230324150743-update-status-and-orgaan-of-st-egidius-van-sint-gillis-waas.sparql @@ -0,0 +1,46 @@ +DELETE DATA { + GRAPH { + + . + } +} +; +INSERT DATA { + GRAPH { + + . + + + a ; + "18d843d47e223409b9bb6e7af8a4d8a8" ; + ; + "2020-04-01T00:00:00"^^xsd:dateTime ; + "2023-03-31T00:00:00"^^xsd:dateTime ; + ; + ; + ; + ; + ; + . + + + a ; + "daac7f53-b020-443d-92a7-9d47be6cb186" ; + . + + + a ; + "3618fc2a-83f7-45dd-969f-06afd0c3277b" ; + . + + + a ; + "e4b05858-c66d-4098-9c43-59af0359e603" ; + . + + + a ; + "5147f880-b0a5-4145-ac01-db8323a5410d" ; + . + } +} diff --git a/docker-compose.yml b/docker-compose.yml index ba2164b..3444fa0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -507,7 +507,7 @@ services: # MU SEARCH ################################################################################ search: - image: semtech/mu-search:0.9.0-beta.2 + image: semtech/mu-search:0.9.0-beta.3 links: - db:database restart: always From 4858aa51f382bfb9aaa40dacda61d89bd7f79d1f Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 29 Mar 2023 09:33:01 +0200 Subject: [PATCH 46/58] Enable new period for organen and create missing data (#259) --- .../20230327162800-merge-organen.sparql | 42 ++++ ...e-missing-organen-in-time-2023-2026.sparql | 46 +++++ ...-central-worship-services-2023-2026.sparql | 152 ++++++++++++++ ...ositions-worship-services-2023-2026.sparql | 186 ++++++++++++++++++ docker-compose.yml | 3 + 5 files changed, 429 insertions(+) create mode 100644 config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327162800-merge-organen.sparql create mode 100644 config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327162900-create-missing-organen-in-time-2023-2026.sparql create mode 100644 config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327163000-create-missing-positions-central-worship-services-2023-2026.sparql create mode 100644 config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327163000-create-missing-positions-worship-services-2023-2026.sparql diff --git a/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327162800-merge-organen.sparql b/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327162800-merge-organen.sparql new file mode 100644 index 0000000..46b8e86 --- /dev/null +++ b/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327162800-merge-organen.sparql @@ -0,0 +1,42 @@ +# We have two bestuursorganen of the same classification for 2 bestuurseenheden. +# Then need to be merged, as we should only have one orgaan (not in time) per classification. + +PREFIX mandaat: +PREFIX ere: +PREFIX besluit: +PREFIX generiek: + +DELETE { + GRAPH { + ?otherBestuursorgaan ?potherBestuursorgaan ?ootherBestuursorgaan . + ?s ?p ?otherBestuursorgaan . + } +} INSERT { + GRAPH { + ?s ?p ?bestuursorgaan . + } +} WHERE { + GRAPH { + VALUES ?type { + ere:BestuurVanDeEredienst + ere:CentraalBestuurVanDeEredienst + } + + ?bestuurseenheid a ?type . + + FILTER NOT EXISTS { + ?bestuurseenheid . + } + + ?bestuursorgaan besluit:bestuurt ?bestuurseenheid ; + ?classif . + + ?otherBestuursorgaan besluit:bestuurt ?bestuurseenheid ; + ?classif ; + ?potherBestuursorgaan ?ootherBestuursorgaan . + + ?s ?p ?otherBestuursorgaan . + + FILTER (str(?bestuursorgaan) > str(?otherBestuursorgaan)) + } +} \ No newline at end of file diff --git a/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327162900-create-missing-organen-in-time-2023-2026.sparql b/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327162900-create-missing-organen-in-time-2023-2026.sparql new file mode 100644 index 0000000..8de3be5 --- /dev/null +++ b/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327162900-create-missing-organen-in-time-2023-2026.sparql @@ -0,0 +1,46 @@ +PREFIX mandaat: +PREFIX ere: +PREFIX besluit: +PREFIX generiek: +PREFIX mu: + +INSERT { + GRAPH { + ?orgaanInTime a besluit:Bestuursorgaan ; + mu:uuid ?uuidOrgaanInTime ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ ; + generiek:isTijdspecialisatieVan ?bestuursorgaan . + } +} +WHERE { + GRAPH { + VALUES ?type { + ere:BestuurVanDeEredienst + ere:CentraalBestuurVanDeEredienst + } + + ?bestuurseenheid a ?type . + + FILTER NOT EXISTS { + ?bestuurseenheid . + } + + ?bestuursorgaan besluit:bestuurt ?bestuurseenheid . + + FILTER NOT EXISTS { + ?missingOrgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan . + ?missingOrgaanInTime mandaat:bindingStart ?start . + FILTER CONTAINS(STR(?start), "2023") + } + + FILTER NOT EXISTS { + ?missingOrgaanInTime generiek:isTijdspecialisatieVan ?bestuursorgaan . + ?missingOrgaanInTime mandaat:bindingEinde ?end . + FILTER CONTAINS(STR(?end), "2026") + } + } + + BIND(MD5(CONCAT(?bestuurseenheid, ?bestuursorgaan, "2023-2026")) as ?uuidOrgaanInTime) + BIND(IRI(CONCAT("http://data.lblod.info/id/bestuursorganen/", ?uuidOrgaanInTime)) AS ?orgaanInTime) +} diff --git a/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327163000-create-missing-positions-central-worship-services-2023-2026.sparql b/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327163000-create-missing-positions-central-worship-services-2023-2026.sparql new file mode 100644 index 0000000..9ae2b54 --- /dev/null +++ b/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327163000-create-missing-positions-central-worship-services-2023-2026.sparql @@ -0,0 +1,152 @@ +# Add missing mandates to central worship services + +## 2023 - 2026 + +PREFIX ere: +PREFIX besluit: +PREFIX org: +PREFIX mandaat: +PREFIX generiek: +PREFIX mu: + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?expert . + + ?expert a mandaat:Mandaat ; + mu:uuid ?uuidExpert ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:CentraalBestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?expert . + ?expert org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Expert 2023-2026")) as ?uuidExpert) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidExpert)) AS ?expert) +} + +; + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?vertegenwoordiger . + + ?vertegenwoordiger a mandaat:Mandaat ; + mu:uuid ?uuidVertegenwoordiger ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:CentraalBestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?vertegenwoordiger . + ?vertegenwoordiger org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Vertegenwoordiger 2023-2026")) as ?uuidVertegenwoordiger) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidVertegenwoordiger)) AS ?vertegenwoordiger) +} + +; + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?secretaris . + + ?secretaris a mandaat:Mandaat ; + mu:uuid ?uuidSecretaris ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:CentraalBestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?secretaris . + ?secretaris org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Secretaris 2023-2026")) as ?uuidSecretaris) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidSecretaris)) AS ?secretaris) +} + +; + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?voorzitter . + + ?voorzitter a mandaat:Mandaat ; + mu:uuid ?uuidVoorzitter ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:CentraalBestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?voorzitter . + ?voorzitter org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Voorzitter 2023-2026")) as ?uuidVoorzitter) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidVoorzitter)) AS ?voorzitter) +} + +; + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?bestuurslid . + + ?bestuurslid a mandaat:Mandaat ; + mu:uuid ?uuidBestuurslid ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:CentraalBestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?worshipService . + } + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?bestuurslid . + ?bestuurslid org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Bestuurslid 2023-2026")) as ?uuidBestuurslid) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidBestuurslid)) AS ?bestuurslid) +} diff --git a/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327163000-create-missing-positions-worship-services-2023-2026.sparql b/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327163000-create-missing-positions-worship-services-2023-2026.sparql new file mode 100644 index 0000000..dc80b96 --- /dev/null +++ b/config/migrations/2023/20230327162700-prepare-change-legislation-period/20230327163000-create-missing-positions-worship-services-2023-2026.sparql @@ -0,0 +1,186 @@ +# Add missing mandates to worship services for period 2023-2026 + +PREFIX ere: +PREFIX besluit: +PREFIX org: +PREFIX mandaat: +PREFIX generiek: +PREFIX mu: + +## Case LidVanRechtswege : shared mandate for all the organen in time + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?lidVanRechtswege . + + ?lidVanRechtswege a mandaat:Mandaat ; + mu:uuid ?uuidLidVanRechtswege ; + org:role . + } +} WHERE { + GRAPH { + VALUES (?start ?end) { + ("2023-04-01T00:00:00"^^ "2026-03-31T00:00:00"^^) + } + + ?worshipService a ere:BestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart ?start ; + mandaat:bindingEinde ?end . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?lidVanRechtswege . + ?lidVanRechtswege org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Rechtswege")) as ?uuidLidVanRechtswege) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidLidVanRechtswege)) AS ?lidVanRechtswege) +} + +## Case Bestuursleden, small or big half : 2023 is the beggining of a KH + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?bestuurslidGH . + + ?bestuurslidGH a mandaat:Mandaat ; + mu:uuid ?uuidBestuurslidGH ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:BestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?bestuurslidGH . + ?bestuurslidGH org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "BestuurslidGH 2020-2026")) as ?uuidBestuurslidGH) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidBestuurslidGH)) AS ?bestuurslidGH) +} + +; + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?bestuurslidKH . + + ?bestuurslidKH a mandaat:Mandaat ; + mu:uuid ?uuidBestuurslidKH ; + org:role . + } +} WHERE { + GRAPH { + VALUES (?start ?end) { + ("2023-04-01T00:00:00"^^ "2026-03-31T00:00:00"^^) + } + + ?worshipService a ere:BestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart ?start ; + mandaat:bindingEinde ?end . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?bestuurslidKH . + ?bestuurslidKH org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "BestuurslidKH 2023-2029")) as ?uuidBestuurslidKH) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidBestuurslidKH)) AS ?bestuurslidKH) +} + +; + +## Other cases are simpler, one mandate is linked to only one orgaan in time. Creating them when missing. + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?voorzitter . + + ?voorzitter a mandaat:Mandaat ; + mu:uuid ?uuidVoorzitter ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:BestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?voorzitter . + ?voorzitter org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Voorzitter 2023-2026")) as ?uuidVoorzitter) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidVoorzitter)) AS ?voorzitter) +} + +; + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?secretaris . + + ?secretaris a mandaat:Mandaat ; + mu:uuid ?uuidSecretaris ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:BestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?secretaris . + ?secretaris org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Secretaris 2023-2026")) as ?uuidSecretaris) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidSecretaris)) AS ?secretaris) +} + +; + +INSERT { + GRAPH { + ?orgaanInTime org:hasPost ?penningmeester . + + ?penningmeester a mandaat:Mandaat ; + mu:uuid ?uuidPenningmeester ; + org:role . + } +} WHERE { + GRAPH { + ?worshipService a ere:BestuurVanDeEredienst . + ?orgaan besluit:bestuurt ?worshipService . + ?orgaanInTime generiek:isTijdspecialisatieVan ?orgaan ; + mandaat:bindingStart "2023-04-01T00:00:00"^^ ; + mandaat:bindingEinde "2026-03-31T00:00:00"^^ . + + FILTER NOT EXISTS { + ?orgaanInTime org:hasPost ?penningmeester . + ?penningmeester org:role . + } + } + + BIND(MD5(CONCAT(?worshipService, ?orgaan, "Penningmeester 2023-2026")) as ?uuidPenningmeester) + BIND(IRI(CONCAT("http://data.lblod.info/id/mandaten/", ?uuidPenningmeester)) AS ?penningmeester) +} diff --git a/docker-compose.yml b/docker-compose.yml index 4d02d8a..45f575b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -218,6 +218,9 @@ services: - ./config/jobs-controller/:/config/ construct-administrative-unit-relationships: image: lblod/construct-administrative-unit-relationships-service:0.1.1 + environment: + START_DATE_WORSHIP_GOVERNING_BODY: "2023-04-01T00:00:00" + END_DATE_WORSHIP_GOVERNING_BODY: "2026-03-31T00:00:00" links: - db:database restart: always From f43ae9f6424dde19e384995107ac5215026db923 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Mon, 3 Apr 2023 10:05:37 +0200 Subject: [PATCH 47/58] migration from in rang to waarneemende (#260) --- ...6-from-imam-in-rang-to-waarneemende.sparql | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 config/migrations/2023/20230331095346-from-imam-in-rang-to-waarneemende.sparql diff --git a/config/migrations/2023/20230331095346-from-imam-in-rang-to-waarneemende.sparql b/config/migrations/2023/20230331095346-from-imam-in-rang-to-waarneemende.sparql new file mode 100644 index 0000000..c7e71c1 --- /dev/null +++ b/config/migrations/2023/20230331095346-from-imam-in-rang-to-waarneemende.sparql @@ -0,0 +1,24 @@ +delete { + graph { + ?position ?function. + } +} + +insert { + graph { + ?position . + } +} +where { + graph { + ?position ?function. + ?worshipService ?position; + ?worshipStatus. + + filter (?function in (, + , + ) + && ?worshipStatus = ) + } + +} From dc0aeec0dc72d7a90c0d59b03467f58c591fcc4f Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Tue, 11 Apr 2023 15:42:54 +0200 Subject: [PATCH 48/58] ignore virtuoso.ini --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3f398c9..ff36b53 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ # Virtuoso files /data/db /data/db.bck/** /data/db/* -!/data/db/virtuoso.ini +/data/db/virtuoso.ini !/data/db/toLoad !/data/db/toLoad/* /data/files/* From 9486643214e447df106f9742262859c6b1caa03e Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Thu, 13 Apr 2023 13:52:44 +0200 Subject: [PATCH 49/58] Export identifiers of ROs in public producer (#261) --- config/delta-producer/public/export.json | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index c298ce4..4e55975 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -223,6 +223,18 @@ "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" ] }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ], + "additionalFilter": "?ro ?subject ; a ." + }, { "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", "graphsFilter": [ @@ -241,6 +253,17 @@ "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" ] }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ], + "additionalFilter": "?ro / ?subject ; a ." + }, { "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", "graphsFilter": [ From d08c88f31252e2ad2d1bc5fb81d053d4bd7988e3 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Tue, 18 Apr 2023 08:14:48 +0000 Subject: [PATCH 50/58] Add missing post type --- ...93500-add-org-post-type-to-mandates.sparql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 config/migrations/2023/20230418093500-add-org-post-type-to-mandates.sparql diff --git a/config/migrations/2023/20230418093500-add-org-post-type-to-mandates.sparql b/config/migrations/2023/20230418093500-add-org-post-type-to-mandates.sparql new file mode 100644 index 0000000..f8e4e24 --- /dev/null +++ b/config/migrations/2023/20230418093500-add-org-post-type-to-mandates.sparql @@ -0,0 +1,19 @@ +INSERT { + GRAPH { + ?s a . + } +} WHERE { + GRAPH { + ?s a . + } +} + +INSERT { + GRAPH { + ?s a . + } +} WHERE { + GRAPH { + ?s a . + } +} \ No newline at end of file From 86e4304b7ea53c7981d9431af422ab4e3c007037 Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Wed, 19 Apr 2023 09:26:04 +0200 Subject: [PATCH 51/58] reset elastic script (#264) --- scripts/reset-elastic.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 scripts/reset-elastic.sh diff --git a/scripts/reset-elastic.sh b/scripts/reset-elastic.sh new file mode 100644 index 0000000..7adbe5c --- /dev/null +++ b/scripts/reset-elastic.sh @@ -0,0 +1,11 @@ +#!/bin/bash +echo "warning this will run queries on the triplestore and delete containers, you have 3 seconds to press ctrl+c" +sleep 3 +docker-compose rm -fs elasticsearch search +sudo rm -rf data/elasticsearch/ +docker-compose exec -T triplestore isql-v < { ?s ?p ?o. } }; +exec('checkpoint'); +exit; +EOF +docker-compose up -d --remove-orphan From e3883db0871f3df2c15864042a3d11bc9a076c60 Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Wed, 19 Apr 2023 11:13:42 +0000 Subject: [PATCH 52/58] 0.6.2 -> 0.6.3 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f5a9922..2c260ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,7 +173,7 @@ services: links: - db:database adressenregister: - image: lblod/adressenregister-fuzzy-search-service:0.6.2 + image: lblod/adressenregister-fuzzy-search-service:0.6.3 labels: - "logging=true" restart: always From 3aadf0960e9802906d3446cd649627038c5d307b Mon Sep 17 00:00:00 2001 From: Nordine Bittich Date: Fri, 21 Apr 2023 07:43:06 +0200 Subject: [PATCH 53/58] bump role service --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2c260ad..7290967 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -530,7 +530,7 @@ services: - db:database roles: - image: lblod/select-role-service + image: lblod/select-role-service:0.1.0 labels: - "logging=true" links: From fefb103cb5763bd0c61a362c2d5d5aed2a37063e Mon Sep 17 00:00:00 2001 From: Boris De Vloed Date: Fri, 21 Apr 2023 11:57:15 +0200 Subject: [PATCH 54/58] Consume mandatarissen, leidinggevenden and worship positions from Loket (#239) * loket filter * cleanup previously migrated data before sync * increase mu-auth timouts * split up deletion migration to avoid timouts * identical utls added catch statements * deletes before inserts * Adapt migrations to splitted graphs * Adapt consumers to the delta-consumer with contexts * Bump consumers and cache * Resolve cache not getting cleared * Flush ministers of non active worship services too --------- Co-authored-by: Claire Lovisa --- .../leidinggevenden-dispatching/config.js | 17 +- .../delta-context-config.js | 11 + .../delta-sync-dispatching.js | 93 ++++---- .../initial-sync-dispatching.js | 68 ++++-- .../leidinggevenden-dispatching/util.js | 119 +++++----- .../config.js | 18 +- .../delta-context-config.js | 11 + .../delta-sync-dispatching.js | 85 ++++++++ .../initial-sync-dispatching.js | 85 ++++++++ .../util.js | 118 +++++----- .../delta-sync-dispatching.js | 70 ------ .../initial-sync-dispatching.js | 57 ----- .../mandate-dispatching/util.js | 153 ------------- .../positions-dispatching/config.js | 51 ----- .../delta-sync-dispatching.js | 88 -------- .../initial-sync-dispatching.js | 75 ------- .../config.js | 40 ++++ .../delta-context-config.js | 11 + .../delta-sync-dispatching.js | 114 ++++++++++ .../initial-sync-dispatching.js | 106 +++++++++ .../util.js | 204 ++++++++++++++++++ ...00-remove-loket-persons-before-sync.sparql | 54 +++++ ...e-loket-functionarissen-before-sync.sparql | 52 +++++ ...et-mandatarissen-before-sync-part-1.sparql | 53 +++++ ...et-mandatarissen-before-sync-part-2.sparql | 53 +++++ ...et-mandatarissen-before-sync-part-3.sparql | 52 +++++ ...move-ministers-that-will-get-synced.sparql | 21 ++ .../reasoner/dl2op/main/inverse_relations.n3 | 1 - config/reasoner/dl2op/main/mapping.n3 | 26 ++- .../dl2op/main/persoon_http_to_https.n3 | 1 - config/reasoner/dl2op/main/query.n3q | 63 ++++++ .../dl2op/main/retract_empty_strings.n3 | 1 - config/reasoner/dl2op/main/rrn.n3 | 2 - config/resources/domain.lisp | 1 + docker-compose.yml | 54 +++-- 35 files changed, 1329 insertions(+), 699 deletions(-) create mode 100644 config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js rename config/loket-consumer/{mandate-dispatching => mandatarissen-dispatching}/config.js (69%) create mode 100644 config/loket-consumer/mandatarissen-dispatching/delta-context-config.js create mode 100644 config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js create mode 100644 config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js rename config/loket-consumer/{positions-dispatching => mandatarissen-dispatching}/util.js (50%) delete mode 100644 config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js delete mode 100644 config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js delete mode 100644 config/loket-consumer/mandate-dispatching/util.js delete mode 100644 config/loket-consumer/positions-dispatching/config.js delete mode 100644 config/loket-consumer/positions-dispatching/delta-sync-dispatching.js delete mode 100644 config/loket-consumer/positions-dispatching/initial-sync-dispatching.js create mode 100644 config/loket-consumer/worship-services-sensitive-dispatching/config.js create mode 100644 config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js create mode 100644 config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js create mode 100644 config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js create mode 100644 config/loket-consumer/worship-services-sensitive-dispatching/util.js create mode 100644 config/migrations/2023/20230330164600-sync-loket-to-op/20230330164600-remove-loket-persons-before-sync.sparql create mode 100644 config/migrations/2023/20230330164600-sync-loket-to-op/20230330164601-remove-loket-functionarissen-before-sync.sparql create mode 100644 config/migrations/2023/20230330164600-sync-loket-to-op/20230330164602-remove-loket-mandatarissen-before-sync-part-1.sparql create mode 100644 config/migrations/2023/20230330164600-sync-loket-to-op/20230330164603-remove-loket-mandatarissen-before-sync-part-2.sparql create mode 100644 config/migrations/2023/20230330164600-sync-loket-to-op/20230330164604-remove-loket-mandatarissen-before-sync-part-3.sparql create mode 100644 config/migrations/2023/20230330164600-sync-loket-to-op/20230330164605-remove-ministers-that-will-get-synced.sparql create mode 100644 config/reasoner/dl2op/main/query.n3q diff --git a/config/loket-consumer/leidinggevenden-dispatching/config.js b/config/loket-consumer/leidinggevenden-dispatching/config.js index 2afb525..b3ebc2a 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/config.js +++ b/config/loket-consumer/leidinggevenden-dispatching/config.js @@ -1,18 +1,20 @@ const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; -const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql'; const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); -const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`; +// Graphs +const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`; +const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`; -if (!process.env.FILE_SYNC_GRAPH) - throw `Expected 'FILE_SYNC_GRAPH' to be provided.`; -const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH; +// ENDPOINTS +const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db'; +const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; module.exports = { BATCH_SIZE, @@ -24,6 +26,7 @@ module.exports = { SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - INGEST_GRAPH, - FILE_SYNC_GRAPH + LANDING_ZONE_GRAPH, + TARGET_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT }; diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js new file mode 100644 index 0000000..a8df81c --- /dev/null +++ b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js @@ -0,0 +1,11 @@ +const contextConfig = { + addTypes: { + scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both + exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta + }, + contextQueries: [] +} + +module.exports = { + contextConfig +}; diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js index 669895b..b309f35 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js +++ b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js @@ -1,11 +1,8 @@ -const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util'); const { - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH -} = require('./config'); + transformStatements, + deleteFromTargetGraph, + insertIntoTargetGraph, +} = require('./util'); /** * Dispatch the fetched information to a target graph. @@ -23,43 +20,61 @@ const { * @return {void} Nothing */ async function dispatch(lib, data) { - const { mu, muAuthSudo, fetch } = lib; - const { termObjectChangeSets } = data; + const { mu, fetch } = lib; + let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; - for (let { deletes, inserts } of termObjectChangeSets) { - const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + // Both arrays are the same length, so we can zip them together for easier processing + zippedChangeSets = termObjectChangeSets.map((o, i) => ({ + original: o, + withContext: termObjectChangeSetsWithContext[i] + })); - if (deleteStatements.length) { - const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements); - if (transformedStatementsToDelete.length) { - await deleteFromAllGraphs( - muAuthSudo.updateSudo, - transformedStatementsToDelete, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - ); + console.log(`Received ${zippedChangeSets.length} change sets`) + + for (let { original, withContext } of zippedChangeSets) { + let originalDeletes, originalInserts, insertsWithContext, deletesWithContext; + try { + originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + } catch (e) { + console.log(`Error while processing change set: ${e}`); + console.log(`Original change set: ${JSON.stringify(original)}`); + throw e; + } + + // Extra context needed for mapping from DL to OP model and filtering based on type. + try { + insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + } catch (e) { + console.log(`Error while processing change set with context: ${e}`); + console.log(`Change set with context: ${JSON.stringify(withContext)}`); + throw e; + } + + if (originalDeletes.length) { + // Map deletes from DL to OP model + const transformedDeletes = await transformStatements(fetch, deletesWithContext); + + if (!transformedDeletes.length) { + console.log(`Warn: Delete statements mapped to empty result.`); + console.log(`Input: ${deletesWithContext}`); + console.log(`Output: ${transformedDeletes}`); + } else { + await deleteFromTargetGraph(lib, transformedDeletes); } } - if (insertStatements.length) { - const transformedStatementsToInsert = await transformStatements(fetch, insertStatements); - if (transformedStatementsToInsert.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatementsToInsert, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "INSERT" - ); + if (originalInserts.length) { + // Map inserts from DL to OP model + const transformedInserts = await transformStatements(fetch, insertsWithContext); + + if (!transformedInserts.length) { + console.log(`Warn: Insert statements mapped to empty result.`); + console.log(`Input: ${insertsWithContext}`); + console.log(`Output: ${transformedInserts}`); + } else { + await insertIntoTargetGraph(lib, transformedInserts); } } } diff --git a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js index 7967073..d5d672f 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js +++ b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js @@ -1,13 +1,17 @@ -const { transformStatements, batchedDbUpdate } = require('./util'); -const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, +const { batchedDbUpdate, transformLandingZoneGraph } = require('./util'); +const { + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, DIRECT_DATABASE_ENDPOINT, MU_CALL_SCOPE_ID_INITIAL_SYNC, BATCH_SIZE, MAX_DB_RETRY_ATTEMPTS, SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH + TARGET_GRAPH, + LANDING_ZONE_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT } = require('./config'); + const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; /** @@ -24,34 +28,58 @@ const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT * @return {void} Nothing */ async function dispatch(lib, data) { - const { mu, muAuthSudo, fetch } = lib; + const { mu, muAuthSudo } = lib; const { termObjects } = data; + // Steps: + // 1. Write the triples to the landing zone graph - withouth any mapping or filtering + // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { console.warn(`Service configured to skip MU_AUTH!`); } console.log(`Using ${endpoint} to insert triples`); if (termObjects.length) { - originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) + const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples); - if (transformedStatementsToInsert.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatementsToInsert, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } + await batchedDbUpdate( + muAuthSudo.updateSudo, + LANDING_ZONE_GRAPH, + originalInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + LANDING_ZONE_DATABASE_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); } } +async function onFinishInitialIngest(lib) { + const { muAuthSudo, fetch } = lib; + + console.log(`!! On-finish triggered !!`); + + const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint); + + console.log(`Transformed ${transformedInsertTriples.length} triples`); + + await batchedDbUpdate( + muAuthSudo.updateSudo, + TARGET_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); +} + module.exports = { - dispatch + dispatch, + onFinishInitialIngest }; diff --git a/config/loket-consumer/leidinggevenden-dispatching/util.js b/config/loket-consumer/leidinggevenden-dispatching/util.js index bdf477b..5eced4a 100644 --- a/config/loket-consumer/leidinggevenden-dispatching/util.js +++ b/config/loket-consumer/leidinggevenden-dispatching/util.js @@ -1,9 +1,16 @@ const { MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + TARGET_GRAPH, + LANDING_ZONE_GRAPH, + BATCH_SIZE, } = require('./config'); -async function batchedDbUpdate(muUpdate, +async function batchedDbUpdate( + muUpdate, graph, triples, extraHeaders, @@ -37,39 +44,6 @@ ${batch} } } -async function deleteFromAllGraphs(muUpdate, - triples, - extraHeaders, - endpoint, - maxAttempts, - sleepBetweenBatches = 1000, - sleepTimeOnFail = 1000, -) { - - for (const triple of triples) { - - console.log(`Deleting a triple from all graphs in triplestore`); - - const deleteCall = async () => { - await muUpdate(` - DELETE { - GRAPH ?g { - ${triple} - } - } WHERE { - GRAPH ?g { - ${triple} - } - } - `, extraHeaders, endpoint); - }; - - await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); - console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); - await new Promise(r => setTimeout(r, sleepBetweenBatches)); - } -} - async function operationWithRetry(callback, attempt, maxAttempts, @@ -110,18 +84,16 @@ function partition(arr, fn) { return { passes, fails }; } - /** * Send triples to reasoning service for conversion * */ -function transformTriples(fetch, triples) { - return operationWithRetry(mainConversion(fetch, triples), 0, +async function transformTriples(fetch, triples, mapping) { + return await operationWithRetry(mainConversion(fetch, triples, mapping), 0, MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); } - -function mainConversion(fetch, triples) { +async function mainConversion(fetch, triples, mapping) { let formdata = new URLSearchParams(); formdata.append("data", triples); @@ -131,23 +103,66 @@ function mainConversion(fetch, triples) { redirect: 'follow' }; - return fetch("http://reasoner/reason/dl2op/main", requestOptions) - .then(response => response.text()); + const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) + return await response.text(); } -function transformStatements(fetch, triples) { - return transformTriples(fetch, triples.join('\n')).then( - graph => { - statements = graph.replace(/\n{2,}/g, '').split('\n') - console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`) - return statements - } - ) +async function transformStatements(fetch, triples, mapping = 'main') { + console.log(`Received ${JSON.stringify(triples)} to transform`); + const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping); + statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : []; + console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`); + return statements; +} + +async function deleteFromTargetGraph(lib, statements) { + console.log(`Deleting ${statements.length} statements from target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + TARGET_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); +} + +async function insertIntoTargetGraph(lib, statements) { + console.log(`Inserting ${statements.length} statements into target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + TARGET_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); +} + +async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') { + console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`); + + const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`); + const text = await response.text(); + const statements = text.replace(/\n{2,}/g, '').split('\n'); + + return statements; } module.exports = { batchedDbUpdate, - deleteFromAllGraphs, partition, - transformStatements + transformStatements, + transformLandingZoneGraph, + deleteFromTargetGraph, + insertIntoTargetGraph }; diff --git a/config/loket-consumer/mandate-dispatching/config.js b/config/loket-consumer/mandatarissen-dispatching/config.js similarity index 69% rename from config/loket-consumer/mandate-dispatching/config.js rename to config/loket-consumer/mandatarissen-dispatching/config.js index 2afb525..13291cf 100644 --- a/config/loket-consumer/mandate-dispatching/config.js +++ b/config/loket-consumer/mandatarissen-dispatching/config.js @@ -1,18 +1,20 @@ const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; -const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql'; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql'; const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); -const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`; +// Graphs +const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`; +const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`; -if (!process.env.FILE_SYNC_GRAPH) - throw `Expected 'FILE_SYNC_GRAPH' to be provided.`; -const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH; +// ENDPOINTS +const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db'; +const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; module.exports = { BATCH_SIZE, @@ -24,6 +26,8 @@ module.exports = { SLEEP_BETWEEN_BATCHES, SLEEP_TIME_AFTER_FAILED_DB_OPERATION, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - INGEST_GRAPH, - FILE_SYNC_GRAPH + LANDING_ZONE_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT, + TARGET_GRAPH }; + diff --git a/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js b/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js new file mode 100644 index 0000000..a8df81c --- /dev/null +++ b/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js @@ -0,0 +1,11 @@ +const contextConfig = { + addTypes: { + scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both + exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta + }, + contextQueries: [] +} + +module.exports = { + contextConfig +}; diff --git a/config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js b/config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js new file mode 100644 index 0000000..b309f35 --- /dev/null +++ b/config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js @@ -0,0 +1,85 @@ +const { + transformStatements, + deleteFromTargetGraph, + insertIntoTargetGraph, +} = require('./util'); + +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, fetch } = lib; + let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; + + // Both arrays are the same length, so we can zip them together for easier processing + zippedChangeSets = termObjectChangeSets.map((o, i) => ({ + original: o, + withContext: termObjectChangeSetsWithContext[i] + })); + + console.log(`Received ${zippedChangeSets.length} change sets`) + + for (let { original, withContext } of zippedChangeSets) { + let originalDeletes, originalInserts, insertsWithContext, deletesWithContext; + try { + originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + } catch (e) { + console.log(`Error while processing change set: ${e}`); + console.log(`Original change set: ${JSON.stringify(original)}`); + throw e; + } + + // Extra context needed for mapping from DL to OP model and filtering based on type. + try { + insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + } catch (e) { + console.log(`Error while processing change set with context: ${e}`); + console.log(`Change set with context: ${JSON.stringify(withContext)}`); + throw e; + } + + if (originalDeletes.length) { + // Map deletes from DL to OP model + const transformedDeletes = await transformStatements(fetch, deletesWithContext); + + if (!transformedDeletes.length) { + console.log(`Warn: Delete statements mapped to empty result.`); + console.log(`Input: ${deletesWithContext}`); + console.log(`Output: ${transformedDeletes}`); + } else { + await deleteFromTargetGraph(lib, transformedDeletes); + } + } + + if (originalInserts.length) { + // Map inserts from DL to OP model + const transformedInserts = await transformStatements(fetch, insertsWithContext); + + if (!transformedInserts.length) { + console.log(`Warn: Insert statements mapped to empty result.`); + console.log(`Input: ${insertsWithContext}`); + console.log(`Output: ${transformedInserts}`); + } else { + await insertIntoTargetGraph(lib, transformedInserts); + } + } + } +} + +module.exports = { + dispatch +}; diff --git a/config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js b/config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js new file mode 100644 index 0000000..d5d672f --- /dev/null +++ b/config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js @@ -0,0 +1,85 @@ +const { batchedDbUpdate, transformLandingZoneGraph } = require('./util'); +const { + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + TARGET_GRAPH, + LANDING_ZONE_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT +} = require('./config'); + +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo } = lib; + const { termObjects } = data; + + // Steps: + // 1. Write the triples to the landing zone graph - withouth any mapping or filtering + // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + await batchedDbUpdate( + muAuthSudo.updateSudo, + LANDING_ZONE_GRAPH, + originalInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + LANDING_ZONE_DATABASE_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } +} + +async function onFinishInitialIngest(lib) { + const { muAuthSudo, fetch } = lib; + + console.log(`!! On-finish triggered !!`); + + const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint); + + console.log(`Transformed ${transformedInsertTriples.length} triples`); + + await batchedDbUpdate( + muAuthSudo.updateSudo, + TARGET_GRAPH, + transformedInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); +} + +module.exports = { + dispatch, + onFinishInitialIngest +}; diff --git a/config/loket-consumer/positions-dispatching/util.js b/config/loket-consumer/mandatarissen-dispatching/util.js similarity index 50% rename from config/loket-consumer/positions-dispatching/util.js rename to config/loket-consumer/mandatarissen-dispatching/util.js index f6411a2..524fe45 100644 --- a/config/loket-consumer/positions-dispatching/util.js +++ b/config/loket-consumer/mandatarissen-dispatching/util.js @@ -1,9 +1,16 @@ const { MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + TARGET_GRAPH, + LANDING_ZONE_GRAPH, + BATCH_SIZE } = require('./config'); -async function batchedDbUpdate(muUpdate, +async function batchedDbUpdate( + muUpdate, graph, triples, extraHeaders, @@ -37,39 +44,6 @@ ${batch} } } -async function deleteFromAllGraphs(muUpdate, - triples, - extraHeaders, - endpoint, - maxAttempts, - sleepBetweenBatches = 1000, - sleepTimeOnFail = 1000, -) { - - for (const triple of triples) { - - console.log(`Deleting a triple from all graphs in triplestore`); - - const deleteCall = async () => { - await muUpdate(` - DELETE { - GRAPH ?g { - ${triple} - } - } WHERE { - GRAPH ?g { - ${triple} - } - } - `, extraHeaders, endpoint); - }; - - await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); - console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); - await new Promise(r => setTimeout(r, sleepBetweenBatches)); - } -} - async function operationWithRetry(callback, attempt, maxAttempts, @@ -110,18 +84,16 @@ function partition(arr, fn) { return { passes, fails }; } - /** * Send triples to reasoning service for conversion * */ -function transformTriples(fetch, triples, mapping) { - return operationWithRetry(mainConversion(fetch, triples, mapping), 0, +async function transformTriples(fetch, triples, mapping) { + return await operationWithRetry(mainConversion(fetch, triples, mapping), 0, MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); } - -function mainConversion(fetch, triples, mapping) { +async function mainConversion(fetch, triples, mapping) { let formdata = new URLSearchParams(); formdata.append("data", triples); @@ -131,24 +103,66 @@ function mainConversion(fetch, triples, mapping) { redirect: 'follow' }; - return fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) - .then(response => response.text()); + const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) + return await response.text(); } +async function transformStatements(fetch, triples, mapping = 'main') { + console.log(`Received ${JSON.stringify(triples)} to transform`); + const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping); + statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : []; + console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`); + return statements; +} -function transformStatements(fetch, triples, mapping) { - return transformTriples(fetch, triples.join('\n'), mapping).then( - graph => { - statements = graph ? graph.split('\n') : []; - console.log(`CONVERSION ${mapping}: FROM ${triples.length} triples to ${statements.length}`) - return statements - } - ) +async function deleteFromTargetGraph(lib, statements) { + console.log(`Deleting ${statements.length} statements from target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + TARGET_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); +} + +async function insertIntoTargetGraph(lib, statements) { + console.log(`Inserting ${statements.length} statements into target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + TARGET_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); +} + +async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') { + console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`); + + const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`); + const text = await response.text(); + const statements = text.replace(/\n{2,}/g, '').split('\n'); + + return statements; } module.exports = { batchedDbUpdate, - deleteFromAllGraphs, partition, - transformStatements + transformStatements, + transformLandingZoneGraph, + deleteFromTargetGraph, + insertIntoTargetGraph }; diff --git a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js deleted file mode 100644 index 669895b..0000000 --- a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js +++ /dev/null @@ -1,70 +0,0 @@ -const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util'); -const { - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH -} = require('./config'); - -/** -* Dispatch the fetched information to a target graph. -* Note: will be ADDED to it's own graph. -* We take only care of adding them, not updating triples, this is a TODO -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, muAuthSudo, fetch } = lib; - const { termObjectChangeSets } = data; - - for (let { deletes, inserts } of termObjectChangeSets) { - const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - - if (deleteStatements.length) { - const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements); - if (transformedStatementsToDelete.length) { - await deleteFromAllGraphs( - muAuthSudo.updateSudo, - transformedStatementsToDelete, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - ); - } - } - - if (insertStatements.length) { - const transformedStatementsToInsert = await transformStatements(fetch, insertStatements); - if (transformedStatementsToInsert.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatementsToInsert, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - "INSERT" - ); - } - } - } -} - -module.exports = { - dispatch -}; diff --git a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js deleted file mode 100644 index 152abda..0000000 --- a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js +++ /dev/null @@ -1,57 +0,0 @@ -const { transformStatements, batchedDbUpdate } = require('./util'); -const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH -} = require('./config'); -const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; - -/** -* Dispatch the fetched information to a target graph. -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjects } data - The fetched quad information, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, muAuthSudo, fetch } = lib; - const { termObjects } = data; - - if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { - console.warn(`Service configured to skip MU_AUTH!`); - } - console.log(`Using ${endpoint} to insert triples`); - - if (termObjects.length) { - originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - - const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples); - if (transformedStatementsToInsert.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatementsToInsert, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ) - } - } -} - -module.exports = { - dispatch -}; diff --git a/config/loket-consumer/mandate-dispatching/util.js b/config/loket-consumer/mandate-dispatching/util.js deleted file mode 100644 index bdf477b..0000000 --- a/config/loket-consumer/mandate-dispatching/util.js +++ /dev/null @@ -1,153 +0,0 @@ -const { - MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION -} = require('./config'); - -async function batchedDbUpdate(muUpdate, - graph, - triples, - extraHeaders, - endpoint, - batchSize, - maxAttempts, - sleepBetweenBatches = 1000, - sleepTimeOnFail = 1000, - operation = 'INSERT' -) { - - for (let i = 0; i < triples.length; i += batchSize) { - console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); - - const batch = triples.slice(i, i + batchSize).join('\n'); - - const insertCall = async () => { - await muUpdate(` -${operation} DATA { -GRAPH <${graph}> { -${batch} -} -} -`, extraHeaders, endpoint); - }; - - await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); - - console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); - await new Promise(r => setTimeout(r, sleepBetweenBatches)); - } -} - -async function deleteFromAllGraphs(muUpdate, - triples, - extraHeaders, - endpoint, - maxAttempts, - sleepBetweenBatches = 1000, - sleepTimeOnFail = 1000, -) { - - for (const triple of triples) { - - console.log(`Deleting a triple from all graphs in triplestore`); - - const deleteCall = async () => { - await muUpdate(` - DELETE { - GRAPH ?g { - ${triple} - } - } WHERE { - GRAPH ?g { - ${triple} - } - } - `, extraHeaders, endpoint); - }; - - await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail); - console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); - await new Promise(r => setTimeout(r, sleepBetweenBatches)); - } -} - -async function operationWithRetry(callback, - attempt, - maxAttempts, - sleepTimeOnFail) { - try { - if (typeof callback === "function") - return await callback(); - else // Catch error from promise - not how I would do it normally, but allows re use of existing code. - return await callback; - } - catch (e) { - console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); - console.log(`Error: ${e}`); - console.log(`Sleeping ${sleepTimeOnFail} ms`); - - if (attempt >= maxAttempts) { - console.log(`Max attempts reached for ${callback.toString()}, giving up`); - throw e; - } - - await new Promise(r => setTimeout(r, sleepTimeOnFail)); - return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); - } -} - -/** - * Splits an array into two parts, a part that passes and a part that fails a predicate function. - * Credits: https://github.com/benjay10 - * @public - * @function partition - * @param {Array} arr - Array to be partitioned - * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. - * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively - */ -function partition(arr, fn) { - let passes = [], fails = []; - arr.forEach((item) => (fn(item) ? passes : fails).push(item)); - return { passes, fails }; -} - - -/** - * Send triples to reasoning service for conversion - * - */ -function transformTriples(fetch, triples) { - return operationWithRetry(mainConversion(fetch, triples), 0, - MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); -} - - -function mainConversion(fetch, triples) { - let formdata = new URLSearchParams(); - formdata.append("data", triples); - - let requestOptions = { - method: 'POST', - body: formdata, - redirect: 'follow' - }; - - return fetch("http://reasoner/reason/dl2op/main", requestOptions) - .then(response => response.text()); -} - -function transformStatements(fetch, triples) { - return transformTriples(fetch, triples.join('\n')).then( - graph => { - statements = graph.replace(/\n{2,}/g, '').split('\n') - console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`) - return statements - } - ) -} - -module.exports = { - batchedDbUpdate, - deleteFromAllGraphs, - partition, - transformStatements -}; diff --git a/config/loket-consumer/positions-dispatching/config.js b/config/loket-consumer/positions-dispatching/config.js deleted file mode 100644 index ab34eca..0000000 --- a/config/loket-consumer/positions-dispatching/config.js +++ /dev/null @@ -1,51 +0,0 @@ -const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; -const MU_CALL_SCOPE_ID_INITIAL_SYNC = - process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || - "http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync"; -const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = - process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == "true" ? true : false; -const DIRECT_DATABASE_ENDPOINT = - process.env.DIRECT_DATABASE_ENDPOINT || "http://virtuoso:8890/sparql"; -const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); -const MAX_REASONING_RETRY_ATTEMPTS = parseInt( - process.env.MAX_REASONING_RETRY_ATTEMPTS || 5 -); -const SLEEP_BETWEEN_BATCHES = parseInt( - process.env.SLEEP_BETWEEN_BATCHES || 1000 -); -const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt( - process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000 -); -const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt( - process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000 -); -const INGEST_GRAPH = - process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/worship-service`; -const PRIVACY_SENSITIVE_GRAPH = - process.env.PRIVACY_SENSITIVE_GRAPH || - `http://mu.semte.ch/graphs/worship-privacy-centric-graph`; -const UNFILTERED_MAPPING = "main"; -const MAIN_INFO_MAPPING = "main-info"; -const PRIVATE_INFO_MAPPING = "private-info"; - -if (!process.env.FILE_SYNC_GRAPH) - throw `Expected 'FILE_SYNC_GRAPH' to be provided.`; -const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH; - -module.exports = { - BATCH_SIZE, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MAX_DB_RETRY_ATTEMPTS, - MAX_REASONING_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, - INGEST_GRAPH, - PRIVACY_SENSITIVE_GRAPH, - UNFILTERED_MAPPING, - MAIN_INFO_MAPPING, - PRIVATE_INFO_MAPPING, - FILE_SYNC_GRAPH, -}; diff --git a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js deleted file mode 100644 index 5384005..0000000 --- a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js +++ /dev/null @@ -1,88 +0,0 @@ -const { transformStatements, batchedDbUpdate, deleteFromAllGraphs } = require('./util'); -const { - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH, - PRIVACY_SENSITIVE_GRAPH, - UNFILTERED_MAPPING, - MAIN_INFO_MAPPING, - PRIVATE_INFO_MAPPING, -} = require('./config'); - -/** -* Dispatch the fetched information to a target graph. -* Note: will be ADDED to it's own graph. -* We take only care of adding them, not updating triples, this is a TODO -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, muAuthSudo, fetch } = lib; - const { termObjectChangeSets } = data; - - for (let { deletes, inserts } of termObjectChangeSets) { - const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); - - if (deleteStatements.length) { - const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING); - if (transformedStatementsToDelete.length) { - await deleteFromAllGraphs( - muAuthSudo.updateSudo, - transformedStatementsToDelete, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - ); - } - } - - if (insertStatements.length) { - const transformedStatementsToInsert = await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING); - if (transformedStatementsToInsert.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatementsToInsert, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } - - const transformedStatementsToInsertPrivate = await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING); - if (transformedStatementsToInsertPrivate.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - PRIVACY_SENSITIVE_GRAPH, - transformedStatementsToInsertPrivate, - { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' }, - process.env.MU_SPARQL_ENDPOINT, // Note: this is the default endpoint through auth - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } - } - } -} - -module.exports = { - dispatch -}; diff --git a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js deleted file mode 100644 index 86f34b7..0000000 --- a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js +++ /dev/null @@ -1,75 +0,0 @@ -const { transformStatements, batchedDbUpdate, } = require('./util'); -const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, - DIRECT_DATABASE_ENDPOINT, - MU_CALL_SCOPE_ID_INITIAL_SYNC, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION, - INGEST_GRAPH, - PRIVACY_SENSITIVE_GRAPH, - MAIN_INFO_MAPPING, - PRIVATE_INFO_MAPPING -} = require('./config'); -const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; - -/** -* Dispatch the fetched information to a target graph. -* @param { mu, muAuthSudo } lib - The provided libraries from the host service. -* @param { termObjects } data - The fetched quad information, which objects of serialized Terms -* [ { -* graph: "", -* subject: "", -* predicate: "", -* object: "^^" -* } -* ] -* @return {void} Nothing -*/ -async function dispatch(lib, data) { - const { mu, muAuthSudo, fetch } = lib; - const { termObjects } = data; - - if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { - console.warn(`Service configured to skip MU_AUTH!`); - } - console.log(`Using ${endpoint} to insert triples`); - - if (termObjects.length) { - originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`) - - const transformedStatementsToInsert = await transformStatements(fetch, originalInsertTriples, MAIN_INFO_MAPPING); - if (transformedStatementsToInsert.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - INGEST_GRAPH, - transformedStatementsToInsert, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } - - const transformedStatementsToInsertPrivate = await transformStatements(fetch, originalInsertTriples, PRIVATE_INFO_MAPPING); - if (transformedStatementsToInsertPrivate.length) { - await batchedDbUpdate( - muAuthSudo.updateSudo, - PRIVACY_SENSITIVE_GRAPH, - transformedStatementsToInsertPrivate, - { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, - endpoint, - BATCH_SIZE, - MAX_DB_RETRY_ATTEMPTS, - SLEEP_BETWEEN_BATCHES, - SLEEP_TIME_AFTER_FAILED_DB_OPERATION - ); - } - } -} - -module.exports = { - dispatch -}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/config.js b/config/loket-consumer/worship-services-sensitive-dispatching/config.js new file mode 100644 index 0000000..e89bca3 --- /dev/null +++ b/config/loket-consumer/worship-services-sensitive-dispatching/config.js @@ -0,0 +1,40 @@ +const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100; +const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync'; +const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false; +const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql'; +const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5); +const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5); +const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000); +const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000); +const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000); + +// Graphs +const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/worship-service`; +const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/worship-privacy-centric-graph`; +const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`; + +// Mappings +const MAIN_INFO_MAPPING = "main-info"; +const PRIVATE_INFO_MAPPING = "private-info"; + +// ENDPOINTS +const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db'; +const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`; + +module.exports = { + BATCH_SIZE, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MAX_DB_RETRY_ATTEMPTS, + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + LANDING_ZONE_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT, + TARGET_GRAPH, + PRIVACY_SENSITIVE_GRAPH, + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING +}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js b/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js new file mode 100644 index 0000000..a8df81c --- /dev/null +++ b/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js @@ -0,0 +1,11 @@ +const contextConfig = { + addTypes: { + scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both + exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta + }, + contextQueries: [] +} + +module.exports = { + contextConfig +}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js b/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js new file mode 100644 index 0000000..ef18e2e --- /dev/null +++ b/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js @@ -0,0 +1,114 @@ +const { + transformStatements, + deleteFromTargetGraph, + insertIntoTargetGraph, + deleteFromPrivateGraph, + insertIntoPrivateGraph +} = require('./util'); + +const { + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING +} = require('./config'); + +/** +* Dispatch the fetched information to a target graph. +* Note: will be ADDED to it's own graph. +* We take only care of adding them, not updating triples, this is a TODO +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, fetch } = lib; + let { termObjectChangeSets, termObjectChangeSetsWithContext } = data; + + // Both arrays are the same length, so we can zip them together for easier processing + zippedChangeSets = termObjectChangeSets.map((o, i) => ({ + original: o, + withContext: termObjectChangeSetsWithContext[i] + })); + + console.log(`Received ${zippedChangeSets.length} change sets`) + + for (let { original, withContext } of zippedChangeSets) { + let originalDeletes, originalInserts, insertsWithContext, deletesWithContext; + try { + originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + } catch (e) { + console.log(`Error while processing change set: ${e}`); + console.log(`Original change set: ${JSON.stringify(original)}`); + throw e; + } + + // Extra context needed for mapping from DL to OP model and filtering based on type. + try { + insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + } catch (e) { + console.log(`Error while processing change set with context: ${e}`); + console.log(`Change set with context: ${JSON.stringify(withContext)}`); + throw e; + } + + if (originalDeletes.length) { + // Map main deletes from DL to OP model + const transformedMainDeletes = await transformStatements(fetch, deletesWithContext, MAIN_INFO_MAPPING); + + if (!transformedMainDeletes.length) { + console.log(`Warn: Delete main statements mapped to empty result.`); + console.log(`Input: ${deletesWithContext}`); + console.log(`Output: ${transformedMainDeletes}`); + } else { + await deleteFromTargetGraph(lib, transformedMainDeletes); + } + + // Map private deletes from DL to OP model + const transformedPrivateDeletes = await transformStatements(fetch, deletesWithContext, PRIVATE_INFO_MAPPING); + + if (!transformedPrivateDeletes.length) { + console.log(`Warn: Delete private statements mapped to empty result.`); + console.log(`Input: ${deletesWithContext}`); + console.log(`Output: ${transformedPrivateDeletes}`); + } else { + await deleteFromPrivateGraph(lib, transformedDeletes); + } + } + + if (originalInserts.length) { + // Map main inserts from DL to OP model + const transformedMainInserts = await transformStatements(fetch, insertsWithContext, MAIN_INFO_MAPPING); + + if (!transformedMainInserts.length) { + console.log(`Warn: Insert statements mapped to empty result.`); + console.log(`Input: ${insertsWithContext}`); + console.log(`Output: ${transformedMainInserts}`); + } else { + await insertIntoTargetGraph(lib, transformedMainInserts); + } + + // Map private inserts from DL to OP model + const transformedPrivateInserts = await transformStatements(fetch, insertsWithContext, PRIVATE_INFO_MAPPING); + + if (!transformedPrivateInserts.length) { + console.log(`Warn: Insert statements mapped to empty result.`); + console.log(`Input: ${insertsWithContext}`); + console.log(`Output: ${transformedPrivateInserts}`); + } else { + await insertIntoPrivateGraph(lib, transformedPrivateInserts); + } + } + } +} + +module.exports = { + dispatch +}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js b/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js new file mode 100644 index 0000000..559250f --- /dev/null +++ b/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js @@ -0,0 +1,106 @@ +const { batchedDbUpdate, transformLandingZoneGraph } = require('./util'); +const { + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES, + DIRECT_DATABASE_ENDPOINT, + MU_CALL_SCOPE_ID_INITIAL_SYNC, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + TARGET_GRAPH, + PRIVACY_SENSITIVE_GRAPH, + LANDING_ZONE_GRAPH, + LANDING_ZONE_DATABASE_ENDPOINT, + MAIN_INFO_MAPPING, + PRIVATE_INFO_MAPPING +} = require('./config'); + +const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT; + +/** +* Dispatch the fetched information to a target graph. +* @param { mu, muAuthSudo } lib - The provided libraries from the host service. +* @param { termObjects } data - The fetched quad information, which objects of serialized Terms +* [ { +* graph: "", +* subject: "", +* predicate: "", +* object: "^^" +* } +* ] +* @return {void} Nothing +*/ +async function dispatch(lib, data) { + const { mu, muAuthSudo } = lib; + const { termObjects } = data; + + // Steps: + // 1. Write the triples to the landing zone graph - withouth any mapping or filtering + // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish) + + if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) { + console.warn(`Service configured to skip MU_AUTH!`); + } + console.log(`Using ${endpoint} to insert triples`); + + if (termObjects.length) { + const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`); + + await batchedDbUpdate( + muAuthSudo.updateSudo, + LANDING_ZONE_GRAPH, + originalInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + LANDING_ZONE_DATABASE_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } +} + +async function onFinishInitialIngest(lib) { + const { muAuthSudo, fetch } = lib; + + console.log(`!! On-finish triggered !!`); + + const transformedMainInsertTriples = await transformLandingZoneGraph(fetch, endpoint, MAIN_INFO_MAPPING); + console.log(`Transformed ${transformedMainInsertTriples.length} main triples`); + + if (transformedMainInsertTriples.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + TARGET_GRAPH, + transformedMainInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } + + const transformedPrivateInsertTriples = await transformLandingZoneGraph(fetch, endpoint, PRIVATE_INFO_MAPPING); + console.log(`Transformed ${transformedPrivateInsertTriples.length} private triples`); + + if (transformedPrivateInsertTriples.length) { + await batchedDbUpdate( + muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + transformedPrivateInsertTriples, + { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC }, + endpoint, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION + ); + } +} + +module.exports = { + dispatch, + onFinishInitialIngest +}; diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/util.js b/config/loket-consumer/worship-services-sensitive-dispatching/util.js new file mode 100644 index 0000000..4b17b75 --- /dev/null +++ b/config/loket-consumer/worship-services-sensitive-dispatching/util.js @@ -0,0 +1,204 @@ +const { + MAX_REASONING_RETRY_ATTEMPTS, + SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + TARGET_GRAPH, + PRIVACY_SENSITIVE_GRAPH, + LANDING_ZONE_GRAPH, + BATCH_SIZE +} = require('./config'); + +async function batchedDbUpdate( + muUpdate, + graph, + triples, + extraHeaders, + endpoint, + batchSize, + maxAttempts, + sleepBetweenBatches = 1000, + sleepTimeOnFail = 1000, + operation = 'INSERT' +) { + + for (let i = 0; i < triples.length; i += batchSize) { + console.log(`Inserting triples in batch: ${i}-${i + batchSize}`); + + const batch = triples.slice(i, i + batchSize).join('\n'); + + const insertCall = async () => { + await muUpdate(` +${operation} DATA { +GRAPH <${graph}> { +${batch} +} +} +`, extraHeaders, endpoint); + }; + + await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail); + + console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`); + await new Promise(r => setTimeout(r, sleepBetweenBatches)); + } +} + +async function operationWithRetry(callback, + attempt, + maxAttempts, + sleepTimeOnFail) { + try { + if (typeof callback === "function") + return await callback(); + else // Catch error from promise - not how I would do it normally, but allows re use of existing code. + return await callback; + } + catch (e) { + console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`); + console.log(`Error: ${e}`); + console.log(`Sleeping ${sleepTimeOnFail} ms`); + + if (attempt >= maxAttempts) { + console.log(`Max attempts reached for ${callback.toString()}, giving up`); + throw e; + } + + await new Promise(r => setTimeout(r, sleepTimeOnFail)); + return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail); + } +} + +/** + * Splits an array into two parts, a part that passes and a part that fails a predicate function. + * Credits: https://github.com/benjay10 + * @public + * @function partition + * @param {Array} arr - Array to be partitioned + * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value. + * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively + */ +function partition(arr, fn) { + let passes = [], fails = []; + arr.forEach((item) => (fn(item) ? passes : fails).push(item)); + return { passes, fails }; +} + +/** + * Send triples to reasoning service for conversion + * + */ +async function transformTriples(fetch, triples, mapping) { + return await operationWithRetry(mainConversion(fetch, triples, mapping), 0, + MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION); +} + +async function mainConversion(fetch, triples, mapping) { + let formdata = new URLSearchParams(); + formdata.append("data", triples); + + let requestOptions = { + method: 'POST', + body: formdata, + redirect: 'follow' + }; + + const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions) + return await response.text(); +} + +async function transformStatements(fetch, triples, mapping = 'main') { + console.log(`Received ${JSON.stringify(triples)} to transform`); + const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping); + statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : []; + console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`); + return statements; +} + +async function deleteFromTargetGraph(lib, statements) { + console.log(`Deleting ${statements.length} statements from target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + TARGET_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); +} + +async function deleteFromPrivateGraph(lib, statements) { + console.log(`Deleting ${statements.length} statements from target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'DELETE'); +} + +async function insertIntoTargetGraph(lib, statements) { + console.log(`Inserting ${statements.length} statements into target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + TARGET_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); +} + +async function insertIntoPrivateGraph(lib, statements) { + console.log(`Inserting ${statements.length} statements into target graph`); + console.log(`Statements: ${JSON.stringify(statements)}`) + + await batchedDbUpdate( + lib.muAuthSudo.updateSudo, + PRIVACY_SENSITIVE_GRAPH, + statements, + {}, + process.env.MU_SPARQL_ENDPOINT, + BATCH_SIZE, + MAX_DB_RETRY_ATTEMPTS, + SLEEP_BETWEEN_BATCHES, + SLEEP_TIME_AFTER_FAILED_DB_OPERATION, + 'INSERT'); +} + +async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') { + console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`); + + const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`); + const text = await response.text(); + const statements = text.replace(/\n{2,}/g, '').split('\n'); + + return statements; +} + +module.exports = { + batchedDbUpdate, + partition, + transformStatements, + transformLandingZoneGraph, + deleteFromTargetGraph, + insertIntoTargetGraph, + deleteFromPrivateGraph, + insertIntoPrivateGraph +}; diff --git a/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164600-remove-loket-persons-before-sync.sparql b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164600-remove-loket-persons-before-sync.sparql new file mode 100644 index 0000000..7f6b7b7 --- /dev/null +++ b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164600-remove-loket-persons-before-sync.sparql @@ -0,0 +1,54 @@ +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX ch: +PREFIX code: +PREFIX dbpedia: +PREFIX dc_terms: +PREFIX ere: +PREFIX euro: +PREFIX euvoc: +PREFIX ext: +PREFIX foaf: +PREFIX generiek: +PREFIX lblodlg: +PREFIX locn: +PREFIX mandaat: +PREFIX org: +PREFIX organisatie: +PREFIX person: +PREFIX persoon: +PREFIX prov: +PREFIX rdfs: +PREFIX regorg: +PREFIX schema: +PREFIX skos: +PREFIX vcard: +PREFIX xsd: + +DELETE { + GRAPH { + ?person + ?p ?o + } +} WHERE { + GRAPH { + ?person a person:Person ; + ?p ?o. + } + GRAPH { + VALUES ?typePosition { + + + } + + ?position mandaat:isBestuurlijkeAliasVan ?person ; + a ?typePosition . + } + + FILTER NOT EXISTS { + GRAPH { + ?worshipPosition ?person . + } + } +} diff --git a/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164601-remove-loket-functionarissen-before-sync.sparql b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164601-remove-loket-functionarissen-before-sync.sparql new file mode 100644 index 0000000..55e716b --- /dev/null +++ b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164601-remove-loket-functionarissen-before-sync.sparql @@ -0,0 +1,52 @@ +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX ch: +PREFIX code: +PREFIX dbpedia: +PREFIX dc_terms: +PREFIX ere: +PREFIX euro: +PREFIX euvoc: +PREFIX ext: +PREFIX foaf: +PREFIX generiek: +PREFIX lblodlg: +PREFIX locn: +PREFIX mandaat: +PREFIX org: +PREFIX organisatie: +PREFIX person: +PREFIX persoon: +PREFIX prov: +PREFIX rdfs: +PREFIX regorg: +PREFIX schema: +PREFIX skos: +PREFIX vcard: +PREFIX xsd: + +DELETE { + GRAPH { + ?functionaris ?p ?o. + } +} WHERE { + GRAPH { + ?functionaris a ; + org:holds ?bestuursFunctie ; + ?p ?o . + } + + ?bestuursorgaanInTime lblodlg:heeftBestuursfunctie ?bestuursFunctie ; + generiek:isTijdspecialisatieVan ?bestuursorgaan . + + ?bestuursorgaan besluit:bestuurt ?bestuurseenheid . + + FILTER NOT EXISTS { + VALUES ?typeBestuur { + + + } + ?bestuurseenheid a ?typeBestuur . + } +} diff --git a/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164602-remove-loket-mandatarissen-before-sync-part-1.sparql b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164602-remove-loket-mandatarissen-before-sync-part-1.sparql new file mode 100644 index 0000000..3fc6737 --- /dev/null +++ b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164602-remove-loket-mandatarissen-before-sync-part-1.sparql @@ -0,0 +1,53 @@ +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX ch: +PREFIX code: +PREFIX dbpedia: +PREFIX dc_terms: +PREFIX ere: +PREFIX euro: +PREFIX euvoc: +PREFIX ext: +PREFIX foaf: +PREFIX generiek: +PREFIX lblodlg: +PREFIX locn: +PREFIX mandaat: +PREFIX org: +PREFIX organisatie: +PREFIX person: +PREFIX persoon: +PREFIX prov: +PREFIX rdfs: +PREFIX regorg: +PREFIX schema: +PREFIX skos: +PREFIX vcard: +PREFIX xsd: + +DELETE { + GRAPH { + ?mandataris ?p ?o. + } +} WHERE { + GRAPH { + ?mandataris a ; + org:holds ?mandaat ; + ?p ?o . + } + + ?bestuursorgaanInTime org:hasPost ?mandaat ; + generiek:isTijdspecialisatieVan ?bestuursorgaan . + + ?bestuursorgaan besluit:bestuurt ?bestuurseenheid . + + FILTER NOT EXISTS { + VALUES ?typeBestuur { + + + } + ?bestuurseenheid a ?typeBestuur . + } +} +LIMIT 1000000 \ No newline at end of file diff --git a/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164603-remove-loket-mandatarissen-before-sync-part-2.sparql b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164603-remove-loket-mandatarissen-before-sync-part-2.sparql new file mode 100644 index 0000000..59d01c2 --- /dev/null +++ b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164603-remove-loket-mandatarissen-before-sync-part-2.sparql @@ -0,0 +1,53 @@ +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX ch: +PREFIX code: +PREFIX dbpedia: +PREFIX dc_terms: +PREFIX ere: +PREFIX euro: +PREFIX euvoc: +PREFIX ext: +PREFIX foaf: +PREFIX generiek: +PREFIX lblodlg: +PREFIX locn: +PREFIX mandaat: +PREFIX org: +PREFIX organisatie: +PREFIX person: +PREFIX persoon: +PREFIX prov: +PREFIX rdfs: +PREFIX regorg: +PREFIX schema: +PREFIX skos: +PREFIX vcard: +PREFIX xsd: + +DELETE { + GRAPH { + ?mandataris ?p ?o. + } +} WHERE { + GRAPH { + ?mandataris a ; + org:holds ?mandaat ; + ?p ?o . + } + + ?bestuursorgaanInTime org:hasPost ?mandaat ; + generiek:isTijdspecialisatieVan ?bestuursorgaan . + + ?bestuursorgaan besluit:bestuurt ?bestuurseenheid . + + FILTER NOT EXISTS { + VALUES ?typeBestuur { + + + } + ?bestuurseenheid a ?typeBestuur . + } +} +LIMIT 1000000 diff --git a/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164604-remove-loket-mandatarissen-before-sync-part-3.sparql b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164604-remove-loket-mandatarissen-before-sync-part-3.sparql new file mode 100644 index 0000000..e7d6d00 --- /dev/null +++ b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164604-remove-loket-mandatarissen-before-sync-part-3.sparql @@ -0,0 +1,52 @@ +PREFIX adms: +PREFIX adres: +PREFIX besluit: +PREFIX ch: +PREFIX code: +PREFIX dbpedia: +PREFIX dc_terms: +PREFIX ere: +PREFIX euro: +PREFIX euvoc: +PREFIX ext: +PREFIX foaf: +PREFIX generiek: +PREFIX lblodlg: +PREFIX locn: +PREFIX mandaat: +PREFIX org: +PREFIX organisatie: +PREFIX person: +PREFIX persoon: +PREFIX prov: +PREFIX rdfs: +PREFIX regorg: +PREFIX schema: +PREFIX skos: +PREFIX vcard: +PREFIX xsd: + +DELETE { + GRAPH { + ?mandataris ?p ?o. + } +} WHERE { + GRAPH { + ?mandataris a ; + org:holds ?mandaat ; + ?p ?o . + } + + ?bestuursorgaanInTime org:hasPost ?mandaat ; + generiek:isTijdspecialisatieVan ?bestuursorgaan . + + ?bestuursorgaan besluit:bestuurt ?bestuurseenheid . + + FILTER NOT EXISTS { + VALUES ?typeBestuur { + + + } + ?bestuurseenheid a ?typeBestuur . + } +} diff --git a/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164605-remove-ministers-that-will-get-synced.sparql b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164605-remove-ministers-that-will-get-synced.sparql new file mode 100644 index 0000000..76dfb29 --- /dev/null +++ b/config/migrations/2023/20230330164600-sync-loket-to-op/20230330164605-remove-ministers-that-will-get-synced.sparql @@ -0,0 +1,21 @@ +PREFIX ere: + +DELETE { + GRAPH { + ?ministerPosition ?p ?o . + ?worshipService ?ministerPosition . + } +} WHERE { + GRAPH { + ?ministerPosition a ere:PositieBedienaar ; + ?p ?o . + + VALUES ?status { + # Actief + # Niet Actief + } + + ?worshipService ?ministerPosition ; + ?status . + } +} diff --git a/config/reasoner/dl2op/main/inverse_relations.n3 b/config/reasoner/dl2op/main/inverse_relations.n3 index bfeda31..ea25c90 100644 --- a/config/reasoner/dl2op/main/inverse_relations.n3 +++ b/config/reasoner/dl2op/main/inverse_relations.n3 @@ -18,7 +18,6 @@ PREFIX skos: PREFIX string: PREFIX xsd: - # Inverse properties { ?s org:holds ?o. diff --git a/config/reasoner/dl2op/main/mapping.n3 b/config/reasoner/dl2op/main/mapping.n3 index 82bebd9..1838441 100644 --- a/config/reasoner/dl2op/main/mapping.n3 +++ b/config/reasoner/dl2op/main/mapping.n3 @@ -18,7 +18,6 @@ PREFIX skos: PREFIX string: PREFIX xsd: - # DL and OP both use persoon:gebruikteVoornaam but with different semantics (!) # Hence the mapping is in two steps. # The first step maps the DL persoon:gebruikteVoornaam to the OP counterpart @@ -92,6 +91,31 @@ PREFIX xsd: ?s mandaat:einde ?eindeDate }. +{ + ?s mandaat:bindingStart ?bindingStartDateTime. + # use backward arrow rule defined below + ?bindingStartDateTime :dateFromDateTime ?bindingStartDate. + { + ?s mandaat:bindingStart ?bindingStartDateTime + } e:becomes { + ?s mandaat:bindingStart ?bindingStartDate + }. +} => { + ?s mandaat:bindingStart ?bindingStartDate +}. + +{ + ?s mandaat:bindingEinde ?bindingEindeDateTime. + # use backward arrow rule defined below + ?bindingEindeDateTime :dateFromDateTime ?bindingEindeDate. + { + ?s mandaat:bindingEinde ?bindingEindeDateTime + } e:becomes { + ?s mandaat:bindingEinde ?bindingEindeDate + }. +} => { + ?s mandaat:bindingEinde ?bindingEindeDate +}. # Backward arrow rule is like a function call. # Removes time info in the returned date diff --git a/config/reasoner/dl2op/main/persoon_http_to_https.n3 b/config/reasoner/dl2op/main/persoon_http_to_https.n3 index 6ced405..3658d1d 100644 --- a/config/reasoner/dl2op/main/persoon_http_to_https.n3 +++ b/config/reasoner/dl2op/main/persoon_http_to_https.n3 @@ -18,7 +18,6 @@ PREFIX skos: PREFIX string: PREFIX xsd: - # Map from http persoon to https persoon namespace. ## classes: diff --git a/config/reasoner/dl2op/main/query.n3q b/config/reasoner/dl2op/main/query.n3q new file mode 100644 index 0000000..9ff9d10 --- /dev/null +++ b/config/reasoner/dl2op/main/query.n3q @@ -0,0 +1,63 @@ +# Filtering classes to ingest + +{ + ?s a ; + ?p ?o . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?p ?o . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?p ?o . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?p ?o . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?p ?o . + } => { + ?s a ; + ?p ?o . +}. + +{ + ?s a ; + ?p ?o . + } => { + ?s a ; + ?p ?o . +}. +{ + ?s a ; + ?p ?o . + } => { + ?s a ; + ?p ?o . +}. +{ + ?s a ; + ?p ?o . + } => { + ?s a ; + ?p ?o . +}. diff --git a/config/reasoner/dl2op/main/retract_empty_strings.n3 b/config/reasoner/dl2op/main/retract_empty_strings.n3 index f8204a7..dfe2ddd 100644 --- a/config/reasoner/dl2op/main/retract_empty_strings.n3 +++ b/config/reasoner/dl2op/main/retract_empty_strings.n3 @@ -18,7 +18,6 @@ PREFIX skos: PREFIX string: PREFIX xsd: - # Retract empty strings { { diff --git a/config/reasoner/dl2op/main/rrn.n3 b/config/reasoner/dl2op/main/rrn.n3 index b45c688..a87e31d 100644 --- a/config/reasoner/dl2op/main/rrn.n3 +++ b/config/reasoner/dl2op/main/rrn.n3 @@ -18,8 +18,6 @@ PREFIX skos: PREFIX string: PREFIX xsd: - - # Rijksregisternummer # Extend identifiers to match OP's model # skos:notation "00010100105". diff --git a/config/resources/domain.lisp b/config/resources/domain.lisp index 2f6b2a8..8b83654 100644 --- a/config/resources/domain.lisp +++ b/config/resources/domain.lisp @@ -9,6 +9,7 @@ (defparameter *cache-model-properties* t) (defparameter *cache-count-queries* t) (defparameter *supply-cache-headers-p* t) +(defparameter sparql:*experimental-no-application-graph-for-sudo-select-queries* t) (read-domain-file "domain.json") (read-domain-file "auth.json") diff --git a/docker-compose.yml b/docker-compose.yml index 7290967..32e0c9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,7 +35,7 @@ services: restart: always logging: *default-logging db: - image: semtech/mu-authorization:feature-service-roam-r1 + image: semtech/mu-authorization:feature-service-roam-r1.1 environment: MU_SPARQL_ENDPOINT: "http://triplestore:8890/sparql" LOG_OUTGOING_SPARQL_QUERIES: "true" @@ -43,6 +43,8 @@ services: LOG_ACCESS_RIGHTS: "true" LOG_OUTGOING_SPARQL_QUERY_ROUNDTRIP: "true" ERROR_ON_UNWRITTEN_DATA: "true" + QUERY_MAX_PROCESSING_TIME: "300000" + QUERY_MAX_EXECUTION_TIME: "300000" volumes: - ./config/authorization:/config labels: @@ -70,7 +72,7 @@ services: restart: always logging: *default-logging resource: - image: semtech/mu-cl-resources:1.21.1 + image: semtech/mu-cl-resources:1.22.0 environment: CACHE_CLEAR_PATH: "http://cache/.mu/clear-keys" links: @@ -82,7 +84,7 @@ services: restart: always logging: *default-logging cache: - image: semtech/mu-cache:2.0.1 + image: semtech/mu-cache:2.0.2 links: - resource:backend labels: @@ -226,7 +228,7 @@ services: restart: always logging: *default-logging reasoner: - image: eyereasoner/reasoning-service:1.0.1 + image: eyereasoner/reasoning-service:1.0.2 volumes: - ./config/reasoner:/config ################################################################################ @@ -418,25 +420,28 @@ services: # DELTA CONSUMERS ################################################################################ ################################################################################ - # LOKET MANDATEN CONSUMERS + # LOKET MANDATARISSEN CONSUMERS ################################################################################ - mandaten-consumer: - image: lblod/delta-consumer:0.0.14 + mandatarissen-consumer: + image: lblod/delta-consumer:0.0.17-rc.7 environment: - DCR_SERVICE_NAME: "mandatendatabank-consumer" + DCR_SERVICE_NAME: "mandatarissen-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/MandatarissenCacheGraphDump" DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/initialSync/mandatarissen" DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/deltaSync/mandatarissen" - DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatendatabank-consumer" + DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/mandatarissen-consumer" DCR_DISABLE_INITIAL_SYNC: "true" BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" + BATCH_SIZE: 1000 DCR_SYNC_FILES_PATH: "/sync/mandatarissen/files" - FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data" - KEEP_DELTA_FILES: "true" + DCR_KEEP_DELTA_FILES: "true" + DCR_ENABLE_DELTA_CONTEXT: "true" + DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/mandatarissen" volumes: - - ./config/loket-consumer/mandate-dispatching:/config/triples-dispatching/custom-dispatching - - ./data/files/consumer-files/mandaten:/consumer-files/ + - ./config/loket-consumer/mandatarissen-dispatching:/config/triples-dispatching/custom-dispatching + - ./data/files/consumer-files/mandatarissen:/consumer-files/ restart: always labels: - "logging=true" @@ -449,7 +454,7 @@ services: # LOKET LEIDINGGEVENDEN CONSUMERS ################################################################################ leidinggevenden-consumer: - image: lblod/delta-consumer:0.0.14 + image: lblod/delta-consumer:0.0.17-rc.7 environment: DCR_SERVICE_NAME: "leidinggevenden-consumer" DCR_SYNC_BASE_URL: "https://loket.lblod.info" # replace with link to Loket API @@ -460,8 +465,11 @@ services: DCR_DISABLE_INITIAL_SYNC: "true" BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" DCR_SYNC_FILES_PATH: "/sync/leidinggevenden/files" - FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data" - KEEP_DELTA_FILES: "true" + DCR_KEEP_DELTA_FILES: "true" + BATCH_SIZE: 1000 + DCR_ENABLE_DELTA_CONTEXT: "true" + DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/leidinggevenden" volumes: - ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching - ./data/files/consumer-files/leidinggevenden:/consumer-files/ @@ -477,22 +485,24 @@ services: # LOKET WORSHIP SENSITIVE CONSUMERS ################################################################################ worship-services-sensitive-consumer: - image: lblod/delta-consumer:0.0.14 + image: lblod/delta-consumer:0.0.17-rc.7 environment: DCR_SYNC_BASE_URL: "https://loket.lblod.info/" DCR_SERVICE_NAME: "worship-services-sensitive-consumer" - DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files" DCR_SYNC_DATASET_SUBJECT: "http://data.lblod.info/datasets/delta-producer/dumps/WorshipServicesSensitiveCacheGraphDump" DCR_INITIAL_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitive" DCR_DELTA_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/consumer/wsSensitiveDeltaFileSyncing" DCR_JOB_CREATOR_URI: "http://data.lblod.info/services/id/worship-services-sensitive-consumer" DCR_DISABLE_INITIAL_SYNC: "true" - FILE_SYNC_GRAPH: "http://mu.semte.ch/graphs/temp/original-physical-files-data" + BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "true" + DCR_SYNC_FILES_PATH: "/sync/worship-services-sensitive/files" + BATCH_SIZE: 1000 DCR_KEEP_DELTA_FILES: "true" - DCR_DELTA_FILE_FOLDER: "/consumer-files" - BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false" + DCR_ENABLE_DELTA_CONTEXT: "true" + DCR_LANDING_ZONE_DATABASE: "triplestore" + DCR_LANDING_ZONE_GRAPH: "http://mu.semte.ch/graphs/landing-zone/worship-services-sensitive" volumes: - - ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching + - ./config/loket-consumer/worship-services-sensitive-dispatching:/config/triples-dispatching/custom-dispatching - ./data/files/consumer-files/worship-services-sensitive:/consumer-files/ restart: always labels: From 02de06921eca7bea96bf6d801eb58a6a5ec28821 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Fri, 21 Apr 2023 16:33:54 +0200 Subject: [PATCH 55/58] Remove conflicting rule If the prefix rule applies before the rule fixing the difference between Loket and OP's names first/last names models, it breaks the flow --- .../reasoner/dl2op/main-info/persoon_http_to_https.n3 | 10 ---------- config/reasoner/dl2op/main/persoon_http_to_https.n3 | 10 ---------- .../dl2op/private-info/persoon_http_to_https.n3 | 10 ---------- 3 files changed, 30 deletions(-) diff --git a/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 b/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 index 6ced405..8afa661 100644 --- a/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 +++ b/config/reasoner/dl2op/main-info/persoon_http_to_https.n3 @@ -293,16 +293,6 @@ PREFIX xsd: ?s persoon:datumVanAfstamming ?o }. -{ - { - ?s persoon_loket:gebruikteVoornaam ?o - } e:becomes { - ?s persoon:gebruikteVoornaam ?o - } -} => { - ?s persoon:gebruikteVoornaam ?o -}. - { { ?s persoon_loket:geslacht ?o diff --git a/config/reasoner/dl2op/main/persoon_http_to_https.n3 b/config/reasoner/dl2op/main/persoon_http_to_https.n3 index 3658d1d..e2f942a 100644 --- a/config/reasoner/dl2op/main/persoon_http_to_https.n3 +++ b/config/reasoner/dl2op/main/persoon_http_to_https.n3 @@ -292,16 +292,6 @@ PREFIX xsd: ?s persoon:datumVanAfstamming ?o }. -{ - { - ?s persoon_loket:gebruikteVoornaam ?o - } e:becomes { - ?s persoon:gebruikteVoornaam ?o - } -} => { - ?s persoon:gebruikteVoornaam ?o -}. - { { ?s persoon_loket:geslacht ?o diff --git a/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 b/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 index 6ced405..8afa661 100644 --- a/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 +++ b/config/reasoner/dl2op/private-info/persoon_http_to_https.n3 @@ -293,16 +293,6 @@ PREFIX xsd: ?s persoon:datumVanAfstamming ?o }. -{ - { - ?s persoon_loket:gebruikteVoornaam ?o - } e:becomes { - ?s persoon:gebruikteVoornaam ?o - } -} => { - ?s persoon:gebruikteVoornaam ?o -}. - { { ?s persoon_loket:geslacht ?o From 4ed997dabdbefe3833b49b10e1ef45320bca047c Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Tue, 2 May 2023 12:01:30 +0200 Subject: [PATCH 56/58] change date change event (#267) --- ...20230427090317-change-date-change-event.sparql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 config/migrations/2023/20230427090317-change-date-change-event.sparql diff --git a/config/migrations/2023/20230427090317-change-date-change-event.sparql b/config/migrations/2023/20230427090317-change-date-change-event.sparql new file mode 100644 index 0000000..4710ded --- /dev/null +++ b/config/migrations/2023/20230427090317-change-date-change-event.sparql @@ -0,0 +1,15 @@ +delete { +graph { + ?date. +} +} +insert { +graph { + "2022-04-22". +} +} +where { +graph { + ?date. +} +} From 4454a95137673e4442cb091b4d4d628db719295b Mon Sep 17 00:00:00 2001 From: Nordine Bittich <3816305+nbittich@users.noreply.github.com> Date: Wed, 3 May 2023 09:10:47 +0200 Subject: [PATCH 57/58] updating end date bestuursorgaan triggers too many deletes (#268) --- config/delta-producer/public/export.json | 34 +++--------------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index 4e55975..c8af817 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -279,48 +279,20 @@ }, { "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", - "strictTypeExport": "true", "graphsFilter": [ "http://mu.semte.ch/graphs/shared", - "http://mu.semte.ch/graphs/worship-service", - "http://mu.semte.ch/graphs/administrative-unit" - ], - "pathToConceptScheme": [ - "http://www.w3.org/ns/org#classification", - "http://www.w3.org/2004/02/skos/core#inScheme" + "http://mu.semte.ch/graphs/worship-service" ], "properties": [ "http://mu.semte.ch/vocabularies/core/uuid", - "http://data.vlaanderen.be/ns/mandaat#bindingEinde", - "http://data.vlaanderen.be/ns/mandaat#bindingStart", "http://data.vlaanderen.be/ns/besluit#bestuurt", "http://www.w3.org/ns/org#classification", - "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", - "http://www.w3.org/ns/org#hasPost" - ] - }, - { - "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", - "strictTypeExport": "true", - "graphsFilter": [ - "http://mu.semte.ch/graphs/shared", - "http://mu.semte.ch/graphs/worship-service", - "http://mu.semte.ch/graphs/administrative-unit" - ], - "pathToConceptScheme": [ - "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", - "http://www.w3.org/ns/org#classification", - "http://www.w3.org/2004/02/skos/core#inScheme" - ], - "properties": [ - "http://mu.semte.ch/vocabularies/core/uuid", "http://data.vlaanderen.be/ns/mandaat#bindingEinde", "http://data.vlaanderen.be/ns/mandaat#bindingStart", - "http://data.vlaanderen.be/ns/besluit#bestuurt", - "http://www.w3.org/ns/org#classification", "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", "http://www.w3.org/ns/org#hasPost" - ] + ], + "additionalFilter": "?subject (//)|(/) ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" }, { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", From b32930f4bb55eca12cac80d15b280eba122f6ca8 Mon Sep 17 00:00:00 2001 From: Claire Lovisa Date: Wed, 3 May 2023 18:52:08 +0200 Subject: [PATCH 58/58] Produce public data as done in prod (#270) --- config/delta-producer/public/export.json | 3 ++- docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json index c8af817..f0c3e85 100644 --- a/config/delta-producer/public/export.json +++ b/config/delta-producer/public/export.json @@ -281,6 +281,7 @@ "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", "graphsFilter": [ "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit", "http://mu.semte.ch/graphs/worship-service" ], "properties": [ @@ -292,7 +293,7 @@ "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", "http://www.w3.org/ns/org#hasPost" ], - "additionalFilter": "?subject (//)|(/) ?bestuurClassification . FILTER (?bestuurClassification IN (, ))" + "additionalFilter": "?subject (//)|(/) ." }, { "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", diff --git a/docker-compose.yml b/docker-compose.yml index 32e0c9c..b60d523 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -325,7 +325,7 @@ services: restart: always logging: *default-logging delta-producer-pub-graph-maintainer-public: - image: lblod/delta-producer-publication-graph-maintainer:0.15.1 + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 environment: MAX_BODY_SIZE: "50mb" JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"