diff --git a/.gitignore b/.gitignore
index f5c1878..caed09e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
# Virtuoso files /data/db
+/data/db.bck/**
/data/db/*
-!/data/db/virtuoso.ini
+/data/db/virtuoso.ini
!/data/db/toLoad
!/data/db/toLoad/*
/data/files/*
diff --git a/config/authorization/config.ex b/config/authorization/config.ex
index e52cb73..1dd43ab 100644
--- a/config/authorization/config.ex
+++ b/config/authorization/config.ex
@@ -8,70 +8,86 @@ alias Acl.GroupSpec.GraphCleanup, as: GraphCleanup
defmodule Acl.UserGroups.Config do
- @error_type [
- "https://docs.oasis-open-projects.org/oslc-op/core/v3.0/os/core-vocab.html#Error"
- ]
+ @error_type [
+ "https://docs.oasis-open-projects.org/oslc-op/core/v3.0/os/core-vocab.html#Error"
+ ]
- @protected_resource_type [
- "http://www.w3.org/ns/org#Organization",
- "http://data.vlaanderen.be/ns/besluit#Besluit",
- "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit",
- "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
+ @worship_type [
+ "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris",
"http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
"http://data.lblod.info/vocabularies/erediensten/EredienstBestuurseenheid",
- "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
"http://data.lblod.info/vocabularies/erediensten/RolBedienaar",
"http://data.lblod.info/vocabularies/erediensten/VoorwaardenBedienaar",
- "https://data.vlaanderen.be/ns/persoon#Geboorte",
- "http://mu.semte.ch/vocabularies/ext/GeslachtCode",
"http://data.lblod.info/vocabularies/erediensten/PositieBedienaar",
- "http://www.w3.org/ns/adms#Identifier",
- "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
- "http://www.w3.org/ns/org#ChangeEvent",
"http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen",
"http://data.lblod.info/vocabularies/erediensten/VerbondenJuridischeStructuren",
- "http://www.w3.org/ns/prov#Location",
- "http://www.w3.org/ns/org#Site",
- "http://www.w3.org/ns/locn#Address",
- "http://www.w3.org/ns/person#Person",
- "http://data.vlaanderen.be/ns/mandaat#Mandaat",
- "http://data.vlaanderen.be/ns/mandaat#Mandataris",
- "http://data.lblod.info/vocabularies/leidinggevenden/FunctionarisStatusCode",
- "http://data.lblod.info/vocabularies/leidinggevenden/Functionaris",
- "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris",
- "http://schema.org/ContactPoint",
- "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie",
"http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst",
"http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan",
+ ]
+
+ @public_type [
"http://www.w3.org/ns/org#Role",
- "http://www.w3.org/ns/org#Post",
- "http://data.lblod.info/vocabularies/contacthub/AgentInPositie",
+ "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
+ "http://xmlns.com/foaf/0.1/Person",
+ "http://xmlns.com/foaf/0.1/OnlineAccount",
+ "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject",
+ "http://www.w3.org/ns/dcat#Dataset",
+ "http://www.w3.org/ns/dcat#Distribution",
+ "http://www.w3.org/ns/dcat#Catalog",
+ "http://mu.semte.ch/vocabularies/ext/GeslachtCode",
+ "http://data.lblod.info/vocabularies/leidinggevenden/FunctionarisStatusCode",
"http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode",
"http://lblod.data.gift/vocabularies/organisatie/BestuursfunctieCode",
"http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode",
- "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",
- "http://lblod.data.gift/vocabularies/organisatie/HelftVerkiezing",
- "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis",
- "http://lblod.data.gift/vocabularies/organisatie/VeranderingsgebeurtenisResultaat",
"http://lblod.data.gift/vocabularies/organisatie/MandatarisStatusCode",
"http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode",
+ "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",
+ "http://www.w3.org/2004/02/skos/core#ConceptScheme",
+ "http://lblod.data.gift/vocabularies/organisatie/BedienaarFinanceringCode",
+ "http://www.w3.org/2004/02/skos/core#Concept",
+ "http://publications.europa.eu/ontology/euvoc#Country",
+ "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging",
+ "http://lblod.data.gift/vocabularies/organisatie/HelftVerkiezing",
+ "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis",
"http://lblod.data.gift/vocabularies/organisatie/TypeBetrokkenheid",
"http://lblod.data.gift/vocabularies/organisatie/VoorwaardenBedienaarCriterium",
"http://lblod.data.gift/vocabularies/organisatie/BedienaarCriteriumBewijsstuk",
"http://lblod.data.gift/vocabularies/organisatie/EredienstBeroepen",
- "http://lblod.data.gift/vocabularies/organisatie/BedienaarFinanceringCode",
- "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging",
- "http://lblod.data.gift/vocabularies/organisatie/Rechtsvormtype",
- "http://www.w3.org/2004/02/skos/core#ConceptScheme",
- "http://www.w3.org/2004/02/skos/core#Concept",
+ "http://lblod.data.gift/vocabularies/organisatie/Rechtsvormtype"
+ ]
+
+ @org_type [
+ "http://www.w3.org/ns/org#Organization",
+ "http://data.vlaanderen.be/ns/besluit#Besluit",
+ "https://data.vlaanderen.be/ns/besluitvorming#Beslissingsactiviteit",
+ "http://www.w3.org/ns/org#Site",
+ "http://www.w3.org/ns/locn#Address",
+ "http://www.w3.org/ns/person#Person",
+ "http://schema.org/ContactPoint",
+ "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
+ "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
+ "http://www.w3.org/ns/adms#Identifier",
+ "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
+ "http://data.vlaanderen.be/ns/mandaat#Mandaat",
+ "http://data.vlaanderen.be/ns/mandaat#Mandataris",
+ "http://data.lblod.info/vocabularies/leidinggevenden/Functionaris",
+ "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie",
+ "http://lblod.data.gift/vocabularies/organisatie/VeranderingsgebeurtenisResultaat",
+ "http://www.w3.org/ns/org#Post",
+ "http://www.w3.org/ns/org#ChangeEvent",
+ "http://data.lblod.info/vocabularies/contacthub/AgentInPositie"
+ ]
+
+ @shared_protected_type [
+ "http://www.w3.org/ns/prov#Location",
"http://xmlns.com/foaf/0.1/Image",
- "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject",
- "http://publications.europa.eu/ontology/euvoc#Country"
- ]
+ "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject"
+ ]
+
defp access_by_role( group_string ) do
%AccessByQuery{
# vars: ["session_group","session_role"], todo we might have to create multiple graph depending on role
- vars: [],
+ vars: [],
query: sparql_query_for_access_role( group_string ) }
end
@@ -80,48 +96,88 @@ defmodule Acl.UserGroups.Config do
PREFIX mu:
SELECT distinct ?session_group ?session_role WHERE {
ext:sessionGroup/mu:uuid ?session_group;
- ext:sessionRole ?session_role.
+ ext:activeSessionRole ?session_role.
FILTER( ?session_role = \"#{group_string}\" )
}"
end
def user_groups do
[
- %GroupSpec{
+
+ %GroupSpec{
name: "acmidm-lezer",
useage: [:read],
access: access_by_role( "ABBOrganisatiePortaalGebruiker-lezer" ),
graphs: [
%GraphSpec{
- graph: "http://mu.semte.ch/graphs/organisatieportaal",
+ graph: "http://mu.semte.ch/graphs/administrative-unit",
constraint: %ResourceConstraint{
- resource_types: @protected_resource_type
+ resource_types: @org_type
}
}
]
},
- %GroupSpec{
+ %GroupSpec{
name: "acmidm-editeerder",
useage: [:read, :write, :read_for_write],
access: access_by_role( "ABBOrganisatiePortaalGebruiker-editeerder" ),
graphs: [
%GraphSpec{
- graph: "http://mu.semte.ch/graphs/organisatieportaal",
+ graph: "http://mu.semte.ch/graphs/administrative-unit",
constraint: %ResourceConstraint{
- resource_types: @protected_resource_type ++ @error_type
+ resource_types: @org_type ++ @error_type
}
}
]
},
- %GroupSpec{
+ %GroupSpec{
name: "acmidm-beheerder",
useage: [:read, :write, :read_for_write],
access: access_by_role( "ABBOrganisatiePortaalGebruiker-beheerder" ),
graphs: [
%GraphSpec{
- graph: "http://mu.semte.ch/graphs/organisatieportaal",
+ graph: "http://mu.semte.ch/graphs/administrative-unit",
constraint: %ResourceConstraint{
- resource_types: @protected_resource_type ++ @error_type
+ resource_types: @org_type ++ @error_type
+ }
+ }
+ ]
+ },
+ %GroupSpec{
+ name: "acmidm-worship-lezer",
+ useage: [:read],
+ access: access_by_role( "ABBOrganisatiePortaalErediensten-lezer" ),
+ graphs: [
+ %GraphSpec{
+ graph: "http://mu.semte.ch/graphs/worship-service",
+ constraint: %ResourceConstraint{
+ resource_types: @org_type ++ @worship_type
+ }
+ }
+ ]
+ },
+ %GroupSpec{
+ name: "acmidm-worship-editeerder",
+ useage: [:read, :write, :read_for_write],
+ access: access_by_role( "ABBOrganisatiePortaalErediensten-editeerder" ),
+ graphs: [
+ %GraphSpec{
+ graph: "http://mu.semte.ch/graphs/worship-service",
+ constraint: %ResourceConstraint{
+ resource_types: @org_type ++ @error_type ++ @worship_type
+ }
+ }
+ ]
+ },
+ %GroupSpec{
+ name: "acmidm-worship-beheerder",
+ useage: [:read, :write, :read_for_write],
+ access: access_by_role( "ABBOrganisatiePortaalErediensten-beheerder" ),
+ graphs: [
+ %GraphSpec{
+ graph: "http://mu.semte.ch/graphs/worship-service",
+ constraint: %ResourceConstraint{
+ resource_types: @org_type ++ @error_type ++ @worship_type
}
}
]
@@ -130,19 +186,21 @@ defmodule Acl.UserGroups.Config do
name: "public",
useage: [:read],
access: %AlwaysAccessible{},
- graphs: [ %GraphSpec{
- graph: "http://mu.semte.ch/graphs/public", # mock login only
+ graphs: [
+ %GraphSpec{
+ graph: "http://mu.semte.ch/graphs/public",
constraint: %ResourceConstraint{
- resource_types: [
- "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
- "http://xmlns.com/foaf/0.1/Person",
- "http://xmlns.com/foaf/0.1/OnlineAccount",
- "http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject",
- "http://www.w3.org/ns/dcat#Dataset",
- "http://www.w3.org/ns/dcat#Distribution",
- "http://www.w3.org/ns/dcat#Catalog"
- ]
- } }]},
+ resource_types: @public_type
+ }
+ },
+ %GraphSpec{
+ graph: "http://mu.semte.ch/graphs/shared",
+ constraint: %ResourceConstraint{
+ resource_types: @shared_protected_type ++ @org_type
+ }
+ }
+
+ ]},
# // CLEANUP
#
diff --git a/config/delta-producer/administrative-units/export.json b/config/delta-producer/administrative-units/export.json
deleted file mode 100644
index d38b7b0..0000000
--- a/config/delta-producer/administrative-units/export.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
- "export": [
- {
- "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode",
- "graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
- ],
- "properties": [
- "http://mu.semte.ch/vocabularies/core/uuid",
- "http://www.w3.org/2004/02/skos/core#prefLabel",
- "http://www.w3.org/2004/02/skos/core#scopeNote"
- ]
- },
- {
- "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
- "graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
- ],
- "properties": [
- "http://mu.semte.ch/vocabularies/core/uuid",
- "http://www.w3.org/2004/02/skos/core#prefLabel",
- "http://www.w3.org/2004/02/skos/core#altLabel",
- "http://data.vlaanderen.be/ns/besluit#werkingsgebied",
- "http://www.w3.org/ns/org#classification"
- ]
- },
- {
- "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
- "graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
- ],
- "properties": [
- "http://mu.semte.ch/vocabularies/core/uuid",
- "http://www.w3.org/2004/02/skos/core#prefLabel",
- "http://data.vlaanderen.be/ns/besluit#bestuurt",
- "http://www.w3.org/ns/org#classification",
- "http://data.vlaanderen.be/ns/mandaat#bindingEinde",
- "http://data.vlaanderen.be/ns/mandaat#bindingStart",
- "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan"
- ]
- },
- {
- "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode",
- "graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
- ],
- "properties": [
- "http://mu.semte.ch/vocabularies/core/uuid",
- "http://www.w3.org/2004/02/skos/core#prefLabel",
- "http://www.w3.org/2004/02/skos/core#scopeNote"
- ]
- }
- ]
-}
diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json
index d8c54f3..42c183f 100644
--- a/config/delta-producer/dump-file-publisher/config.json
+++ b/config/delta-producer/dump-file-publisher/config.json
@@ -1,20 +1,4 @@
{
- "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units": {
- "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump",
- "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units",
- "fileBaseName": "dump-administrative-units",
- "targetDcatGraph": "http://mu.semte.ch/graphs/public",
- "targetFilesGraph": "http://mu.semte.ch/graphs/public",
- "cleanupOldDumps": true
- },
- "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units": {
- "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump",
- "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units",
- "fileBaseName": "dump-administrative-units",
- "targetDcatGraph": "http://mu.semte.ch/graphs/public",
- "targetFilesGraph": "http://mu.semte.ch/graphs/public",
- "cleanupOldDumps": true
- },
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations": {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
@@ -47,7 +31,7 @@
"targetFilesGraph": "http://mu.semte.ch/graphs/public",
"cleanupOldDumps": true
},
- "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts" : {
+ "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/worship-posts": {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts",
"fileBaseName": "dump-worship-posts",
@@ -55,7 +39,7 @@
"targetFilesGraph": "http://mu.semte.ch/graphs/public",
"cleanupOldDumps": true
},
- "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts" : {
+ "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/worship-posts": {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts",
"fileBaseName": "dump-worship-posts",
diff --git a/config/delta-producer/organizations/export.json b/config/delta-producer/organizations/export.json
index bb86cfb..510c48b 100644
--- a/config/delta-producer/organizations/export.json
+++ b/config/delta-producer/organizations/export.json
@@ -3,9 +3,11 @@
{
"type": "http://www.w3.org/ns/person#Person",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
- "properties": [
+ "properties": [
"http://xmlns.com/foaf/0.1/givenName",
"http://xmlns.com/foaf/0.1/familyName",
"https://data.vlaanderen.be/ns/persoon#gebruikteVoornaam",
@@ -16,7 +18,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/MandatarisStatusCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -26,7 +28,8 @@
"type": "http://data.lblod.info/vocabularies/erediensten/EredienstMandataris",
"strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared"
],
"properties": [
"http://data.lblod.info/vocabularies/contacthub/startdatum",
@@ -46,7 +49,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/HelftVerkiezing",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -55,7 +58,8 @@
{
"type": "http://schema.org/ContactPoint",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://schema.org/email",
@@ -67,7 +71,9 @@
{
"type": "http://data.vlaanderen.be/ns/mandaat#Mandaat",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://www.w3.org/ns/org#role",
@@ -78,7 +84,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuursfunctieCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -87,7 +93,7 @@
{
"type": "http://www.w3.org/ns/org#Role",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -96,7 +102,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -105,7 +111,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -115,7 +121,8 @@
"type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
"strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel",
@@ -142,7 +149,8 @@
"type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst",
"strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel",
@@ -166,7 +174,8 @@
"type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan",
"strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel",
@@ -187,7 +196,9 @@
"type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
"strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://data.vlaanderen.be/ns/mandaat#bindingEinde",
@@ -201,7 +212,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -210,7 +221,8 @@
{
"type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://data.lblod.info/vocabularies/erediensten/financieringspercentage",
@@ -221,7 +233,9 @@
{
"type": "http://www.w3.org/ns/adms#Identifier",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/administrative-unit",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#notation",
@@ -231,7 +245,9 @@
{
"type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/administrative-unit",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
@@ -240,7 +256,9 @@
{
"type": "http://www.w3.org/ns/locn#Address",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer",
@@ -258,7 +276,8 @@
{
"type": "http://www.w3.org/ns/org#Site",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://www.w3.org/ns/org#siteAddress",
@@ -269,7 +288,8 @@
{
"type": "http://www.w3.org/ns/org#ChangeEvent",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://purl.org/dc/terms/date",
@@ -282,7 +302,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -291,7 +311,7 @@
{
"type": "http://www.w3.org/2004/02/skos/core#Concept",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -300,7 +320,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -309,16 +329,17 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeBetrokkenheid",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
- "type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar",
+ "type": "http://data.lblod.info/vocabularies/erediensten/RolBedienaar",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://data.lblod.info/vocabularies/contacthub/startdatum",
@@ -333,26 +354,30 @@
{
"type": "http://data.lblod.info/vocabularies/erediensten/VoorwaardenBedienaar",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://data.lblod.info/vocabularies/erediensten/voldaan",
"http://data.lblod.info/vocabularies/erediensten/criterium",
"http://data.lblod.info/vocabularies/erediensten/criteriumbewijsstuktype"
]
- },{
+ },
+ {
"type": "http://publications.europa.eu/ontology/euvoc#Country",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/2000/01/rdf-schema#label"
]
- },{
+ },
+ {
"type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://www.w3.org/ns/org#role",
@@ -364,7 +389,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/EredienstBeroepen",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -373,7 +398,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BedienaarFinanceringCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -382,7 +407,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/VoorwaardenBedienaarCriterium",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -391,7 +416,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BedienaarCriteriumBewijsstuk",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
@@ -400,7 +425,7 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json
index 49d2931..f0c3e85 100644
--- a/config/delta-producer/public/export.json
+++ b/config/delta-producer/public/export.json
@@ -5,13 +5,15 @@
"type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
"strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"pathToConceptScheme": [
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/2004/02/skos/core#altLabel",
"http://www.w3.org/ns/adms#identifier",
@@ -24,8 +26,6 @@
"http://www.w3.org/ns/org#linkedTo",
"http://www.w3.org/ns/org#hasSubOrganization",
"http://www.w3.org/ns/org#classification",
- "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur",
- "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau",
"http://data.lblod.info/vocabularies/erediensten/denominatie",
"http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend",
"http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor",
@@ -36,13 +36,15 @@
"type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst",
"strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"pathToConceptScheme": [
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/2004/02/skos/core#altLabel",
"http://www.w3.org/ns/adms#identifier",
@@ -55,23 +57,98 @@
"http://www.w3.org/ns/org#linkedTo",
"http://www.w3.org/ns/org#hasSubOrganization",
"http://www.w3.org/ns/org#classification",
- "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur",
- "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau",
"http://data.lblod.info/vocabularies/erediensten/typeEredienst"
]
},
+ {
+ "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
+ "strictTypeExport": "true",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit"
+ ],
+ "pathToConceptScheme": [
+ "http://www.w3.org/ns/org#classification",
+ "http://www.w3.org/2004/02/skos/core#inScheme"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur",
+ "http://data.lblod.info/vocabularies/erediensten/denominatie",
+ "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend",
+ "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
+ "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor",
+ "http://data.vlaanderen.be/ns/besluit#werkingsgebied",
+ "http://www.opengis.net/ont/geosparql#sfWithin",
+ "http://www.w3.org/2002/07/owl#sameAs",
+ "http://www.w3.org/2004/02/skos/core#altLabel",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://www.w3.org/ns/adms#identifier",
+ "http://www.w3.org/ns/org#changedBy",
+ "http://www.w3.org/ns/org#classification",
+ "http://www.w3.org/ns/org#hasPost",
+ "http://www.w3.org/ns/org#hasPrimarySite",
+ "http://www.w3.org/ns/org#hasSite",
+ "http://www.w3.org/ns/org#hasSubOrganization",
+ "http://www.w3.org/ns/org#linkedTo",
+ "http://www.w3.org/ns/org#resultedFrom",
+ "http://www.w3.org/ns/regorg#orgStatus"
+ ]
+ },
+ {
+ "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://www.w3.org/ns/org#linkedTo",
+ "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
+ "http://www.w3.org/ns/adms#identifier"
+ ]
+ },
+ {
+ "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://data.lblod.info/vocabularies/erediensten/typebetrokkenheid",
+ "http://www.w3.org/ns/org#organization",
+ "http://data.lblod.info/vocabularies/erediensten/financieringspercentage"
+ ]
+ },
+ {
+ "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/ns/org#role",
+ "http://data.lblod.info/vocabularies/erediensten/functie"
+ ]
+ },
{
"type": "http://www.w3.org/ns/org#Site",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"pathToConceptScheme": [
"^http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
-
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/ns/org#siteAddress",
"https://data.vlaanderen.be/ns/organisatie#bestaatUit",
"http://data.lblod.info/vocabularies/erediensten/vestigingstype"
@@ -80,16 +157,18 @@
{
"type": "http://www.w3.org/ns/locn#Address",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"pathToConceptScheme": [
"^https://data.vlaanderen.be/ns/organisatie#bestaatUit",
"^http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
-
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer",
"https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer",
"http://www.w3.org/ns/locn#thoroughfare",
@@ -105,7 +184,9 @@
{
"type": "http://schema.org/ContactPoint",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"pathToConceptScheme": [
"^http://www.w3.org/ns/org#siteAddress",
@@ -122,13 +203,14 @@
"http://xmlns.com/foaf/0.1/page",
"http://schema.org/email",
"http://schema.org/telephone"
-
]
},
{
"type": "http://www.w3.org/ns/adms#Identifier",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"pathToConceptScheme": [
"^http://www.w3.org/ns/adms#identifier",
@@ -136,14 +218,29 @@
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#notation",
"https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator"
]
},
+ {
+ "type": "http://www.w3.org/ns/adms#Identifier",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#notation",
+ "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator"
+ ],
+ "additionalFilter": "?ro ?subject ; a ."
+ },
{
"type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"pathToConceptScheme": [
"^https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator",
@@ -152,73 +249,91 @@
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
]
},
+ {
+ "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
+ ],
+ "additionalFilter": "?ro / ?subject ; a ."
+ },
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
- "strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
- ],
- "pathToConceptScheme": [
- "http://www.w3.org/ns/org#classification",
- "http://www.w3.org/2004/02/skos/core#inScheme"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
- "http://data.vlaanderen.be/ns/mandaat#bindingEinde",
- "http://data.vlaanderen.be/ns/mandaat#bindingStart",
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://data.vlaanderen.be/ns/besluit#bestuurt",
"http://www.w3.org/ns/org#classification",
- "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan"
- ]
+ "http://data.vlaanderen.be/ns/mandaat#bindingEinde",
+ "http://data.vlaanderen.be/ns/mandaat#bindingStart",
+ "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
+ "http://www.w3.org/ns/org#hasPost"
+ ],
+ "additionalFilter": "?subject (//)|(/) ."
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://www.w3.org/ns/org#ChangeEvent",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"pathToConceptScheme": [
"http://data.lblod.info/vocabularies/contacthub/typeWijziging",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://purl.org/dc/terms/date",
"http://purl.org/dc/terms/description",
"http://data.lblod.info/vocabularies/contacthub/typeWijziging",
@@ -233,36 +348,76 @@
{
"type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/public"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
+ },
+ {
+ "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
+ ],
+ "pathToConceptScheme": [
+ "^http://www.w3.org/ns/org#hasPost",
+ "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
+ "http://www.w3.org/ns/org#classification",
+ "http://www.w3.org/2004/02/skos/core#inScheme"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/ns/org#role"
+ ]
+ },
+ {
+ "type": "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/administrative-unit"
+ ],
+ "pathToConceptScheme": [
+ "^http://data.lblod.info/vocabularies/leidinggevenden/heeftBestuursfunctie",
+ "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
+ "http://www.w3.org/ns/org#classification",
+ "http://www.w3.org/2004/02/skos/core#inScheme"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/ns/org#role",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
}
diff --git a/config/delta-producer/worship-posts/export.json b/config/delta-producer/worship-posts/export.json
index 9c4b9ad..0f540bd 100644
--- a/config/delta-producer/worship-posts/export.json
+++ b/config/delta-producer/worship-posts/export.json
@@ -3,7 +3,8 @@
{
"type": "http://data.vlaanderen.be/ns/mandaat#Mandaat",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
@@ -12,36 +13,282 @@
"additionalFilter": "?orgaan ?subject ; / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))"
},
{
- "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
- "strictTypeExport": "true",
+ "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
- "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor"
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://www.w3.org/2004/02/skos/core#altLabel",
+ "http://www.w3.org/ns/regorg#orgStatus",
+ "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
+ "http://www.w3.org/ns/org#classification",
+ "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor",
+ "http://www.w3.org/ns/org#hasPrimarySite",
+ "http://www.w3.org/ns/org#hasSite",
+ "http://www.w3.org/ns/adms#identifier",
+ "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur"
+ ],
+ "additionalFilter": "?subject ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))"
+ },
+ {
+ "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://www.w3.org/2004/02/skos/core#altLabel",
+ "http://www.w3.org/ns/regorg#orgStatus",
+ "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
+ "http://www.w3.org/ns/org#classification",
+ "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor",
+ "http://www.w3.org/ns/org#hasPrimarySite",
+ "http://www.w3.org/ns/org#hasSite",
+ "http://www.w3.org/ns/adms#identifier"
+ ]
+ },
+ {
+ "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst",
+ "strictTypeExport": "true",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://www.w3.org/2004/02/skos/core#altLabel",
+ "http://www.w3.org/ns/regorg#orgStatus",
+ "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
+ "http://www.w3.org/ns/org#classification",
+ "http://www.w3.org/ns/org#hasPrimarySite",
+ "http://www.w3.org/ns/org#hasSite",
+ "http://www.w3.org/ns/adms#identifier"
]
},
{
"type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
- "strictTypeExport": "true",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://data.vlaanderen.be/ns/besluit#bestuurt",
+ "http://www.w3.org/ns/org#classification",
+ "http://data.vlaanderen.be/ns/mandaat#bindingEinde",
+ "http://data.vlaanderen.be/ns/mandaat#bindingStart",
+ "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
"http://www.w3.org/ns/org#hasPost"
],
- "additionalFilter": "?subject / ?bestuurseenheid . ?bestuurseenheid a ?type. FILTER (?type IN (, ))"
+ "additionalFilter": "?subject (//)|(/) ?bestuurClassification . FILTER (?bestuurClassification IN (, , , ))"
},
{
"type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar",
"graphsFilter": [
- "http://mu.semte.ch/graphs/organisatieportaal"
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/ns/org#role",
"http://data.lblod.info/vocabularies/erediensten/functie"
]
+ },
+ {
+ "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://data.lblod.info/vocabularies/erediensten/typebetrokkenheid",
+ "http://www.w3.org/ns/org#organization"
+ ]
+ },
+ {
+ "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://www.w3.org/2004/02/skos/core#altLabel",
+ "http://www.w3.org/ns/regorg#orgStatus",
+ "http://www.w3.org/ns/org#hasPrimarySite",
+ "http://www.w3.org/ns/org#hasSite",
+ "http://www.w3.org/ns/org#linkedTo",
+ "http://data.lblod.info/vocabularies/erediensten/typeEredienst",
+ "http://www.w3.org/ns/adms#identifier"
+ ]
+ },
+ {
+ "type": "http://www.w3.org/ns/adms#Identifier",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#notation",
+ "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator"
+ ],
+ "additionalFilter": "?subject \"KBO nummer\" ."
+ },
+ {
+ "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/worship-service",
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/administrative-unit"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
+ ],
+ "additionalFilter": "?subject ^/ \"KBO nummer\" ."
+ },
+ {
+ "type": "http://www.w3.org/ns/org#Site",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "https://data.vlaanderen.be/ns/organisatie#bestaatUit",
+ "http://data.lblod.info/vocabularies/erediensten/vestigingstype"
+ ]
+ },
+ {
+ "type": "http://www.w3.org/ns/locn#Address",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/shared",
+ "http://mu.semte.ch/graphs/worship-service"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer",
+ "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer",
+ "http://www.w3.org/ns/locn#thoroughfare",
+ "http://www.w3.org/ns/locn#postCode",
+ "https://data.vlaanderen.be/ns/adres#gemeentenaam",
+ "http://www.w3.org/ns/locn#adminUnitL2",
+ "http://www.w3.org/ns/locn#fullAddress"
+ ],
+ "additionalFilter": "?subject ^ ?site. ?site a ."
+ },
+ {
+ "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://www.w3.org/2004/02/skos/core#scopeNote"
+ ]
+ },
+ {
+ "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel",
+ "http://www.w3.org/2004/02/skos/core#scopeNote"
+ ]
+ },
+ {
+ "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
+ },
+ {
+ "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
+ },
+ {
+ "type": "http://lblod.data.gift/vocabularies/organisatie/TypeBetrokkenheid",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
+ },
+ {
+ "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
+ },
+ {
+ "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursfunctieCode",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
+ },
+ {
+ "type": "http://lblod.data.gift/vocabularies/organisatie/EredienstBeroepen",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
+ },
+ {
+ "type": "http://mu.semte.ch/vocabularies/ext/GeslachtCode",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
+ },
+ {
+ "type": "http://publications.europa.eu/ontology/euvoc#Country",
+ "graphsFilter": [
+ "http://mu.semte.ch/graphs/public"
+ ],
+ "properties": [
+ "http://mu.semte.ch/vocabularies/core/uuid",
+ "http://www.w3.org/2000/01/rdf-schema#label",
+ "http://www.w3.org/2004/02/skos/core#prefLabel"
+ ]
}
]
}
diff --git a/config/delta/rules.js b/config/delta/rules.js
index 0c98f11..e262b64 100644
--- a/config/delta/rules.js
+++ b/config/delta/rules.js
@@ -3,7 +3,7 @@ export default [
match: {
graph: {
type: "uri",
- value: "http://mu.semte.ch/graphs/organisatieportaal"
+ value: "http://mu.semte.ch/graphs/administrative-unit"
}
},
callback: {
@@ -20,7 +20,25 @@ export default [
match: {
graph: {
type: "uri",
- value: "http://mu.semte.ch/graphs/organisatieportaal"
+ value: "http://mu.semte.ch/graphs/worship-service"
+ }
+ },
+ callback: {
+ url: 'http://search/update',
+ method: 'POST'
+ },
+ options: {
+ resourceFormat: "v0.0.1",
+ gracePeriod: 10000,
+ ignoreFromSelf: true
+ }
+ },
+
+ {
+ match: {
+ graph: {
+ type: "uri",
+ value: "http://mu.semte.ch/graphs/administrative-unit"
}
},
callback: {
@@ -33,6 +51,25 @@ export default [
ignoreFromSelf: true
}
},
+
+ {
+ match: {
+ graph: {
+ type: "uri",
+ value: "http://mu.semte.ch/graphs/worship-service"
+ }
+ },
+ callback: {
+ url: 'http://resource/.mu/delta',
+ method: 'POST'
+ },
+ options: {
+ resourceFormat: "v0.0.1",
+ gracePeriod: 250,
+ ignoreFromSelf: true
+ }
+ },
+
{
match: {
predicate: {
@@ -48,42 +85,7 @@ export default [
resourceFormat: 'v0.0.1',
gracePeriod: 1000,
ignoreFromSelf: true,
- optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
- }
- },
- {
- match: {
- },
- callback: {
- url: 'http://delta-producer-pub-graph-maintainer-administrative-units/delta',
- method: 'POST'
- },
- options: {
- resourceFormat: 'v0.0.1',
- gracePeriod: 1000,
- ignoreFromSelf: true,
- optOutMuScopeIds: [
- "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
- "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
- ]
- }
- },
- {
- match: {
- graph: {
- type: 'uri',
- value: 'http://redpencil.data.gift/id/deltas/producer/administrative-units'
- }
- },
- callback: {
- url: 'http://delta-producer-json-diff-publisher-administrative-units/delta',
- method: 'POST'
- },
- options: {
- resourceFormat: 'v0.0.1',
- gracePeriod: 1000,
- ignoreFromSelf: true,
- optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
+ optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"]
}
},
{
@@ -98,9 +100,9 @@ export default [
gracePeriod: 1000,
ignoreFromSelf: true,
optOutMuScopeIds: [
- "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
- "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
- ]
+ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
+ "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
+ ]
}
},
{
@@ -118,7 +120,7 @@ export default [
resourceFormat: 'v0.0.1',
gracePeriod: 1000,
ignoreFromSelf: true,
- optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
+ optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"]
}
},
{
@@ -133,9 +135,9 @@ export default [
gracePeriod: 1000,
ignoreFromSelf: true,
optOutMuScopeIds: [
- "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
- "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
- ]
+ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
+ "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
+ ]
}
},
{
@@ -153,7 +155,7 @@ export default [
resourceFormat: 'v0.0.1',
gracePeriod: 1000,
ignoreFromSelf: true,
- optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
+ optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"]
}
},
{
@@ -169,9 +171,9 @@ export default [
gracePeriod: 1000,
ignoreFromSelf: true,
optOutMuScopeIds: [
- "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
- "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
- ]
+ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
+ "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
+ ]
}
},
{
@@ -193,7 +195,7 @@ export default [
resourceFormat: 'v0.0.1',
gracePeriod: 1000,
ignoreFromSelf: true,
- optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
+ optOutMuScopeIds: ["http://redpencil.data.gift/id/concept/muScope/deltas/initialSync"]
}
},
{
diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex
index 763fc83..acad7b8 100644
--- a/config/dispatcher/dispatcher.ex
+++ b/config/dispatcher/dispatcher.ex
@@ -77,7 +77,7 @@ defmodule Dispatcher do
end
match "/administrative-units/*path", %{ accept: [:json], layer: :api} do
- Proxy.forward conn, path, "http://cache/administrative-units/"
+ Proxy.forward conn, path, "http://resource/administrative-units/"
end
match "/administrative-unit-classification-codes/*path", %{ accept: [:json], layer: :api} do
@@ -85,7 +85,7 @@ defmodule Dispatcher do
end
match "/worship-administrative-units/*path", %{ accept: [:json], layer: :api} do
- Proxy.forward conn, path, "http://cache/worship-administrative-units/"
+ Proxy.forward conn, path, "http://resource/worship-administrative-units/"
end
match "/worship-services/*path", %{ accept: [:json], layer: :api} do
@@ -220,11 +220,14 @@ defmodule Dispatcher do
###############
match "/accounts", %{ accept: [:json], layer: :api} do
- Proxy.forward conn, [], "http://resource/accounts/"
+ Proxy.forward conn, [], "http://resource/accounts/"
end
match "/accounts/*path", %{ accept: [:json], layer: :api} do
Proxy.forward conn, path, "http://accountdetail/accounts/"
end
+ match "/active-role/*path", %{ accept: [:json], layer: :api} do
+ Proxy.forward conn, path, "http://roles/"
+ end
match "/groups/*path", %{ accept: [:json], layer: :api} do
Proxy.forward conn, path, "http://resource/groups/"
end
@@ -242,7 +245,7 @@ defmodule Dispatcher do
###############################################################
match "/search/*path", %{ accept: %{ json: true }, layer: :api_services} do
- IO.puts("hey")
+ IO.puts("hey")
Proxy.forward conn, path, "http://search/"
end
@@ -262,20 +265,36 @@ defmodule Dispatcher do
forward conn, path, "http://adressenregister/"
end
+ match "/worship/person-information-updates/*path", %{ layer: :api_services, accept: %{ json: true } } do
+ forward conn, path, "http://privacy-worship/person-information-updates"
+ end
+
+ match "/worship/person-information-requests/*path", %{ layer: :api_services, accept: %{ json: true } } do
+ forward conn, path, "http://privacy-worship/person-information-requests"
+ end
+
+ match "/worship/person-information-ask/*path", %{ layer: :api_services, accept: %{ json: true } } do
+ forward conn, path, "http://privacy-worship/person-information-ask/"
+ end
+
+ match "/worship/person-information-validate-ssn/*path", %{ layer: :api_services, accept: %{ json: true } } do
+ forward conn, path, "http://privacy-worship/person-information-validate-ssn/"
+ end
+
match "/person-information-updates/*path", %{ layer: :api_services, accept: %{ json: true } } do
- forward conn, path, "http://privacy/person-information-updates"
+ forward conn, path, "http://privacy-unit/person-information-updates"
end
match "/person-information-requests/*path", %{ layer: :api_services, accept: %{ json: true } } do
- forward conn, path, "http://privacy/person-information-requests"
+ forward conn, path, "http://privacy-unit/person-information-requests"
end
match "/person-information-ask/*path", %{ layer: :api_services, accept: %{ json: true } } do
- forward conn, path, "http://privacy/person-information-ask/"
+ forward conn, path, "http://privacy-unit/person-information-ask/"
end
match "/person-information-validate-ssn/*path", %{ layer: :api_services, accept: %{ json: true } } do
- forward conn, path, "http://privacy/person-information-validate-ssn/"
+ forward conn, path, "http://privacy-unit/person-information-validate-ssn/"
end
get "/uri-info/*path", %{ layer: :api_services, accept: %{ json: true } } do
@@ -307,7 +326,7 @@ defmodule Dispatcher do
end
get "/files/*path" , %{ layer: :api_services, accept: %{ json: true } } do
- Proxy.forward conn, path, "http://cache/files/"
+ Proxy.forward conn, path, "http://resource/files/"
end
#################################################################
@@ -315,11 +334,11 @@ defmodule Dispatcher do
#################################################################
get "/datasets/*path", %{ layer: :api_services, accept: %{ json: true } } do
- Proxy.forward conn, path, "http://cache/datasets/"
+ Proxy.forward conn, path, "http://resource/datasets/"
end
get "/distributions/*path", %{ layer: :api_services, accept: %{ json: true } } do
- Proxy.forward conn, path, "http://cache/distributions/"
+ Proxy.forward conn, path, "http://resource/distributions/"
end
@@ -339,6 +358,14 @@ defmodule Dispatcher do
Proxy.forward conn, path, "http://delta-producer-json-diff-file-publisher-organizations/files/"
end
+ #################################################################
+ # DELTA: public
+ #################################################################
+
+ get "/sync/public/files/*path" do
+ Proxy.forward conn, path, "http://delta-producer-json-diff-publisher-public/files/"
+ end
+
#################################################################
# DELTA: posts
#################################################################
diff --git a/config/loket-consumer/leidinggevenden-dispatching/config.js b/config/loket-consumer/leidinggevenden-dispatching/config.js
index e280733..b3ebc2a 100644
--- a/config/loket-consumer/leidinggevenden-dispatching/config.js
+++ b/config/loket-consumer/leidinggevenden-dispatching/config.js
@@ -1,18 +1,20 @@
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
-const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
+const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
-const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`;
+// Graphs
+const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
+const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
-if (!process.env.FILE_SYNC_GRAPH)
- throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
-const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
+// ENDPOINTS
+const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
+const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
module.exports = {
BATCH_SIZE,
@@ -24,6 +26,7 @@ module.exports = {
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
- INGEST_GRAPH,
- FILE_SYNC_GRAPH
-};
\ No newline at end of file
+ LANDING_ZONE_GRAPH,
+ TARGET_GRAPH,
+ LANDING_ZONE_DATABASE_ENDPOINT
+};
diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js
new file mode 100644
index 0000000..a8df81c
--- /dev/null
+++ b/config/loket-consumer/leidinggevenden-dispatching/delta-context-config.js
@@ -0,0 +1,11 @@
+const contextConfig = {
+ addTypes: {
+ scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
+ exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
+ },
+ contextQueries: []
+}
+
+module.exports = {
+ contextConfig
+};
diff --git a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js
index c8d8ba1..b309f35 100644
--- a/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js
+++ b/config/loket-consumer/leidinggevenden-dispatching/delta-sync-dispatching.js
@@ -1,14 +1,8 @@
-const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util');
-const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
- DIRECT_DATABASE_ENDPOINT,
- MU_CALL_SCOPE_ID_INITIAL_SYNC,
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- INGEST_GRAPH,
- FILE_SYNC_GRAPH
-} = require('./config');
+const {
+ transformStatements,
+ deleteFromTargetGraph,
+ insertIntoTargetGraph,
+} = require('./util');
/**
* Dispatch the fetched information to a target graph.
@@ -26,51 +20,66 @@ const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
* @return {void} Nothing
*/
async function dispatch(lib, data) {
- const { mu, muAuthSudo, fetch } = lib;
- const { termObjectChangeSets } = data;
+ const { mu, fetch } = lib;
+ let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
- for (let { deletes, inserts } of termObjectChangeSets) {
- const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
- const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ // Both arrays are the same length, so we can zip them together for easier processing
+ zippedChangeSets = termObjectChangeSets.map((o, i) => ({
+ original: o,
+ withContext: termObjectChangeSetsWithContext[i]
+ }));
- if (deleteStatements.length) {
- await transformStatements(fetch, deleteStatements).then(
- transformedStatements => {
- deleteFromAllGraphs(
- muAuthSudo.updateSudo,
- transformedStatements,
- { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
- process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- );
- }
- )
+ console.log(`Received ${zippedChangeSets.length} change sets`)
+
+ for (let { original, withContext } of zippedChangeSets) {
+ let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
+ try {
+ originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ } catch (e) {
+ console.log(`Error while processing change set: ${e}`);
+ console.log(`Original change set: ${JSON.stringify(original)}`);
+ throw e;
}
+ // Extra context needed for mapping from DL to OP model and filtering based on type.
+ try {
+ insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ } catch (e) {
+ console.log(`Error while processing change set with context: ${e}`);
+ console.log(`Change set with context: ${JSON.stringify(withContext)}`);
+ throw e;
+ }
- if (insertStatements.length) {
- await transformStatements(fetch, insertStatements).then(
- transformedStatements => {
- batchedDbUpdate(
- muAuthSudo.updateSudo,
- INGEST_GRAPH,
- transformedStatements,
- { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
- process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- "INSERT"
- );
- }
- )
+ if (originalDeletes.length) {
+ // Map deletes from DL to OP model
+ const transformedDeletes = await transformStatements(fetch, deletesWithContext);
+
+ if (!transformedDeletes.length) {
+ console.log(`Warn: Delete statements mapped to empty result.`);
+ console.log(`Input: ${deletesWithContext}`);
+ console.log(`Output: ${transformedDeletes}`);
+ } else {
+ await deleteFromTargetGraph(lib, transformedDeletes);
+ }
+ }
+
+ if (originalInserts.length) {
+ // Map inserts from DL to OP model
+ const transformedInserts = await transformStatements(fetch, insertsWithContext);
+
+ if (!transformedInserts.length) {
+ console.log(`Warn: Insert statements mapped to empty result.`);
+ console.log(`Input: ${insertsWithContext}`);
+ console.log(`Output: ${transformedInserts}`);
+ } else {
+ await insertIntoTargetGraph(lib, transformedInserts);
+ }
}
}
}
module.exports = {
dispatch
-};
\ No newline at end of file
+};
diff --git a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js
index 88017c1..d5d672f 100644
--- a/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js
+++ b/config/loket-consumer/leidinggevenden-dispatching/initial-sync-dispatching.js
@@ -1,14 +1,17 @@
-const { transformStatements, batchedDbUpdate, partition } = require('./util');
-const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
+const {
+ BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
DIRECT_DATABASE_ENDPOINT,
MU_CALL_SCOPE_ID_INITIAL_SYNC,
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- INGEST_GRAPH,
- FILE_SYNC_GRAPH
+ TARGET_GRAPH,
+ LANDING_ZONE_GRAPH,
+ LANDING_ZONE_DATABASE_ENDPOINT
} = require('./config');
+
const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
/**
@@ -25,35 +28,58 @@ const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT
* @return {void} Nothing
*/
async function dispatch(lib, data) {
- const { mu, muAuthSudo, fetch } = lib;
+ const { mu, muAuthSudo } = lib;
const { termObjects } = data;
+ // Steps:
+ // 1. Write the triples to the landing zone graph - withouth any mapping or filtering
+ // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
+
if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
console.warn(`Service configured to skip MU_AUTH!`);
}
console.log(`Using ${endpoint} to insert triples`);
if (termObjects.length) {
- originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
+ const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
- await transformStatements(fetch, originalInsertTriples).then(
- transformedInsertTriples => {
- batchedDbUpdate(
- muAuthSudo.updateSudo,
- INGEST_GRAPH,
- transformedInsertTriples,
- { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
- endpoint,
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION
- )
- }
- )
+ await batchedDbUpdate(
+ muAuthSudo.updateSudo,
+ LANDING_ZONE_GRAPH,
+ originalInsertTriples,
+ { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+ LANDING_ZONE_DATABASE_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+ );
}
}
+async function onFinishInitialIngest(lib) {
+ const { muAuthSudo, fetch } = lib;
+
+ console.log(`!! On-finish triggered !!`);
+
+ const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint);
+
+ console.log(`Transformed ${transformedInsertTriples.length} triples`);
+
+ await batchedDbUpdate(
+ muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ transformedInsertTriples,
+ { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+ endpoint,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+ );
+}
+
module.exports = {
- dispatch
-};
\ No newline at end of file
+ dispatch,
+ onFinishInitialIngest
+};
diff --git a/config/loket-consumer/leidinggevenden-dispatching/util.js b/config/loket-consumer/leidinggevenden-dispatching/util.js
index ff3932f..5eced4a 100644
--- a/config/loket-consumer/leidinggevenden-dispatching/util.js
+++ b/config/loket-consumer/leidinggevenden-dispatching/util.js
@@ -1,9 +1,16 @@
const {
MAX_REASONING_RETRY_ATTEMPTS,
- SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
+ SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ TARGET_GRAPH,
+ LANDING_ZONE_GRAPH,
+ BATCH_SIZE,
} = require('./config');
-async function batchedDbUpdate(muUpdate,
+async function batchedDbUpdate(
+ muUpdate,
graph,
triples,
extraHeaders,
@@ -37,39 +44,6 @@ ${batch}
}
}
-async function deleteFromAllGraphs(muUpdate,
- triples,
- extraHeaders,
- endpoint,
- maxAttempts,
- sleepBetweenBatches = 1000,
- sleepTimeOnFail = 1000,
-) {
-
- for (const triple of triples) {
-
- console.log(`Deleting a triple from all graphs in triplestore`);
-
- const deleteCall = async () => {
- await muUpdate(`
- DELETE {
- GRAPH ?g {
- ${triple}
- }
- } WHERE {
- GRAPH ?g {
- ${triple}
- }
- }
- `, extraHeaders, endpoint);
- };
-
- await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
- console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
- await new Promise(r => setTimeout(r, sleepBetweenBatches));
- }
-}
-
async function operationWithRetry(callback,
attempt,
maxAttempts,
@@ -110,18 +84,16 @@ function partition(arr, fn) {
return { passes, fails };
}
-
/**
* Send triples to reasoning service for conversion
*
*/
-function transformTriples(fetch, triples) {
- return operationWithRetry(mainConversion(fetch, triples), 0,
+async function transformTriples(fetch, triples, mapping) {
+ return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
}
-
-function mainConversion(fetch, triples) {
+async function mainConversion(fetch, triples, mapping) {
let formdata = new URLSearchParams();
formdata.append("data", triples);
@@ -131,23 +103,66 @@ function mainConversion(fetch, triples) {
redirect: 'follow'
};
- return fetch("http://reasoner/reason/dl2op/main", requestOptions)
- .then(response => response.text());
+ const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
+ return await response.text();
}
-function transformStatements(fetch, triples) {
- return transformTriples(fetch, triples.join('\n')).then(
- graph => {
- statements = graph.replace(/\n{2,}/g, '').split('\n')
- console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`)
- return statements
- }
- )
+async function transformStatements(fetch, triples, mapping = 'main') {
+ console.log(`Received ${JSON.stringify(triples)} to transform`);
+ const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
+ statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
+ console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
+ return statements;
+}
+
+async function deleteFromTargetGraph(lib, statements) {
+ console.log(`Deleting ${statements.length} statements from target graph`);
+ console.log(`Statements: ${JSON.stringify(statements)}`)
+ await batchedDbUpdate(
+ lib.muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ statements,
+ {},
+ process.env.MU_SPARQL_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ 'DELETE');
+}
+
+async function insertIntoTargetGraph(lib, statements) {
+ console.log(`Inserting ${statements.length} statements into target graph`);
+ console.log(`Statements: ${JSON.stringify(statements)}`)
+
+ await batchedDbUpdate(
+ lib.muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ statements,
+ {},
+ process.env.MU_SPARQL_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ 'INSERT');
+}
+
+async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
+ console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
+
+ const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
+ const text = await response.text();
+ const statements = text.replace(/\n{2,}/g, '').split('\n');
+
+ return statements;
}
module.exports = {
batchedDbUpdate,
- deleteFromAllGraphs,
partition,
- transformStatements
-};
\ No newline at end of file
+ transformStatements,
+ transformLandingZoneGraph,
+ deleteFromTargetGraph,
+ insertIntoTargetGraph
+};
diff --git a/config/loket-consumer/mandate-dispatching/config.js b/config/loket-consumer/mandatarissen-dispatching/config.js
similarity index 67%
rename from config/loket-consumer/mandate-dispatching/config.js
rename to config/loket-consumer/mandatarissen-dispatching/config.js
index e280733..13291cf 100644
--- a/config/loket-consumer/mandate-dispatching/config.js
+++ b/config/loket-consumer/mandatarissen-dispatching/config.js
@@ -1,18 +1,20 @@
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
-const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
+const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
-const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`;
+// Graphs
+const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/administrative-unit`;
+const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
-if (!process.env.FILE_SYNC_GRAPH)
- throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
-const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
+// ENDPOINTS
+const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
+const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
module.exports = {
BATCH_SIZE,
@@ -24,6 +26,8 @@ module.exports = {
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
- INGEST_GRAPH,
- FILE_SYNC_GRAPH
-};
\ No newline at end of file
+ LANDING_ZONE_GRAPH,
+ LANDING_ZONE_DATABASE_ENDPOINT,
+ TARGET_GRAPH
+};
+
diff --git a/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js b/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js
new file mode 100644
index 0000000..a8df81c
--- /dev/null
+++ b/config/loket-consumer/mandatarissen-dispatching/delta-context-config.js
@@ -0,0 +1,11 @@
+const contextConfig = {
+ addTypes: {
+ scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
+ exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
+ },
+ contextQueries: []
+}
+
+module.exports = {
+ contextConfig
+};
diff --git a/config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js b/config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js
new file mode 100644
index 0000000..b309f35
--- /dev/null
+++ b/config/loket-consumer/mandatarissen-dispatching/delta-sync-dispatching.js
@@ -0,0 +1,85 @@
+const {
+ transformStatements,
+ deleteFromTargetGraph,
+ insertIntoTargetGraph,
+} = require('./util');
+
+/**
+* Dispatch the fetched information to a target graph.
+* Note: will be ADDED to it's own graph.
+* We take only care of adding them, not updating triples, this is a TODO
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
+* [ {
+* graph: "",
+* subject: "",
+* predicate: "",
+* object: "^^"
+* }
+* ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+ const { mu, fetch } = lib;
+ let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
+
+ // Both arrays are the same length, so we can zip them together for easier processing
+ zippedChangeSets = termObjectChangeSets.map((o, i) => ({
+ original: o,
+ withContext: termObjectChangeSetsWithContext[i]
+ }));
+
+ console.log(`Received ${zippedChangeSets.length} change sets`)
+
+ for (let { original, withContext } of zippedChangeSets) {
+ let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
+ try {
+ originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ } catch (e) {
+ console.log(`Error while processing change set: ${e}`);
+ console.log(`Original change set: ${JSON.stringify(original)}`);
+ throw e;
+ }
+
+ // Extra context needed for mapping from DL to OP model and filtering based on type.
+ try {
+ insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ } catch (e) {
+ console.log(`Error while processing change set with context: ${e}`);
+ console.log(`Change set with context: ${JSON.stringify(withContext)}`);
+ throw e;
+ }
+
+ if (originalDeletes.length) {
+ // Map deletes from DL to OP model
+ const transformedDeletes = await transformStatements(fetch, deletesWithContext);
+
+ if (!transformedDeletes.length) {
+ console.log(`Warn: Delete statements mapped to empty result.`);
+ console.log(`Input: ${deletesWithContext}`);
+ console.log(`Output: ${transformedDeletes}`);
+ } else {
+ await deleteFromTargetGraph(lib, transformedDeletes);
+ }
+ }
+
+ if (originalInserts.length) {
+ // Map inserts from DL to OP model
+ const transformedInserts = await transformStatements(fetch, insertsWithContext);
+
+ if (!transformedInserts.length) {
+ console.log(`Warn: Insert statements mapped to empty result.`);
+ console.log(`Input: ${insertsWithContext}`);
+ console.log(`Output: ${transformedInserts}`);
+ } else {
+ await insertIntoTargetGraph(lib, transformedInserts);
+ }
+ }
+ }
+}
+
+module.exports = {
+ dispatch
+};
diff --git a/config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js b/config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js
new file mode 100644
index 0000000..d5d672f
--- /dev/null
+++ b/config/loket-consumer/mandatarissen-dispatching/initial-sync-dispatching.js
@@ -0,0 +1,85 @@
+const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
+const {
+ BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+ DIRECT_DATABASE_ENDPOINT,
+ MU_CALL_SCOPE_ID_INITIAL_SYNC,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ TARGET_GRAPH,
+ LANDING_ZONE_GRAPH,
+ LANDING_ZONE_DATABASE_ENDPOINT
+} = require('./config');
+
+const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
+
+/**
+* Dispatch the fetched information to a target graph.
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
+* [ {
+* graph: "",
+* subject: "",
+* predicate: "",
+* object: "^^"
+* }
+* ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+ const { mu, muAuthSudo } = lib;
+ const { termObjects } = data;
+
+ // Steps:
+ // 1. Write the triples to the landing zone graph - withouth any mapping or filtering
+ // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
+
+ if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
+ console.warn(`Service configured to skip MU_AUTH!`);
+ }
+ console.log(`Using ${endpoint} to insert triples`);
+
+ if (termObjects.length) {
+ const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+
+ await batchedDbUpdate(
+ muAuthSudo.updateSudo,
+ LANDING_ZONE_GRAPH,
+ originalInsertTriples,
+ { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+ LANDING_ZONE_DATABASE_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+ );
+ }
+}
+
+async function onFinishInitialIngest(lib) {
+ const { muAuthSudo, fetch } = lib;
+
+ console.log(`!! On-finish triggered !!`);
+
+ const transformedInsertTriples = await transformLandingZoneGraph(fetch, endpoint);
+
+ console.log(`Transformed ${transformedInsertTriples.length} triples`);
+
+ await batchedDbUpdate(
+ muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ transformedInsertTriples,
+ { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+ endpoint,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+ );
+}
+
+module.exports = {
+ dispatch,
+ onFinishInitialIngest
+};
diff --git a/config/loket-consumer/mandate-dispatching/util.js b/config/loket-consumer/mandatarissen-dispatching/util.js
similarity index 50%
rename from config/loket-consumer/mandate-dispatching/util.js
rename to config/loket-consumer/mandatarissen-dispatching/util.js
index ff3932f..524fe45 100644
--- a/config/loket-consumer/mandate-dispatching/util.js
+++ b/config/loket-consumer/mandatarissen-dispatching/util.js
@@ -1,9 +1,16 @@
const {
MAX_REASONING_RETRY_ATTEMPTS,
- SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
+ SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ TARGET_GRAPH,
+ LANDING_ZONE_GRAPH,
+ BATCH_SIZE
} = require('./config');
-async function batchedDbUpdate(muUpdate,
+async function batchedDbUpdate(
+ muUpdate,
graph,
triples,
extraHeaders,
@@ -37,39 +44,6 @@ ${batch}
}
}
-async function deleteFromAllGraphs(muUpdate,
- triples,
- extraHeaders,
- endpoint,
- maxAttempts,
- sleepBetweenBatches = 1000,
- sleepTimeOnFail = 1000,
-) {
-
- for (const triple of triples) {
-
- console.log(`Deleting a triple from all graphs in triplestore`);
-
- const deleteCall = async () => {
- await muUpdate(`
- DELETE {
- GRAPH ?g {
- ${triple}
- }
- } WHERE {
- GRAPH ?g {
- ${triple}
- }
- }
- `, extraHeaders, endpoint);
- };
-
- await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
- console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
- await new Promise(r => setTimeout(r, sleepBetweenBatches));
- }
-}
-
async function operationWithRetry(callback,
attempt,
maxAttempts,
@@ -110,18 +84,16 @@ function partition(arr, fn) {
return { passes, fails };
}
-
/**
* Send triples to reasoning service for conversion
*
*/
-function transformTriples(fetch, triples) {
- return operationWithRetry(mainConversion(fetch, triples), 0,
+async function transformTriples(fetch, triples, mapping) {
+ return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
}
-
-function mainConversion(fetch, triples) {
+async function mainConversion(fetch, triples, mapping) {
let formdata = new URLSearchParams();
formdata.append("data", triples);
@@ -131,23 +103,66 @@ function mainConversion(fetch, triples) {
redirect: 'follow'
};
- return fetch("http://reasoner/reason/dl2op/main", requestOptions)
- .then(response => response.text());
+ const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
+ return await response.text();
}
-function transformStatements(fetch, triples) {
- return transformTriples(fetch, triples.join('\n')).then(
- graph => {
- statements = graph.replace(/\n{2,}/g, '').split('\n')
- console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`)
- return statements
- }
- )
+async function transformStatements(fetch, triples, mapping = 'main') {
+ console.log(`Received ${JSON.stringify(triples)} to transform`);
+ const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
+ statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
+ console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
+ return statements;
+}
+
+async function deleteFromTargetGraph(lib, statements) {
+ console.log(`Deleting ${statements.length} statements from target graph`);
+ console.log(`Statements: ${JSON.stringify(statements)}`)
+ await batchedDbUpdate(
+ lib.muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ statements,
+ {},
+ process.env.MU_SPARQL_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ 'DELETE');
+}
+
+async function insertIntoTargetGraph(lib, statements) {
+ console.log(`Inserting ${statements.length} statements into target graph`);
+ console.log(`Statements: ${JSON.stringify(statements)}`)
+
+ await batchedDbUpdate(
+ lib.muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ statements,
+ {},
+ process.env.MU_SPARQL_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ 'INSERT');
+}
+
+async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
+ console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
+
+ const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
+ const text = await response.text();
+ const statements = text.replace(/\n{2,}/g, '').split('\n');
+
+ return statements;
}
module.exports = {
batchedDbUpdate,
- deleteFromAllGraphs,
partition,
- transformStatements
-};
\ No newline at end of file
+ transformStatements,
+ transformLandingZoneGraph,
+ deleteFromTargetGraph,
+ insertIntoTargetGraph
+};
diff --git a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js
deleted file mode 100644
index c8d8ba1..0000000
--- a/config/loket-consumer/mandate-dispatching/delta-sync-dispatching.js
+++ /dev/null
@@ -1,76 +0,0 @@
-const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util');
-const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
- DIRECT_DATABASE_ENDPOINT,
- MU_CALL_SCOPE_ID_INITIAL_SYNC,
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- INGEST_GRAPH,
- FILE_SYNC_GRAPH
-} = require('./config');
-
-/**
-* Dispatch the fetched information to a target graph.
-* Note: will be ADDED to it's own graph.
-* We take only care of adding them, not updating triples, this is a TODO
-* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
-* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
-* [ {
-* graph: "",
-* subject: "",
-* predicate: "",
-* object: "^^"
-* }
-* ]
-* @return {void} Nothing
-*/
-async function dispatch(lib, data) {
- const { mu, muAuthSudo, fetch } = lib;
- const { termObjectChangeSets } = data;
-
- for (let { deletes, inserts } of termObjectChangeSets) {
- const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
- const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
-
- if (deleteStatements.length) {
- await transformStatements(fetch, deleteStatements).then(
- transformedStatements => {
- deleteFromAllGraphs(
- muAuthSudo.updateSudo,
- transformedStatements,
- { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
- process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- );
- }
- )
- }
-
-
- if (insertStatements.length) {
- await transformStatements(fetch, insertStatements).then(
- transformedStatements => {
- batchedDbUpdate(
- muAuthSudo.updateSudo,
- INGEST_GRAPH,
- transformedStatements,
- { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
- process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- "INSERT"
- );
- }
- )
- }
- }
-}
-
-module.exports = {
- dispatch
-};
\ No newline at end of file
diff --git a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js b/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js
deleted file mode 100644
index 88017c1..0000000
--- a/config/loket-consumer/mandate-dispatching/initial-sync-dispatching.js
+++ /dev/null
@@ -1,59 +0,0 @@
-const { transformStatements, batchedDbUpdate, partition } = require('./util');
-const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
- DIRECT_DATABASE_ENDPOINT,
- MU_CALL_SCOPE_ID_INITIAL_SYNC,
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- INGEST_GRAPH,
- FILE_SYNC_GRAPH
-} = require('./config');
-const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
-
-/**
-* Dispatch the fetched information to a target graph.
-* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
-* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
-* [ {
-* graph: "",
-* subject: "",
-* predicate: "",
-* object: "^^"
-* }
-* ]
-* @return {void} Nothing
-*/
-async function dispatch(lib, data) {
- const { mu, muAuthSudo, fetch } = lib;
- const { termObjects } = data;
-
- if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
- console.warn(`Service configured to skip MU_AUTH!`);
- }
- console.log(`Using ${endpoint} to insert triples`);
-
- if (termObjects.length) {
- originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
-
- await transformStatements(fetch, originalInsertTriples).then(
- transformedInsertTriples => {
- batchedDbUpdate(
- muAuthSudo.updateSudo,
- INGEST_GRAPH,
- transformedInsertTriples,
- { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
- endpoint,
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION
- )
- }
- )
- }
-}
-
-module.exports = {
- dispatch
-};
\ No newline at end of file
diff --git a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js
deleted file mode 100644
index d826b2e..0000000
--- a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js
+++ /dev/null
@@ -1,88 +0,0 @@
-const { transformStatements, batchedDbUpdate, partition, deleteFromAllGraphs } = require('./util');
-const {
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- INGEST_GRAPH,
- PRIVACY_SENSITIVE_GRAPH,
- UNFILTERED_MAPPING,
- MAIN_INFO_MAPPING,
- PRIVATE_INFO_MAPPING,
-} = require('./config');
-
-/**
-* Dispatch the fetched information to a target graph.
-* Note: will be ADDED to it's own graph.
-* We take only care of adding them, not updating triples, this is a TODO
-* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
-* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
-* [ {
-* graph: "",
-* subject: "",
-* predicate: "",
-* object: "^^"
-* }
-* ]
-* @return {void} Nothing
-*/
-async function dispatch(lib, data) {
- const { mu, muAuthSudo, fetch } = lib;
- const { termObjectChangeSets } = data;
-
- for (let { deletes, inserts } of termObjectChangeSets) {
- const deleteStatements = deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
- const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
-
- if (deleteStatements.length) {
- const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING);
- if (transformedStatementsToDelete.length) {
- await deleteFromAllGraphs(
- muAuthSudo.updateSudo,
- transformedStatementsToDelete,
- { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
- process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- );
- }
- }
-
- if (insertStatements.length) {
- const transformedStatementsToInsert = await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING);
- if (transformedStatementsToInsert.length) {
- await batchedDbUpdate(
- muAuthSudo.updateSudo,
- INGEST_GRAPH,
- transformedStatementsToInsert,
- { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
- process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION
- );
- }
-
- const transformedStatementsToInsertPrivate = await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING);
- if (transformedStatementsToInsertPrivate.length) {
- await batchedDbUpdate(
- muAuthSudo.updateSudo,
- PRIVACY_SENSITIVE_GRAPH,
- transformedStatementsToInsertPrivate,
- { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
- process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION
- );
- }
- }
- }
-}
-
-module.exports = {
- dispatch
-};
\ No newline at end of file
diff --git a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js b/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js
deleted file mode 100644
index f5fedc8..0000000
--- a/config/loket-consumer/positions-dispatching/initial-sync-dispatching.js
+++ /dev/null
@@ -1,80 +0,0 @@
-const { transformStatements, batchedDbUpdate, } = require('./util');
-const { BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
- DIRECT_DATABASE_ENDPOINT,
- MU_CALL_SCOPE_ID_INITIAL_SYNC,
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
- INGEST_GRAPH,
- PRIVACY_SENSITIVE_GRAPH,
- MAIN_INFO_MAPPING,
- PRIVATE_INFO_MAPPING
-} = require('./config');
-const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
-
-/**
-* Dispatch the fetched information to a target graph.
-* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
-* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
-* [ {
-* graph: "",
-* subject: "",
-* predicate: "",
-* object: "^^"
-* }
-* ]
-* @return {void} Nothing
-*/
-async function dispatch(lib, data) {
- const { mu, muAuthSudo, fetch } = lib;
- const { termObjects } = data;
-
- if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
- console.warn(`Service configured to skip MU_AUTH!`);
- }
- console.log(`Using ${endpoint} to insert triples`);
-
- if (termObjects.length) {
- originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`)
-
- await transformStatements(fetch, originalInsertTriples, MAIN_INFO_MAPPING).then(
- transformedInsertTriples => {
- if (transformedInsertTriples.length) {
- batchedDbUpdate(
- muAuthSudo.updateSudo,
- INGEST_GRAPH,
- transformedInsertTriples,
- { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
- endpoint,
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION
- )
- }
- }
- );
- await transformStatements(fetch, originalInsertTriples, PRIVATE_INFO_MAPPING).then(
- transformedInsertTriples => {
- if (transformedInsertTriples.length) {
- batchedDbUpdate(
- muAuthSudo.updateSudo,
- PRIVACY_SENSITIVE_GRAPH,
- transformedInsertTriples,
- { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
- endpoint,
- BATCH_SIZE,
- MAX_DB_RETRY_ATTEMPTS,
- SLEEP_BETWEEN_BATCHES,
- SLEEP_TIME_AFTER_FAILED_DB_OPERATION
- )
- }
- }
- );
- }
-}
-
-module.exports = {
- dispatch
-};
\ No newline at end of file
diff --git a/config/loket-consumer/positions-dispatching/util.js b/config/loket-consumer/positions-dispatching/util.js
deleted file mode 100644
index ce38369..0000000
--- a/config/loket-consumer/positions-dispatching/util.js
+++ /dev/null
@@ -1,154 +0,0 @@
-const {
- MAX_REASONING_RETRY_ATTEMPTS,
- SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION
-} = require('./config');
-
-async function batchedDbUpdate(muUpdate,
- graph,
- triples,
- extraHeaders,
- endpoint,
- batchSize,
- maxAttempts,
- sleepBetweenBatches = 1000,
- sleepTimeOnFail = 1000,
- operation = 'INSERT'
-) {
-
- for (let i = 0; i < triples.length; i += batchSize) {
- console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
-
- const batch = triples.slice(i, i + batchSize).join('\n');
-
- const insertCall = async () => {
- await muUpdate(`
-${operation} DATA {
-GRAPH <${graph}> {
-${batch}
-}
-}
-`, extraHeaders, endpoint);
- };
-
- await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
-
- console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
- await new Promise(r => setTimeout(r, sleepBetweenBatches));
- }
-}
-
-async function deleteFromAllGraphs(muUpdate,
- triples,
- extraHeaders,
- endpoint,
- maxAttempts,
- sleepBetweenBatches = 1000,
- sleepTimeOnFail = 1000,
-) {
-
- for (const triple of triples) {
-
- console.log(`Deleting a triple from all graphs in triplestore`);
-
- const deleteCall = async () => {
- await muUpdate(`
- DELETE {
- GRAPH ?g {
- ${triple}
- }
- } WHERE {
- GRAPH ?g {
- ${triple}
- }
- }
- `, extraHeaders, endpoint);
- };
-
- await operationWithRetry(deleteCall, 0, maxAttempts, sleepTimeOnFail);
- console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
- await new Promise(r => setTimeout(r, sleepBetweenBatches));
- }
-}
-
-async function operationWithRetry(callback,
- attempt,
- maxAttempts,
- sleepTimeOnFail) {
- try {
- if (typeof callback === "function")
- return await callback();
- else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
- return await callback;
- }
- catch (e) {
- console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
- console.log(`Error: ${e}`);
- console.log(`Sleeping ${sleepTimeOnFail} ms`);
-
- if (attempt >= maxAttempts) {
- console.log(`Max attempts reached for ${callback.toString()}, giving up`);
- throw e;
- }
-
- await new Promise(r => setTimeout(r, sleepTimeOnFail));
- return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
- }
-}
-
-/**
- * Splits an array into two parts, a part that passes and a part that fails a predicate function.
- * Credits: https://github.com/benjay10
- * @public
- * @function partition
- * @param {Array} arr - Array to be partitioned
- * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
- * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
- */
-function partition(arr, fn) {
- let passes = [], fails = [];
- arr.forEach((item) => (fn(item) ? passes : fails).push(item));
- return { passes, fails };
-}
-
-
-/**
- * Send triples to reasoning service for conversion
- *
- */
-function transformTriples(fetch, triples, mapping) {
- return operationWithRetry(mainConversion(fetch, triples, mapping), 0,
- MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
-}
-
-
-function mainConversion(fetch, triples, mapping) {
- let formdata = new URLSearchParams();
- formdata.append("data", triples);
-
- let requestOptions = {
- method: 'POST',
- body: formdata,
- redirect: 'follow'
- };
-
- return fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
- .then(response => response.text());
-}
-
-
-function transformStatements(fetch, triples, mapping) {
- return transformTriples(fetch, triples.join('\n'), mapping).then(
- graph => {
- statements = graph ? graph.split('\n') : [];
- console.log(`CONVERSION ${mapping}: FROM ${triples.length} triples to ${statements.length}`)
- return statements
- }
- )
-}
-
-module.exports = {
- batchedDbUpdate,
- deleteFromAllGraphs,
- partition,
- transformStatements
-};
\ No newline at end of file
diff --git a/config/loket-consumer/positions-dispatching/config.js b/config/loket-consumer/worship-services-sensitive-dispatching/config.js
similarity index 63%
rename from config/loket-consumer/positions-dispatching/config.js
rename to config/loket-consumer/worship-services-sensitive-dispatching/config.js
index 911277b..e89bca3 100644
--- a/config/loket-consumer/positions-dispatching/config.js
+++ b/config/loket-consumer/worship-services-sensitive-dispatching/config.js
@@ -1,22 +1,25 @@
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE) || 100;
const MU_CALL_SCOPE_ID_INITIAL_SYNC = process.env.MU_CALL_SCOPE_ID_INITIAL_SYNC || 'http://redpencil.data.gift/id/concept/muScope/deltas/consumer/initialSync';
const BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES = process.env.BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES == 'true' ? true : false;
-const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://virtuoso:8890/sparql';
+const DIRECT_DATABASE_ENDPOINT = process.env.DIRECT_DATABASE_ENDPOINT || 'http://triplestore:8890/sparql';
const MAX_DB_RETRY_ATTEMPTS = parseInt(process.env.MAX_DB_RETRY_ATTEMPTS || 5);
const MAX_REASONING_RETRY_ATTEMPTS = parseInt(process.env.MAX_REASONING_RETRY_ATTEMPTS || 5);
const SLEEP_BETWEEN_BATCHES = parseInt(process.env.SLEEP_BETWEEN_BATCHES || 1000);
const SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION || 10000);
const SLEEP_TIME_AFTER_FAILED_DB_OPERATION = parseInt(process.env.SLEEP_TIME_AFTER_FAILED_DB_OPERATION || 60000);
-const INGEST_GRAPH = process.env.INGEST_GRAPH || `http://mu.semte.ch/graphs/organisatieportaal`;
-const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/privacy-centric-graph`;
-const UNFILTERED_MAPPING = 'main';
-const MAIN_INFO_MAPPING = 'main-info';
-const PRIVATE_INFO_MAPPING = 'private-info';
+// Graphs
+const TARGET_GRAPH = process.env.TARGET_GRAPH || `http://mu.semte.ch/graphs/worship-service`;
+const PRIVACY_SENSITIVE_GRAPH = process.env.PRIVACY_SENSITIVE_GRAPH || `http://mu.semte.ch/graphs/worship-privacy-centric-graph`;
+const LANDING_ZONE_GRAPH = process.env.DCR_LANDING_ZONE_GRAPH || `http://mu.semte.ch/graphs/landing-zone/leidinggevenden`;
-if (!process.env.FILE_SYNC_GRAPH)
- throw `Expected 'FILE_SYNC_GRAPH' to be provided.`;
-const FILE_SYNC_GRAPH = process.env.FILE_SYNC_GRAPH;
+// Mappings
+const MAIN_INFO_MAPPING = "main-info";
+const PRIVATE_INFO_MAPPING = "private-info";
+
+// ENDPOINTS
+const LANDING_ZONE_DATABASE = process.env.DCR_LANDING_ZONE_DATABASE || 'db';
+const LANDING_ZONE_DATABASE_ENDPOINT = process.env.DCR_LANDING_ZONE_DATABASE_ENDPOINT || `http://${LANDING_ZONE_DATABASE}:8890/sparql`;
module.exports = {
BATCH_SIZE,
@@ -28,9 +31,10 @@ module.exports = {
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
- INGEST_GRAPH,
+ LANDING_ZONE_GRAPH,
+ LANDING_ZONE_DATABASE_ENDPOINT,
+ TARGET_GRAPH,
PRIVACY_SENSITIVE_GRAPH,
- UNFILTERED_MAPPING,
MAIN_INFO_MAPPING,
- PRIVATE_INFO_MAPPING,
-};
\ No newline at end of file
+ PRIVATE_INFO_MAPPING
+};
diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js b/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js
new file mode 100644
index 0000000..a8df81c
--- /dev/null
+++ b/config/loket-consumer/worship-services-sensitive-dispatching/delta-context-config.js
@@ -0,0 +1,11 @@
+const contextConfig = {
+ addTypes: {
+ scope: 'all', // 'inserts, 'deletes', 'all' or none. To add rdf:type to subjects of inserts, deletes or both
+ exhausitive: false, // true or false: find all types for a subject, even if one is already present in delta
+ },
+ contextQueries: []
+}
+
+module.exports = {
+ contextConfig
+};
diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js b/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js
new file mode 100644
index 0000000..ef18e2e
--- /dev/null
+++ b/config/loket-consumer/worship-services-sensitive-dispatching/delta-sync-dispatching.js
@@ -0,0 +1,114 @@
+const {
+ transformStatements,
+ deleteFromTargetGraph,
+ insertIntoTargetGraph,
+ deleteFromPrivateGraph,
+ insertIntoPrivateGraph
+} = require('./util');
+
+const {
+ MAIN_INFO_MAPPING,
+ PRIVATE_INFO_MAPPING
+} = require('./config');
+
+/**
+* Dispatch the fetched information to a target graph.
+* Note: will be ADDED to it's own graph.
+* We take only care of adding them, not updating triples, this is a TODO
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjectChangeSets: { deletes, inserts } } data - The fetched changes sets, which objects of serialized Terms
+* [ {
+* graph: "",
+* subject: "",
+* predicate: "",
+* object: "^^"
+* }
+* ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+ const { mu, fetch } = lib;
+ let { termObjectChangeSets, termObjectChangeSetsWithContext } = data;
+
+ // Both arrays are the same length, so we can zip them together for easier processing
+ zippedChangeSets = termObjectChangeSets.map((o, i) => ({
+ original: o,
+ withContext: termObjectChangeSetsWithContext[i]
+ }));
+
+ console.log(`Received ${zippedChangeSets.length} change sets`)
+
+ for (let { original, withContext } of zippedChangeSets) {
+ let originalDeletes, originalInserts, insertsWithContext, deletesWithContext;
+ try {
+ originalInserts = original.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ originalDeletes = original.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ } catch (e) {
+ console.log(`Error while processing change set: ${e}`);
+ console.log(`Original change set: ${JSON.stringify(original)}`);
+ throw e;
+ }
+
+ // Extra context needed for mapping from DL to OP model and filtering based on type.
+ try {
+ insertsWithContext = withContext.inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ deletesWithContext = withContext.deletes.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+ } catch (e) {
+ console.log(`Error while processing change set with context: ${e}`);
+ console.log(`Change set with context: ${JSON.stringify(withContext)}`);
+ throw e;
+ }
+
+ if (originalDeletes.length) {
+ // Map main deletes from DL to OP model
+ const transformedMainDeletes = await transformStatements(fetch, deletesWithContext, MAIN_INFO_MAPPING);
+
+ if (!transformedMainDeletes.length) {
+ console.log(`Warn: Delete main statements mapped to empty result.`);
+ console.log(`Input: ${deletesWithContext}`);
+ console.log(`Output: ${transformedMainDeletes}`);
+ } else {
+ await deleteFromTargetGraph(lib, transformedMainDeletes);
+ }
+
+ // Map private deletes from DL to OP model
+ const transformedPrivateDeletes = await transformStatements(fetch, deletesWithContext, PRIVATE_INFO_MAPPING);
+
+ if (!transformedPrivateDeletes.length) {
+ console.log(`Warn: Delete private statements mapped to empty result.`);
+ console.log(`Input: ${deletesWithContext}`);
+ console.log(`Output: ${transformedPrivateDeletes}`);
+ } else {
+ await deleteFromPrivateGraph(lib, transformedDeletes);
+ }
+ }
+
+ if (originalInserts.length) {
+ // Map main inserts from DL to OP model
+ const transformedMainInserts = await transformStatements(fetch, insertsWithContext, MAIN_INFO_MAPPING);
+
+ if (!transformedMainInserts.length) {
+ console.log(`Warn: Insert statements mapped to empty result.`);
+ console.log(`Input: ${insertsWithContext}`);
+ console.log(`Output: ${transformedMainInserts}`);
+ } else {
+ await insertIntoTargetGraph(lib, transformedMainInserts);
+ }
+
+ // Map private inserts from DL to OP model
+ const transformedPrivateInserts = await transformStatements(fetch, insertsWithContext, PRIVATE_INFO_MAPPING);
+
+ if (!transformedPrivateInserts.length) {
+ console.log(`Warn: Insert statements mapped to empty result.`);
+ console.log(`Input: ${insertsWithContext}`);
+ console.log(`Output: ${transformedPrivateInserts}`);
+ } else {
+ await insertIntoPrivateGraph(lib, transformedPrivateInserts);
+ }
+ }
+ }
+}
+
+module.exports = {
+ dispatch
+};
diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js b/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js
new file mode 100644
index 0000000..559250f
--- /dev/null
+++ b/config/loket-consumer/worship-services-sensitive-dispatching/initial-sync-dispatching.js
@@ -0,0 +1,106 @@
+const { batchedDbUpdate, transformLandingZoneGraph } = require('./util');
+const {
+ BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES,
+ DIRECT_DATABASE_ENDPOINT,
+ MU_CALL_SCOPE_ID_INITIAL_SYNC,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ TARGET_GRAPH,
+ PRIVACY_SENSITIVE_GRAPH,
+ LANDING_ZONE_GRAPH,
+ LANDING_ZONE_DATABASE_ENDPOINT,
+ MAIN_INFO_MAPPING,
+ PRIVATE_INFO_MAPPING
+} = require('./config');
+
+const endpoint = BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES ? DIRECT_DATABASE_ENDPOINT : process.env.MU_SPARQL_ENDPOINT;
+
+/**
+* Dispatch the fetched information to a target graph.
+* @param { mu, muAuthSudo } lib - The provided libraries from the host service.
+* @param { termObjects } data - The fetched quad information, which objects of serialized Terms
+* [ {
+* graph: "",
+* subject: "",
+* predicate: "",
+* object: "^^"
+* }
+* ]
+* @return {void} Nothing
+*/
+async function dispatch(lib, data) {
+ const { mu, muAuthSudo } = lib;
+ const { termObjects } = data;
+
+ // Steps:
+ // 1. Write the triples to the landing zone graph - withouth any mapping or filtering
+ // 2. One-time reasoning run with the full landing zone graph and write the results to the target graph (on-finish)
+
+ if (BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES) {
+ console.warn(`Service configured to skip MU_AUTH!`);
+ }
+ console.log(`Using ${endpoint} to insert triples`);
+
+ if (termObjects.length) {
+ const originalInsertTriples = termObjects.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
+
+ await batchedDbUpdate(
+ muAuthSudo.updateSudo,
+ LANDING_ZONE_GRAPH,
+ originalInsertTriples,
+ { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+ LANDING_ZONE_DATABASE_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+ );
+ }
+}
+
+async function onFinishInitialIngest(lib) {
+ const { muAuthSudo, fetch } = lib;
+
+ console.log(`!! On-finish triggered !!`);
+
+ const transformedMainInsertTriples = await transformLandingZoneGraph(fetch, endpoint, MAIN_INFO_MAPPING);
+ console.log(`Transformed ${transformedMainInsertTriples.length} main triples`);
+
+ if (transformedMainInsertTriples.length) {
+ await batchedDbUpdate(
+ muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ transformedMainInsertTriples,
+ { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+ endpoint,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+ );
+ }
+
+ const transformedPrivateInsertTriples = await transformLandingZoneGraph(fetch, endpoint, PRIVATE_INFO_MAPPING);
+ console.log(`Transformed ${transformedPrivateInsertTriples.length} private triples`);
+
+ if (transformedPrivateInsertTriples.length) {
+ await batchedDbUpdate(
+ muAuthSudo.updateSudo,
+ PRIVACY_SENSITIVE_GRAPH,
+ transformedPrivateInsertTriples,
+ { 'mu-call-scope-id': MU_CALL_SCOPE_ID_INITIAL_SYNC },
+ endpoint,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+ );
+ }
+}
+
+module.exports = {
+ dispatch,
+ onFinishInitialIngest
+};
diff --git a/config/loket-consumer/worship-services-sensitive-dispatching/util.js b/config/loket-consumer/worship-services-sensitive-dispatching/util.js
new file mode 100644
index 0000000..4b17b75
--- /dev/null
+++ b/config/loket-consumer/worship-services-sensitive-dispatching/util.js
@@ -0,0 +1,204 @@
+const {
+ MAX_REASONING_RETRY_ATTEMPTS,
+ SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ TARGET_GRAPH,
+ PRIVACY_SENSITIVE_GRAPH,
+ LANDING_ZONE_GRAPH,
+ BATCH_SIZE
+} = require('./config');
+
+async function batchedDbUpdate(
+ muUpdate,
+ graph,
+ triples,
+ extraHeaders,
+ endpoint,
+ batchSize,
+ maxAttempts,
+ sleepBetweenBatches = 1000,
+ sleepTimeOnFail = 1000,
+ operation = 'INSERT'
+) {
+
+ for (let i = 0; i < triples.length; i += batchSize) {
+ console.log(`Inserting triples in batch: ${i}-${i + batchSize}`);
+
+ const batch = triples.slice(i, i + batchSize).join('\n');
+
+ const insertCall = async () => {
+ await muUpdate(`
+${operation} DATA {
+GRAPH <${graph}> {
+${batch}
+}
+}
+`, extraHeaders, endpoint);
+ };
+
+ await operationWithRetry(insertCall, 0, maxAttempts, sleepTimeOnFail);
+
+ console.log(`Sleeping before next query execution: ${sleepBetweenBatches}`);
+ await new Promise(r => setTimeout(r, sleepBetweenBatches));
+ }
+}
+
+async function operationWithRetry(callback,
+ attempt,
+ maxAttempts,
+ sleepTimeOnFail) {
+ try {
+ if (typeof callback === "function")
+ return await callback();
+ else // Catch error from promise - not how I would do it normally, but allows re use of existing code.
+ return await callback;
+ }
+ catch (e) {
+ console.log(`Operation failed for ${callback.toString()}, attempt: ${attempt} of ${maxAttempts}`);
+ console.log(`Error: ${e}`);
+ console.log(`Sleeping ${sleepTimeOnFail} ms`);
+
+ if (attempt >= maxAttempts) {
+ console.log(`Max attempts reached for ${callback.toString()}, giving up`);
+ throw e;
+ }
+
+ await new Promise(r => setTimeout(r, sleepTimeOnFail));
+ return operationWithRetry(callback, ++attempt, maxAttempts, sleepTimeOnFail);
+ }
+}
+
+/**
+ * Splits an array into two parts, a part that passes and a part that fails a predicate function.
+ * Credits: https://github.com/benjay10
+ * @public
+ * @function partition
+ * @param {Array} arr - Array to be partitioned
+ * @param {Function} fn - Function that accepts single argument: an element of the array, and should return a truthy or falsy value.
+ * @returns {Object} Object that contains keys passes and fails, each representing an array with elemets that pass or fail the predicate respectively
+ */
+function partition(arr, fn) {
+ let passes = [], fails = [];
+ arr.forEach((item) => (fn(item) ? passes : fails).push(item));
+ return { passes, fails };
+}
+
+/**
+ * Send triples to reasoning service for conversion
+ *
+ */
+async function transformTriples(fetch, triples, mapping) {
+ return await operationWithRetry(mainConversion(fetch, triples, mapping), 0,
+ MAX_REASONING_RETRY_ATTEMPTS, SLEEP_TIME_AFTER_FAILED_REASONING_OPERATION);
+}
+
+async function mainConversion(fetch, triples, mapping) {
+ let formdata = new URLSearchParams();
+ formdata.append("data", triples);
+
+ let requestOptions = {
+ method: 'POST',
+ body: formdata,
+ redirect: 'follow'
+ };
+
+ const response = await fetch(`http://reasoner/reason/dl2op/${mapping}`, requestOptions)
+ return await response.text();
+}
+
+async function transformStatements(fetch, triples, mapping = 'main') {
+ console.log(`Received ${JSON.stringify(triples)} to transform`);
+ const transformedTriples = await transformTriples(fetch, triples.join('\n'), mapping);
+ statements = transformedTriples ? transformedTriples.replace(/\n{2,}/g, '').split('\n') : [];
+ console.log(`CONVERSION: FROM ${triples.length} triples to ${statements.length}`);
+ return statements;
+}
+
+async function deleteFromTargetGraph(lib, statements) {
+ console.log(`Deleting ${statements.length} statements from target graph`);
+ console.log(`Statements: ${JSON.stringify(statements)}`)
+ await batchedDbUpdate(
+ lib.muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ statements,
+ {},
+ process.env.MU_SPARQL_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ 'DELETE');
+}
+
+async function deleteFromPrivateGraph(lib, statements) {
+ console.log(`Deleting ${statements.length} statements from target graph`);
+ console.log(`Statements: ${JSON.stringify(statements)}`)
+ await batchedDbUpdate(
+ lib.muAuthSudo.updateSudo,
+ PRIVACY_SENSITIVE_GRAPH,
+ statements,
+ {},
+ process.env.MU_SPARQL_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ 'DELETE');
+}
+
+async function insertIntoTargetGraph(lib, statements) {
+ console.log(`Inserting ${statements.length} statements into target graph`);
+ console.log(`Statements: ${JSON.stringify(statements)}`)
+
+ await batchedDbUpdate(
+ lib.muAuthSudo.updateSudo,
+ TARGET_GRAPH,
+ statements,
+ {},
+ process.env.MU_SPARQL_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ 'INSERT');
+}
+
+async function insertIntoPrivateGraph(lib, statements) {
+ console.log(`Inserting ${statements.length} statements into target graph`);
+ console.log(`Statements: ${JSON.stringify(statements)}`)
+
+ await batchedDbUpdate(
+ lib.muAuthSudo.updateSudo,
+ PRIVACY_SENSITIVE_GRAPH,
+ statements,
+ {},
+ process.env.MU_SPARQL_ENDPOINT,
+ BATCH_SIZE,
+ MAX_DB_RETRY_ATTEMPTS,
+ SLEEP_BETWEEN_BATCHES,
+ SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+ 'INSERT');
+}
+
+async function transformLandingZoneGraph(fetch, endpoint, mapping = 'main') {
+ console.log(`Transforming landing zone graph: ${LANDING_ZONE_GRAPH}`);
+
+ const response = await fetch(`http://reasoner/reason/dl2op/${mapping}?data=${encodeURIComponent(`${endpoint}?default-graph-uri=&query=CONSTRUCT+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D+WHERE+%7B%0D%0A+GRAPH+<${LANDING_ZONE_GRAPH}>+%7B%0D%0A%3Fs+%3Fp+%3Fo%0D%0A%7D%0D%0A%7D&should-sponge=&format=text%2Fplain&timeout=0&run=+Run+Query`)}`);
+ const text = await response.text();
+ const statements = text.replace(/\n{2,}/g, '').split('\n');
+
+ return statements;
+}
+
+module.exports = {
+ batchedDbUpdate,
+ partition,
+ transformStatements,
+ transformLandingZoneGraph,
+ deleteFromTargetGraph,
+ insertIntoTargetGraph,
+ deleteFromPrivateGraph,
+ insertIntoPrivateGraph
+};
diff --git a/config/migrations-triggering-indexing/2023/20230324150743-update-status-and-orgaan-of-st-egidius-van-sint-gillis-waas.sparql b/config/migrations-triggering-indexing/2023/20230324150743-update-status-and-orgaan-of-st-egidius-van-sint-gillis-waas.sparql
new file mode 100644
index 0000000..060b4be
--- /dev/null
+++ b/config/migrations-triggering-indexing/2023/20230324150743-update-status-and-orgaan-of-st-egidius-van-sint-gillis-waas.sparql
@@ -0,0 +1,46 @@
+DELETE DATA {
+ GRAPH {
+
+ .
+ }
+}
+;
+INSERT DATA {
+ GRAPH {
+
+ .
+
+
+ a ;
+ "18d843d47e223409b9bb6e7af8a4d8a8" ;
+ ;
+ "2020-04-01T00:00:00"^^xsd:dateTime ;
+ "2023-03-31T00:00:00"^^xsd:dateTime ;
+ ;
+ ;
+ ;
+ ;
+ ;
+ .
+
+
+ a ;
+ "daac7f53-b020-443d-92a7-9d47be6cb186" ;
+ .
+
+
+ a ;
+ "3618fc2a-83f7-45dd-969f-06afd0c3277b" ;
+ .
+
+
+ a ;
+ "e4b05858-c66d-4098-9c43-59af0359e603" ;
+ .
+
+
+ a ;
+ "5147f880-b0a5-4145-ac01-db8323a5410d" ;
+ .
+ }
+}
diff --git a/config/migrations/2023/20230213113324-add-new-codelist.sparql b/config/migrations/2023/20230213113324-add-new-codelist.sparql
new file mode 100644
index 0000000..ca4a6d3
--- /dev/null
+++ b/config/migrations/2023/20230213113324-add-new-codelist.sparql
@@ -0,0 +1,18 @@
+
+PREFIX org:
+PREFIX ere:
+PREFIX skos:
+PREFIX mu:
+
+
+INSERT DATA {
+ graph {
+ a ;
+ mu:uuid "7d8e0c15-7726-4196-9416-6dc40ed858bb";
+ skos:prefLabel "Waarnemende Imam";
+ ;
+ .
+ }
+}
+
+
diff --git a/config/migrations/2023/20230214163952-clear-admin-unit-producer-data.sparql b/config/migrations/2023/20230214163952-clear-admin-unit-producer-data.sparql
new file mode 100644
index 0000000..e5938f6
--- /dev/null
+++ b/config/migrations/2023/20230214163952-clear-admin-unit-producer-data.sparql
@@ -0,0 +1,61 @@
+DELETE {
+ GRAPH ?g {
+ ?s ?p ?o .
+ }
+} WHERE {
+ GRAPH ?g {
+ ?s a ;
+ ;
+ ?p ?o .
+ }
+}
+;
+DELETE {
+ GRAPH ?g {
+ ?s ?p ?o .
+ }
+} WHERE {
+ GRAPH ?g {
+ ?s a ;
+ ;
+ ?p ?o .
+ }
+}
+;
+DELETE {
+ GRAPH ?g {
+ ?s ?p ?o .
+ }
+} WHERE {
+ GRAPH ?g {
+ ?s a ;
+ ;
+ ?p ?o .
+ }
+}
+;
+DELETE {
+ GRAPH ?g {
+ ?s ?p ?o .
+ }
+} WHERE {
+ GRAPH ?g {
+ ?s a ;
+ ;
+ ?p ?o .
+ }
+}
+;
+DELETE {
+ GRAPH ?g {
+ ?s ?p ?o .
+ }
+} WHERE {
+ GRAPH ?g {
+ ?s a ;
+ ;
+ ?p ?o .
+ }
+}
+;
+CLEAR GRAPH
diff --git a/config/migrations/2023/20230215094055-add-missing-position.graph b/config/migrations/2023/20230215094055-add-missing-position.graph
new file mode 100644
index 0000000..00b1762
--- /dev/null
+++ b/config/migrations/2023/20230215094055-add-missing-position.graph
@@ -0,0 +1 @@
+http://mu.semte.ch/graphs/organisatieportaal
diff --git a/config/migrations/2023/20230215094055-add-missing-position.ttl b/config/migrations/2023/20230215094055-add-missing-position.ttl
new file mode 100644
index 0000000..f385e01
--- /dev/null
+++ b/config/migrations/2023/20230215094055-add-missing-position.ttl
@@ -0,0 +1,613 @@
+@prefix ns0: .
+@prefix ns1: .
+@prefix ns2: .
+@prefix rdf: .
+@prefix ns4: .
+@prefix ns5: .
+
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:e06b3e3d005d25e4a23459dd5773c00e .
+
+ ns0:wordtBediendDoor ns1:a3d7cda7f4acad34682b8086d13ea397 .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:a6735a9f75e9f30f8d31198e18e39192 .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:e39958a6e2d38e23b5669a3b9ae1e719 .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:c1efedae090d2c4be97d529c73b5ae87 .
+
+ ns0:wordtBediendDoor ns1:ef9ebb3d9429b24c87a6576bfe302d85 .
+
+ ns0:wordtBediendDoor .
+ns2:ef77bb3094fdc4a7ed7e09551500acfe
+ ns0:wordtBediendDoor .
+ns2:a8faa182c59167da7551f40979dc5022
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:c924a44f9163ec6c946505f5f687708d .
+ns2:ca0daee18a29e187017bf236e26f1ec2
+ ns0:wordtBediendDoor ns1:cb4bf613bd82f99785df7aedb65b3367 .
+ns2:f95b761af55091b491fd62bd6bed11dc
+ ns0:wordtBediendDoor ns1:d1dd10ca84303e547546f7e7a502dc6c .
+
+ ns0:wordtBediendDoor .
+ns2:c1c334a2cc56ac501badce5e879aec65
+ ns0:wordtBediendDoor ns1:b74b9e77b49bbe8f5be5766be677c9bb .
+
+ ns0:wordtBediendDoor ns1:bfe1868d6a41e073f114dd2efdcf51a5 .
+
+ ns0:wordtBediendDoor .
+ns2:a6e0531c6c4b5885552604163cfabfbc
+ ns0:wordtBediendDoor ns1:cdb151adeceb026fc5e81ea80d729391 .
+
+ ns0:wordtBediendDoor ns1:c94d252366917f598a19df33680bbb29 .
+ns2:cd7d4208e8e6291d75b33d89e3a0a439
+ ns0:wordtBediendDoor .
+ns2:a8d9f1a45c11c6761306bdf680349593
+ ns0:wordtBediendDoor ns1:fad9626cd2ab433ce884ef75d06e840b .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:d5ef4ca7efebfdafc4d28d2d8ab019b1 .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+ns2:a8880f3d8b482788c5f3adbf58f01b4f
+ ns0:wordtBediendDoor .
+ns2:adeaab8a02815bf2c5816165a45ad4c5
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+ns2:c7d6827eb88cd977146e72583461071b
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+ns2:b81199eddd02f291bd4c1a0e29e5b9c8
+ ns0:wordtBediendDoor ns1:cf320c8fb2e9db3a488b0ac016440f33 .
+ns2:db46915e77fbf8f083e9861c802b2b13
+ ns0:wordtBediendDoor ns1:b6c9aeea4bf9daa1520156b9fe5a49e5 .
+ns2:fd4ae252fffca29852354b12ef37b086
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+ns2:b540096ac8be2c9c72dafd59e61080c5
+ ns0:wordtBediendDoor ns1:f9a134e17b7ac4a42a66b7b66d86d00b .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:d4505963c15a4dd1cc77b34977b727c0 .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:f90bc6678d032b1203c3e003b05a903e .
+
+ ns0:wordtBediendDoor ns1:a1a4364ce6767907e600d8025c2f8cb9 .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:baa288870012628b450bd134175cb91a .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor .
+
+ ns0:wordtBediendDoor ns1:f1ac3ad7a3cf88917b6ff453bcdae693 .
+
+ ns0:wordtBediendDoor ns1:e3c51e910902a9b4087f9d2dfe8629a2 .
+
+ ns0:wordtBediendDoor .
+