diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 66391ca..5f0fd27 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -2,21 +2,37 @@ "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", - "fileBaseName": "dump-administrative-units" + "fileBaseName": "dump-administrative-units", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", - "fileBaseName": "dump-administrative-units" + "fileBaseName": "dump-administrative-units", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", - "fileBaseName": "dump-organizations" + "fileBaseName": "dump-organizations", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", - "fileBaseName": "dump-organizations" + "fileBaseName": "dump-organizations", + "compression": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" : { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "fileBaseName": "dump-public", + "compression": false + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" : { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", + "fileBaseName": "dump-public", + "compression": false } } diff --git a/config/delta-producer/public/export.json b/config/delta-producer/public/export.json new file mode 100644 index 0000000..49d2931 --- /dev/null +++ b/config/delta-producer/public/export.json @@ -0,0 +1,270 @@ +{ + "conceptScheme": "http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b", + "export": [ + { + "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", + "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau", + "http://data.lblod.info/vocabularies/erediensten/denominatie", + "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", + "http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://www.w3.org/ns/org#Site", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + + ], + "properties": [ + "http://www.w3.org/ns/org#siteAddress", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://data.lblod.info/vocabularies/erediensten/vestigingstype" + ] + }, + { + "type": "http://www.w3.org/ns/locn#Address", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "^http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + + ], + "properties": [ + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer", + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer", + "http://www.w3.org/ns/locn#thoroughfare", + "http://www.w3.org/ns/locn#postCode", + "https://data.vlaanderen.be/ns/adres#gemeentenaam", + "http://www.w3.org/ns/locn#adminUnitL2", + "https://data.vlaanderen.be/ns/adres#land", + "http://www.w3.org/ns/locn#fullAddress", + "https://data.vlaanderen.be/ns/adres#verwijstNaar", + "http://purl.org/dc/terms/source" + ] + }, + { + "type": "http://schema.org/ContactPoint", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^http://www.w3.org/ns/org#siteAddress", + "^http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/locn#address", + "http://xmlns.com/foaf/0.1/page", + "http://schema.org/faxNumber", + "http://schema.org/contactType", + "http://xmlns.com/foaf/0.1/page", + "http://schema.org/email", + "http://schema.org/telephone" + + ] + }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ] + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "^https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator", + "^http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://data.vlaanderen.be/ns/mandaat#bindingEinde", + "http://data.vlaanderen.be/ns/mandaat#bindingStart", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#classification", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://www.w3.org/ns/org#ChangeEvent", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://data.lblod.info/vocabularies/contacthub/typeWijziging", + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://purl.org/dc/terms/date", + "http://purl.org/dc/terms/description", + "http://data.lblod.info/vocabularies/contacthub/typeWijziging", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#changedBy", + "http://lblod.data.gift/vocabularies/organisatie/veranderingsgebeurtenisResultaat", + "http://www.w3.org/ns/org#originalOrganization", + "http://www.w3.org/ns/org#resultingOrganization", + "http://data.europa.eu/m8g/hasFormalFramework" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", + "graphsFilter": [ + "http://mu.semte.ch/graphs/organisatieportaal" + ], + "pathToConceptScheme": [ + "http://www.w3.org/2004/02/skos/core#inScheme" + ], + "properties": [ + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + } + ] +} diff --git a/config/delta/rules.js b/config/delta/rules.js index 7f3265b..15a2309 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -121,6 +121,41 @@ export default [ optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] } }, + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-public/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, + { + match: { + graph: { + type: 'uri', + value: 'http://redpencil.data.gift/id/deltas/producer/public' + } + }, + callback: { + url: 'http://delta-producer-json-diff-publisher-public/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ] + } + }, { match: { predicate: { diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 317c490..928128b 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -62,5 +62,37 @@ "nextIndex": "0" } ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextIndex": "0" + }, + { + "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "1" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "0" + } + ] } } diff --git a/config/migrations/2022/20220407110000-add-export-concept-scheme.sparql b/config/migrations/2022/20220407110000-add-export-concept-scheme.sparql new file mode 100644 index 0000000..3f76325 --- /dev/null +++ b/config/migrations/2022/20220407110000-add-export-concept-scheme.sparql @@ -0,0 +1,43 @@ +PREFIX mu: +PREFIX skos: + +INSERT { + GRAPH { + + a + skos:ConceptScheme ; + mu:uuid + "10caf4a4-1ff5-4c05-917e-56489910d68b" ; + skos:prefLabel + "Public OP Export" . + ?code + skos:inScheme + . + } +} WHERE { + VALUES ?type { + + + + + + } + GRAPH { + ?code + a + ?type . + } +} + +INSERT { + GRAPH { + + skos:inScheme + . + + + skos:inScheme + . + } +} + diff --git a/config/resources/dcat.json b/config/resources/dcat.json index eba75ea..4945a66 100644 --- a/config/resources/dcat.json +++ b/config/resources/dcat.json @@ -133,6 +133,10 @@ "byte-size": { "type": "number", "predicate": "dcat:byteSize" + }, + "compression": { + "type": "url", + "predicate": "dcat:compressFormat" } }, "relationships": { diff --git a/docker-compose.yml b/docker-compose.yml index 59b5e1c..c80080b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -202,7 +202,7 @@ services: # DELTA GENERAL ################################################################################ delta-producer-dump-file-publisher: - image: bdevloed/delta-producer-dump-file-publisher:0.5.0-beta + image: bdevloed/delta-producer-dump-file-publisher:0.8.0-beta volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share @@ -337,6 +337,69 @@ services: ################################################################################ +################################################################################ +# DELTA PUBLIC INFORMATION: START +################################################################################ + delta-producer-bg-jobs-initiator-public: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public' + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public' + HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public' + CRON_PATTERN_DUMP_JOB: '11 */4 * * *' # At minute 11 every 4 hours + CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM + START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-public: + image: lblod/delta-producer-publication-graph-maintainer:0.15.1 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + RELATIVE_FILE_PATH: "deltas/public" + PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/public' + HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public' + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public' + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: 'true' + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + volumes: + - ./config/delta-producer/public:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + logging: *default-logging + delta-producer-json-diff-publisher-public: + image: lblod/delta-producer-json-diff-file-publisher:0.2.3 + environment: + MAX_BODY_SIZE: "50mb" + ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal" + RELATIVE_FILE_PATH: "deltas/public" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public" + PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/public' + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + volumes: + - ./data/files:/share + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging +################################################################################ +# DELTA PUBLIC INFORMATION: END +################################################################################ + + ################################################################################ # MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH ################################################################################