felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #136 from lblod/feature/public-export

Browse Source 
publish public info
This commit is contained in:
Boris De Vloed
2022-04-22 09:28:08 +02:00
committed by GitHub
parent c1a361b338 8afaf03854
commit f5d611a5b5
7 changed files with 468 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
config/delta-producer/dump-file-publisher/config.json
View File

@@ -2,21 +2,37 @@
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" : {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units",
"fileBaseName": "dump-administrative-units"
"fileBaseName": "dump-administrative-units",
"compression": true
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" : {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units",
"fileBaseName": "dump-administrative-units"
"fileBaseName": "dump-administrative-units",
"compression": true
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" : {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
"fileBaseName": "dump-organizations"
"fileBaseName": "dump-organizations",
"compression": true
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" : {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
"fileBaseName": "dump-organizations"
"fileBaseName": "dump-organizations",
"compression": true
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" : {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/public",
"fileBaseName": "dump-public",
"compression": false
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" : {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/public",
"fileBaseName": "dump-public",
"compression": false
}
}

270
config/delta-producer/public/export.json Normal file
View File

@@ -0,0 +1,270 @@
{
"conceptScheme": "http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b",
"export": [
{
"type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
"strictTypeExport": "true",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/2004/02/skos/core#altLabel",
"http://www.w3.org/ns/adms#identifier",
"http://www.w3.org/ns/regorg#orgStatus",
"http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#hasSite",
"http://www.w3.org/ns/org#changedBy",
"http://www.w3.org/ns/org#resultedFrom",
"http://www.w3.org/ns/org#hasPost",
"http://www.w3.org/ns/org#linkedTo",
"http://www.w3.org/ns/org#hasSubOrganization",
"http://www.w3.org/ns/org#classification",
"http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur",
"http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau",
"http://data.lblod.info/vocabularies/erediensten/denominatie",
"http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend",
"http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor",
"http://data.lblod.info/vocabularies/erediensten/typeEredienst"
]
},
{
"type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst",
"strictTypeExport": "true",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/2004/02/skos/core#altLabel",
"http://www.w3.org/ns/adms#identifier",
"http://www.w3.org/ns/regorg#orgStatus",
"http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#hasSite",
"http://www.w3.org/ns/org#changedBy",
"http://www.w3.org/ns/org#resultedFrom",
"http://www.w3.org/ns/org#hasPost",
"http://www.w3.org/ns/org#linkedTo",
"http://www.w3.org/ns/org#hasSubOrganization",
"http://www.w3.org/ns/org#classification",
"http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur",
"http://mu.semte.ch/vocabularies/ext/werkingsgebiedNiveau",
"http://data.lblod.info/vocabularies/erediensten/typeEredienst"
]
},
{
"type": "http://www.w3.org/ns/org#Site",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"^http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/ns/org#siteAddress",
"https://data.vlaanderen.be/ns/organisatie#bestaatUit",
"http://data.lblod.info/vocabularies/erediensten/vestigingstype"
]
},
{
"type": "http://www.w3.org/ns/locn#Address",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"^https://data.vlaanderen.be/ns/organisatie#bestaatUit",
"^http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer",
"https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer",
"http://www.w3.org/ns/locn#thoroughfare",
"http://www.w3.org/ns/locn#postCode",
"https://data.vlaanderen.be/ns/adres#gemeentenaam",
"http://www.w3.org/ns/locn#adminUnitL2",
"https://data.vlaanderen.be/ns/adres#land",
"http://www.w3.org/ns/locn#fullAddress",
"https://data.vlaanderen.be/ns/adres#verwijstNaar",
"http://purl.org/dc/terms/source"
]
},
{
"type": "http://schema.org/ContactPoint",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"^http://www.w3.org/ns/org#siteAddress",
"^http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/ns/locn#address",
"http://xmlns.com/foaf/0.1/page",
"http://schema.org/faxNumber",
"http://schema.org/contactType",
"http://xmlns.com/foaf/0.1/page",
"http://schema.org/email",
"http://schema.org/telephone"
]
},
{
"type": "http://www.w3.org/ns/adms#Identifier",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"^http://www.w3.org/ns/adms#identifier",
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#notation",
"https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator"
]
},
{
"type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"^https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator",
"^http://www.w3.org/ns/adms#identifier",
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
"strictTypeExport": "true",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://data.vlaanderen.be/ns/mandaat#bindingEinde",
"http://data.vlaanderen.be/ns/mandaat#bindingStart",
"http://data.vlaanderen.be/ns/besluit#bestuurt",
"http://www.w3.org/ns/org#classification",
"https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://www.w3.org/ns/org#ChangeEvent",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://data.lblod.info/vocabularies/contacthub/typeWijziging",
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://purl.org/dc/terms/date",
"http://purl.org/dc/terms/description",
"http://data.lblod.info/vocabularies/contacthub/typeWijziging",
"http://www.w3.org/ns/org#resultedFrom",
"http://www.w3.org/ns/org#changedBy",
"http://lblod.data.gift/vocabularies/organisatie/veranderingsgebeurtenisResultaat",
"http://www.w3.org/ns/org#originalOrganization",
"http://www.w3.org/ns/org#resultingOrganization",
"http://data.europa.eu/m8g/hasFormalFramework"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging",
"graphsFilter": [
"http://mu.semte.ch/graphs/organisatieportaal"
],
"pathToConceptScheme": [
"http://www.w3.org/2004/02/skos/core#inScheme"
],
"properties": [
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
}
]
}

35
config/delta/rules.js
View File

@@ -121,6 +121,41 @@ export default [
optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
}
},
{
match: {
},
callback: {
url: 'http://delta-producer-pub-graph-maintainer-public/delta',
method: 'POST'
},
options: {
resourceFormat: 'v0.0.1',
gracePeriod: 1000,
ignoreFromSelf: true,
optOutMuScopeIds: [
"http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
"http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
]
}
},
{
match: {
graph: {
type: 'uri',
value: 'http://redpencil.data.gift/id/deltas/producer/public'
}
},
callback: {
url: 'http://delta-producer-json-diff-publisher-public/delta',
method: 'POST'
},
options: {
resourceFormat: 'v0.0.1',
gracePeriod: 1000,
ignoreFromSelf: true,
optOutMuScopeIds: [ "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync" ]
}
},
{
match: {
predicate: {

32
config/jobs-controller/config.json
View File

@@ -62,5 +62,37 @@
"nextIndex": "0"
}
]
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": {
"tasksConfiguration": [
{
"currentOperation": null,
"nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing",
"nextIndex": "0"
},
{
"currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing",
"nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation",
"nextIndex": "1"
}
]
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public": {
"tasksConfiguration": [
{
"currentOperation": null,
"nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph",
"nextIndex": "0"
}
]
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": {
"tasksConfiguration": [
{
"currentOperation": null,
"nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation",
"nextIndex": "0"
}
]
}
}

43
config/migrations/2022/20220407110000-add-export-concept-scheme.sparql Normal file
View File

@@ -0,0 +1,43 @@
PREFIX mu: <http://mu.semte.ch/vocabularies/core/>
PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
INSERT {
GRAPH <http://mu.semte.ch/graphs/organisatieportaal> {
<http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b>
a
skos:ConceptScheme ;
mu:uuid
"10caf4a4-1ff5-4c05-917e-56489910d68b" ;
skos:prefLabel
"Public OP Export" .
?code
skos:inScheme
<http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b> .
}
} WHERE {
VALUES ?type {
<http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode>
<http://lblod.data.gift/vocabularies/organisatie/TypeEredienst>
<http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis>
<http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode>
<http://lblod.data.gift/vocabularies/organisatie/TypeVestiging>
}
GRAPH <http://mu.semte.ch/graphs/organisatieportaal> {
?code
a
?type .
}
}
INSERT {
GRAPH <http://mu.semte.ch/graphs/organisatieportaal> {
<http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/f9cac08a-13c1-49da-9bcb-f650b0604054>
skos:inScheme
<http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b> .
<http://data.vlaanderen.be/id/concept/BestuurseenheidClassificatieCode/66ec74fd-8cfc-4e16-99c6-350b35012e86>
skos:inScheme
<http://lblod.data.gift/concept-schemes/10caf4a4-1ff5-4c05-917e-56489910d68b> .
}
}

4
config/resources/dcat.json
View File

@@ -133,6 +133,10 @@
"byte-size": {
"type": "number",
"predicate": "dcat:byteSize"
},
"compression": {
"type": "url",
"predicate": "dcat:compressFormat"
}
},
"relationships": {

65
docker-compose.yml
View File

@@ -202,7 +202,7 @@ services:
# DELTA GENERAL
################################################################################
delta-producer-dump-file-publisher:
image: bdevloed/delta-producer-dump-file-publisher:0.5.0-beta
image: bdevloed/delta-producer-dump-file-publisher:0.8.0-beta
volumes:
- ./config/delta-producer/dump-file-publisher:/config
- ./data/files:/share
@@ -337,6 +337,69 @@ services:
################################################################################
################################################################################
# DELTA PUBLIC INFORMATION: START
################################################################################
delta-producer-bg-jobs-initiator-public:
image: lblod/delta-producer-background-jobs-initiator:0.3.1
environment:
JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal"
DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public'
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public'
HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public'
CRON_PATTERN_DUMP_JOB: '11 */4 * * *' # At minute 11 every 4 hours
CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM
START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts.
labels:
- "logging=true"
links:
- db:database
restart: always
logging: *default-logging
delta-producer-pub-graph-maintainer-public:
image: lblod/delta-producer-publication-graph-maintainer:0.15.1
environment:
MAX_BODY_SIZE: "50mb"
JOBS_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal"
RELATIVE_FILE_PATH: "deltas/public"
PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/public'
HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public'
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public'
QUEUE_POLL_INTERVAL: 3000
USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
SKIP_MU_AUTH_INITIAL_SYNC: 'true'
HEALING_PATCH_GRAPH_BATCH_SIZE: 500
volumes:
- ./config/delta-producer/public:/config
- ./data/files/:/share
links:
- db:database
- triplestore:virtuoso
restart: always
logging: *default-logging
delta-producer-json-diff-publisher-public:
image: lblod/delta-producer-json-diff-file-publisher:0.2.3
environment:
MAX_BODY_SIZE: "50mb"
ERROR_GRAPH: "http://mu.semte.ch/graphs/organisatieportaal"
RELATIVE_FILE_PATH: "deltas/public"
PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/public"
PUBLICATION_GRAPH: 'http://redpencil.data.gift/id/deltas/producer/public'
DELTA_INTERVAL_MS: 10000
PRETTY_PRINT_DIFF_JSON: "true"
volumes:
- ./data/files:/share
labels:
- "logging=true"
links:
- db:database
restart: always
logging: *default-logging
################################################################################
# DELTA PUBLIC INFORMATION: END
################################################################################
################################################################################
# MU SEARCH - TODO USE THE CORRECT IMAGE ONCE INHERITANCE IS IN MU SEARCH
################################################################################