felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add publication triplestore and organizations public info producer (#356)

Browse Source
This commit is contained in:
Claire Lovisa
2023-12-06 14:09:38 +01:00
committed by GitHub
parent 2a66359ebd
commit 79d2dda77a
9 changed files with 1004 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
config/delta-producer/dump-file-publisher/config.json
View File

@@ -3,6 +3,7 @@
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
"fileBaseName": "dump-organizations",
"publicationGraphEndpoint": "http://triplestore:8890/sparql",
"targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
"targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
"cleanupOldDumps": true
@@ -11,14 +12,34 @@
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations",
"fileBaseName": "dump-organizations",
"publicationGraphEndpoint": "http://triplestore:8890/sparql",
"targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
"targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas",
"cleanupOldDumps": true
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info",
"fileBaseName": "dump-organizations-public-info",
"publicationGraphEndpoint": "http://publication-triplestore:8890/sparql",
"targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas",
"targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas",
"cleanupOldDumps": true
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info",
"fileBaseName": "dump-organizations-public-info",
"publicationGraphEndpoint": "http://publication-triplestore:8890/sparql",
"targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas",
"targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas",
"cleanupOldDumps": true
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": {
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/public",
"fileBaseName": "dump-public",
"publicationGraphEndpoint": "http://triplestore:8890/sparql",
"targetDcatGraph": "http://mu.semte.ch/graphs/public",
"targetFilesGraph": "http://mu.semte.ch/graphs/public",
"cleanupOldDumps": true
@@ -27,6 +48,7 @@
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/public",
"fileBaseName": "dump-public",
"publicationGraphEndpoint": "http://triplestore:8890/sparql",
"targetDcatGraph": "http://mu.semte.ch/graphs/public",
"targetFilesGraph": "http://mu.semte.ch/graphs/public",
"cleanupOldDumps": true
@@ -35,6 +57,7 @@
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts",
"fileBaseName": "dump-worship-posts",
"publicationGraphEndpoint": "http://triplestore:8890/sparql",
"targetDcatGraph": "http://mu.semte.ch/graphs/public",
"targetFilesGraph": "http://mu.semte.ch/graphs/public",
"cleanupOldDumps": true
@@ -43,6 +66,7 @@
"dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump",
"targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts",
"fileBaseName": "dump-worship-posts",
"publicationGraphEndpoint": "http://triplestore:8890/sparql",
"targetDcatGraph": "http://mu.semte.ch/graphs/public",
"targetFilesGraph": "http://mu.semte.ch/graphs/public",
"cleanupOldDumps": true

349
config/delta-producer/organizations-public-info/export.json Normal file
View File

@@ -0,0 +1,349 @@
{
"export": [
{
"type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst",
"strictTypeExport": "true",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/2004/02/skos/core#altLabel",
"http://www.w3.org/ns/adms#identifier",
"http://www.w3.org/ns/regorg#orgStatus",
"http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#hasSite",
"http://www.w3.org/ns/org#changedBy",
"http://www.w3.org/ns/org#resultedFrom",
"http://www.w3.org/ns/org#hasPost",
"http://www.w3.org/ns/org#linkedTo",
"http://www.w3.org/ns/org#hasSubOrganization",
"http://www.w3.org/ns/org#classification",
"http://data.lblod.info/vocabularies/erediensten/denominatie",
"http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend",
"http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor",
"http://data.lblod.info/vocabularies/erediensten/typeEredienst"
]
},
{
"type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst",
"strictTypeExport": "true",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/2004/02/skos/core#altLabel",
"http://www.w3.org/ns/adms#identifier",
"http://www.w3.org/ns/regorg#orgStatus",
"http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#hasSite",
"http://www.w3.org/ns/org#changedBy",
"http://www.w3.org/ns/org#resultedFrom",
"http://www.w3.org/ns/org#hasPost",
"http://www.w3.org/ns/org#linkedTo",
"http://www.w3.org/ns/org#hasSubOrganization",
"http://www.w3.org/ns/org#classification",
"http://data.lblod.info/vocabularies/erediensten/typeEredienst"
]
},
{
"type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid",
"strictTypeExport": "true",
"graphsFilter": [
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur",
"http://data.lblod.info/vocabularies/erediensten/denominatie",
"http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend",
"http://data.lblod.info/vocabularies/erediensten/typeEredienst",
"http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor",
"http://data.vlaanderen.be/ns/besluit#werkingsgebied",
"http://www.opengis.net/ont/geosparql#sfWithin",
"http://www.w3.org/2002/07/owl#sameAs",
"http://www.w3.org/2004/02/skos/core#altLabel",
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/ns/adms#identifier",
"http://www.w3.org/ns/org#changedBy",
"http://www.w3.org/ns/org#classification",
"http://www.w3.org/ns/org#hasPost",
"http://www.w3.org/ns/org#hasPrimarySite",
"http://www.w3.org/ns/org#hasSite",
"http://www.w3.org/ns/org#hasSubOrganization",
"http://www.w3.org/ns/org#linkedTo",
"http://www.w3.org/ns/org#resultedFrom",
"http://www.w3.org/ns/regorg#orgStatus"
]
},
{
"type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel",
"http://www.w3.org/ns/org#linkedTo",
"http://data.lblod.info/vocabularies/erediensten/typeEredienst",
"http://www.w3.org/ns/adms#identifier"
]
},
{
"type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://data.lblod.info/vocabularies/erediensten/typebetrokkenheid",
"http://www.w3.org/ns/org#organization",
"http://data.lblod.info/vocabularies/erediensten/financieringspercentage"
]
},
{
"type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/ns/org#role",
"http://data.lblod.info/vocabularies/erediensten/functie"
]
},
{
"type": "http://www.w3.org/ns/org#Site",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/ns/org#siteAddress",
"https://data.vlaanderen.be/ns/organisatie#bestaatUit",
"http://data.lblod.info/vocabularies/erediensten/vestigingstype"
]
},
{
"type": "http://www.w3.org/ns/locn#Address",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer",
"https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer",
"http://www.w3.org/ns/locn#thoroughfare",
"http://www.w3.org/ns/locn#postCode",
"https://data.vlaanderen.be/ns/adres#gemeentenaam",
"http://www.w3.org/ns/locn#adminUnitL2",
"https://data.vlaanderen.be/ns/adres#land",
"http://www.w3.org/ns/locn#fullAddress",
"https://data.vlaanderen.be/ns/adres#verwijstNaar",
"http://purl.org/dc/terms/source"
]
},
{
"type": "http://schema.org/ContactPoint",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/ns/locn#address",
"http://xmlns.com/foaf/0.1/page",
"http://schema.org/faxNumber",
"http://schema.org/contactType",
"http://xmlns.com/foaf/0.1/page",
"http://schema.org/email",
"http://schema.org/telephone"
]
},
{
"type": "http://www.w3.org/ns/adms#Identifier",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#notation",
"https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator"
]
},
{
"type": "http://www.w3.org/ns/adms#Identifier",
"graphsFilter": [
"http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#notation",
"https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator"
],
"additionalFilter": "?ro <http://www.w3.org/ns/adms#identifier> ?subject ; a <http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan> ."
},
{
"type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
]
},
{
"type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator",
"graphsFilter": [
"http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"https://data.vlaanderen.be/ns/generiek#lokaleIdentificator"
],
"additionalFilter": "?ro <http://www.w3.org/ns/adms#identifier>/<https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator> ?subject ; a <http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan> ."
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode",
"graphsFilter": [
"http://mu.semte.ch/graphs/public"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/administrative-unit",
"http://mu.semte.ch/graphs/worship-service"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://data.vlaanderen.be/ns/besluit#bestuurt",
"http://www.w3.org/ns/org#classification",
"http://data.vlaanderen.be/ns/mandaat#bindingEinde",
"http://data.vlaanderen.be/ns/mandaat#bindingStart",
"https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan",
"http://www.w3.org/ns/org#hasPost"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst",
"graphsFilter": [
"http://mu.semte.ch/graphs/public"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode",
"graphsFilter": [
"http://mu.semte.ch/graphs/public"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://www.w3.org/ns/org#ChangeEvent",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://purl.org/dc/terms/date",
"http://purl.org/dc/terms/description",
"http://data.lblod.info/vocabularies/contacthub/typeWijziging",
"http://www.w3.org/ns/org#resultedFrom",
"http://www.w3.org/ns/org#changedBy",
"http://lblod.data.gift/vocabularies/organisatie/veranderingsgebeurtenisResultaat",
"http://www.w3.org/ns/org#originalOrganization",
"http://www.w3.org/ns/org#resultingOrganization",
"http://data.europa.eu/m8g/hasFormalFramework"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis",
"graphsFilter": [
"http://mu.semte.ch/graphs/public"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode",
"graphsFilter": [
"http://mu.semte.ch/graphs/public"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging",
"graphsFilter": [
"http://mu.semte.ch/graphs/public"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
},
{
"type": "http://data.vlaanderen.be/ns/mandaat#Mandaat",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/ns/org#role"
]
},
{
"type": "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie",
"graphsFilter": [
"http://mu.semte.ch/graphs/shared",
"http://mu.semte.ch/graphs/worship-service",
"http://mu.semte.ch/graphs/administrative-unit"
],
"properties": [
"http://mu.semte.ch/vocabularies/core/uuid",
"http://www.w3.org/ns/org#role",
"http://www.w3.org/2004/02/skos/core#prefLabel"
]
}
]
}

17
config/delta/rules.js
View File

@@ -105,6 +105,23 @@ export default [
]
}
},
{
match: {
},
callback: {
url: 'http://delta-producer-pub-graph-maintainer-organizations-public-info/delta',
method: 'POST'
},
options: {
resourceFormat: 'v0.0.1',
gracePeriod: 1000,
ignoreFromSelf: true,
optOutMuScopeIds: [
"http://redpencil.data.gift/id/concept/muScope/deltas/initialSync",
"http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance"
]
}
},
{
match: {
},

8
config/dispatcher/dispatcher.ex
View File

@@ -357,6 +357,14 @@ defmodule Dispatcher do
Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/"
end
#################################################################
# DELTA: organizations-public-info
#################################################################
get "/sync/organizations-public-info/files/*path" do
Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations-public-info/files/"
end
#################################################################
# DELTA: public
#################################################################

32
config/jobs-controller/config.json
View File

@@ -63,6 +63,38 @@
}
]
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": {
"tasksConfiguration": [
{
"currentOperation": null,
"nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing",
"nextIndex": "0"
},
{
"currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing",
"nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation",
"nextIndex": "1"
}
]
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info": {
"tasksConfiguration": [
{
"currentOperation": null,
"nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph",
"nextIndex": "0"
}
]
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": {
"tasksConfiguration": [
{
"currentOperation": null,
"nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation",
"nextIndex": "0"
}
]
},
"http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": {
"tasksConfiguration": [
{

253
config/publication-triplestore/virtuoso-production.ini Normal file
View File

@@ -0,0 +1,253 @@
;
; virtuoso.ini
;
; Configuration file for the OpenLink Virtuoso VDBMS Server
;
; To learn more about this product, or any other product in our
; portfolio, please check out our web site at:
;
; http://virtuoso.openlinksw.com/
;
; or contact us at:
;
; general.information@openlinksw.com
;
; If you have any technical questions, please contact our support
; staff at:
;
; technical.support@openlinksw.com
;
;
; Database setup
;
[Database]
DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.db
ErrorLogFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.log
LockFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.lck
TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.trx
xa_persistent_file = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.pxa
ErrorLogLevel = 7
FileExtend = 200
MaxCheckpointRemap = 17000 ;;niels 1/4 of the number of buffers (assuming db uses most of the available buffers, you can check with `status()` in isql-v
Striping = 0
TempStorage = TempDatabase
Syslog = 1 ; write to syslog in case of failures where writing to log is not possible
[TempDatabase]
DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.db
TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.trx
MaxCheckpointRemap = 2000
Striping = 0
;
; Server parameters
;
[Parameters]
ServerPort = 1111
LiteMode = 0
DisableUnixSocket = 1
DisableTcpSocket = 0
;SSLServerPort = 2111
;SSLCertificate = cert.pem
;SSLPrivateKey = pk.pem
;X509ClientVerify = 0
;X509ClientVerifyDepth = 0
;X509ClientVerifyCAFile = ca.pem
MaxClientConnections = 10
CheckpointInterval = 60
TraceOn = errors ; write traces
O_DIRECT = 0
CaseMode = 2
MaxStaticCursorRows = 5000
CheckpointAuditTrail = 0
AllowOSCalls = 0
SchedulerInterval = 10
DirsAllowed = ., /usr/local/virtuoso-opensource/share/virtuoso/vad
ThreadCleanupInterval = 10
ThreadThreshold = 10
ResourcesCleanupInterval = 10
FreeTextBatchSize = 100000
SingleCPU = 0
VADInstallDir = /usr/local/virtuoso-opensource/share/virtuoso/vad/
PrefixResultNames = 0
RdfFreeTextRulesSize = 100
IndexTreeMaps = 256
MaxMemPoolSize = 200000000
PrefixResultNames = 0
MacSpotlight = 0
MaxQueryMem = 4G ; memory allocated to query processor ;;Niels said to 4g
VectorSize = 1000 ; initial parallel query vector (array of query operations) size
MaxVectorSize = 1000000 ; query vector size threshold.
AdjustVectorSize = 0
ThreadsPerQuery = 4
AsyncQueueMaxThreads = 10
TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient consumption is 450MB (roll forward + roll back)
;;
;; When running with large data sets, one should configure the Virtuoso
;; process to use between 2/3 to 3/5 of free system memory and to stripe
;; storage on all available disks.
;;
;; Uncomment next two lines if there is 2 GB system memory free
;;NumberOfBuffers = 170000
;;MaxDirtyBuffers = 130000
;; Uncomment next two lines if there is 4 GB system memory free
;;NumberOfBuffers = 340000
;;MaxDirtyBuffers = 250000
;; Uncomment next two lines if there is 8 GB system memory free
NumberOfBuffers = 680000
MaxDirtyBuffers = 500000
;; Uncomment next two lines if there is 16 GB system memory free
;NumberOfBuffers = 1360000
;MaxDirtyBuffers = 1000000
;; Uncomment next two lines if there is 32 GB system memory free
;NumberOfBuffers = 2720000
;MaxDirtyBuffers = 2000000
;; Uncomment next two lines if there is 48 GB system memory free
;NumberOfBuffers = 4000000
;MaxDirtyBuffers = 3000000
;; Uncomment next two lines if there is 64 GB system memory free
;NumberOfBuffers = 5450000
;MaxDirtyBuffers = 4000000
;;
;; Note the default settings will take very little memory
;; but will not result in very good performance
;;
;NumberOfBuffers = 10000
;MaxDirtyBuffers = 6000
;; Introduced after bug with export service
MaxSortedTopRows = 1000000 ; 1 million!
[HTTPServer]
ServerPort = 8890
ServerRoot = /usr/local/virtuoso-opensource/var/lib/virtuoso/vsp
MaxClientConnections = 10
DavRoot = DAV
EnabledDavVSP = 0
HTTPProxyEnabled = 0
TempASPXDir = 0
DefaultMailServer = localhost:25
ServerThreads = 10
MaxKeepAlives = 10
KeepAliveTimeout = 10
MaxCachedProxyConnections = 10
ProxyConnectionCacheTimeout = 15
HTTPThreadSize = 280000
HttpPrintWarningsInOutput = 0
Charset = UTF-8
;;HTTPLogFile = http.log
MaintenancePage = atomic.html
EnabledGzipContent = 1
[AutoRepair]
BadParentLinks = 0
[Client]
SQL_PREFETCH_ROWS = 100
SQL_PREFETCH_BYTES = 16000
SQL_QUERY_TIMEOUT = 0
SQL_TXN_TIMEOUT = 0
;SQL_NO_CHAR_C_ESCAPE = 1
;SQL_UTF8_EXECS = 0
;SQL_NO_SYSTEM_TABLES = 0
;SQL_BINARY_TIMESTAMP = 1
;SQL_ENCRYPTION_ON_PASSWORD = -1
[VDB]
ArrayOptimization = 0
NumArrayParameters = 10
VDBDisconnectTimeout = 1000
KeepConnectionOnFixedThread = 0
[Replication]
ServerName = db-D602566B774E
ServerEnable = 1
QueueMax = 50000
;
; Striping setup
;
; These parameters have only effect when Striping is set to 1 in the
; [Database] section, in which case the DatabaseFile parameter is ignored.
;
; With striping, the database is spawned across multiple segments
; where each segment can have multiple stripes.
;
; Format of the lines below:
; Segment<number> = <size>, <stripe file name> [, <stripe file name> .. ]
;
; <number> must be ordered from 1 up.
;
; The <size> is the total size of the segment which is equally divided
; across all stripes forming the segment. Its specification can be in
; gigabytes (g), megabytes (m), kilobytes (k) or in database blocks
; (b, the default)
;
; Note that the segment size must be a multiple of the database page size
; which is currently 8k. Also, the segment size must be divisible by the
; number of stripe files forming the segment.
;
; The example below creates a 200 meg database striped on two segments
; with two stripes of 50 meg and one of 100 meg.
;
; You can always add more segments to the configuration, but once
; added, do not change the setup.
;
[Striping]
Segment1 = 100M, db-seg1-1.db, db-seg1-2.db
Segment2 = 100M, db-seg2-1.db
;...
;[TempStriping]
;Segment1 = 100M, db-seg1-1.db, db-seg1-2.db
;Segment2 = 100M, db-seg2-1.db
;...
;[Ucms]
;UcmPath = <path>
;Ucm1 = <file>
;Ucm2 = <file>
;...
[Zero Config]
ServerName = virtuoso (D602566B774E)
;ServerDSN = ZDSN
;SSLServerName =
;SSLServerDSN =
[Mono]
;MONO_TRACE = Off
;MONO_PATH = <path_here>
;MONO_ROOT = <path_here>
;MONO_CFG_DIR = <path_here>
;virtclr.dll =
[URIQA]
DynamicLocal = 0
DefaultHost = localhost:8890
[SPARQL]
;ExternalQuerySource = 1
;ExternalXsltSource = 1
;DefaultGraph = http://localhost:8890/dataspace
;ImmutableGraphs = http://localhost:8890/dataspace
ResultSetMaxRows = 1000000 ; 1 million
;MaxQueryCostEstimationTime = 4000 ; in seconds
MaxQueryExecutionTime = 1200 ; in seconds
DefaultQuery = select distinct ?Concept where {[] a ?Concept} LIMIT 100
DeferInferenceRulesInit = 0 ; controls inference rules loading
;PingService = http://rpc.pingthesemanticweb.com/
[Plugins]
LoadPath = /usr/local/virtuoso-opensource/lib/virtuoso/hosting
;Load1 = plain, wikiv
;Load2 = plain, mediawiki
;Load3 = plain, creolewiki
;Load4 = plain, im
;Load5 = plain, wbxml2
;Load6 = plain, hslookup
;Load7 = attach, libphp5.so
;Load8 = Hosting, hosting_php.so
;Load9 = Hosting,hosting_perl.so
;Load10 = Hosting,hosting_python.so
;Load11 = Hosting,hosting_ruby.so
;Load12 = msdtc,msdtc_sample

253
config/publication-triplestore/virtuoso.ini Executable file
View File

@@ -0,0 +1,253 @@
;
; virtuoso.ini
;
; Configuration file for the OpenLink Virtuoso VDBMS Server
;
; To learn more about this product, or any other product in our
; portfolio, please check out our web site at:
;
; http://virtuoso.openlinksw.com/
;
; or contact us at:
;
; general.information@openlinksw.com
;
; If you have any technical questions, please contact our support
; staff at:
;
; technical.support@openlinksw.com
;
;
; Database setup
;
[Database]
DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.db
ErrorLogFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.log
LockFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.lck
TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.trx
xa_persistent_file = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.pxa
ErrorLogLevel = 7
FileExtend = 200
MaxCheckpointRemap = 8500 ;;niels 1/4 of the number of buffers (assuming db uses most of the available buffers, you can check with `status()` in isql-v
Striping = 0
TempStorage = TempDatabase
Syslog = 1 ; write to syslog in case of failures where writing to log is not possible
[TempDatabase]
DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.db
TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.trx
MaxCheckpointRemap = 2000
Striping = 0
;
; Server parameters
;
[Parameters]
ServerPort = 1111
LiteMode = 0
DisableUnixSocket = 1
DisableTcpSocket = 0
;SSLServerPort = 2111
;SSLCertificate = cert.pem
;SSLPrivateKey = pk.pem
;X509ClientVerify = 0
;X509ClientVerifyDepth = 0
;X509ClientVerifyCAFile = ca.pem
MaxClientConnections = 10
CheckpointInterval = 60
TraceOn = errors ; write traces
O_DIRECT = 0
CaseMode = 2
MaxStaticCursorRows = 5000
CheckpointAuditTrail = 0
AllowOSCalls = 0
SchedulerInterval = 10
DirsAllowed = ., /usr/local/virtuoso-opensource/share/virtuoso/vad
ThreadCleanupInterval = 10
ThreadThreshold = 10
ResourcesCleanupInterval = 10
FreeTextBatchSize = 100000
SingleCPU = 0
VADInstallDir = /usr/local/virtuoso-opensource/share/virtuoso/vad/
PrefixResultNames = 0
RdfFreeTextRulesSize = 100
IndexTreeMaps = 256
MaxMemPoolSize = 200000000
PrefixResultNames = 0
MacSpotlight = 0
MaxQueryMem = 2G
VectorSize = 1000 ; initial parallel query vector (array of query operations) size
MaxVectorSize = 1000000 ; query vector size threshold.
AdjustVectorSize = 0
ThreadsPerQuery = 4
AsyncQueueMaxThreads = 10
TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient consumption is 450MB (roll forward + roll back)
;;
;; When running with large data sets, one should configure the Virtuoso
;; process to use between 2/3 to 3/5 of free system memory and to stripe
;; storage on all available disks.
;;
;; Uncomment next two lines if there is 2 GB system memory free
;;NumberOfBuffers = 170000
;;MaxDirtyBuffers = 130000
;; Uncomment next two lines if there is 4 GB system memory free
NumberOfBuffers = 340000
MaxDirtyBuffers = 250000
;; Uncomment next two lines if there is 8 GB system memory free
;NumberOfBuffers = 680000
;MaxDirtyBuffers = 500000
;; Uncomment next two lines if there is 16 GB system memory free
;NumberOfBuffers = 1360000
;MaxDirtyBuffers = 1000000
;; Uncomment next two lines if there is 32 GB system memory free
;NumberOfBuffers = 2720000
;MaxDirtyBuffers = 2000000
;; Uncomment next two lines if there is 48 GB system memory free
;NumberOfBuffers = 4000000
;MaxDirtyBuffers = 3000000
;; Uncomment next two lines if there is 64 GB system memory free
;NumberOfBuffers = 5450000
;MaxDirtyBuffers = 4000000
;;
;; Note the default settings will take very little memory
;; but will not result in very good performance
;;
;NumberOfBuffers = 10000
;MaxDirtyBuffers = 6000
;; Introduced after bug with export service
MaxSortedTopRows = 1000000 ; 1 million!
[HTTPServer]
ServerPort = 8890
ServerRoot = /usr/local/virtuoso-opensource/var/lib/virtuoso/vsp
MaxClientConnections = 10
DavRoot = DAV
EnabledDavVSP = 0
HTTPProxyEnabled = 0
TempASPXDir = 0
DefaultMailServer = localhost:25
ServerThreads = 10
MaxKeepAlives = 10
KeepAliveTimeout = 10
MaxCachedProxyConnections = 10
ProxyConnectionCacheTimeout = 15
HTTPThreadSize = 280000
HttpPrintWarningsInOutput = 0
Charset = UTF-8
;;HTTPLogFile = http.log
MaintenancePage = atomic.html
EnabledGzipContent = 1
[AutoRepair]
BadParentLinks = 0
[Client]
SQL_PREFETCH_ROWS = 100
SQL_PREFETCH_BYTES = 16000
SQL_QUERY_TIMEOUT = 0
SQL_TXN_TIMEOUT = 0
;SQL_NO_CHAR_C_ESCAPE = 1
;SQL_UTF8_EXECS = 0
;SQL_NO_SYSTEM_TABLES = 0
;SQL_BINARY_TIMESTAMP = 1
;SQL_ENCRYPTION_ON_PASSWORD = -1
[VDB]
ArrayOptimization = 0
NumArrayParameters = 10
VDBDisconnectTimeout = 1000
KeepConnectionOnFixedThread = 0
[Replication]
ServerName = db-D602566B774E
ServerEnable = 1
QueueMax = 50000
;
; Striping setup
;
; These parameters have only effect when Striping is set to 1 in the
; [Database] section, in which case the DatabaseFile parameter is ignored.
;
; With striping, the database is spawned across multiple segments
; where each segment can have multiple stripes.
;
; Format of the lines below:
; Segment<number> = <size>, <stripe file name> [, <stripe file name> .. ]
;
; <number> must be ordered from 1 up.
;
; The <size> is the total size of the segment which is equally divided
; across all stripes forming the segment. Its specification can be in
; gigabytes (g), megabytes (m), kilobytes (k) or in database blocks
; (b, the default)
;
; Note that the segment size must be a multiple of the database page size
; which is currently 8k. Also, the segment size must be divisible by the
; number of stripe files forming the segment.
;
; The example below creates a 200 meg database striped on two segments
; with two stripes of 50 meg and one of 100 meg.
;
; You can always add more segments to the configuration, but once
; added, do not change the setup.
;
[Striping]
Segment1 = 100M, db-seg1-1.db, db-seg1-2.db
Segment2 = 100M, db-seg2-1.db
;...
;[TempStriping]
;Segment1 = 100M, db-seg1-1.db, db-seg1-2.db
;Segment2 = 100M, db-seg2-1.db
;...
;[Ucms]
;UcmPath = <path>
;Ucm1 = <file>
;Ucm2 = <file>
;...
[Zero Config]
ServerName = virtuoso (D602566B774E)
;ServerDSN = ZDSN
;SSLServerName =
;SSLServerDSN =
[Mono]
;MONO_TRACE = Off
;MONO_PATH = <path_here>
;MONO_ROOT = <path_here>
;MONO_CFG_DIR = <path_here>
;virtclr.dll =
[URIQA]
DynamicLocal = 0
DefaultHost = localhost:8890
[SPARQL]
;ExternalQuerySource = 1
;ExternalXsltSource = 1
;DefaultGraph = http://localhost:8890/dataspace
;ImmutableGraphs = http://localhost:8890/dataspace
ResultSetMaxRows = 1000000 ; 1 million
;MaxQueryCostEstimationTime = 4000 ; in seconds
MaxQueryExecutionTime = 1200 ; in seconds
DefaultQuery = select distinct ?Concept where {[] a ?Concept} LIMIT 100
DeferInferenceRulesInit = 0 ; controls inference rules loading
;PingService = http://rpc.pingthesemanticweb.com/
[Plugins]
LoadPath = /usr/local/virtuoso-opensource/lib/virtuoso/hosting
;Load1 = plain, wikiv
;Load2 = plain, mediawiki
;Load3 = plain, creolewiki
;Load4 = plain, im
;Load5 = plain, wbxml2
;Load6 = plain, hslookup
;Load7 = attach, libphp5.so
;Load8 = Hosting, hosting_php.so
;Load9 = Hosting,hosting_perl.so
;Load10 = Hosting,hosting_python.so
;Load11 = Hosting,hosting_ruby.so
;Load12 = msdtc,msdtc_sample

4
docker-compose.dev.yml
View File

@@ -63,6 +63,10 @@ services:
restart: "no"
file:
restart: "no"
publication-triplestore:
restart: "no"
ports:
- "8891:8890"
delta-producer-dump-file-publisher:
restart: "no"
organizations-sync-with-sharepoint-jobs-initiator:

65
docker-compose.yml
View File

@@ -302,11 +302,22 @@ services:
- "logging=true"
restart: always
logging: *default-logging
################################################################################
# DELTA GENERAL
################################################################################
publication-triplestore:
image: redpencil/virtuoso:1.2.0-rc.1
environment:
SPARQL_UPDATE: "true"
DEFAULT_GRAPH: "http://mu.semte.ch/application"
volumes:
- ./data/publication-triplestore:/data
- ./config/publication-triplestore/virtuoso.ini:/data/virtuoso.ini
restart: always
logging: *default-logging
delta-producer-dump-file-publisher:
image: lblod/delta-producer-dump-file-publisher:0.9.1
image: lblod/delta-producer-dump-file-publisher:0.10.2
volumes:
- ./config/delta-producer/dump-file-publisher:/config
- ./data/files:/share
@@ -314,6 +325,7 @@ services:
- db:database
restart: always
logging: *default-logging
################################################################################
# DELTA ORGANIZATIONS: START
################################################################################
@@ -363,6 +375,57 @@ services:
# DELTA ORGANIZATIONS: END
################################################################################
################################################################################
# DELTA ORGANIZATIONS PUBLIC INFO: START
################################################################################
delta-producer-bg-jobs-initiator-organizations-public-info:
image: lblod/delta-producer-background-jobs-initiator:0.3.1
environment:
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info"
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info"
HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info"
CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours
CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM
START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync
labels:
- "logging=true"
links:
- db:database
restart: always
logging: *default-logging
delta-producer-pub-graph-maintainer-organizations-public-info:
image: lblod/delta-producer-publication-graph-maintainer:0.15.4
environment:
MAX_BODY_SIZE: "50mb"
JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs"
RELATIVE_FILE_PATH: "deltas/organizations"
PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info"
HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info"
INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info"
QUEUE_POLL_INTERVAL: 3000
USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive
SKIP_MU_AUTH_INITIAL_SYNC: "true"
HEALING_PATCH_GRAPH_BATCH_SIZE: 500
SERVE_DELTA_FILES: "true"
PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info"
DELTA_INTERVAL_MS: 10000
PRETTY_PRINT_DIFF_JSON: "true"
PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql"
PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql"
ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs"
volumes:
- ./config/delta-producer/organizations-public-info:/config
- ./data/files/:/share
links:
- db:database
- triplestore:virtuoso
restart: always
logging: *default-logging
################################################################################
# DELTA ORGANIZATIONS PUBLIC INFO: END
################################################################################
################################################################################
# DELTA PUBLIC INFORMATION: START
################################################################################