diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 42c183f..83076a1 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -3,6 +3,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true @@ -11,14 +12,34 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", "fileBaseName": "dump-organizations", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-deltas", "cleanupOldDumps": true }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "fileBaseName": "dump-organizations-public-info", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", + "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas", + "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas", + "cleanupOldDumps": true + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { + "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsPublicInfoCacheGraphDump", + "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info", + "fileBaseName": "dump-organizations-public-info", + "publicationGraphEndpoint": "http://publication-triplestore:8890/sparql", + "targetDcatGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas", + "targetFilesGraph": "http://redpencil.data.gift/id/deltas/producer/organizations-public-info-deltas", + "cleanupOldDumps": true + }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://mu.semte.ch/graphs/public", "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true @@ -27,6 +48,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", "fileBaseName": "dump-public", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://mu.semte.ch/graphs/public", "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true @@ -35,6 +57,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", "fileBaseName": "dump-worship-posts", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://mu.semte.ch/graphs/public", "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true @@ -43,6 +66,7 @@ "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/WorshipPostsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/worship-posts", "fileBaseName": "dump-worship-posts", + "publicationGraphEndpoint": "http://triplestore:8890/sparql", "targetDcatGraph": "http://mu.semte.ch/graphs/public", "targetFilesGraph": "http://mu.semte.ch/graphs/public", "cleanupOldDumps": true diff --git a/config/delta-producer/organizations-public-info/export.json b/config/delta-producer/organizations-public-info/export.json new file mode 100644 index 0000000..f3bb667 --- /dev/null +++ b/config/delta-producer/organizations-public-info/export.json @@ -0,0 +1,349 @@ +{ + "export": [ + { + "type": "http://data.lblod.info/vocabularies/erediensten/BestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/denominatie", + "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/CentraalBestuurVanDeEredienst", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/regorg#orgStatus", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#classification", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuurseenheid", + "strictTypeExport": "true", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://data.lblod.info/vocabularies/erediensten/betrokkenBestuur", + "http://data.lblod.info/vocabularies/erediensten/denominatie", + "http://data.lblod.info/vocabularies/erediensten/grensoverschrijdend", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://data.lblod.info/vocabularies/erediensten/wordtBediendDoor", + "http://data.vlaanderen.be/ns/besluit#werkingsgebied", + "http://www.opengis.net/ont/geosparql#sfWithin", + "http://www.w3.org/2002/07/owl#sameAs", + "http://www.w3.org/2004/02/skos/core#altLabel", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/adms#identifier", + "http://www.w3.org/ns/org#changedBy", + "http://www.w3.org/ns/org#classification", + "http://www.w3.org/ns/org#hasPost", + "http://www.w3.org/ns/org#hasPrimarySite", + "http://www.w3.org/ns/org#hasSite", + "http://www.w3.org/ns/org#hasSubOrganization", + "http://www.w3.org/ns/org#linkedTo", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/regorg#orgStatus" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/RepresentatiefOrgaan", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel", + "http://www.w3.org/ns/org#linkedTo", + "http://data.lblod.info/vocabularies/erediensten/typeEredienst", + "http://www.w3.org/ns/adms#identifier" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/BetrokkenLokaleBesturen", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://data.lblod.info/vocabularies/erediensten/typebetrokkenheid", + "http://www.w3.org/ns/org#organization", + "http://data.lblod.info/vocabularies/erediensten/financieringspercentage" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/erediensten/PositieBedienaar", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role", + "http://data.lblod.info/vocabularies/erediensten/functie" + ] + }, + { + "type": "http://www.w3.org/ns/org#Site", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#siteAddress", + "https://data.vlaanderen.be/ns/organisatie#bestaatUit", + "http://data.lblod.info/vocabularies/erediensten/vestigingstype" + ] + }, + { + "type": "http://www.w3.org/ns/locn#Address", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.huisnummer", + "https://data.vlaanderen.be/ns/adres#Adresvoorstelling.busnummer", + "http://www.w3.org/ns/locn#thoroughfare", + "http://www.w3.org/ns/locn#postCode", + "https://data.vlaanderen.be/ns/adres#gemeentenaam", + "http://www.w3.org/ns/locn#adminUnitL2", + "https://data.vlaanderen.be/ns/adres#land", + "http://www.w3.org/ns/locn#fullAddress", + "https://data.vlaanderen.be/ns/adres#verwijstNaar", + "http://purl.org/dc/terms/source" + ] + }, + { + "type": "http://schema.org/ContactPoint", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/locn#address", + "http://xmlns.com/foaf/0.1/page", + "http://schema.org/faxNumber", + "http://schema.org/contactType", + "http://xmlns.com/foaf/0.1/page", + "http://schema.org/email", + "http://schema.org/telephone" + ] + }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ] + }, + { + "type": "http://www.w3.org/ns/adms#Identifier", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#notation", + "https://data.vlaanderen.be/ns/generiek#gestructureerdeIdentificator" + ], + "additionalFilter": "?ro ?subject ; a ." + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ] + }, + { + "type": "https://data.vlaanderen.be/ns/generiek#GestructureerdeIdentificator", + "graphsFilter": [ + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "https://data.vlaanderen.be/ns/generiek#lokaleIdentificator" + ], + "additionalFilter": "?ro / ?subject ; a ." + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuurseenheidClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/besluit#Bestuursorgaan", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/administrative-unit", + "http://mu.semte.ch/graphs/worship-service" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://data.vlaanderen.be/ns/besluit#bestuurt", + "http://www.w3.org/ns/org#classification", + "http://data.vlaanderen.be/ns/mandaat#bindingEinde", + "http://data.vlaanderen.be/ns/mandaat#bindingStart", + "https://data.vlaanderen.be/ns/generiek#isTijdspecialisatieVan", + "http://www.w3.org/ns/org#hasPost" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeEredienst", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/BestuursorgaanClassificatieCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://www.w3.org/ns/org#ChangeEvent", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://purl.org/dc/terms/date", + "http://purl.org/dc/terms/description", + "http://data.lblod.info/vocabularies/contacthub/typeWijziging", + "http://www.w3.org/ns/org#resultedFrom", + "http://www.w3.org/ns/org#changedBy", + "http://lblod.data.gift/vocabularies/organisatie/veranderingsgebeurtenisResultaat", + "http://www.w3.org/ns/org#originalOrganization", + "http://www.w3.org/ns/org#resultingOrganization", + "http://data.europa.eu/m8g/hasFormalFramework" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/Veranderingsgebeurtenis", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/OrganisatieStatusCode", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://lblod.data.gift/vocabularies/organisatie/TypeVestiging", + "graphsFilter": [ + "http://mu.semte.ch/graphs/public" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + }, + { + "type": "http://data.vlaanderen.be/ns/mandaat#Mandaat", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role" + ] + }, + { + "type": "http://data.lblod.info/vocabularies/leidinggevenden/Bestuursfunctie", + "graphsFilter": [ + "http://mu.semte.ch/graphs/shared", + "http://mu.semte.ch/graphs/worship-service", + "http://mu.semte.ch/graphs/administrative-unit" + ], + "properties": [ + "http://mu.semte.ch/vocabularies/core/uuid", + "http://www.w3.org/ns/org#role", + "http://www.w3.org/2004/02/skos/core#prefLabel" + ] + } + ] +} diff --git a/config/delta/rules.js b/config/delta/rules.js index a5d3e12..3a24e37 100644 --- a/config/delta/rules.js +++ b/config/delta/rules.js @@ -105,6 +105,23 @@ export default [ ] } }, + { + match: { + }, + callback: { + url: 'http://delta-producer-pub-graph-maintainer-organizations-public-info/delta', + method: 'POST' + }, + options: { + resourceFormat: 'v0.0.1', + gracePeriod: 1000, + ignoreFromSelf: true, + optOutMuScopeIds: [ + "http://redpencil.data.gift/id/concept/muScope/deltas/initialSync", + "http://redpencil.data.gift/id/concept/muScope/deltas/publicationGraphMaintenance" + ] + } + }, { match: { }, diff --git a/config/dispatcher/dispatcher.ex b/config/dispatcher/dispatcher.ex index cce1b56..7c1c085 100644 --- a/config/dispatcher/dispatcher.ex +++ b/config/dispatcher/dispatcher.ex @@ -357,6 +357,14 @@ defmodule Dispatcher do Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations/files/" end + ################################################################# + # DELTA: organizations-public-info + ################################################################# + + get "/sync/organizations-public-info/files/*path" do + Proxy.forward conn, path, "http://delta-producer-pub-graph-maintainer-organizations-public-info/files/" + end + ################################################################# # DELTA: public ################################################################# diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index e22a768..4115501 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -63,6 +63,38 @@ } ] }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextIndex": "0" + }, + { + "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "1" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "0" + } + ] + }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { "tasksConfiguration": [ { diff --git a/config/publication-triplestore/virtuoso-production.ini b/config/publication-triplestore/virtuoso-production.ini new file mode 100644 index 0000000..c074796 --- /dev/null +++ b/config/publication-triplestore/virtuoso-production.ini @@ -0,0 +1,253 @@ +; +; virtuoso.ini +; +; Configuration file for the OpenLink Virtuoso VDBMS Server +; +; To learn more about this product, or any other product in our +; portfolio, please check out our web site at: +; +; http://virtuoso.openlinksw.com/ +; +; or contact us at: +; +; general.information@openlinksw.com +; +; If you have any technical questions, please contact our support +; staff at: +; +; technical.support@openlinksw.com +; +; +; Database setup +; +[Database] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.db +ErrorLogFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.log +LockFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.lck +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.trx +xa_persistent_file = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.pxa +ErrorLogLevel = 7 +FileExtend = 200 +MaxCheckpointRemap = 17000 ;;niels 1/4 of the number of buffers (assuming db uses most of the available buffers, you can check with `status()` in isql-v +Striping = 0 +TempStorage = TempDatabase +Syslog = 1 ; write to syslog in case of failures where writing to log is not possible + +[TempDatabase] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.db +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.trx +MaxCheckpointRemap = 2000 +Striping = 0 + +; +; Server parameters +; +[Parameters] +ServerPort = 1111 +LiteMode = 0 +DisableUnixSocket = 1 +DisableTcpSocket = 0 +;SSLServerPort = 2111 +;SSLCertificate = cert.pem +;SSLPrivateKey = pk.pem +;X509ClientVerify = 0 +;X509ClientVerifyDepth = 0 +;X509ClientVerifyCAFile = ca.pem +MaxClientConnections = 10 +CheckpointInterval = 60 +TraceOn = errors ; write traces +O_DIRECT = 0 +CaseMode = 2 +MaxStaticCursorRows = 5000 +CheckpointAuditTrail = 0 +AllowOSCalls = 0 +SchedulerInterval = 10 +DirsAllowed = ., /usr/local/virtuoso-opensource/share/virtuoso/vad +ThreadCleanupInterval = 10 +ThreadThreshold = 10 +ResourcesCleanupInterval = 10 +FreeTextBatchSize = 100000 +SingleCPU = 0 +VADInstallDir = /usr/local/virtuoso-opensource/share/virtuoso/vad/ +PrefixResultNames = 0 +RdfFreeTextRulesSize = 100 +IndexTreeMaps = 256 +MaxMemPoolSize = 200000000 +PrefixResultNames = 0 +MacSpotlight = 0 +MaxQueryMem = 4G ; memory allocated to query processor ;;Niels said to 4g +VectorSize = 1000 ; initial parallel query vector (array of query operations) size +MaxVectorSize = 1000000 ; query vector size threshold. +AdjustVectorSize = 0 +ThreadsPerQuery = 4 +AsyncQueueMaxThreads = 10 +TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient consumption is 450MB (roll forward + roll back) +;; +;; When running with large data sets, one should configure the Virtuoso +;; process to use between 2/3 to 3/5 of free system memory and to stripe +;; storage on all available disks. +;; +;; Uncomment next two lines if there is 2 GB system memory free +;;NumberOfBuffers = 170000 +;;MaxDirtyBuffers = 130000 + +;; Uncomment next two lines if there is 4 GB system memory free +;;NumberOfBuffers = 340000 +;;MaxDirtyBuffers = 250000 +;; Uncomment next two lines if there is 8 GB system memory free +NumberOfBuffers = 680000 +MaxDirtyBuffers = 500000 +;; Uncomment next two lines if there is 16 GB system memory free +;NumberOfBuffers = 1360000 +;MaxDirtyBuffers = 1000000 +;; Uncomment next two lines if there is 32 GB system memory free +;NumberOfBuffers = 2720000 +;MaxDirtyBuffers = 2000000 +;; Uncomment next two lines if there is 48 GB system memory free +;NumberOfBuffers = 4000000 +;MaxDirtyBuffers = 3000000 +;; Uncomment next two lines if there is 64 GB system memory free +;NumberOfBuffers = 5450000 +;MaxDirtyBuffers = 4000000 +;; +;; Note the default settings will take very little memory +;; but will not result in very good performance +;; +;NumberOfBuffers = 10000 +;MaxDirtyBuffers = 6000 + +;; Introduced after bug with export service +MaxSortedTopRows = 1000000 ; 1 million! + +[HTTPServer] +ServerPort = 8890 +ServerRoot = /usr/local/virtuoso-opensource/var/lib/virtuoso/vsp +MaxClientConnections = 10 +DavRoot = DAV +EnabledDavVSP = 0 +HTTPProxyEnabled = 0 +TempASPXDir = 0 +DefaultMailServer = localhost:25 +ServerThreads = 10 +MaxKeepAlives = 10 +KeepAliveTimeout = 10 +MaxCachedProxyConnections = 10 +ProxyConnectionCacheTimeout = 15 +HTTPThreadSize = 280000 +HttpPrintWarningsInOutput = 0 +Charset = UTF-8 +;;HTTPLogFile = http.log +MaintenancePage = atomic.html +EnabledGzipContent = 1 + +[AutoRepair] +BadParentLinks = 0 + +[Client] +SQL_PREFETCH_ROWS = 100 +SQL_PREFETCH_BYTES = 16000 +SQL_QUERY_TIMEOUT = 0 +SQL_TXN_TIMEOUT = 0 +;SQL_NO_CHAR_C_ESCAPE = 1 +;SQL_UTF8_EXECS = 0 +;SQL_NO_SYSTEM_TABLES = 0 +;SQL_BINARY_TIMESTAMP = 1 +;SQL_ENCRYPTION_ON_PASSWORD = -1 + +[VDB] +ArrayOptimization = 0 +NumArrayParameters = 10 +VDBDisconnectTimeout = 1000 +KeepConnectionOnFixedThread = 0 + +[Replication] +ServerName = db-D602566B774E +ServerEnable = 1 +QueueMax = 50000 + +; +; Striping setup +; +; These parameters have only effect when Striping is set to 1 in the +; [Database] section, in which case the DatabaseFile parameter is ignored. +; +; With striping, the database is spawned across multiple segments +; where each segment can have multiple stripes. +; +; Format of the lines below: +; Segment = , [, .. ] +; +; must be ordered from 1 up. +; +; The is the total size of the segment which is equally divided +; across all stripes forming the segment. Its specification can be in +; gigabytes (g), megabytes (m), kilobytes (k) or in database blocks +; (b, the default) +; +; Note that the segment size must be a multiple of the database page size +; which is currently 8k. Also, the segment size must be divisible by the +; number of stripe files forming the segment. +; +; The example below creates a 200 meg database striped on two segments +; with two stripes of 50 meg and one of 100 meg. +; +; You can always add more segments to the configuration, but once +; added, do not change the setup. +; +[Striping] +Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +Segment2 = 100M, db-seg2-1.db +;... +;[TempStriping] +;Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +;Segment2 = 100M, db-seg2-1.db +;... +;[Ucms] +;UcmPath = +;Ucm1 = +;Ucm2 = +;... + +[Zero Config] +ServerName = virtuoso (D602566B774E) +;ServerDSN = ZDSN +;SSLServerName = +;SSLServerDSN = + +[Mono] +;MONO_TRACE = Off +;MONO_PATH = +;MONO_ROOT = +;MONO_CFG_DIR = +;virtclr.dll = + +[URIQA] +DynamicLocal = 0 +DefaultHost = localhost:8890 + +[SPARQL] +;ExternalQuerySource = 1 +;ExternalXsltSource = 1 +;DefaultGraph = http://localhost:8890/dataspace +;ImmutableGraphs = http://localhost:8890/dataspace +ResultSetMaxRows = 1000000 ; 1 million +;MaxQueryCostEstimationTime = 4000 ; in seconds +MaxQueryExecutionTime = 1200 ; in seconds +DefaultQuery = select distinct ?Concept where {[] a ?Concept} LIMIT 100 +DeferInferenceRulesInit = 0 ; controls inference rules loading +;PingService = http://rpc.pingthesemanticweb.com/ + +[Plugins] +LoadPath = /usr/local/virtuoso-opensource/lib/virtuoso/hosting +;Load1 = plain, wikiv +;Load2 = plain, mediawiki +;Load3 = plain, creolewiki +;Load4 = plain, im +;Load5 = plain, wbxml2 +;Load6 = plain, hslookup +;Load7 = attach, libphp5.so +;Load8 = Hosting, hosting_php.so +;Load9 = Hosting,hosting_perl.so +;Load10 = Hosting,hosting_python.so +;Load11 = Hosting,hosting_ruby.so +;Load12 = msdtc,msdtc_sample diff --git a/config/publication-triplestore/virtuoso.ini b/config/publication-triplestore/virtuoso.ini new file mode 100755 index 0000000..fb33835 --- /dev/null +++ b/config/publication-triplestore/virtuoso.ini @@ -0,0 +1,253 @@ +; +; virtuoso.ini +; +; Configuration file for the OpenLink Virtuoso VDBMS Server +; +; To learn more about this product, or any other product in our +; portfolio, please check out our web site at: +; +; http://virtuoso.openlinksw.com/ +; +; or contact us at: +; +; general.information@openlinksw.com +; +; If you have any technical questions, please contact our support +; staff at: +; +; technical.support@openlinksw.com +; +; +; Database setup +; +[Database] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.db +ErrorLogFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.log +LockFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.lck +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.trx +xa_persistent_file = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso.pxa +ErrorLogLevel = 7 +FileExtend = 200 +MaxCheckpointRemap = 8500 ;;niels 1/4 of the number of buffers (assuming db uses most of the available buffers, you can check with `status()` in isql-v +Striping = 0 +TempStorage = TempDatabase +Syslog = 1 ; write to syslog in case of failures where writing to log is not possible + +[TempDatabase] +DatabaseFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.db +TransactionFile = /usr/local/virtuoso-opensource/var/lib/virtuoso/db/virtuoso-temp.trx +MaxCheckpointRemap = 2000 +Striping = 0 + +; +; Server parameters +; +[Parameters] +ServerPort = 1111 +LiteMode = 0 +DisableUnixSocket = 1 +DisableTcpSocket = 0 +;SSLServerPort = 2111 +;SSLCertificate = cert.pem +;SSLPrivateKey = pk.pem +;X509ClientVerify = 0 +;X509ClientVerifyDepth = 0 +;X509ClientVerifyCAFile = ca.pem +MaxClientConnections = 10 +CheckpointInterval = 60 +TraceOn = errors ; write traces +O_DIRECT = 0 +CaseMode = 2 +MaxStaticCursorRows = 5000 +CheckpointAuditTrail = 0 +AllowOSCalls = 0 +SchedulerInterval = 10 +DirsAllowed = ., /usr/local/virtuoso-opensource/share/virtuoso/vad +ThreadCleanupInterval = 10 +ThreadThreshold = 10 +ResourcesCleanupInterval = 10 +FreeTextBatchSize = 100000 +SingleCPU = 0 +VADInstallDir = /usr/local/virtuoso-opensource/share/virtuoso/vad/ +PrefixResultNames = 0 +RdfFreeTextRulesSize = 100 +IndexTreeMaps = 256 +MaxMemPoolSize = 200000000 +PrefixResultNames = 0 +MacSpotlight = 0 +MaxQueryMem = 2G +VectorSize = 1000 ; initial parallel query vector (array of query operations) size +MaxVectorSize = 1000000 ; query vector size threshold. +AdjustVectorSize = 0 +ThreadsPerQuery = 4 +AsyncQueueMaxThreads = 10 +TransactionAfterImageLimit = 150000000 ; max 150MB transaction, total transient consumption is 450MB (roll forward + roll back) +;; +;; When running with large data sets, one should configure the Virtuoso +;; process to use between 2/3 to 3/5 of free system memory and to stripe +;; storage on all available disks. +;; +;; Uncomment next two lines if there is 2 GB system memory free +;;NumberOfBuffers = 170000 +;;MaxDirtyBuffers = 130000 + +;; Uncomment next two lines if there is 4 GB system memory free +NumberOfBuffers = 340000 +MaxDirtyBuffers = 250000 +;; Uncomment next two lines if there is 8 GB system memory free +;NumberOfBuffers = 680000 +;MaxDirtyBuffers = 500000 +;; Uncomment next two lines if there is 16 GB system memory free +;NumberOfBuffers = 1360000 +;MaxDirtyBuffers = 1000000 +;; Uncomment next two lines if there is 32 GB system memory free +;NumberOfBuffers = 2720000 +;MaxDirtyBuffers = 2000000 +;; Uncomment next two lines if there is 48 GB system memory free +;NumberOfBuffers = 4000000 +;MaxDirtyBuffers = 3000000 +;; Uncomment next two lines if there is 64 GB system memory free +;NumberOfBuffers = 5450000 +;MaxDirtyBuffers = 4000000 +;; +;; Note the default settings will take very little memory +;; but will not result in very good performance +;; +;NumberOfBuffers = 10000 +;MaxDirtyBuffers = 6000 + +;; Introduced after bug with export service +MaxSortedTopRows = 1000000 ; 1 million! + +[HTTPServer] +ServerPort = 8890 +ServerRoot = /usr/local/virtuoso-opensource/var/lib/virtuoso/vsp +MaxClientConnections = 10 +DavRoot = DAV +EnabledDavVSP = 0 +HTTPProxyEnabled = 0 +TempASPXDir = 0 +DefaultMailServer = localhost:25 +ServerThreads = 10 +MaxKeepAlives = 10 +KeepAliveTimeout = 10 +MaxCachedProxyConnections = 10 +ProxyConnectionCacheTimeout = 15 +HTTPThreadSize = 280000 +HttpPrintWarningsInOutput = 0 +Charset = UTF-8 +;;HTTPLogFile = http.log +MaintenancePage = atomic.html +EnabledGzipContent = 1 + +[AutoRepair] +BadParentLinks = 0 + +[Client] +SQL_PREFETCH_ROWS = 100 +SQL_PREFETCH_BYTES = 16000 +SQL_QUERY_TIMEOUT = 0 +SQL_TXN_TIMEOUT = 0 +;SQL_NO_CHAR_C_ESCAPE = 1 +;SQL_UTF8_EXECS = 0 +;SQL_NO_SYSTEM_TABLES = 0 +;SQL_BINARY_TIMESTAMP = 1 +;SQL_ENCRYPTION_ON_PASSWORD = -1 + +[VDB] +ArrayOptimization = 0 +NumArrayParameters = 10 +VDBDisconnectTimeout = 1000 +KeepConnectionOnFixedThread = 0 + +[Replication] +ServerName = db-D602566B774E +ServerEnable = 1 +QueueMax = 50000 + +; +; Striping setup +; +; These parameters have only effect when Striping is set to 1 in the +; [Database] section, in which case the DatabaseFile parameter is ignored. +; +; With striping, the database is spawned across multiple segments +; where each segment can have multiple stripes. +; +; Format of the lines below: +; Segment = , [, .. ] +; +; must be ordered from 1 up. +; +; The is the total size of the segment which is equally divided +; across all stripes forming the segment. Its specification can be in +; gigabytes (g), megabytes (m), kilobytes (k) or in database blocks +; (b, the default) +; +; Note that the segment size must be a multiple of the database page size +; which is currently 8k. Also, the segment size must be divisible by the +; number of stripe files forming the segment. +; +; The example below creates a 200 meg database striped on two segments +; with two stripes of 50 meg and one of 100 meg. +; +; You can always add more segments to the configuration, but once +; added, do not change the setup. +; +[Striping] +Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +Segment2 = 100M, db-seg2-1.db +;... +;[TempStriping] +;Segment1 = 100M, db-seg1-1.db, db-seg1-2.db +;Segment2 = 100M, db-seg2-1.db +;... +;[Ucms] +;UcmPath = +;Ucm1 = +;Ucm2 = +;... + +[Zero Config] +ServerName = virtuoso (D602566B774E) +;ServerDSN = ZDSN +;SSLServerName = +;SSLServerDSN = + +[Mono] +;MONO_TRACE = Off +;MONO_PATH = +;MONO_ROOT = +;MONO_CFG_DIR = +;virtclr.dll = + +[URIQA] +DynamicLocal = 0 +DefaultHost = localhost:8890 + +[SPARQL] +;ExternalQuerySource = 1 +;ExternalXsltSource = 1 +;DefaultGraph = http://localhost:8890/dataspace +;ImmutableGraphs = http://localhost:8890/dataspace +ResultSetMaxRows = 1000000 ; 1 million +;MaxQueryCostEstimationTime = 4000 ; in seconds +MaxQueryExecutionTime = 1200 ; in seconds +DefaultQuery = select distinct ?Concept where {[] a ?Concept} LIMIT 100 +DeferInferenceRulesInit = 0 ; controls inference rules loading +;PingService = http://rpc.pingthesemanticweb.com/ + +[Plugins] +LoadPath = /usr/local/virtuoso-opensource/lib/virtuoso/hosting +;Load1 = plain, wikiv +;Load2 = plain, mediawiki +;Load3 = plain, creolewiki +;Load4 = plain, im +;Load5 = plain, wbxml2 +;Load6 = plain, hslookup +;Load7 = attach, libphp5.so +;Load8 = Hosting, hosting_php.so +;Load9 = Hosting,hosting_perl.so +;Load10 = Hosting,hosting_python.so +;Load11 = Hosting,hosting_ruby.so +;Load12 = msdtc,msdtc_sample \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f4ed919..0906798 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -63,6 +63,10 @@ services: restart: "no" file: restart: "no" + publication-triplestore: + restart: "no" + ports: + - "8891:8890" delta-producer-dump-file-publisher: restart: "no" organizations-sync-with-sharepoint-jobs-initiator: diff --git a/docker-compose.yml b/docker-compose.yml index 3b4de11..ce52526 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -302,11 +302,22 @@ services: - "logging=true" restart: always logging: *default-logging + ################################################################################ # DELTA GENERAL ################################################################################ + publication-triplestore: + image: redpencil/virtuoso:1.2.0-rc.1 + environment: + SPARQL_UPDATE: "true" + DEFAULT_GRAPH: "http://mu.semte.ch/application" + volumes: + - ./data/publication-triplestore:/data + - ./config/publication-triplestore/virtuoso.ini:/data/virtuoso.ini + restart: always + logging: *default-logging delta-producer-dump-file-publisher: - image: lblod/delta-producer-dump-file-publisher:0.9.1 + image: lblod/delta-producer-dump-file-publisher:0.10.2 volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share @@ -314,6 +325,7 @@ services: - db:database restart: always logging: *default-logging + ################################################################################ # DELTA ORGANIZATIONS: START ################################################################################ @@ -363,6 +375,57 @@ services: # DELTA ORGANIZATIONS: END ################################################################################ + ################################################################################ + # DELTA ORGANIZATIONS PUBLIC INFO: START + ################################################################################ + delta-producer-bg-jobs-initiator-organizations-public-info: + image: lblod/delta-producer-background-jobs-initiator:0.3.1 + environment: + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + DUMP_FILE_CREATION_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations-public-info" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + CRON_PATTERN_DUMP_JOB: "27 */4 * * *" # At minute 27 every 4 hours + CRON_PATTERN_HEALING_JOB: "0 0 4 * * *" # everyday at 4 AM + START_INITIAL_SYNC: "false" # Enable in override when ready to initial sync + labels: + - "logging=true" + links: + - db:database + restart: always + logging: *default-logging + delta-producer-pub-graph-maintainer-organizations-public-info: + image: lblod/delta-producer-publication-graph-maintainer:0.15.4 + environment: + MAX_BODY_SIZE: "50mb" + JOBS_GRAPH: "http://mu.semte.ch/graphs/jobs" + RELATIVE_FILE_PATH: "deltas/organizations" + PUBLICATION_GRAPH: "http://redpencil.data.gift/id/deltas/producer/organizations-public-info" + HEALING_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/organizations-public-info" + INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations-public-info" + QUEUE_POLL_INTERVAL: 3000 + USE_VIRTUOSO_FOR_EXPENSIVE_SELECTS: "true" # some queries are just too expensive + SKIP_MU_AUTH_INITIAL_SYNC: "true" + HEALING_PATCH_GRAPH_BATCH_SIZE: 500 + SERVE_DELTA_FILES: "true" + PUBLISHER_URI: "http://data.lblod.info/services/delta-production-json-diff-file-manager/organizations-public-info" + DELTA_INTERVAL_MS: 10000 + PRETTY_PRINT_DIFF_JSON: "true" + PUBLICATION_VIRTUOSO_ENDPOINT: "http://publication-triplestore:8890/sparql" + PUBLICATION_MU_AUTH_ENDPOINT: "http://publication-triplestore:8890/sparql" + ERROR_GRAPH: "http://mu.semte.ch/graphs/jobs" + volumes: + - ./config/delta-producer/organizations-public-info:/config + - ./data/files/:/share + links: + - db:database + - triplestore:virtuoso + restart: always + logging: *default-logging + ################################################################################ + # DELTA ORGANIZATIONS PUBLIC INFO: END + ################################################################################ + ################################################################################ # DELTA PUBLIC INFORMATION: START ################################################################################