diff --git a/config/delta-producer/dump-file-publisher/config.json b/config/delta-producer/dump-file-publisher/config.json index 8d33759..5f0fd27 100644 --- a/config/delta-producer/dump-file-publisher/config.json +++ b/config/delta-producer/dump-file-publisher/config.json @@ -2,31 +2,37 @@ "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/administrative-units" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", - "fileBaseName": "dump-administrative-units" + "fileBaseName": "dump-administrative-units", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/administrative-units" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/AdministrativeUnitsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/administrative-units", - "fileBaseName": "dump-administrative-units" + "fileBaseName": "dump-administrative-units", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/organizations" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", - "fileBaseName": "dump-organizations" + "fileBaseName": "dump-organizations", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/organizations" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/OrganizationsCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/organizations", - "fileBaseName": "dump-organizations" + "fileBaseName": "dump-organizations", + "compression": true }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", - "fileBaseName": "dump-public" + "fileBaseName": "dump-public", + "compression": false }, "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public" : { "dcatDataSetSubject": "http://data.lblod.info/datasets/delta-producer/dumps/PublicCacheGraphDump", "targetGraph": "http://redpencil.data.gift/id/deltas/producer/public", - "fileBaseName": "dump-public" + "fileBaseName": "dump-public", + "compression": false } } diff --git a/config/jobs-controller/config.json b/config/jobs-controller/config.json index 317c490..928128b 100644 --- a/config/jobs-controller/config.json +++ b/config/jobs-controller/config.json @@ -62,5 +62,37 @@ "nextIndex": "0" } ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextIndex": "0" + }, + { + "currentOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/initialPublicationGraphSyncing", + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "1" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/healing/patchPublicationGraph", + "nextIndex": "0" + } + ] + }, + "http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public": { + "tasksConfiguration": [ + { + "currentOperation": null, + "nextOperation": "http://redpencil.data.gift/id/jobs/concept/TaskOperation/deltas/deltaDumpFileCreation", + "nextIndex": "0" + } + ] } } diff --git a/config/resources/dcat.json b/config/resources/dcat.json index eba75ea..4945a66 100644 --- a/config/resources/dcat.json +++ b/config/resources/dcat.json @@ -133,6 +133,10 @@ "byte-size": { "type": "number", "predicate": "dcat:byteSize" + }, + "compression": { + "type": "url", + "predicate": "dcat:compressFormat" } }, "relationships": { diff --git a/docker-compose.yml b/docker-compose.yml index 94f3a0a..8f3a5af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -200,7 +200,7 @@ services: # DELTA GENERAL ################################################################################ delta-producer-dump-file-publisher: - image: bdevloed/delta-producer-dump-file-publisher:0.5.0-beta + image: bdevloed/delta-producer-dump-file-publisher:0.8.0-beta volumes: - ./config/delta-producer/dump-file-publisher:/config - ./data/files:/share @@ -345,7 +345,7 @@ services: DUMP_FILE_CREATION_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/deltaDumpFileCreation/public' INITIAL_PUBLICATION_GRAPH_SYNC_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/initialPublicationGraphSyncing/public' HEALING_JOB_OPERATION: 'http://redpencil.data.gift/id/jobs/concept/JobOperation/deltas/healingOperation/public' - CRON_PATTERN_DUMP_JOB: '42 * * * *' # At minute 42 every hour + CRON_PATTERN_DUMP_JOB: '11 */4 * * *' # At minute 11 every 4 hours CRON_PATTERN_HEALING_JOB: '0 0 4 * * *' # everyday at 4 AM START_INITIAL_SYNC: 'false' # prefer to disable it now. We want to have feeling with how the DB acts. labels: