felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ensure deletes and inserts happen in the correct order

Browse Source
This commit is contained in:
Claire Lovisa
2023-01-12 18:24:46 +01:00
parent 94fc9ac78e
commit c4c8f2332b
2 changed files with 44 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
config/loket-consumer/positions-dispatching/delta-sync-dispatching.js
View File

@@ -35,57 +35,50 @@ async function dispatch(lib, data) {
const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
if (deleteStatements.length) {
await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING).then(
transformedStatements => {
deleteFromAllGraphs(
muAuthSudo.updateSudo,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
);
}
)
const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING);
if (transformedStatementsToDelete.length) {
await deleteFromAllGraphs(
muAuthSudo.updateSudo,
transformedStatementsToDelete,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
);
}
}
if (insertStatements.length) {
await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING).then(
transformedStatements => {
if (transformedStatements.length) {
batchedDbUpdate(
muAuthSudo.updateSudo,
INGEST_GRAPH,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
);
}
}
);
await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING).then(
transformedStatements => {
if (transformedStatements.length) {
batchedDbUpdate(
muAuthSudo.updateSudo,
PRIVACY_SENSITIVE_GRAPH,
transformedStatements,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
);
}
}
);
const transformedStatementsToInsert = await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING);
if (transformedStatementsToInsert.length) {
await batchedDbUpdate(
muAuthSudo.updateSudo,
INGEST_GRAPH,
transformedStatementsToInsert,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
);
}
const transformedStatementsToInsertPrivate = await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING);
if (transformedStatementsToInsertPrivate.length) {
await batchedDbUpdate(
muAuthSudo.updateSudo,
PRIVACY_SENSITIVE_GRAPH,
transformedStatementsToInsertPrivate,
{ 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
BATCH_SIZE,
MAX_DB_RETRY_ATTEMPTS,
SLEEP_BETWEEN_BATCHES,
SLEEP_TIME_AFTER_FAILED_DB_OPERATION
);
}
}
}
}

3
docker-compose.yml
View File

@@ -475,6 +475,7 @@ services:
KEEP_DELTA_FILES: "true"
volumes:
- ./config/loket-consumer/mandate-dispatching:/config/triples-dispatching/custom-dispatching
- ./data/files/consumer-files/mandaten:/consumer-files/
restart: always
labels:
- "logging=true"
@@ -502,6 +503,7 @@ services:
KEEP_DELTA_FILES: "true"
volumes:
- ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching
- ./data/files/consumer-files/leidinggevenden:/consumer-files/
restart: always
labels:
- "logging=true"
@@ -530,6 +532,7 @@ services:
BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false"
volumes:
- ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching
- ./data/files/consumer-files/worship-services-sensitive:/consumer-files/
restart: always
labels:
- "logging=true"