From c4c8f2332b25d96ead08c068b104351abf1cbdb4 Mon Sep 17 00:00:00 2001
From: Claire Lovisa <lovisa.claire@gmail.com>
Date: Thu, 12 Jan 2023 18:24:46 +0100
Subject: [PATCH] Ensure deletes and inserts happen in the correct order

---
 .../delta-sync-dispatching.js                 | 89 +++++++++----------
 docker-compose.yml                            |  3 +
 2 files changed, 44 insertions(+), 48 deletions(-)

diff --git a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js
index 42c997d..d826b2e 100644
--- a/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js
+++ b/config/loket-consumer/positions-dispatching/delta-sync-dispatching.js
@@ -35,57 +35,50 @@ async function dispatch(lib, data) {
     const insertStatements = inserts.map(o => `${o.subject} ${o.predicate} ${o.object}.`);
 
     if (deleteStatements.length) {
-      await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING).then(
-        transformedStatements => {
-          deleteFromAllGraphs(
-            muAuthSudo.updateSudo,
-            transformedStatements,
-            { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-            process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-            MAX_DB_RETRY_ATTEMPTS,
-            SLEEP_BETWEEN_BATCHES,
-            SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
-          );
-        }
-      )
+      const transformedStatementsToDelete = await transformStatements(fetch, deleteStatements, UNFILTERED_MAPPING);
+      if (transformedStatementsToDelete.length) {
+        await deleteFromAllGraphs(
+          muAuthSudo.updateSudo,
+          transformedStatementsToDelete,
+          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
+          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
+          MAX_DB_RETRY_ATTEMPTS,
+          SLEEP_BETWEEN_BATCHES,
+          SLEEP_TIME_AFTER_FAILED_DB_OPERATION,
+        );
+      }
     }
 
-
     if (insertStatements.length) {
-      await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING).then(
-        transformedStatements => {
-          if (transformedStatements.length) {
-            batchedDbUpdate(
-              muAuthSudo.updateSudo,
-              INGEST_GRAPH,
-              transformedStatements,
-              { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-              process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-              BATCH_SIZE,
-              MAX_DB_RETRY_ATTEMPTS,
-              SLEEP_BETWEEN_BATCHES,
-              SLEEP_TIME_AFTER_FAILED_DB_OPERATION
-            );
-          }
-        }
-      );
-      await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING).then(
-        transformedStatements => {
-          if (transformedStatements.length) {
-            batchedDbUpdate(
-              muAuthSudo.updateSudo,
-              PRIVACY_SENSITIVE_GRAPH,
-              transformedStatements,
-              { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
-              process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
-              BATCH_SIZE,
-              MAX_DB_RETRY_ATTEMPTS,
-              SLEEP_BETWEEN_BATCHES,
-              SLEEP_TIME_AFTER_FAILED_DB_OPERATION
-            );
-          }
-        }
-      );
+      const transformedStatementsToInsert = await transformStatements(fetch, insertStatements, MAIN_INFO_MAPPING);
+      if (transformedStatementsToInsert.length) {
+        await batchedDbUpdate(
+          muAuthSudo.updateSudo,
+          INGEST_GRAPH,
+          transformedStatementsToInsert,
+          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
+          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
+          BATCH_SIZE,
+          MAX_DB_RETRY_ATTEMPTS,
+          SLEEP_BETWEEN_BATCHES,
+          SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+        );
+      }
+
+      const transformedStatementsToInsertPrivate = await transformStatements(fetch, insertStatements, PRIVATE_INFO_MAPPING);
+      if (transformedStatementsToInsertPrivate.length) {
+        await batchedDbUpdate(
+          muAuthSudo.updateSudo,
+          PRIVACY_SENSITIVE_GRAPH,
+          transformedStatementsToInsertPrivate,
+          { 'mu-call-scope-id': 'http://redpencil.data.gift/id/concept/muScope/deltas/write-for-dispatch' },
+          process.env.MU_SPARQL_ENDPOINT, //Note: this is the default endpoint through auth
+          BATCH_SIZE,
+          MAX_DB_RETRY_ATTEMPTS,
+          SLEEP_BETWEEN_BATCHES,
+          SLEEP_TIME_AFTER_FAILED_DB_OPERATION
+        );
+      }
     }
   }
 }
diff --git a/docker-compose.yml b/docker-compose.yml
index bc9266d..2446211 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -475,6 +475,7 @@ services:
       KEEP_DELTA_FILES: "true"
     volumes:
       - ./config/loket-consumer/mandate-dispatching:/config/triples-dispatching/custom-dispatching
+      - ./data/files/consumer-files/mandaten:/consumer-files/
     restart: always
     labels:
       - "logging=true"
@@ -502,6 +503,7 @@ services:
       KEEP_DELTA_FILES: "true"
     volumes:
       - ./config/loket-consumer/leidinggevenden-dispatching:/config/triples-dispatching/custom-dispatching
+      - ./data/files/consumer-files/leidinggevenden:/consumer-files/
     restart: always
     labels:
       - "logging=true"
@@ -530,6 +532,7 @@ services:
       BYPASS_MU_AUTH_FOR_EXPENSIVE_QUERIES: "false"
     volumes:
       - ./config/loket-consumer/positions-dispatching:/config/triples-dispatching/custom-dispatching
+      - ./data/files/consumer-files/worship-services-sensitive:/consumer-files/
     restart: always
     labels:
       - "logging=true"