diff --git a/config/search-migrations/.gitignore b/config/search-migrations/.gitignore new file mode 100644 index 0000000..282b503 --- /dev/null +++ b/config/search-migrations/.gitignore @@ -0,0 +1 @@ +.migration_history diff --git a/config/search-migrations/.gitkeep b/config/search-migrations/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml index 2d3b4f0..ea4f4e7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -566,3 +566,13 @@ services: image: lblod/error-report-service:0.1.0 links: - db:database + + +################################################################################### +# MU SCRIPTS +# ################################################################################# + project-scripts: + image: semtech/mu-scripts:1.0.0 + volumes: + - ./scripts/project:/app/scripts/ + restart: "no" diff --git a/scripts/project/config.json b/scripts/project/config.json new file mode 100644 index 0000000..2a8ae6f --- /dev/null +++ b/scripts/project/config.json @@ -0,0 +1,21 @@ +{ + "version": "0.1", + "scripts": [ + { + "documentation": { + "command": "mu-search-delta", + "description": "produce delta for mu-search", + "arguments": [] + }, + "environment": { + "image": "node:16", + "interactive": true, + "script": "mu-search-delta/build.sh", + "join_networks": true + }, + "mounts": { + "app": "/data/app/" + } + } + ] +} diff --git a/scripts/project/mu-search-delta/.gitignore b/scripts/project/mu-search-delta/.gitignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/scripts/project/mu-search-delta/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/scripts/project/mu-search-delta/README.md b/scripts/project/mu-search-delta/README.md new file mode 100644 index 0000000..4c4dc8f --- /dev/null +++ b/scripts/project/mu-search-delta/README.md @@ -0,0 +1,43 @@ +#### Scripts mu-search-delta +* make the script executable: `chmod a+x scripts/project/mu-search-delta/build.sh` +* add your delta messages in json format in `config/search-migrations`, e.g: + ```js + // config/search-migrations/2023/001-delete-bedinaeren.json + [ + { + "deletes": [ + { + "subject": { + "type": "uri", + "value": "http://data.lblod.info/id/mandatarissen/807a3e07c95fe40962d64695069af946" + }, + "predicate": { + "type": "uri", + "value": "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan" + }, + "object": { + "type": "uri", + "value": "http://data.lblod.info/id/personen/620523C451BFC225E4ECFA97" + } + }, + { + "subject": { + "type": "uri", + "value": "http://data.lblod.info/id/mandatarissen/40ed3cb19f9fa33e47b39016780dce1d" + }, + "predicate": { + "type": "uri", + "value": "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan" + }, + "object": { + "type": "uri", + "value": "http://data.lblod.info/id/personen/620523C451BFC225E4ECFA97" + } + } + ], + "inserts": [] + } + ] + ``` +* make sure virtuoso, mu-auth, elasticsearch & mu search are running +* run `mu script project-scripts mu-search-delta` diff --git a/scripts/project/mu-search-delta/build.sh b/scripts/project/mu-search-delta/build.sh new file mode 100755 index 0000000..ea8e121 --- /dev/null +++ b/scripts/project/mu-search-delta/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +mkdir -p /data/app/config/search-migrations +npm i +node script.mjs diff --git a/scripts/project/mu-search-delta/package-lock.json b/scripts/project/mu-search-delta/package-lock.json new file mode 100644 index 0000000..98598b8 --- /dev/null +++ b/scripts/project/mu-search-delta/package-lock.json @@ -0,0 +1,144 @@ +{ + "name": "mu-search-delta", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "mu-search-delta", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "node-fetch": "^3.3.0" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz", + "integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==", + "engines": { + "node": ">= 8" + } + } + }, + "dependencies": { + "data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" + }, + "fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "requires": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + } + }, + "formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "requires": { + "fetch-blob": "^3.1.2" + } + }, + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" + }, + "node-fetch": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz", + "integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==", + "requires": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + } + }, + "web-streams-polyfill": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz", + "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==" + } + } +} diff --git a/scripts/project/mu-search-delta/package.json b/scripts/project/mu-search-delta/package.json new file mode 100644 index 0000000..87120e8 --- /dev/null +++ b/scripts/project/mu-search-delta/package.json @@ -0,0 +1,14 @@ +{ + "name": "mu-search-delta", + "version": "1.0.0", + "description": "", + "main": "script.mjs", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Nordine Bittich", + "license": "MIT", + "dependencies": { + "node-fetch": "^3.3.0" + } +} diff --git a/scripts/project/mu-search-delta/script.mjs b/scripts/project/mu-search-delta/script.mjs new file mode 100644 index 0000000..1461ebd --- /dev/null +++ b/scripts/project/mu-search-delta/script.mjs @@ -0,0 +1,82 @@ +import fs from 'fs'; +import fetch from 'node-fetch'; +import path from 'path'; + +const CONFIG_PATH = process.env.SEARCH_CONFIG_PATH || '/data/app/config/search-migrations'; +const HISTORY_PATH = process.env.HISTORY_PATH || CONFIG_PATH + '/.migration_history'; +const MU_SEARCH_DELTA_URL = process.env.MU_SEARCH_DELTA_URL || 'http://search/update'; +const SEARCH_MIGRATIONS_DIR = process.env.SEARCH_MIGRATIONS_DIR || CONFIG_PATH; +const CONTENT_TYPE_JSON_API = { 'Content-Type': 'application/vnd.api+json' }; + +function read(fd, len) { + const buff = Buffer.alloc(len), + pos = 0, offset = 0; + fs.readSync(fd, buff, offset, len, pos); + return buff.toString(); +} + +function readDirRecursive(dir, filterHistory = [], filterExtensions = ["json"]) { + const files = fs.readdirSync(dir); + + const resultFiles = []; + + for (const file of files) { + const filePath = path.join(dir, file); + const stat = fs.statSync(filePath); + if (stat.isDirectory()) { + resultFiles.push(...(readDirRecursive(`${dir}/${file}`, filterHistory))); + } else if (filterExtensions.some((ext) => filePath.endsWith(ext)) && !filterHistory.includes(filePath)) { + resultFiles.push({ + time: stat.atimeMs, + file: filePath, + }); + } else { + console.log(`filter file ${filePath}`); + } + } + + return resultFiles.sort((a, b) => a.time - b.time); +} + +// main + +async function main() { + const historyFileDescriptor = fs.openSync(HISTORY_PATH, 'a+'); + const historyFileLen = fs.statSync(HISTORY_PATH).size; + + const history = read(historyFileDescriptor, historyFileLen) + .split(/\r?\n/) + .map((f) => f.trim()) + .filter((f) => f?.length); + + const fileNames = readDirRecursive(SEARCH_MIGRATIONS_DIR, history); + console.log(`processing ${fileNames.length} migration(s)`); + for (const { file } of fileNames) { + console.log(`Current file: ${file}`); + const payload = fs.readFileSync(file); + + try { + JSON.parse(payload); + } catch (e) { + console.error("Verify your migration. Exit..."); + process.exit(1); + } + + const r = await fetch(MU_SEARCH_DELTA_URL, { + method: "post", + body: payload, + headers: CONTENT_TYPE_JSON_API, + }); + console.log(`"[Delta]=> Status: ${r.status} `); + if (r.status == 200) { + const responseBody = await r.json(); + console.log(`response ${JSON.stringify(responseBody)}`); + fs.appendFileSync(HISTORY_PATH, file + '\n'); + } else { + console.error("an error occurred. check your migration, make sure mu search is running."); + process.exit(1); + } + } +} + +main().then(() => console.log("Done."));