felix.ruizdearcaute/app-organization-portal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mu search delta script (#229)

Browse Source
This commit is contained in:
Nordine Bittich
2023-01-31 11:33:53 +01:00
committed by GitHub
parent bb74c52267
commit 7b8890b1ff
10 changed files with 320 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
scripts/project/config.json Normal file
View File

@@ -0,0 +1,21 @@
{
"version": "0.1",
"scripts": [
{
"documentation": {
"command": "mu-search-delta",
"description": "produce delta for mu-search",
"arguments": []
},
"environment": {
"image": "node:16",
"interactive": true,
"script": "mu-search-delta/build.sh",
"join_networks": true
},
"mounts": {
"app": "/data/app/"
}
}
]
}

1
scripts/project/mu-search-delta/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
node_modules

43
scripts/project/mu-search-delta/README.md Normal file
View File

@@ -0,0 +1,43 @@
#### Scripts mu-search-delta
* make the script executable: `chmod a+x scripts/project/mu-search-delta/build.sh`
* add your delta messages in json format in `config/search-migrations`, e.g:
```js
// config/search-migrations/2023/001-delete-bedinaeren.json
[
{
"deletes": [
{
"subject": {
"type": "uri",
"value": "http://data.lblod.info/id/mandatarissen/807a3e07c95fe40962d64695069af946"
},
"predicate": {
"type": "uri",
"value": "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan"
},
"object": {
"type": "uri",
"value": "http://data.lblod.info/id/personen/620523C451BFC225E4ECFA97"
}
},
{
"subject": {
"type": "uri",
"value": "http://data.lblod.info/id/mandatarissen/40ed3cb19f9fa33e47b39016780dce1d"
},
"predicate": {
"type": "uri",
"value": "http://data.vlaanderen.be/ns/mandaat#isBestuurlijkeAliasVan"
},
"object": {
"type": "uri",
"value": "http://data.lblod.info/id/personen/620523C451BFC225E4ECFA97"
}
}
],
"inserts": []
}
]
```
* make sure virtuoso, mu-auth, elasticsearch & mu search are running
* run `mu script project-scripts mu-search-delta`

4
scripts/project/mu-search-delta/build.sh Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/bash
mkdir -p /data/app/config/search-migrations
npm i
node script.mjs

144
scripts/project/mu-search-delta/package-lock.json generated Normal file
View File

@@ -0,0 +1,144 @@
{
"name": "mu-search-delta",
"version": "1.0.0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "mu-search-delta",
"version": "1.0.0",
"license": "MIT",
"dependencies": {
"node-fetch": "^3.3.0"
}
},
"node_modules/data-uri-to-buffer": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
"engines": {
"node": ">= 12"
}
},
"node_modules/fetch-blob": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "paypal",
"url": "https://paypal.me/jimmywarting"
}
],
"dependencies": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
},
"engines": {
"node": "^12.20 || >= 14.13"
}
},
"node_modules/formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"dependencies": {
"fetch-blob": "^3.1.2"
},
"engines": {
"node": ">=12.20.0"
}
},
"node_modules/node-domexception": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "github",
"url": "https://paypal.me/jimmywarting"
}
],
"engines": {
"node": ">=10.5.0"
}
},
"node_modules/node-fetch": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz",
"integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==",
"dependencies": {
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
},
"engines": {
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/node-fetch"
}
},
"node_modules/web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
"engines": {
"node": ">= 8"
}
}
},
"dependencies": {
"data-uri-to-buffer": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="
},
"fetch-blob": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"requires": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
}
},
"formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"requires": {
"fetch-blob": "^3.1.2"
}
},
"node-domexception": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="
},
"node-fetch": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.0.tgz",
"integrity": "sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==",
"requires": {
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
}
},
"web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
}
}
}

14
scripts/project/mu-search-delta/package.json Normal file
View File

@@ -0,0 +1,14 @@
{
"name": "mu-search-delta",
"version": "1.0.0",
"description": "",
"main": "script.mjs",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "Nordine Bittich",
"license": "MIT",
"dependencies": {
"node-fetch": "^3.3.0"
}
}

82
scripts/project/mu-search-delta/script.mjs Normal file
View File

@@ -0,0 +1,82 @@
import fs from 'fs';
import fetch from 'node-fetch';
import path from 'path';
const CONFIG_PATH = process.env.SEARCH_CONFIG_PATH || '/data/app/config/search-migrations';
const HISTORY_PATH = process.env.HISTORY_PATH || CONFIG_PATH + '/.migration_history';
const MU_SEARCH_DELTA_URL = process.env.MU_SEARCH_DELTA_URL || 'http://search/update';
const SEARCH_MIGRATIONS_DIR = process.env.SEARCH_MIGRATIONS_DIR || CONFIG_PATH;
const CONTENT_TYPE_JSON_API = { 'Content-Type': 'application/vnd.api+json' };
function read(fd, len) {
const buff = Buffer.alloc(len),
pos = 0, offset = 0;
fs.readSync(fd, buff, offset, len, pos);
return buff.toString();
}
function readDirRecursive(dir, filterHistory = [], filterExtensions = ["json"]) {
const files = fs.readdirSync(dir);
const resultFiles = [];
for (const file of files) {
const filePath = path.join(dir, file);
const stat = fs.statSync(filePath);
if (stat.isDirectory()) {
resultFiles.push(...(readDirRecursive(`${dir}/${file}`, filterHistory)));
} else if (filterExtensions.some((ext) => filePath.endsWith(ext)) && !filterHistory.includes(filePath)) {
resultFiles.push({
time: stat.atimeMs,
file: filePath,
});
} else {
console.log(`filter file ${filePath}`);
}
}
return resultFiles.sort((a, b) => a.time - b.time);
}
// main
async function main() {
const historyFileDescriptor = fs.openSync(HISTORY_PATH, 'a+');
const historyFileLen = fs.statSync(HISTORY_PATH).size;
const history = read(historyFileDescriptor, historyFileLen)
.split(/\r?\n/)
.map((f) => f.trim())
.filter((f) => f?.length);
const fileNames = readDirRecursive(SEARCH_MIGRATIONS_DIR, history);
console.log(`processing ${fileNames.length} migration(s)`);
for (const { file } of fileNames) {
console.log(`Current file: ${file}`);
const payload = fs.readFileSync(file);
try {
JSON.parse(payload);
} catch (e) {
console.error("Verify your migration. Exit...");
process.exit(1);
}
const r = await fetch(MU_SEARCH_DELTA_URL, {
method: "post",
body: payload,
headers: CONTENT_TYPE_JSON_API,
});
console.log(`"[Delta]=> Status: ${r.status} `);
if (r.status == 200) {
const responseBody = await r.json();
console.log(`response ${JSON.stringify(responseBody)}`);
fs.appendFileSync(HISTORY_PATH, file + '\n');
} else {
console.error("an error occurred. check your migration, make sure mu search is running.");
process.exit(1);
}
}
}
main().then(() => console.log("Done."));