felix.ruizdearcaute/natural-language-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup

Browse Source
This commit is contained in:
felix
2026-09-27 00:15:18 +02:00
parent ad278dd680
commit 9b0b8a85c9
25 changed files with 588 additions and 1959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@@ -44,6 +44,7 @@ Two documents drive everything:
A **profile** is a SHACL shapes graph: one `sh:NodeShape` per entity, fields A **profile** is a SHACL shapes graph: one `sh:NodeShape` per entity, fields
as `sh:property`, code lists via `sh:class`, links via `sh:node`. The service as `sh:property`, code lists via `sh:class`, links via `sh:node`. The service
ships no profile and knows no vocabulary; profiles live in the app repo. ships no profile and knows no vocabulary; profiles live in the app repo.
Profiles load at boot; a broken profile stops the service.
A **spec** is `rep:ReportSpec` + `sh:NodeShape`: `sh:property` selects the A **spec** is `rep:ReportSpec` + `sh:NodeShape`: `sh:property` selects the
subjects, `rep:columns` says what the CSV holds. The two never interact. The subjects, `rep:columns` says what the CSV holds. The two never interact. The
@@ -107,13 +108,11 @@ The scope URI is `SERVICE_SCOPE` (default
POST /assistant/conversations/:id/turns { content } → 202 { id } POST /assistant/conversations/:id/turns { content } → 202 { id }
``` ```
This is the chat. The turn module in This is the chat. The route is in `app.js`: it records the question, answers
`src/chat/` is generic — it imports nothing from `src/llm/` or `src/runner/` `202` with the message id, then classifies the turn and refines or executes
and moves between services without edits — and `src/report-assistant.js` is (`src/report-assistant.js`) with the conversation's history. The
this service's one `answer` hook. It records the question, answers `202` with conversation, message and bijlage reads and writes are in `src/chat.js`. The
the message id, then classifies the turn and refines or executes with the answer and its bijlagen are written when the turn ends. The browser polls
conversation's history. The answer and its bijlagen are written when the turn
ends. The browser polls
`/chat-conversations/:id?include=messages.attachments` and finds the answer `/chat-conversations/:id?include=messages.attachments` and finds the answer
that way. Failure is a message, never a silent loader. that way. Failure is a message, never a silent loader.

69
app.js
View File

@@ -1,25 +1,66 @@
import { app, errorHandler } from 'mu'; import { app, errorHandler } from 'mu';
import bodyParser from 'body-parser';
import { loadProfiles } from './src/runner/profile.js'; import { loadProfiles } from './src/runner/profile.js';
import { mountChat } from './src/chat/index.js'; import { readConversation, readHistory, findAssistant, writeMessage } from './src/chat.js';
import { reportAssistant } from './src/report-assistant.js'; import { wantsExecution } from './src/llm/mode.js';
import { executeSpec, refineSpec } from './src/report-assistant.js';
const PROFILE_DIR = process.env.PROFILE_DIR || '/config/profiles'; const PROFILE_DIR = process.env.PROFILE_DIR || '/config/profiles';
const CHAT_ASSISTANT_URI = process.env.CHAT_ASSISTANT_URI;
const TITLE_LENGTH = 80;
const FAILED = 'Er ging iets mis. Probeer het opnieuw.';
// Profiles load at boot; handlers await the same promise. A failed load // Profiles load at boot. A broken profile stops the service.
// answers an error per request instead of crashing boot. const profiles = loadProfiles(PROFILE_DIR);
let profilesPromise = loadProfiles(PROFILE_DIR);
profilesPromise.catch(e => console.error('[profiles] loading failed:', e.message));
async function whenProfiles() { // One chat turn. Records the question, answers 202, then writes the
// assistant's answer; the frontend polls the conversation for it. mu's
// query/update run as the caller for the whole turn, also after the 202.
// The template only parses application/vnd.api+json, the chat sends
// application/json.
app.post('/assistant/conversations/:id/turns', bodyParser.json(), async function(req, res) {
const content = String(req.body?.content ?? '').trim();
if (!content) return res.status(400).json({ error: 'content is required' });
let conversation, assistant, history, question;
try { try {
return await profilesPromise; // the access check: the caller's session decides what is readable
conversation = await readConversation(req.params.id);
if (!conversation) return res.status(404).json({ error: 'no such conversation' });
assistant = CHAT_ASSISTANT_URI || await findAssistant();
history = await readHistory(conversation.uri, assistant);
question = await writeMessage({
conversationUri: conversation.uri,
content,
maker: conversation.creator,
title: conversation.title ? undefined : content.slice(0, TITLE_LENGTH),
});
} catch (e) { } catch (e) {
throw new Error('profiles are not available: ' + e.message); console.error('[chat] could not record the question:', e);
return res.status(500).json({ error: e.message });
} }
}
// The chat. reportAssistant is the answer hook: it refines the spec with the res.status(202).json({ id: question.id });
// LLM, or runs the agreed spec without it.
mountChat(app, { path: '/assistant', answer: reportAssistant(whenProfiles) });
app.use(errorHandler); try {
const turns = [...history, { role: 'user', content }];
const answer = await wantsExecution(turns)
? await executeSpec(history, conversation.title, profiles)
: await refineSpec(turns, profiles);
await writeMessage({
conversationUri: conversation.uri,
content: answer.text,
maker: assistant,
attachments: answer.attachments,
});
} catch (e) {
console.error('[chat] turn failed:', e);
try {
await writeMessage({ conversationUri: conversation.uri, content: FAILED, maker: assistant });
} catch (e2) {
console.error('[chat] could not write the failure message:', e2);
}
}
});
app.use(errorHandler);

871
package-lock.json generated
View File

@@ -9,7 +9,6 @@
"version": "0.2.0", "version": "0.2.0",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"express": "^5.2.1",
"n3": "^1.17.0" "n3": "^1.17.0"
} }
}, },
@@ -25,19 +24,6 @@
"node": ">=6.5" "node": ">=6.5"
} }
}, },
"node_modules/accepts": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz",
"integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==",
"license": "MIT",
"dependencies": {
"mime-types": "^3.0.0",
"negotiator": "^1.0.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/base64-js": { "node_modules/base64-js": {
"version": "1.5.1", "version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -58,43 +44,6 @@
], ],
"license": "MIT" "license": "MIT"
}, },
"node_modules/body-parser": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz",
"integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==",
"license": "MIT",
"dependencies": {
"bytes": "^3.1.2",
"content-type": "^2.0.0",
"debug": "^4.4.3",
"http-errors": "^2.0.1",
"iconv-lite": "^0.7.2",
"on-finished": "^2.4.1",
"qs": "^6.15.2",
"raw-body": "^3.0.2",
"type-is": "^2.1.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/body-parser/node_modules/content-type": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz",
"integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/buffer": { "node_modules/buffer": {
"version": "6.0.3", "version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
@@ -119,184 +68,6 @@
"ieee754": "^1.2.1" "ieee754": "^1.2.1"
} }
}, },
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
"integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/call-bound": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz",
"integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"get-intrinsic": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/content-disposition": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz",
"integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/content-type": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
"integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie-signature": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz",
"integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==",
"license": "MIT",
"engines": {
"node": ">=6.6.0"
}
},
"node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
"integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
"license": "MIT"
},
"node_modules/encodeurl": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
"integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-errors": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
"integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-object-atoms": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz",
"integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
"license": "MIT"
},
"node_modules/etag": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
"integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/event-target-shim": { "node_modules/event-target-shim": {
"version": "5.0.1", "version": "5.0.1",
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -315,206 +86,6 @@
"node": ">=0.8.x" "node": ">=0.8.x"
} }
}, },
"node_modules/express": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz",
"integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==",
"license": "MIT",
"dependencies": {
"accepts": "^2.0.0",
"body-parser": "^2.2.1",
"content-disposition": "^1.0.0",
"content-type": "^1.0.5",
"cookie": "^0.7.1",
"cookie-signature": "^1.2.1",
"debug": "^4.4.0",
"depd": "^2.0.0",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"etag": "^1.8.1",
"finalhandler": "^2.1.0",
"fresh": "^2.0.0",
"http-errors": "^2.0.0",
"merge-descriptors": "^2.0.0",
"mime-types": "^3.0.0",
"on-finished": "^2.4.1",
"once": "^1.4.0",
"parseurl": "^1.3.3",
"proxy-addr": "^2.0.7",
"qs": "^6.14.0",
"range-parser": "^1.2.1",
"router": "^2.2.0",
"send": "^1.1.0",
"serve-static": "^2.2.0",
"statuses": "^2.0.1",
"type-is": "^2.0.1",
"vary": "^1.1.2"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/finalhandler": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz",
"integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==",
"license": "MIT",
"dependencies": {
"debug": "^4.4.0",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"on-finished": "^2.4.1",
"parseurl": "^1.3.3",
"statuses": "^2.0.1"
},
"engines": {
"node": ">= 18.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
"integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/fresh": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz",
"integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-intrinsic": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"license": "MIT",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/gopd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-symbols": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hasown": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz",
"integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==",
"license": "MIT",
"dependencies": {
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/http-errors": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
"integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
"license": "MIT",
"dependencies": {
"depd": "~2.0.0",
"inherits": "~2.0.4",
"setprototypeof": "~1.2.0",
"statuses": "~2.0.2",
"toidentifier": "~1.0.1"
},
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/iconv-lite": {
"version": "0.7.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz",
"integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==",
"license": "MIT",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/ieee754": { "node_modules/ieee754": {
"version": "1.2.1", "version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
@@ -535,92 +106,6 @@
], ],
"license": "BSD-3-Clause" "license": "BSD-3-Clause"
}, },
"node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"license": "ISC"
},
"node_modules/ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
"integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
"license": "MIT",
"engines": {
"node": ">= 0.10"
}
},
"node_modules/is-promise": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz",
"integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==",
"license": "MIT"
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/media-typer": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz",
"integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/merge-descriptors": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz",
"integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/mime-db": {
"version": "1.54.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz",
"integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz",
"integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==",
"license": "MIT",
"dependencies": {
"mime-db": "^1.54.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT"
},
"node_modules/n3": { "node_modules/n3": {
"version": "1.26.0", "version": "1.26.0",
"resolved": "https://registry.npmjs.org/n3/-/n3-1.26.0.tgz", "resolved": "https://registry.npmjs.org/n3/-/n3-1.26.0.tgz",
@@ -634,87 +119,6 @@
"node": ">=12.0" "node": ">=12.0"
} }
}, },
"node_modules/negotiator": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz",
"integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==",
"license": "MIT",
"dependencies": {
"content-type": "^2.1.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/negotiator/node_modules/content-type": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz",
"integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/object-inspect": {
"version": "1.13.4",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
"integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
"integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
"license": "MIT",
"dependencies": {
"ee-first": "1.1.1"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
"license": "ISC",
"dependencies": {
"wrappy": "1"
}
},
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/path-to-regexp": {
"version": "8.4.2",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz",
"integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==",
"license": "MIT",
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/process": { "node_modules/process": {
"version": "0.11.10", "version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
@@ -724,63 +128,6 @@
"node": ">= 0.6.0" "node": ">= 0.6.0"
} }
}, },
"node_modules/proxy-addr": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
"integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
"license": "MIT",
"dependencies": {
"forwarded": "0.2.0",
"ipaddr.js": "1.9.1"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/qs": {
"version": "6.15.3",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz",
"integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==",
"license": "BSD-3-Clause",
"dependencies": {
"es-define-property": "^1.0.1",
"side-channel": "^1.1.1"
},
"engines": {
"node": ">=0.6"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/range-parser": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz",
"integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/raw-body": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz",
"integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==",
"license": "MIT",
"dependencies": {
"bytes": "~3.1.2",
"http-errors": "~2.0.1",
"iconv-lite": "~0.7.0",
"unpipe": "~1.0.0"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/readable-stream": { "node_modules/readable-stream": {
"version": "4.7.0", "version": "4.7.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz",
@@ -797,22 +144,6 @@
"node": "^12.22.0 || ^14.17.0 || >=16.0.0" "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
} }
}, },
"node_modules/router": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz",
"integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==",
"license": "MIT",
"dependencies": {
"debug": "^4.4.0",
"depd": "^2.0.0",
"is-promise": "^4.0.0",
"parseurl": "^1.3.3",
"path-to-regexp": "^8.0.0"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/safe-buffer": { "node_modules/safe-buffer": {
"version": "5.2.1", "version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -833,144 +164,6 @@
], ],
"license": "MIT" "license": "MIT"
}, },
"node_modules/safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT"
},
"node_modules/send": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz",
"integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==",
"license": "MIT",
"dependencies": {
"debug": "^4.4.3",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"etag": "^1.8.1",
"fresh": "^2.0.0",
"http-errors": "^2.0.1",
"mime-types": "^3.0.2",
"ms": "^2.1.3",
"on-finished": "^2.4.1",
"range-parser": "^1.2.1",
"statuses": "^2.0.2"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/serve-static": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz",
"integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==",
"license": "MIT",
"dependencies": {
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"parseurl": "^1.3.3",
"send": "^1.2.0"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/setprototypeof": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
"license": "ISC"
},
"node_modules/side-channel": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz",
"integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"object-inspect": "^1.13.4",
"side-channel-list": "^1.0.1",
"side-channel-map": "^1.0.1",
"side-channel-weakmap": "^1.0.2"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-list": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz",
"integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"object-inspect": "^1.13.4"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-map": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz",
"integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==",
"license": "MIT",
"dependencies": {
"call-bound": "^1.0.2",
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.5",
"object-inspect": "^1.13.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-weakmap": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz",
"integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==",
"license": "MIT",
"dependencies": {
"call-bound": "^1.0.2",
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.5",
"object-inspect": "^1.13.3",
"side-channel-map": "^1.0.1"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/string_decoder": { "node_modules/string_decoder": {
"version": "1.3.0", "version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -979,70 +172,6 @@
"dependencies": { "dependencies": {
"safe-buffer": "~5.2.0" "safe-buffer": "~5.2.0"
} }
},
"node_modules/toidentifier": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
"integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
"license": "MIT",
"engines": {
"node": ">=0.6"
}
},
"node_modules/type-is": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz",
"integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==",
"license": "MIT",
"dependencies": {
"content-type": "^2.0.0",
"media-typer": "^1.1.0",
"mime-types": "^3.0.0"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/type-is/node_modules/content-type": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz",
"integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
"integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
"license": "ISC"
} }
} }
} }

1
package.json
View File

@@ -9,7 +9,6 @@
"start": "node app.js" "start": "node app.js"
}, },
"dependencies": { "dependencies": {
"express": "^5.2.1",
"n3": "^1.17.0" "n3": "^1.17.0"
} }
} }

122
src/chat/store.js → src/chat.js
View File

@@ -1,15 +1,14 @@
// The chat's reads and writes. Nothing here knows the service it lives in. // The chat's reads and writes: conversations, messages and their bijlagen.
// Full URIs everywhere: the auth layer rejects a prefixed name with no // Full URIs everywhere: the auth layer rejects a prefixed name with no
// PREFIX line with an opaque 500. // PREFIX line with an opaque 500 (the xsd: types of mu's escape helpers are
// fine).
// //
// A bijlage is a real file: a logical nfo:FileDataObject the file service // A bijlage is a real file: a logical nfo:FileDataObject the file service
// serves at /files/<uuid>/download, over a physical share:// object with // serves at /files/<uuid>/download, over a physical share:// object with
// nie:dataSource back to the logical one. Every attachment carries the // nie:dataSource back to the logical one.
// file's logical URI (see chat/attachment-file.js how those files are import { readFileSync, writeFileSync, statSync } from 'fs';
// made); nothing here writes files. import { query, update, uuid,
import { randomUUID } from 'crypto'; sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt, sparqlEscapeDateTime } from 'mu';
import { query as muQuery, update as muUpdate,
sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
const HISTORY_LIMIT = Number(process.env.CHAT_HISTORY_LIMIT || 20); const HISTORY_LIMIT = Number(process.env.CHAT_HISTORY_LIMIT || 20);
@@ -27,14 +26,28 @@ const T = {
maker: 'http://xmlns.com/foaf/0.1/maker', maker: 'http://xmlns.com/foaf/0.1/maker',
attachment: 'https://www.w3.org/ns/activitystreams#attachment', attachment: 'https://www.w3.org/ns/activitystreams#attachment',
softwareAgent: 'http://www.w3.org/ns/prov#SoftwareAgent', softwareAgent: 'http://www.w3.org/ns/prov#SoftwareAgent',
file: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject',
fileName: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName',
fileSize: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize',
fileCreated: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated',
format: 'http://purl.org/dc/terms/format',
fileType: 'http://purl.org/dc/terms/type',
dataSource: 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource',
}; };
const MESSAGE_BASE = 'http://data.lblod.info/id/chat-messages/'; const MESSAGE_BASE = 'http://data.lblod.info/id/chat-messages/';
const FILE_BASE = 'http://data.lblod.info/files/';
const SHARE_DIR = process.env.SHARE_DIR || '/share';
export const SPEC_MEDIA_TYPE = 'text/turtle';
// The bijlage types. Match the add-chat-attachment-types-codelist migration
// in the portal.
export const SPEC_TYPE = 'http://lblod.data.gift/concepts/6693014e-7b7d-448c-bb24-80c9758521f3';
export const RESULT_TYPE = 'http://lblod.data.gift/concepts/28d97698-e219-4a92-bdbd-7ab367a7524e';
// Every spec file is called like this; readSpecFile opens nothing else.
const SPEC_FILE_NAME = /^specificatie-[0-9a-f-]{36}\.ttl$/;
const u = sparqlEscapeUri; const u = sparqlEscapeUri;
const s = sparqlEscapeString; const s = sparqlEscapeString;
// Full URIs, so spell the dateTime type out.
const dt = (value) => sparqlEscapeDateTime(value)
.replace('xsd:dateTime', '<http://www.w3.org/2001/XMLSchema#dateTime>');
const bindings = (json) => json.results.bindings; const bindings = (json) => json.results.bindings;
@@ -42,7 +55,7 @@ const bindings = (json) => json.results.bindings;
// The conversation by uuid, null when the caller may not read it. // The conversation by uuid, null when the caller may not read it.
export async function readConversation(id) { export async function readConversation(id) {
const json = await muQuery(` const json = await query(`
SELECT ?conversation ?title ?creator WHERE { SELECT ?conversation ?title ?creator WHERE {
?conversation ${u(T.type)} ${u(T.thread)} ; ?conversation ${u(T.type)} ${u(T.thread)} ;
${u(T.uuid)} ${s(id)} . ${u(T.uuid)} ${s(id)} .
@@ -57,7 +70,7 @@ export async function readConversation(id) {
// The last HISTORY_LIMIT messages, oldest first, as // The last HISTORY_LIMIT messages, oldest first, as
// { role, content, attachments: [{ uri, name, mediaType, type }] }. // { role, content, attachments: [{ uri, name, mediaType, type }] }.
export async function readHistory(conversationUri, assistantUri) { export async function readHistory(conversationUri, assistantUri) {
const json = await muQuery(` const json = await query(`
SELECT ?message ?content ?maker ?created ?doc ?docName ?docFormat ?docType WHERE { SELECT ?message ?content ?maker ?created ?doc ?docName ?docFormat ?docType WHERE {
?message ${u(T.hasContainer)} ${u(conversationUri)} ; ?message ${u(T.hasContainer)} ${u(conversationUri)} ;
${u(T.content)} ?content ; ${u(T.content)} ?content ;
@@ -65,9 +78,9 @@ export async function readHistory(conversationUri, assistantUri) {
${u(T.created)} ?created . ${u(T.created)} ?created .
OPTIONAL { OPTIONAL {
?message ${u(T.attachment)} ?doc . ?message ${u(T.attachment)} ?doc .
OPTIONAL { ?doc ${u('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ?docName . } OPTIONAL { ?doc ${u(T.fileName)} ?docName . }
OPTIONAL { ?doc ${u('http://purl.org/dc/terms/format')} ?docFormat . } OPTIONAL { ?doc ${u(T.format)} ?docFormat . }
OPTIONAL { ?doc ${u('http://purl.org/dc/terms/type')} ?docType . } OPTIONAL { ?doc ${u(T.fileType)} ?docType . }
} }
} ORDER BY DESC(?created) LIMIT ${HISTORY_LIMIT * 4}`); } ORDER BY DESC(?created) LIMIT ${HISTORY_LIMIT * 4}`);
const messages = []; const messages = [];
@@ -98,7 +111,7 @@ export async function readHistory(conversationUri, assistantUri) {
// The one assistant the caller can read. Throws unless there is exactly one. // The one assistant the caller can read. Throws unless there is exactly one.
export async function findAssistant() { export async function findAssistant() {
const json = await muQuery(` const json = await query(`
SELECT ?agent WHERE { ?agent ${u(T.type)} ${u(T.softwareAgent)} . } LIMIT 2`); SELECT ?agent WHERE { ?agent ${u(T.type)} ${u(T.softwareAgent)} . } LIMIT 2`);
const rows = bindings(json); const rows = bindings(json);
if (rows.length !== 1) { if (rows.length !== 1) {
@@ -111,12 +124,12 @@ export async function findAssistant() {
// One message, both types, its bijlagen, and the conversation touched, in // One message, both types, its bijlagen, and the conversation touched, in
// one update. `title` is only written when given (the first question names // one update. `title` is only written when given (the first question names
// an unnamed conversation). Every attachment is the logical uri of a file // an unnamed conversation). attachments are the logical uris of files that
// that already exists. // already exist.
export async function writeMessage({ conversationUri, content, maker, attachments = [], title }) { export async function writeMessage({ conversationUri, content, maker, attachments = [], title }) {
const id = randomUUID(); const id = uuid();
const uri = `${MESSAGE_BASE}${id}`; const uri = `${MESSAGE_BASE}${id}`;
const now = dt(new Date()); const now = sparqlEscapeDateTime(new Date());
// Predicate pairs joined with ';', never a trailing one: it breaks the Turtle. // Predicate pairs joined with ';', never a trailing one: it breaks the Turtle.
const pairs = [ const pairs = [
@@ -128,15 +141,14 @@ export async function writeMessage({ conversationUri, content, maker, attachment
`${u(T.created)} ${now}`, `${u(T.created)} ${now}`,
]; ];
if (maker) pairs.push(`${u(T.maker)} ${u(maker)}`); if (maker) pairs.push(`${u(T.maker)} ${u(maker)}`);
for (const attachment of attachments) { for (const fileUri of attachments) {
if (!attachment.uri) throw new Error('an attachment without a file uri'); pairs.push(`${u(T.attachment)} ${u(fileUri)}`);
pairs.push(`${u(T.attachment)} ${u(attachment.uri)}`);
} }
const conversationPairs = [`${u(T.lastActivity)} ${now}`]; const conversationPairs = [`${u(T.lastActivity)} ${now}`];
if (title) conversationPairs.push(`${u(T.title)} ${s(title)}`); if (title) conversationPairs.push(`${u(T.title)} ${s(title)}`);
await muUpdate(` await update(`
DELETE { DELETE {
${u(conversationUri)} ${u(T.lastActivity)} ?old . ${u(conversationUri)} ${u(T.lastActivity)} ?old .
} }
@@ -150,3 +162,63 @@ export async function writeMessage({ conversationUri, content, maker, attachment
return { uri, id }; return { uri, id };
} }
// --- files --------------------------------------------------------------------
export function writeShareFile(fileName, content) {
writeFileSync(`${SHARE_DIR}/${fileName}`, content, 'utf8');
}
// storeSpecFile(spec) → the file uri of a new spec bijlage. A spec file
// keeps one name for the logical and the physical file: readSpecFile finds
// it by the name in the history.
export async function storeSpecFile(spec) {
const fileName = `specificatie-${uuid()}.ttl`;
writeShareFile(fileName, spec);
return registerFile(fileName, SPEC_MEDIA_TYPE, SPEC_TYPE);
}
// readSpecFile(fileName) → the Turtle of a spec bijlage, null when there is
// no such spec file.
export function readSpecFile(fileName) {
if (!SPEC_FILE_NAME.test(fileName || '')) return null;
try {
return readFileSync(`${SHARE_DIR}/${fileName}`, 'utf8');
} catch {
return null;
}
}
// registerFile(shareName, format, type, name) → the logical file uri. One
// logical file called `name` over the physical share file `shareName`, both
// nfo:FileDataObject, both typed with dct:type. A share name must be
// unique: two files on one share name overwrite each other.
export async function registerFile(shareName, format, type, name = shareName) {
const id = uuid();
const physicalId = uuid();
const logical = `${FILE_BASE}${id}`;
const physical = `share://${shareName}`;
const size = statSync(`${SHARE_DIR}/${shareName}`).size;
const now = sparqlEscapeDateTime(new Date());
await update(`
INSERT DATA {
${u(logical)} ${u(T.type)} ${u(T.file)} ;
${u(T.uuid)} ${s(id)} ;
${u(T.fileName)} ${s(name)} ;
${u(T.format)} ${s(format)} ;
${u(T.fileSize)} ${sparqlEscapeInt(size)} ;
${u(T.created)} ${now} ;
${u(T.fileType)} ${u(type)} .
${u(physical)} ${u(T.type)} ${u(T.file)} ;
${u(T.uuid)} ${s(physicalId)} ;
${u(T.fileName)} ${s(shareName)} ;
${u(T.format)} ${s(format)} ;
${u(T.fileSize)} ${sparqlEscapeInt(size)} ;
${u(T.fileCreated)} ${now} ;
${u(T.dataSource)} ${u(logical)} ;
${u(T.fileType)} ${u(type)} .
}`);
return logical;
}

81
src/chat/attachment-file.js
View File

@@ -1,81 +0,0 @@
// A file made to sit on a message: written to the share, registered as a
// logical nfo:FileDataObject over a physical share:// object, typed with
// dct:type. The file service serves /files/<uuid>/download for it.
import { writeFileSync } from 'fs';
import { randomUUID } from 'crypto';
import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString,
sparqlEscapeInt, sparqlEscapeDateTime } from '../db.js';
import { ATTACHMENT_TYPES, SPEC_MEDIA_TYPE } from './vocab.js';
const SHARE_DIR = process.env.SHARE_DIR || '/share';
const FILE_BASE = 'http://data.lblod.info/files/';
const SHARE_BASE = 'share://';
const UUID = 'http://mu.semte.ch/vocabularies/core/uuid';
const NFO = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#';
const FILE = `${NFO}FileDataObject`;
const FILE_NAME = `${NFO}fileName`;
const FILE_SIZE = `${NFO}fileSize`;
const FILE_CREATED = `${NFO}fileCreated`;
const FORMAT = 'http://purl.org/dc/terms/format';
const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type';
const FILE_TYPE = 'http://purl.org/dc/terms/type';
const DATA_SOURCE = 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource';
const file = sparqlEscapeUri;
const str = sparqlEscapeString;
// storeSpecFile(update, content) → { uri, name }. One spec, one unique file:
// spec bijlagen keep their name apart from the report's ttl-file. Every
// spec file starts with "specificatie-", the read_spec tool opens only
// those.
export async function storeSpecFile(update, content) {
const fileName = `specificatie-${randomUUID()}.ttl`;
writeFileSync(`${SHARE_DIR}/${fileName}`, content, 'utf8');
return registerFileObject(update, {
fileName,
format: SPEC_MEDIA_TYPE,
size: Buffer.byteLength(content),
type: ATTACHMENT_TYPES.spec,
});
}
// registerFileObject(update, { fileName, format, size, type }) → { uri, name }.
// One logical file over one physical share:// file, both nfo:FileDataObject,
// both carrying the dct:type when given.
export async function registerFileObject(update, { fileName, format, size, type }) {
const id = randomUUID();
const physicalId = randomUUID();
const logical = `${FILE_BASE}${id}`;
const physical = `${SHARE_BASE}${fileName}`;
const now = sparqlEscapeDateTime(new Date()) // full URI, xsd spelled out
.replace('xsd:dateTime', '<http://www.w3.org/2001/XMLSchema#dateTime>');
const logicalPairs = [
`${file(RDF_TYPE)} ${file(FILE)}`,
`${file(UUID)} ${str(id)}`,
`${file(FILE_NAME)} ${str(fileName)}`,
`${file(FORMAT)} ${str(format)}`,
`${file(FILE_SIZE)} ${sparqlEscapeInt(size)}`,
`${file('http://purl.org/dc/terms/created')} ${now}`,
...(type ? [`${file(FILE_TYPE)} ${file(type)}`] : []),
];
const physicalPairs = [
`${file(RDF_TYPE)} ${file(FILE)}`,
`${file(UUID)} ${str(physicalId)}`,
`${file(FILE_NAME)} ${str(fileName)}`,
`${file(FORMAT)} ${str(format)}`,
`${file(FILE_SIZE)} ${sparqlEscapeInt(size)}`,
`${file(FILE_CREATED)} ${now}`,
`${file(DATA_SOURCE)} ${file(logical)}`,
...(type ? [`${file(FILE_TYPE)} ${file(type)}`] : []),
];
await (update || sessionUpdate)(`
INSERT DATA {
${file(logical)} ${logicalPairs.join(' ;\n ')} .
${file(physical)} ${physicalPairs.join(' ;\n ')} .
}`);
return { uri: logical, name: fileName };
}

86
src/chat/index.js
View File

@@ -1,86 +0,0 @@
// The turn endpoint. Mount it with mountChat(app, { path, answer });
// answer(turn) is the only thing the mounting service writes.
import express from 'express';
import { query as muQuery, update as muUpdate } from 'mu';
import {
readConversation, readHistory, findAssistant,
writeMessage,
} from './store.js';
const TEXT = {
failed: 'Er ging iets mis. Probeer het opnieuw.',
};
const TITLE_LENGTH = 80;
export function mountChat(app, { path = '/assistant', answer }) {
if (typeof answer !== 'function') throw new Error('mountChat needs an answer(turn) function');
app.post(`${path}/conversations/:id/turns`, express.json(), async (req, res) => {
const content = String(req.body?.content ?? '').trim();
if (!content) return res.status(400).json({ error: 'content is required' });
// 1. the access check: the caller's session decides what is readable
let conversation;
try {
conversation = await readConversation(req.params.id);
} catch (e) {
console.error('[chat] conversation read failed:', e);
return res.status(500).json({ error: e.message });
}
if (!conversation) return res.status(404).json({ error: 'no such conversation' });
let assistant, history, userMessage;
try {
assistant = process.env.CHAT_ASSISTANT_URI || await findAssistant();
history = await readHistory(conversation.uri, assistant);
userMessage = await writeMessage({
conversationUri: conversation.uri,
content,
maker: conversation.creator,
title: conversation.title ? undefined : content.slice(0, TITLE_LENGTH),
});
} catch (e) {
console.error('[chat] could not record the question:', e);
return res.status(500).json({ error: e.message });
}
// 2. answer 202 now; the assistant's message comes when it comes. The
// continuation still runs as the caller.
res.status(202).json({ id: userMessage.id });
const turn = {
conversation: { uri: conversation.uri, id: conversation.id, title: conversation.title },
history,
content,
query: muQuery,
update: muUpdate,
finished: false,
say: (text, attachments = []) => writeMessage({
conversationUri: conversation.uri, content: text, maker: assistant, attachments,
}),
};
try {
// answer may return a string, or { text, attachments } for answers the
// assistant writes with an attachment, such as the report spec.
const result = await answer(turn);
const text = typeof result === 'string' ? result : result?.text;
const attachments
= typeof result === 'string' ? [] : (result?.attachments ?? []);
if (typeof text === 'string' && text.trim()) {
await turn.say(text.trim(), attachments);
}
} catch (e) {
console.error('[chat] turn failed:', e);
try {
await turn.say(TEXT.failed);
} catch (e2) {
console.error('[chat] could not write the failure message:', e2);
}
} finally {
turn.finished = true;
}
});
console.log(`[chat] turns at POST ${path}/conversations/:id/turns`);
}

8
src/chat/vocab.js
View File

@@ -1,8 +0,0 @@
// The bijlage types of the report assistant. Match the
// add-chat-attachment-types-codelist migration in the portal.
export const ATTACHMENT_TYPES = {
spec: 'http://lblod.data.gift/concepts/6693014e-7b7d-448c-bb24-80c9758521f3',
result: 'http://lblod.data.gift/concepts/28d97698-e219-4a92-bdbd-7ab367a7524e',
};
export const SPEC_BASE_NAME = 'specificatie';
export const SPEC_MEDIA_TYPE = 'text/turtle';

37
src/db.js
View File

@@ -1,37 +0,0 @@
// Thin re-export of mu's SPARQL helpers so the runner files share one import
// site. mu's query/update attach the caller's session from the request
// context, and mu-authorization decides graphs and visibility; the service
// never touches an auth header.
import {
query as muQuery,
update as muUpdate,
sparqlEscapeUri,
sparqlEscapeString,
sparqlEscapeInt,
sparqlEscapeDateTime,
sparqlEscapeDate,
sparqlEscapeBool,
} from 'mu';
export const sessionUpdate = muUpdate;
export {
sparqlEscapeUri,
sparqlEscapeString,
sparqlEscapeInt,
sparqlEscapeDateTime,
sparqlEscapeDate,
sparqlEscapeBool,
};
// The service's own scope. Reads the LLM drives during refinement (code
// lists, lookup_values) run under this scope, not the caller's session. The
// sparql-parser config grants this scope read access to the public graph
// only, so refinement reads only public data. Execution still runs as the
// caller. Set to match the with-scope grant in config.lisp.
export const SERVICE_SCOPE =
process.env.SERVICE_SCOPE || 'http://services.semantic.works/natural-language-report';
// Wrap a mu query/update so it carries the service scope. mu's query/update
// accept { scope } on mu-javascript-template >= 1.9.0.
export function scopedQuery(q) { return muQuery(q, { scope: SERVICE_SCOPE }); }
export function scopedUpdate(q) { return muUpdate(q, { scope: SERVICE_SCOPE }); }

70
src/llm/describe.js
View File

@@ -1,59 +1,9 @@
// A profile as a plain text menu: entities, fields, code lists. Code lists up // A profile as a plain text menu: entities, fields, code lists. Code lists up
// to INLINE_VALUES_MAX are inlined and cached for VALUES_TTL, so the LLM // to INLINE_VALUES_MAX are inlined (and cached), so the LLM never has to
// never has to search the short ones. // search the short ones.
import { codeList, INLINE_VALUES_MAX } from './lookup.js';
import { fieldsOf, entityForClass } from '../runner/profile.js'; export async function describeProfile(profile) {
const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50);
const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000;
// In-memory cache: profile+field → { values, at }. Refreshed lazily when stale.
const cache = new Map();
// codeListValues(profile, queryFn) → map "shapeUri|predicate|inverse"
// → [{ uri, label }]. queryFn runs under the service scope (public graph), so
// code lists read the same regardless of the caller's rights.
export async function codeListValues(profile, queryFn) {
const out = {};
for (const shape of profile.shapes) {
for (const f of shape.fields) {
if (!f.class) continue;
const key = `${shape.uri}|${f.path}|${f.inverse ? 1 : 0}`;
out[key] = await loadCodeList(f, key, queryFn);
}
}
return out;
}
async function loadCodeList(f, key, queryFn) {
if (!f.class) return [];
const hit = cache.get(key);
if (hit && Date.now() - hit.at < VALUES_TTL) return hit.values;
let values = [];
try {
values = await queryCodeList(f.class, queryFn);
} catch (e) {
// the read failed; report nothing, the LLM will use rep:anyOf with words
values = [];
}
cache.set(key, { values, at: Date.now() });
return values;
}
async function queryCodeList(type, queryFn) {
const q = `SELECT ?uri ?label WHERE {
?uri a <${type}> .
OPTIONAL { ?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label . }
} ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`;
const r = await queryFn(q);
const bindings = r.results.bindings;
return bindings.slice(0, INLINE_VALUES_MAX).map(b => ({
uri: b.uri.value,
label: b.label?.value || b.uri.value,
}));
}
export function describeProfile(profile, codeLists = {}) {
const lines = []; const lines = [];
lines.push('A spec is Turtle, in exactly this shape (copy these prefixes):'); lines.push('A spec is Turtle, in exactly this shape (copy these prefixes):');
lines.push('@prefix rep: <http://mu.semte.ch/vocabularies/reporting/> .'); lines.push('@prefix rep: <http://mu.semte.ch/vocabularies/reporting/> .');
@@ -80,13 +30,12 @@ export function describeProfile(profile, codeLists = {}) {
for (const shape of profile.shapes) { for (const shape of profile.shapes) {
lines.push(`${shape.label || shape.targetClass} — sh:targetClass <${shape.targetClass}> ; rep:entity <${shape.uri}>`); lines.push(`${shape.label || shape.targetClass} — sh:targetClass <${shape.targetClass}> ; rep:entity <${shape.uri}>`);
for (const f of shape.fields) { for (const f of shape.fields) {
const key = `${shape.uri}|${f.path}|${f.inverse ? 1 : 0}`;
const kind = f.datatype ? datatypeName(f.datatype) const kind = f.datatype ? datatypeName(f.datatype)
: f.class ? `→ ${conceptLabel(f.class, codeLists[key])}` : f.class ? `→ ${conceptLabel(f.class, await codeList(f.class))}`
: f.node ? `→ ${linkedLabel(profile, f.node)}` : f.node ? `→ ${linkedLabel(profile, f.node)}`
: '?'; : '?';
const dir = f.inverse ? ' (inverse)' : ''; const dir = f.inverse ? ' (inverse)' : '';
const pathShown = typeof f.path === 'string' ? f.path : '?'; const pathShown = f.path || '?';
lines.push(` ${f.name || pathShown} sh:path <${pathShown}>${dir} ${kind}`); lines.push(` ${f.name || pathShown} sh:path <${pathShown}>${dir} ${kind}`);
} }
lines.push(''); lines.push('');
@@ -95,11 +44,10 @@ export function describeProfile(profile, codeLists = {}) {
} }
function conceptLabel(type, values) { function conceptLabel(type, values) {
const head = values.slice(0, INLINE_VALUES_MAX); if (!values.length) return short(type);
if (!head.length) return short(type); const names = values.map(v => v.label).join(', ');
const names = head.map(v => v.label).join(', ');
const more = values.length >= INLINE_VALUES_MAX ? ', …' : ''; const more = values.length >= INLINE_VALUES_MAX ? ', …' : '';
return `${short(type)} — ${head.length} waarden: ${names}${more}`; return `${short(type)} — ${values.length} waarden: ${names}${more}`;
} }
function linkedLabel(profile, nodeUri) { function linkedLabel(profile, nodeUri) {

134
src/llm/lookup.js
View File

@@ -1,98 +1,118 @@
// Search a code list too long to inline: regex on str(?label), so // The LLM's own reads: code lists. They run under the service scope, not
// language-tagged labels match. Returns the matches, the total, and an // the caller's session; the sparql-parser config grants that scope read
// `exact` flag when a label equals the term (case aside). // access to the public graph only, so they read the same for every caller.
// Execution still runs as the caller.
// //
// This is a suggest helper, not a spec check. It reads candidate values the // lookup_values is a suggest helper, not a spec check. It reads candidate
// LLM can put in a filter; it never runs the spec and never sees report // values the LLM can put in a filter; it never runs the spec and never sees
// rows. queryFn runs under the service scope (public graph only), so the // report rows.
// suggestions come from the same data set the profile describes, regardless import { query, sparqlEscapeString } from 'mu';
// of who calls.
import { sparqlEscapeString } from '../db.js';
// Must match the with-scope grant in the app's config.lisp.
const SERVICE_SCOPE = process.env.SERVICE_SCOPE || 'http://services.semantic.works/natural-language-report';
export const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50);
const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000;
const LIMIT = 25; const LIMIT = 25;
// lookupValues(profile, term, fieldSpec, queryFn) → // class → { values, at }. Refreshed lazily when stale.
// { total, exact, matches } or { error }. const cache = new Map();
// field is "shapeLabel.fieldLabel" as the LLM writes it, or a predicate URI.
// We resolve it to the field's sh:class to know what type to search. function publicQuery(q) {
export async function lookupValues(profile, term, fieldSpec, queryFn) { return query(q, { scope: SERVICE_SCOPE });
const type = resolveClass(profile, fieldSpec); }
if (!type) return { total: 0, exact: null, matches: [], error: `field "${fieldSpec}" not found in profile "${profile.title}"` };
// codeList(type) → up to INLINE_VALUES_MAX [{ uri, label }], cached for
// VALUES_TTL. Empty when the read fails; the LLM then uses rep:anyOf with
// words.
export async function codeList(type) {
const hit = cache.get(type);
if (hit && Date.now() - hit.at < VALUES_TTL) return hit.values;
let values = [];
try {
const r = await publicQuery(`SELECT ?uri ?label WHERE {
?uri a <${type}> .
OPTIONAL { ?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label . }
} ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`);
values = r.results.bindings.slice(0, INLINE_VALUES_MAX).map(b => ({
uri: b.uri.value,
label: b.label?.value || b.uri.value,
}));
} catch (e) {
values = [];
}
cache.set(type, { values, at: Date.now() });
return values;
}
// lookupValues(profile, field, term) → the matches as text for the LLM: up
// to LIMIT labels and URIs, an exact label (case aside) first. field is
// "shapeLabel.fieldLabel" as the LLM writes it; it resolves to the class to
// search.
export async function lookupValues(profile, field, term) {
const type = resolveClass(profile, field);
if (!type) return `field "${field}" not found in profile "${profile.title}"`;
// Search within the class first. A common word ("gemeente") also sits in // Search within the class first. A common word ("gemeente") also sits in
// hundreds of unrelated labels; an unscoped scan drowns the code. // hundreds of unrelated labels; an unscoped scan drowns the code.
let rows = await labelQuery(term, type, queryFn); let rows = await labelQuery(term, type);
// Code lists sometimes type their entries differently than the profile // Code lists sometimes type their entries differently than the profile
// says (MAR codes are ext:supervision/Nomenclature, the profile says // says (MAR codes are ext:supervision/Nomenclature, the profile says
// skos:Concept). Fall back to a scan over everything. // skos:Concept). Fall back to a scan over everything.
if (!rows.length) rows = await labelQuery(term, null, queryFn); if (!rows.length) rows = await labelQuery(term, null);
// Order: an exact label (case aside) first, then the rest as the store // "Gent" must surface before "AGB Erfgoed Gent".
// ordered them. "Gent" must surface before "AGB Erfgoed Gent".
const lower = term.toLowerCase(); const lower = term.toLowerCase();
const exactRow = rows.find(b => b.label.toLowerCase() === lower); const exactRow = rows.find(r => r.label.toLowerCase() === lower);
const rest = rows.filter(b => b !== exactRow); let matches = (exactRow ? [exactRow, ...rows.filter(r => r !== exactRow)] : rows).slice(0, LIMIT);
const ordered = exactRow ? [exactRow, ...rest] : rest; if (!matches.length) matches = await codeQuery(term);
let matches = ordered.slice(0, LIMIT); if (!matches.length) return `no value matches "${term}" in ${field}.`;
if (!matches.length) {
const byCode = await codeQuery(profile, type, term, queryFn); const lines = matches.map(m => `${m.label} <${m.uri}>`);
matches = byCode; if (matches.length >= LIMIT) lines.push(`(${LIMIT}+ matches, narrow the term)`);
} const exact = matches.find(m => m.label.toLowerCase() === lower);
const exact = matches.find(m => m.label.toLowerCase() === term.toLowerCase()); if (exact) lines.push(`exact: <${exact.uri}>`);
return { return lines.join('\n');
total: matches.length >= LIMIT ? `${LIMIT}+` : matches.length,
exact: exact ? exact.uri : null,
matches,
};
} }
async function labelQuery(term, type, queryFn) { // Regex on str(?label), so language-tagged labels match.
async function labelQuery(term, type) {
const q = `SELECT DISTINCT ?uri ?label WHERE { const q = `SELECT DISTINCT ?uri ?label WHERE {
?uri a ${type ? `<${type}>` : '?anyType'} ; ?uri a ${type ? `<${type}>` : '?anyType'} ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?label . <http://www.w3.org/2004/02/skos/core#prefLabel> ?label .
FILTER(REGEX(str(?label), ${sparqlEscapeString(escapeRegex(term))}, "i")) FILTER(REGEX(str(?label), ${sparqlEscapeString(escapeRegex(term))}, "i"))
} ORDER BY ?label LIMIT ${LIMIT * 8}`; } ORDER BY ?label LIMIT ${LIMIT * 8}`;
const r = await queryFn(q); const r = await publicQuery(q);
return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value })); return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value }));
} }
// MAR codes: the label says "MAR7300 - …" but the searchable code sits on // MAR codes: the label says "MAR7300 - …" but the searchable code sits on
// ext:supervision/nomenclatureCode. One extra query, only when the label // ext:supervision/nomenclatureCode. One extra query, only when the label
// search came up empty. // search came up empty.
async function codeQuery(profile, type, term, queryFn) { async function codeQuery(term) {
const q = `SELECT ?uri ?label WHERE { const q = `SELECT ?uri ?label WHERE {
?uri <http://mu.semte.ch/vocabularies/ext/supervision/nomenclatureCode> ?code ; ?uri <http://mu.semte.ch/vocabularies/ext/supervision/nomenclatureCode> ?code ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?label . <http://www.w3.org/2004/02/skos/core#prefLabel> ?label .
FILTER(CONTAINS(LCASE(str(?code)), LCASE(${sparqlEscapeString(term)}))) FILTER(CONTAINS(LCASE(str(?code)), LCASE(${sparqlEscapeString(term)})))
} ORDER BY ?label LIMIT ${LIMIT}`; } ORDER BY ?label LIMIT ${LIMIT}`;
try { try {
const r = await queryFn(q); const r = await publicQuery(q);
return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value })); return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value }));
} catch { } catch {
return []; return [];
} }
} }
function resolveClass(profile, fieldSpec) { // "shapeLabel.fieldLabel" → the class to search: the field's sh:class, or
if (fieldSpec.includes('/')) return null; // for a link (sh:node) the linked shape's sh:targetClass
// "shapeLabel.fieldLabel" → find the field carrying sh:class // ("melding.bestuurseenheid" → besluit:Bestuurseenheid).
const dot = fieldSpec.lastIndexOf('.'); function resolveClass(profile, field) {
if (dot < 0) { if (field.includes('/')) return null;
// bare predicate URI const dot = field.lastIndexOf('.');
for (const shape of profile.shapes) { if (dot < 0) return null;
for (const f of shape.fields) { const shapeLabel = field.slice(0, dot);
if (f.path === fieldSpec && f.class) return f.class; const fieldLabel = field.slice(dot + 1);
}
}
return null;
}
const shapeLabel = fieldSpec.slice(0, dot);
const fieldLabel = fieldSpec.slice(dot + 1);
const shape = profile.shapes.find(s => s.label === shapeLabel); const shape = profile.shapes.find(s => s.label === shapeLabel);
if (!shape) return null; if (!shape) return null;
const f = shape.fields.find(x => x.name === fieldLabel); const f = shape.fields.find(x => x.name === fieldLabel);
// sh:node marks a link, not a code list; the searchable type is the linked
// shape's sh:targetClass ("melding.bestuurseenheid" → besluit:Bestuurseenheid).
if (f?.class) return f.class; if (f?.class) return f.class;
if (f?.node) { if (f?.node) {
const linked = profile.shapes.find(s => s.uri === f.node); const linked = profile.shapes.find(s => s.uri === f.node);
@@ -114,4 +134,4 @@ function shortType(cls) {
function escapeRegex(s) { function escapeRegex(s) {
return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
} }

114
src/llm/loop.js
View File

@@ -8,7 +8,8 @@
// confirm before anything executes; execution runs the agreed spec without // confirm before anything executes; execution runs the agreed spec without
// the LLM (../report-assistant.js). // the LLM (../report-assistant.js).
import { buildTools } from './tools.js'; import { TOOLS, runTool } from './tools.js';
import { SPEC_MEDIA_TYPE } from '../chat.js';
const MAX_ROUNDS = Number(process.env.MAX_ROUNDS || 12); const MAX_ROUNDS = Number(process.env.MAX_ROUNDS || 12);
const LLM_BASE_URL = process.env.LLM_BASE_URL; const LLM_BASE_URL = process.env.LLM_BASE_URL;
@@ -77,70 +78,49 @@ what it says, up to three rounds. Then propose and stop.
Reply to the user in Dutch. Earlier turns are context only. Answer the last one.`; Reply to the user in Dutch. Earlier turns are context only. Answer the last one.`;
// ask(questionOrMessages, profiles, session) → { text } // ask(turns, profiles) → { text, spec }. turns is the conversation so far,
// A string is one question; an array is the conversation so far, // [{ role, content, attachments }], newest last. spec is the last spec
// [{ role, content }], newest last. The system prompt is always this file's // validate_spec accepted in this turn, or null.
// own; a system message in the input is dropped. export async function ask(turns, profiles) {
export async function ask(questionOrMessages, profiles, session) { const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns.map(withSpecHint)];
const turns = typeof questionOrMessages === 'string' let spec = null;
? [{ role: 'user', content: questionOrMessages }]
: questionOrMessages.filter(m => m.role !== 'system');
return runLoop(withAttachmentSpecs(turns), profiles, session); for (let round = 0; round < MAX_ROUNDS; round++) {
const { message } = await chat(messages, TOOLS);
messages.push(message);
const calls = message.tool_calls || [];
if (!calls.length) return { text: message.content || '', spec };
for (const call of calls) {
const args = parseArgs(call.function.arguments);
let result;
try {
result = await runTool(call.function.name, args, profiles);
} catch (e) {
result = `tool failed: ${String(e.message || e).split('\n')[0]}`;
}
// validate_spec answers "ok" (plus notes) when the spec checks out;
// that spec becomes the bijlage of the answer.
if (call.function.name === 'validate_spec' && /^ok(\n|$)/.test(result)) spec = args.spec;
messages.push({ role: 'tool', tool_call_id: call.id, content: result });
}
}
return { text: 'Sorry, dit rapport is te moeilijk. Contacteer de developers.', spec };
} }
// The history carries the spec as a bijlage. Paste one hint line per spec // The history carries the spec as a bijlage. Paste one hint line per spec
// under the message text, so the model knows a spec is there and opens it // under the message text, so the model knows a spec is there and opens it
// with read_spec; the chat itself still keeps the Turtle out of the text. // with read_spec; the chat itself still keeps the Turtle out of the text.
function withAttachmentSpecs(turns) { function withSpecHint(turn) {
return turns.map(t => { const specs = (turn.attachments || []).filter(a => a.mediaType === SPEC_MEDIA_TYPE);
const specs = (t.attachments || []).filter(a => a.mediaType === 'text/turtle' && a.uri); if (!specs.length) return turn;
if (!specs.length) return t; const hints = specs.map(a => `Bijlage "${a.name}", a spec file (read it with read_spec).`);
return { return { ...turn, content: `${turn.content}\n\n${hints.join('\n\n')}` };
...t,
content: `${t.content}\n\n${specs.map(a =>
`Bijlage "${a.name}", a spec file (read it with read_spec): ${a.uri}`).join('\n\n')}`,
};
});
} }
// runLoop drives the agent loop with the refinement prompt and tool set. // chat(messages, tools) → { message }. The one place that talks to the
async function runLoop(turns, profiles, session) { // provider. Without tools it is a plain completion (the mode check).
const { handlers } = buildTools(profiles, session); export async function chat(messages, tools = null) {
const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns];
for (let round = 0; round < MAX_ROUNDS; round++) {
const reply = await chat(messages, { tools: toolDefs() });
messages.push(reply.message);
const calls = reply.message.tool_calls || [];
if (!calls.length) {
return { text: reply.message.content || '' };
}
for (const call of calls) {
const args = parseArgs(call.function.arguments);
const fn = handlers.get(call.function.name);
let resultText;
if (!fn) {
resultText = `unknown tool "${call.function.name}"`;
} else {
try {
const out = await fn(args);
resultText = out.content?.[0]?.text ?? JSON.stringify(out);
} catch (e) {
resultText = `tool failed: ${String(e.message || e).split('\n')[0]}`;
}
}
messages.push({ role: 'tool', tool_call_id: call.id, content: resultText });
}
}
return { text: '(loop did not finish)' };
}
// chat(messages, { tools, toolChoice }) → { message }
// The one place that talks to the provider. tools null (the default) means a
// plain completion (the mode classifier); runLoop passes the tool
// definitions; toolChoice defaults to 'auto'.
export async function chat(messages, { tools = null, toolChoice = 'auto' } = {}) {
// Send only the fields every provider accepts back. // Send only the fields every provider accepts back.
const wire = messages.map(m => { const wire = messages.map(m => {
const out = { role: m.role, content: m.content ?? null }; const out = { role: m.role, content: m.content ?? null };
@@ -154,13 +134,10 @@ export async function chat(messages, { tools = null, toolChoice = 'auto' } = {})
if (m.role === 'tool' && m.tool_call_id) out.tool_call_id = m.tool_call_id; if (m.role === 'tool' && m.tool_call_id) out.tool_call_id = m.tool_call_id;
return out; return out;
}); });
const body = { const body = { model: LLM_MODEL, messages: wire };
model: LLM_MODEL,
messages: wire,
};
if (tools) { if (tools) {
body.tools = tools; body.tools = tools;
body.tool_choice = toolChoice === 'auto' ? 'auto' : toolChoice; body.tool_choice = 'auto';
} }
if (LOG_LLM) { if (LOG_LLM) {
console.log(`[llm] POST ${LLM_BASE_URL}/chat/completions request:\n${ console.log(`[llm] POST ${LLM_BASE_URL}/chat/completions request:\n${
@@ -191,16 +168,3 @@ function parseArgs(s) {
if (!s) return {}; if (!s) return {};
try { return JSON.parse(s); } catch { return {}; } try { return JSON.parse(s); } catch { return {}; }
} }
// The tool definitions sent to the model.
import { toolSchemas } from './schemas.js';
function toolDefs() {
return toolSchemas().map(t => ({
type: 'function',
function: {
name: t.name,
description: t.description,
parameters: t.inputSchema,
},
}));
}

43
src/llm/mode.js
View File

@@ -25,43 +25,18 @@ Rules:
Reply with exactly one token: true or false. No other words, no punctuation, Reply with exactly one token: true or false. No other words, no punctuation,
no markdown.`; no markdown.`;
// classifyMode(turns) → boolean (true = execute now) // wantsExecution(turns) → true when the user's last message says "run it
// turns: [{ role, content }], newest last. A leading system message is // now". turns: [{ role, content }], newest last; the last few are enough.
// dropped; the assistant's own system prompt is never sent to the model. export async function wantsExecution(turns) {
export async function classifyMode(turns) { let message;
const convo = (turns || [])
.filter(m => m.role !== 'system')
.map(m => ({ role: m.role === 'assistant' ? 'assistant' : 'user', content: m.content ?? '' }));
if (!convo.length) return false;
// Keep the last few turns; the decision is about the latest message.
const recent = convo.slice(-6);
const messages = [{ role: 'system', content: MODE_PROMPT }, ...recent];
let raw;
try { try {
const reply = await chat(messages, { tools: null }); ({ message } = await chat([{ role: 'system', content: MODE_PROMPT }, ...turns.slice(-6)]));
raw = firstText(reply.message);
} catch (e) { } catch (e) {
// The classifier call failed (no LLM, provider down). Default to refine: // The call failed (no LLM, provider down). Refine: safer than running a
// safer than running a report on a guess, and the refine loop will // report on a guess, and the refine loop will surface the real error.
// surface the real error to the user.
return false; return false;
} }
const text = stripMarkdown(raw).trim().toLowerCase(); return stripMarkdown(firstText(message)).trim().toLowerCase().startsWith('true');
if (!text) return false;
if (text.startsWith('true')) return true;
if (text.startsWith('false')) return false;
// A JSON {"execute": ...} shape some models emit.
try {
const m = text.match(/\{[\s\S]*\}/);
if (m) {
const v = JSON.parse(m[0]);
if (typeof v === 'boolean') return v;
if (v && typeof v.execute === 'boolean') return v.execute;
if (v && typeof v.refine === 'boolean') return !v.refine;
}
} catch { /* ignore */ }
return false;
} }
// Reasoning models may leave `content` empty and put the visible text in a // Reasoning models may leave `content` empty and put the visible text in a
@@ -78,4 +53,4 @@ function firstText(message) {
function stripMarkdown(s) { function stripMarkdown(s) {
return String(s).replace(/[*_`>#]/g, ' '); return String(s).replace(/[*_`>#]/g, ' ');
} }

57
src/llm/schemas.js
View File

@@ -1,57 +0,0 @@
// The JSON schemas of the tools. The descriptions are what the LLM reads to
// decide which tool to call. Refinement only: list_profiles,
// describe_profile, validate_spec, read_spec and lookup_values.
const COMMON = [
{
name: 'list_profiles',
description: 'List the report profiles available, with their id and title.',
inputSchema: { type: 'object', properties: {}, required: [] },
},
{
name: 'describe_profile',
description: 'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.',
inputSchema: {
type: 'object',
properties: { profile_id: { type: 'string', description: 'The profile URI returned by list_profiles' } },
required: ['profile_id'],
},
},
{
name: 'validate_spec',
description: 'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.',
inputSchema: {
type: 'object',
properties: { spec: { type: 'string', description: 'The spec as Turtle text' } },
required: ['spec'],
},
},
];
const LOOKUP_VALUES = {
name: 'lookup_values',
description: 'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.',
inputSchema: {
type: 'object',
properties: {
profile_id: { type: 'string', description: 'The profile URI returned by list_profiles' },
field: { type: 'string', description: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"' },
term: { type: 'string', description: 'The Dutch word or name to search for' },
},
required: ['profile_id', 'field', 'term'],
},
};
const READ_SPEC = {
name: 'read_spec',
description: 'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.',
inputSchema: {
type: 'object',
properties: { file_uri: { type: 'string', description: 'The file URI of the spec bijlage, e.g. http://data.lblod.info/files/<uuid>' } },
required: ['file_uri'],
},
};
export function toolSchemas() {
return [...COMMON, READ_SPEC, LOOKUP_VALUES];
}

171
src/llm/tools.js
View File

@@ -1,94 +1,91 @@
// The tools the LLM calls, refinement only: list_profiles, describe_profile, // The tools the LLM calls, refinement only: list_profiles,
// validate_spec, lookup_values, read_spec. lookup_values searches a code list // describe_profile, validate_spec, read_spec and lookup_values. Each returns
// under the service scope (public graph) and returns candidate values, so the // plain text for the model. No report is created, no spec is evaluated;
// LLM can suggest concrete filter values. No report is created, no spec is // the code-list reads run under the service scope (public graph).
// evaluated.
//
// The shape every tool returns is { content: [{ type: 'text', text }] },
// which the loop passes straight back to the model.
import { checkSpec, whereNotes } from '../runner/check.js';
import { parseSpec } from '../runner/spec.js'; import { parseSpec } from '../runner/spec.js';
import { describeProfile, codeListValues } from './describe.js'; import { checkSpec, whereNotes, profileError } from '../runner/check.js';
import { describeProfile } from './describe.js';
import { lookupValues } from './lookup.js'; import { lookupValues } from './lookup.js';
import { scopedQuery } from '../db.js'; import { readSpecFile } from '../chat.js';
import { readFileSync } from 'fs';
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8); // The tool definitions sent to the model. The descriptions are what the LLM
// reads to decide which tool to call.
export const TOOLS = [
tool('list_profiles',
'List the report profiles available, with their id and title.',
{}),
tool('describe_profile',
'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.',
{ profile_id: 'The profile URI returned by list_profiles' }),
tool('validate_spec',
'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.',
{ spec: 'The spec as Turtle text' }),
tool('read_spec',
'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.',
{ file_name: 'The file name of the spec bijlage, e.g. specificatie-<uuid>.ttl' }),
tool('lookup_values',
'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.',
{
profile_id: 'The profile URI returned by list_profiles',
field: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"',
term: 'The Dutch word or name to search for',
}),
];
// buildTools(profiles, session) → { handlers } function tool(name, description, params) {
export function buildTools(profiles, session) { const properties = {};
const profileList = () => [...profiles.values()].map(p => ({ id: p.uri, title: p.title })); for (const [key, text] of Object.entries(params)) {
properties[key] = { type: 'string', description: text };
const findProfile = (id) => profiles.get(id);
async function describe_profile({ profile_id }) {
const p = findProfile(profile_id);
if (!p) return text(`no profile <${profile_id}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
const inlined = await codeListValues(p, scopedQuery);
return text(describeProfile(p, inlined));
} }
return {
async function validate_spec({ spec }) { type: 'function',
let parsed; function: {
try { parsed = parseSpec(spec); } name,
catch (e) { return text(`the spec did not parse: ${e.message}`); } description,
const p = findProfile(parsed.profileUri); parameters: { type: 'object', properties, required: Object.keys(params) },
if (!p) return text(`no profile <${parsed.profileUri}>. Available: ${profileList().map(x => x.title).join(', ')}.`); },
const errors = checkSpec(parsed, p, MAX_PATH_DEPTH, profiles); };
if (errors.length) return text(errors[0]);
// The spec that checks out becomes the attached proposal.
session.onSpecValidated?.(spec);
return text(['ok', ...whereNotes(parsed, p)].join('\n'));
}
async function list_profiles() {
return text(profileList().map(x => `${x.title} <${x.id}>`).join('\n'));
}
async function lookup_values({ profile_id, field, term }) {
const p = findProfile(profile_id);
if (!p) return text(`no profile <${profile_id}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
if (!term) return text('no term to search for.');
try {
const out = await lookupValues(p, term, field, scopedQuery);
if (out.error) return text(out.error);
if (!out.matches.length) return text(`no value matches "${term}" in ${field}.`);
const lines = out.matches.map(m => `${m.label} <${m.uri}>`);
if (out.total === '25+') lines.push('(25+ matches, narrow the term)');
if (out.exact) lines.push(`exact: <${out.exact}>`);
return text(lines.join('\n'));
} catch (e) {
return text(`the lookup failed: ${String(e.message || e).split('\n')[0]}`);
}
}
// A spec bijlage next to a message. The model opens it with read_spec:
// only files this service made are readable, hte spec files under share.
async function read_spec({ file_uri }) {
const fileName = String(file_uri || '').split('/').pop();
if (!fileName || !/^specificatie-[0-9a-f]{8}-[0-9a-f-]{27}\.ttl$/.test(fileName)) {
return text('no such spec file.');
}
const shareDir = process.env.SHARE_DIR || '/share';
try {
return text(readFileSync(`${shareDir}/${fileName}`, 'utf8'));
} catch (e) {
return text(`the spec failed to load: ${String(e.message || e).split('\n')[0]}`);
}
}
const handlers = new Map(
[
['list_profiles', list_profiles],
['describe_profile', describe_profile],
['validate_spec', validate_spec],
['read_spec', read_spec],
['lookup_values', lookup_values],
],
);
return { handlers };
} }
function text(s) { return { content: [{ type: 'text', text: s }] }; } export async function runTool(name, args, profiles) {
switch (name) {
case 'list_profiles': return listProfiles(profiles);
case 'describe_profile': return describe(args, profiles);
case 'validate_spec': return validateSpec(args, profiles);
case 'read_spec': return readSpec(args);
case 'lookup_values': return lookup(args, profiles);
default: return `unknown tool "${name}"`;
}
}
function listProfiles(profiles) {
return [...profiles.values()].map(p => `${p.title} <${p.uri}>`).join('\n');
}
async function describe({ profile_id }, profiles) {
return profileError(profiles, profile_id) || describeProfile(profiles.get(profile_id));
}
function validateSpec({ spec }, profiles) {
let parsed;
try {
parsed = parseSpec(spec);
} catch (e) {
return `the spec did not parse: ${e.message}`;
}
const error = checkSpec(parsed, profiles);
if (error) return error;
return ['ok', ...whereNotes(parsed, profiles.get(parsed.profileUri))].join('\n');
}
// Only spec files this service made are readable.
function readSpec({ file_name }) {
return readSpecFile(file_name) || 'no such spec file.';
}
async function lookup({ profile_id, field, term }, profiles) {
const error = profileError(profiles, profile_id);
if (error) return error;
if (!term) return 'no term to search for.';
return lookupValues(profiles.get(profile_id), field, term);
}

158
src/report-assistant.js
View File

@@ -1,129 +1,71 @@
// What this service answers a chat turn with. The service is synchronous // What the assistant answers a chat turn with: { text, attachments }. When
// per turn: when the answer is written, every bijlage already exists. // the answer is written, every bijlage already exists.
// //
// Execute turns skip the LLM loop. The mode classifier says the user wants // Refine: the LLM proposes a spec; the last spec that checked out goes on
// to run the report now; the agreed spec already sits in the conversation // the answer as a bijlage. The prompt never carries the Turtle.
// as a bijlage, so it is read from disk and fed straight to the runner. //
// The answer is hard-coded: no extra prompt runs for it. When the run cannot // Execute: no LLM runs. The last spec bijlage in the conversation is the
// happen or fails, the answer is a hard-coded failure message: nothing is // agreed spec; it is read from the share and fed straight to the runner.
// retried and no spec is rewritten. // The answer is fixed text. When the run cannot happen or fails, the answer
import { classifyMode } from './llm/mode.js'; // is a fixed failure message: nothing is retried and no spec is rewritten.
import { uuid } from 'mu';
import { ask } from './llm/loop.js'; import { ask } from './llm/loop.js';
import { parseSpec } from './runner/spec.js'; import { parseSpec } from './runner/spec.js';
import { checkSpec } from './runner/check.js'; import { checkSpec } from './runner/check.js';
import { run, slug } from './runner/run.js'; import { run, registerReport, slug } from './runner/run.js';
import { readFileSync } from 'fs'; import { storeSpecFile, readSpecFile, writeShareFile, registerFile,
import { storeSpecFile, registerFileObject } from './chat/attachment-file.js'; SPEC_MEDIA_TYPE, SPEC_TYPE, RESULT_TYPE } from './chat.js';
import { SPEC_MEDIA_TYPE, ATTACHMENT_TYPES } from './chat/vocab.js';
const TEXT = { const TEXT = {
executed: 'Het rapport is uitgevoerd. De bijlagen staan erbij.', executed: 'Het rapport is uitgevoerd. De bijlagen staan erbij.',
noSpec: 'Er is nog geen voorstel om uit te voeren. Beschrijf eerst welk rapport je wilt.', noSpec: 'Er is nog geen voorstel om uit te voeren. Beschrijf eerst welk rapport je wilt.',
invalid: 'Het voorstel is niet meer geldig. Vraag een nieuw voorstel.', invalid: 'Het voorstel is niet meer geldig. Vraag een nieuw voorstel.',
failed: 'Het rapport kon niet uitgevoerd worden. Probeer het later opnieuw.', failed: 'Het rapport kon niet uitgevoerd worden. Probeer het later opnieuw.',
noAnswer: 'Er kwam geen antwoord. Probeer het opnieuw.',
}; };
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8); // An empty answer (a reasoning model can end with no text) gets a fixed
const SHARE_DIR = process.env.SHARE_DIR || '/share'; // text and no spec: only a spec the user saw explained may run.
export async function refineSpec(turns, profiles) {
// The chat mounts before the profiles are in; whenProfiles waits for the load. const { text, spec } = await ask(turns, profiles);
export function reportAssistant(whenProfiles) { if (!text.trim()) return { text: TEXT.noAnswer };
return async function answer(turn) { const attachments = spec ? [await storeSpecFile(spec)] : [];
const turns = [...turn.history, { role: 'user', content: turn.content }]; return { text: text.trim(), attachments };
const profiles = await whenProfiles();
if (await classifyMode(turns)) {
try {
return await runStoredSpec(turn, profiles);
} catch (e) {
console.error('[reports] execution failed:', e);
return TEXT.failed;
}
}
return llmAnswer(turn, turns, profiles);
};
} }
// Execution: the last spec bijlage in the conversation is the agreed spec. export async function executeSpec(history, conversationTitle, profiles) {
// No prompt runs; the runner gets it straight away. const attachment = lastSpecAttachment(history);
async function runStoredSpec(turn, profiles) {
const attachment = lastSpecAttachment(turn.history);
const content = attachment && readSpecFile(attachment.name); const content = attachment && readSpecFile(attachment.name);
if (!content) return TEXT.noSpec; if (!content) return { text: TEXT.noSpec };
const parsed = parseSpec(content); try {
const profile = profiles.get(parsed.profileUri); const spec = parseSpec(content);
if (!profile) { const error = checkSpec(spec, profiles);
console.error(`[reports] no profile <${parsed.profileUri}>`); if (error) {
return TEXT.invalid; console.error(`[reports] the agreed spec is not valid: ${error}`);
return { text: TEXT.invalid };
}
// The conversation's title (named by the first question) names the
// file; the share name is unique, so a second run never overwrites the
// first. The report itself keeps the dct:title of the spec.
const shareName = `${uuid()}.csv`;
writeShareFile(shareName, await run(spec, profiles.get(spec.profileUri)));
const csvFile = await registerFile(shareName, 'text/csv', RESULT_TYPE, `${slug(conversationTitle)}.csv`);
await registerReport(spec.title, csvFile);
// Same physical spec file, registered as a bijlage for this message.
const specFile = await registerFile(attachment.name, SPEC_MEDIA_TYPE, SPEC_TYPE);
return { text: TEXT.executed, attachments: [csvFile, specFile] };
} catch (e) {
console.error('[reports] execution failed:', e);
return { text: TEXT.failed };
} }
const errors = checkSpec(parsed, profile, MAX_PATH_DEPTH, profiles);
if (errors.length) {
console.error(`[reports] the agreed spec is not valid: ${errors[0]}`);
return TEXT.invalid;
}
// The conversation's title (named by the first question) names the file.
// The report itself keeps the dct:title of the spec.
const { fileUri } = await run(turn.query, turn.update, parsed, profile,
parsed.title, {
spec: content,
fileType: ATTACHMENT_TYPES.result,
fileName: `${slug(turn.conversation.title)}.csv`,
});
// Same physical spec file, registered as a bijlage for this message.
const specFile = await registerFileObject(turn.update, {
fileName: attachment.name,
format: SPEC_MEDIA_TYPE,
size: Buffer.byteLength(content),
type: ATTACHMENT_TYPES.spec,
});
return {
text: TEXT.executed,
attachments: [
{ uri: fileUri },
{ ...specFile, mediaType: SPEC_MEDIA_TYPE },
],
};
} }
function lastSpecAttachment(history) { function lastSpecAttachment(history) {
for (const message of [...(history || [])].reverse()) { const specs = history.flatMap(message => message.attachments)
const specs = (message.attachments || []).filter(a => .filter(a => a.mediaType === SPEC_MEDIA_TYPE && a.name?.startsWith('specificatie-'));
a.mediaType === SPEC_MEDIA_TYPE return specs.at(-1) || null;
&& /^specificatie-[0-9a-f-]+\.ttl$/.test(a.name || ''));
if (specs.length) return specs[specs.length - 1];
}
return null;
}
function readSpecFile(fileName) {
try {
return readFileSync(`${SHARE_DIR}/${fileName}`, 'utf8');
} catch {
return null;
}
}
// The LLM loop, refinement only. The spec that checked out goes on the
// answer as a bijlage. The prompt never carries the Turtle.
async function llmAnswer(turn, turns, profiles) {
// The last spec that checked out; stored as a file for this turn's
// final message.
let spec = null;
const session = {
onSpecValidated(validated) { spec = validated; },
};
const { text } = await ask(turns, profiles, session);
const attachments = [];
if (spec) {
const stored = await storeSpecFile(turn.update, spec);
attachments.push({ ...stored, mediaType: SPEC_MEDIA_TYPE });
}
return { text, attachments };
} }

20
src/runner/assemble.js
View File

@@ -14,7 +14,6 @@ export const NUMERIC_DATATYPES = new Set([
'nonNegativeInteger', 'positiveInteger', 'nonPositiveInteger', 'negativeInteger', 'nonNegativeInteger', 'positiveInteger', 'nonPositiveInteger', 'negativeInteger',
'unsignedLong', 'unsignedInt', 'unsignedShort', 'unsignedByte', 'unsignedLong', 'unsignedInt', 'unsignedShort', 'unsignedByte',
].map(d => XSD + d)); ].map(d => XSD + d));
export const TEMPORAL_DATATYPES = new Set([XSD + 'date', XSD + 'dateTime']);
export function assemble(subjects, values, spec) { export function assemble(subjects, values, spec) {
const sorted = [...subjects].sort(); const sorted = [...subjects].sort();
@@ -57,19 +56,18 @@ function collect(terms, col) {
} }
} }
// min/max over values that share one datatype. Numbers compare as numbers, // min/max. Numbers compare as numbers. Dates and times compare as text:
// dates and times as instants. Mixed datatypes throw: check.js should have // xsd:date and xsd:dateTime are ISO strings, so text order is date order.
// caught it, and a silently wrong cell is worse than a failed job. // Numbers mixed with other values throw: check.js should have caught it,
// and a silently wrong cell is worse than a failed job. The terms are SPARQL
// JSON bindings, so the datatype is a plain string.
function extreme(terms, col, direction) { function extreme(terms, col, direction) {
if (!terms.length) return ''; if (!terms.length) return '';
const datatypes = [...new Set(terms.map(t => t.datatype?.value || ''))]; const numbers = terms.filter(t => NUMERIC_DATATYPES.has(t.datatype)).length;
if (datatypes.length > 1) { if (numbers && numbers < terms.length) {
throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but its values carry ${datatypes.length} datatypes (${datatypes.join(', ')}).`); throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but mixes numbers with other values.`);
} }
const dt = datatypes[0]; const key = numbers ? t => Number(t.value) : t => t.value;
const key = NUMERIC_DATATYPES.has(dt) ? t => Number(t.value)
: TEMPORAL_DATATYPES.has(dt) ? t => new Date(t.value).getTime()
: t => t.value;
const cmp = (a, b) => a < b ? -1 : a > b ? 1 : 0; const cmp = (a, b) => a < b ? -1 : a > b ? 1 : 0;
const best = terms.reduce((a, b) => (cmp(key(b), key(a)) * direction > 0 ? b : a)); const best = terms.reduce((a, b) => (cmp(key(b), key(a)) * direction > 0 ? b : a));
return best.value; return best.value;

137
src/runner/check.js
View File

@@ -1,4 +1,4 @@
import { fieldsOf } from './profile.js'; import { startShape, fieldsOf, field } from './profile.js';
import { NUMERIC_DATATYPES } from './assemble.js'; import { NUMERIC_DATATYPES } from './assemble.js';
import { sharedSteps } from './spec.js'; import { sharedSteps } from './spec.js';
@@ -9,23 +9,18 @@ const XSD_DATE = XSD + 'date';
const XSD_DATETIME = XSD + 'dateTime'; const XSD_DATETIME = XSD + 'dateTime';
// The validator. checkSpec runs the checks in order and returns the first // The validator. checkSpec runs the checks in order and returns the first
// error as a one-element array (empty if ok). Every message names the thing // error, or null when the spec is fine. Every message names the thing that
// that is wrong in the profile's own words and says what to write instead. // is wrong in the profile's own words and says what to write instead. No
// No database call anywhere in this file. // database call anywhere in this file.
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
const ANYOF_MAX_TERMS = 24; const ANYOF_MAX_TERMS = 24;
const ANYOF_MAX_TERM_LENGTH = 100; const ANYOF_MAX_TERM_LENGTH = 100;
// The spec names a profile we have. knownProfiles is the profile Map loaded // null when the profile exists, else what to write instead.
// at boot; the check is skipped when it is omitted. export function profileError(profiles, uri) {
function profileNamed(spec, knownProfiles) { if (profiles.has(uri)) return null;
if (!spec.profileUri) { return `no profile <${uri}>. Available: ${[...profiles.values()].map(p => p.title).join(', ')}.`;
return `no rep:profile named. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'} .`;
}
if (knownProfiles && !knownProfiles.has(spec.profileUri)) {
return `no profile <${spec.profileUri}>. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'}.`;
}
return null;
} }
// The spec names an entity we have: by rep:entity, or by a targetClass only // The spec names an entity we have: by rep:entity, or by a targetClass only
@@ -74,16 +69,6 @@ function conditions(spec) {
return out; return out;
} }
// The shape a spec starts from, after startEntity has approved it: rep:entity
// wins, otherwise the one shape carrying the targetClass.
function startShapeOf(spec, profile) {
if (spec.entity) {
return profile.shapes.find(s => s.uri === spec.entity)?.uri || null;
}
const matches = profile.shapes.filter(s => s.targetClass === spec.targetClass);
return matches.length === 1 ? matches[0].uri : null;
}
// Columns exist // Columns exist
function columnsPresent(spec) { function columnsPresent(spec) {
if (!spec.columns.length) return 'no columns. Add at least one rep:columns entry.'; if (!spec.columns.length) return 'no columns. Add at least one rep:columns entry.';
@@ -100,26 +85,26 @@ function columnLabel(spec) {
// Labels are unique // Labels are unique
function columnLabelsUnique(spec) { function columnLabelsUnique(spec) {
const seen = new Map(); const seen = new Set();
for (const col of spec.columns) { for (const col of spec.columns) {
if (seen.has(col.label)) { if (seen.has(col.label)) {
return `two columns are called "${col.label}". Labels must differ.`; return `two columns are called "${col.label}". Labels must differ.`;
} }
seen.set(col.label, true); seen.add(col.label);
} }
return null; return null;
} }
// Every path resolves, columns end on a value, depth capped // Every path resolves, columns end on a value, depth capped
function pathsResolve(spec, profile, maxPathDepth) { function pathsResolve(spec, profile) {
const startShapeUri = startShapeOf(spec, profile); const startShapeUri = startShape(profile, spec).uri;
for (const col of spec.columns) { for (const col of spec.columns) {
const err = walk(profile, startShapeUri, col.path, maxPathDepth, const err = walk(profile, startShapeUri, col.path,
`column "${col.label}"`, true, col.nodeKind === SH + 'IRI'); `column "${col.label}"`, true, col.nodeKind === SH + 'IRI');
if (err) return err; if (err) return err;
} }
for (const [i, filter] of spec.filters.entries()) { for (const [i, filter] of spec.filters.entries()) {
const err = walk(profile, startShapeUri, filter.path, maxPathDepth, const err = walk(profile, startShapeUri, filter.path,
`filter ${i + 1}`, false); `filter ${i + 1}`, false);
if (err) return err; if (err) return err;
} }
@@ -128,7 +113,7 @@ function pathsResolve(spec, profile, maxPathDepth) {
// sh:min/sh:max only on numbers and dates // sh:min/sh:max only on numbers and dates
function collectMinMaxTyped(spec, profile) { function collectMinMaxTyped(spec, profile) {
const startShapeUri = startShapeOf(spec, profile); const startShapeUri = startShape(profile, spec).uri;
for (const col of spec.columns) { for (const col of spec.columns) {
if (col.collect !== SH + 'min' && col.collect !== SH + 'max') continue; if (col.collect !== SH + 'min' && col.collect !== SH + 'max') continue;
const name = col.collect === SH + 'min' ? 'sh:min' : 'sh:max'; const name = col.collect === SH + 'min' ? 'sh:min' : 'sh:max';
@@ -136,8 +121,7 @@ function collectMinMaxTyped(spec, profile) {
const last = col.path[col.path.length - 1]; const last = col.path[col.path.length - 1];
const shapeUri = shapeAfter(profile, startShapeUri, col.path); const shapeUri = shapeAfter(profile, startShapeUri, col.path);
if (!shapeUri) continue; // pathsResolve already reported the real error if (!shapeUri) continue; // pathsResolve already reported the real error
const f = fieldsOf(profile, shapeUri) const f = field(profile, shapeUri, last);
.find(x => x.path === last.predicate && !!x.inverse === !!last.inverse);
if (!f) continue; if (!f) continue;
const kind = f.datatype === XSD_DATETIME || f.datatype === XSD_DATE ? 'date' const kind = f.datatype === XSD_DATETIME || f.datatype === XSD_DATE ? 'date'
: NUMERIC_DATATYPES.has(f.datatype) ? 'number' : NUMERIC_DATATYPES.has(f.datatype) ? 'number'
@@ -205,10 +189,10 @@ function filterConstraint(spec) {
function maxCountZero(spec) { function maxCountZero(spec) {
for (const { cond } of conditions(spec)) { for (const { cond } of conditions(spec)) {
const c = cond.constraints; const c = cond.constraints;
if (c.maxCount !== null && c.maxCount !== undefined && c.maxCount > 0) { if (c.maxCount > 0) {
return `sh:maxCount ${c.maxCount} needs counting, which this service cannot do. Use sh:maxCount 0 for "has none", or drop it.`; return `sh:maxCount ${c.maxCount} needs counting, which this service cannot do. Use sh:maxCount 0 for "has none", or drop it.`;
} }
if (c.minCount !== null && c.minCount !== undefined && c.minCount > 2) { if (c.minCount > 2) {
return `sh:minCount ${c.minCount} needs counting. Use sh:minCount 1 (at least one) or sh:minCount 2 (two distinct values); anything higher is beyond this service.`; return `sh:minCount ${c.minCount} needs counting. Use sh:minCount 1 (at least one) or sh:minCount 2 (two distinct values); anything higher is beyond this service.`;
} }
} }
@@ -272,14 +256,14 @@ function literalsParse(spec) {
// rep:where: walks from the row through the profile, shares at least one // rep:where: walks from the row through the profile, shares at least one
// step with its filter or column, and holds no rep:where of its own. // step with its filter or column, and holds no rep:where of its own.
function whereConditions(spec, profile, maxPathDepth) { function whereConditions(spec, profile) {
const startShapeUri = startShapeOf(spec, profile); const startShapeUri = startShape(profile, spec).uri;
for (const { what, cond, host } of conditions(spec)) { for (const { what, cond, host } of conditions(spec)) {
if (!host) continue; if (!host) continue;
if (cond.where.length) { if (cond.where.length) {
return `${what} holds a rep:where of its own. Put every rep:where directly on the filter or column.`; return `${what} holds a rep:where of its own. Put every rep:where directly on the filter or column.`;
} }
const err = walk(profile, startShapeUri, cond.path, maxPathDepth, what, false); const err = walk(profile, startShapeUri, cond.path, what, false);
if (err) return err; if (err) return err;
const k = sharedSteps(host.path, cond.path); const k = sharedSteps(host.path, cond.path);
if (!k) { if (!k) {
@@ -302,14 +286,13 @@ function whereConditions(spec, profile, maxPathDepth) {
// code) it compares a URI and never matches, so its path must end on a field // code) it compares a URI and never matches, so its path must end on a field
// with a datatype. // with a datatype.
function rangeOnValue(spec, profile) { function rangeOnValue(spec, profile) {
const startShapeUri = startShapeOf(spec, profile); const startShapeUri = startShape(profile, spec).uri;
for (const { what, cond } of conditions(spec)) { for (const { what, cond } of conditions(spec)) {
const key = ['minInclusive', 'maxInclusive', 'minExclusive', 'maxExclusive'] const key = ['minInclusive', 'maxInclusive', 'minExclusive', 'maxExclusive']
.find(k => cond.constraints[k]); .find(k => cond.constraints[k]);
if (!key || !cond.path.length) continue; if (!key || !cond.path.length) continue;
const last = cond.path[cond.path.length - 1]; const last = cond.path[cond.path.length - 1];
const f = fieldsOf(profile, shapeAfter(profile, startShapeUri, cond.path)) const f = field(profile, shapeAfter(profile, startShapeUri, cond.path), last);
.find(x => x.path === last.predicate && !!x.inverse === !!last.inverse);
if (!f || f.datatype) continue; // pathsResolve reports a missing step if (!f || f.datatype) continue; // pathsResolve reports a missing step
const target = f.node const target = f.node
? `"${profile.shapes.find(s => s.uri === f.node)?.label || f.node}"` ? `"${profile.shapes.find(s => s.uri === f.node)?.label || f.node}"`
@@ -326,7 +309,7 @@ function rangeOnValue(spec, profile) {
// Where each rep:where applies, in the profile's words, so the proposal can // Where each rep:where applies, in the profile's words, so the proposal can
// say it. Profile only, no database. // say it. Profile only, no database.
export function whereNotes(spec, profile) { export function whereNotes(spec, profile) {
const startShapeUri = startShapeOf(spec, profile); const startShapeUri = startShape(profile, spec).uri;
return conditions(spec).filter(x => x.host).map(({ what, cond, host }) => { return conditions(spec).filter(x => x.host).map(({ what, cond, host }) => {
const k = sharedSteps(host.path, cond.path); const k = sharedSteps(host.path, cond.path);
return `${what} applies at ${nodeName(profile, startShapeUri, host.path.slice(0, k))}, after step ${k} of ${host.what}.`; return `${what} applies at ${nodeName(profile, startShapeUri, host.path.slice(0, k))}, after step ${k} of ${host.what}.`;
@@ -337,7 +320,7 @@ export function whereNotes(spec, profile) {
function nodeName(profile, startShapeUri, hops) { function nodeName(profile, startShapeUri, hops) {
let shapeUri = startShapeUri; let shapeUri = startShapeUri;
for (const [i, hop] of hops.entries()) { for (const [i, hop] of hops.entries()) {
const f = fieldsOf(profile, shapeUri).find(x => x.path === hop.predicate && !!x.inverse === !!hop.inverse); const f = field(profile, shapeUri, hop);
if (!f) return `step ${hops.length}`; if (!f) return `step ${hops.length}`;
if (!f.node) return i === hops.length - 1 ? `the value "${f.name}"` : `step ${hops.length}`; if (!f.node) return i === hops.length - 1 ? `the value "${f.name}"` : `step ${hops.length}`;
shapeUri = f.node; shapeUri = f.node;
@@ -351,33 +334,37 @@ function parsesAsDate(value, withTime) {
return withTime ? value.includes('T') : /^\d{4}-\d{2}-\d{2}$/.test(value); return withTime ? value.includes('T') : /^\d{4}-\d{2}-\d{2}$/.test(value);
} }
// In this order: startEntity comes first, the checks after it rely on a
// start shape.
const CHECKS = [ const CHECKS = [
['no entity or unknown class', startEntity], startEntity,
['no columns', columnsPresent], columnsPresent,
['column without rdfs:label', columnLabel], columnLabel,
['two columns with one label', columnLabelsUnique], columnLabelsUnique,
['column carries constraints', columnPure], columnPure,
['filter carries rdfs:label', filterLabel], filterLabel,
['filter without a condition', filterConstraint], filterConstraint,
['sh:maxCount above 0', maxCountZero], maxCountZero,
['rep:anyOf too big', anyOfLimits], anyOfLimits,
['sh:in mixes URIs and literals', inHomogeneous], inHomogeneous,
['typed literal does not parse', literalsParse], literalsParse,
['two rep:row columns', oneRowColumn], oneRowColumn,
['path does not resolve', pathsResolve], pathsResolve,
['rep:where does not fit', whereConditions], whereConditions,
['date or number condition on a link', rangeOnValue], rangeOnValue,
['sh:min/sh:max on a non-number', collectMinMaxTyped], collectMinMaxTyped,
]; ];
export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) { // checkSpec(spec, profiles) → the first error, or null.
const profileErr = profileNamed(spec, knownProfiles); export function checkSpec(spec, profiles) {
if (profileErr) return [profileErr]; const error = profileError(profiles, spec.profileUri);
for (const [name, check] of CHECKS) { if (error) return error;
const err = check(spec, profile, maxPathDepth); const profile = profiles.get(spec.profileUri);
if (err) return [err]; for (const check of CHECKS) {
const err = check(spec, profile);
if (err) return err;
} }
return []; return null;
} }
// The shape a path lands in: follow sh:node through every hop but the // The shape a path lands in: follow sh:node through every hop but the
@@ -386,17 +373,16 @@ export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) {
function shapeAfter(profile, startShapeUri, hops) { function shapeAfter(profile, startShapeUri, hops) {
let shapeUri = startShapeUri; let shapeUri = startShapeUri;
for (let i = 0; i < hops.length - 1; i++) { for (let i = 0; i < hops.length - 1; i++) {
const fields = fieldsOf(profile, shapeUri); const f = field(profile, shapeUri, hops[i]);
const f = fields.find(x => x.path === hops[i].predicate && !!x.inverse === !!hops[i].inverse);
if (!f || !f.node) return null; if (!f || !f.node) return null;
shapeUri = f.node; shapeUri = f.node;
} }
return shapeUri; return shapeUri;
} }
function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsIri = false) { function walk(profile, startShapeUri, hops, what, isColumn, wantsIri = false) {
if (hops.length > maxPathDepth) { if (hops.length > MAX_PATH_DEPTH) {
return `${what} walks ${hops.length} hops. The limit is ${maxPathDepth}.`; return `${what} walks ${hops.length} hops. The limit is ${MAX_PATH_DEPTH}.`;
} }
if (!hops.length) return null; if (!hops.length) return null;
let shapeUri = startShapeUri; let shapeUri = startShapeUri;
@@ -416,20 +402,15 @@ function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsI
shapeUri = f.node; shapeUri = f.node;
} else if (isColumn) { } else if (isColumn) {
if (wantsIri && f.datatype) { if (wantsIri && f.datatype) {
return `column "${colLabel(what)}" has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`; return `${what} has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`;
} }
if (!wantsIri && !f.datatype && !f.class) { if (!wantsIri && !f.datatype && !f.class) {
const valueFields = fieldsOf(profile, f.node) const valueFields = fieldsOf(profile, f.node)
.filter(x => x.datatype || x.class) .filter(x => x.datatype || x.class)
.map(x => `${x.name} (${x.path})`); .map(x => `${x.name} (${x.path})`);
return `column "${colLabel(what)}" ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`; return `${what} ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`;
} }
} }
} }
return null; return null;
}
function colLabel(what) {
const m = what.match(/"([^"]+)"/);
return m ? m[1] : what;
} }

26
src/runner/columns.js
View File

@@ -1,5 +1,5 @@
import { sparqlEscapeUri } from '../db.js'; import { query, sparqlEscapeUri } from 'mu';
import { wherePatterns } from './seed.js'; import { hopsPattern, wherePatterns } from './seed.js';
// Fetch column values for known subjects. Columns are grouped by their // Fetch column values for known subjects. Columns are grouped by their
// shared non-leaf hops; two columns share a query only when every hop // shared non-leaf hops; two columns share a query only when every hop
@@ -8,12 +8,14 @@ import { wherePatterns } from './seed.js';
const SUBJECT_CHUNK_SIZE = Math.max(1, Number(process.env.SUBJECT_CHUNK_SIZE) || 100); const SUBJECT_CHUNK_SIZE = Math.max(1, Number(process.env.SUBJECT_CHUNK_SIZE) || 100);
export async function fetchColumns(sessionQuery, subjects, spec, values = new Map()) { // fetchColumns(subjects, spec) → subject → column index → [term]
export async function fetchColumns(subjects, spec) {
const values = new Map();
const groups = groupColumns(spec.columns); const groups = groupColumns(spec.columns);
for (const group of groups) { for (const group of groups) {
for (let i = 0; i < subjects.length; i += SUBJECT_CHUNK_SIZE) { for (let i = 0; i < subjects.length; i += SUBJECT_CHUNK_SIZE) {
const chunk = subjects.slice(i, i + SUBJECT_CHUNK_SIZE); const chunk = subjects.slice(i, i + SUBJECT_CHUNK_SIZE);
const result = await sessionQuery(groupQuery(group, chunk)); const result = await query(groupQuery(group, chunk));
collect(result, group, values); collect(result, group, values);
} }
} }
@@ -41,21 +43,11 @@ export function groupColumns(columns) {
export function groupQuery(group, chunk) { export function groupQuery(group, chunk) {
const subjects = chunk.map(sparqlEscapeUri).join(' '); const subjects = chunk.map(sparqlEscapeUri).join(' ');
const lines = [];
let prev = '?s';
// every hop except the last gets its own variable; the leaf is ?v // every hop except the last gets its own variable; the leaf is ?v
group.hops.slice(0, -1).forEach((hop, i) => { const lines = [hopsPattern(group.hops, 'x', '?s', '?v')];
const v = `?x${i}`; // rep:where hangs on the column's own nodes: ?x_<k> after k hops, ?v at
if (hop.inverse) lines.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
else lines.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`);
prev = v;
});
const leaf = group.hops[group.hops.length - 1];
if (leaf.inverse) lines.push(`?v ${sparqlEscapeUri(leaf.predicate)} ${prev} .`);
else lines.push(`${prev} ${sparqlEscapeUri(leaf.predicate)} ?v .`);
// rep:where hangs on the column's own nodes: ?x<k-1> after k hops, ?v at
// the end. // the end.
const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x${k - 1}`); const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x_${k}`);
lines.push(...wherePatterns(group.where, group.hops, nodeVar, 'c')); lines.push(...wherePatterns(group.where, group.hops, nodeVar, 'c'));
return `SELECT ?s ?v WHERE {\n VALUES ?s { ${subjects} }\n ${lines.join('\n ')}\n}`; return `SELECT ?s ?v WHERE {\n VALUES ?s { ${subjects} }\n ${lines.join('\n ')}\n}`;
} }

35
src/runner/csv.js
View File

@@ -1,35 +0,0 @@
import { mkdirSync, writeFileSync } from 'fs';
// Rows to a CSV file. CSV_SEPARATOR is ';' by default. A cell is quoted
// only when it holds the separator, a quote or a newline; a quote inside a
// cell is doubled, per RFC 4180.
const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
const SHARE_DIR = process.env.SHARE_DIR || '/share';
export function rowsToCsv(rows) {
return rows.map(row => row.map(escapeCell).join(CSV_SEPARATOR)).join('\n');
}
function escapeCell(cell) {
const s = String(cell ?? '');
return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s)
? `"${s.replace(/"/g, '""')}"`
: s;
}
export function writeCsv(name, csv) {
mkdirSync(SHARE_DIR, { recursive: true });
const file = `${SHARE_DIR}/${name}`;
writeFileSync(file, csv, 'utf8');
return file;
}
// The spec of the run, next to its CSV (same base name, .ttl), so a report
// can be debugged afterwards. The store keeps only the pointer.
export function writeSpec(csvName, spec) {
mkdirSync(SHARE_DIR, { recursive: true });
const name = csvName.replace(/\.csv$/, '') + '.ttl';
writeFileSync(`${SHARE_DIR}/${name}`, spec, 'utf8');
return `share://${name}`;
}

95
src/runner/profile.js
View File

@@ -1,5 +1,5 @@
import N3 from 'n3'; import N3 from 'n3';
import { readFileSync } from 'fs'; import { readFileSync, readdirSync } from 'fs';
// Parse a profile (a SHACL shapes graph) and answer questions about it. // Parse a profile (a SHACL shapes graph) and answer questions about it.
@@ -8,13 +8,11 @@ const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
const OWL_ONTOLOGY = 'http://www.w3.org/2002/07/owl#Ontology'; const OWL_ONTOLOGY = 'http://www.w3.org/2002/07/owl#Ontology';
const REP = 'http://mu.semte.ch/vocabularies/reporting/'; const REP = 'http://mu.semte.ch/vocabularies/reporting/';
export async function loadProfiles(dir) { export function loadProfiles(dir) {
const fs = await import('fs'); const files = readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
const path = await import('path');
const files = fs.readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
const profiles = new Map(); const profiles = new Map();
for (const file of files) { for (const file of files) {
const profile = await loadProfile(path.join(dir, file)); const profile = loadProfile(`${dir}/${file}`);
profiles.set(profile.uri, profile); profiles.set(profile.uri, profile);
console.log(`[profile] "${profile.title}" (${file}): ${profile.shapes.length} entities, ` + console.log(`[profile] "${profile.title}" (${file}): ${profile.shapes.length} entities, ` +
`${profile.shapes.reduce((n, s) => n + s.fields.length, 0)} fields`); `${profile.shapes.reduce((n, s) => n + s.fields.length, 0)} fields`);
@@ -23,29 +21,19 @@ export async function loadProfiles(dir) {
return profiles; return profiles;
} }
export async function loadProfile(file) { function loadProfile(file) {
const parser = new N3.Parser(); let store;
const store = new N3.Store(); try {
await new Promise((resolve, reject) => { store = new N3.Store(new N3.Parser().parse(readFileSync(file, 'utf8')));
parser.parse(readFileSync(file, 'utf8'), (err, quad) => { } catch (e) {
if (err) return reject(new Error(`${file}: ${err.message}`)); throw new Error(`${file}: ${e.message}`);
if (quad) store.addQuad(quad); else resolve(); }
});
});
const ontology = store.getQuads(null, RDF + 'type', OWL_ONTOLOGY)[0]; const ontology = store.getQuads(null, RDF + 'type', OWL_ONTOLOGY)[0];
if (!ontology) throw new Error(`${file}: no owl:Ontology. A profile file must name itself.`); if (!ontology) throw new Error(`${file}: no owl:Ontology. A profile file must name itself.`);
const uri = ontology.subject.value; const uri = ontology.subject.value;
const title = store.getQuads(ontology.subject, 'http://purl.org/dc/terms/title', null)[0]?.object.value || uri; const title = store.getQuads(ontology.subject, 'http://purl.org/dc/terms/title', null)[0]?.object.value || uri;
const prefixQuads = store.getQuads(null, SH + 'declare', null);
const prefixes = {};
for (const q of prefixQuads) {
const p = store.getQuads(q.object, SH + 'prefix', null)[0]?.object.value;
const ns = store.getQuads(q.object, SH + 'namespace', null)[0]?.object.value;
if (p && ns) prefixes[p] = ns;
}
const shapes = store.getQuads(null, RDF + 'type', SH + 'NodeShape').map(q => q.subject); const shapes = store.getQuads(null, RDF + 'type', SH + 'NodeShape').map(q => q.subject);
if (!shapes.length) throw new Error(`${file}: no sh:NodeShape. A profile without entities is useless.`); if (!shapes.length) throw new Error(`${file}: no sh:NodeShape. A profile without entities is useless.`);
@@ -57,30 +45,26 @@ export async function loadProfile(file) {
return { uri: shape.value, targetClass, label, fields, discriminator }; return { uri: shape.value, targetClass, label, fields, discriminator };
}); });
return { uri, title, prefixes, shapes: shapeList, store }; return { uri, title, shapes: shapeList };
}
// A field's sh:path: one predicate, or [ sh:inversePath <predicate> ].
function readPath(store, node) {
const pathNode = store.getQuads(node, SH + 'path', null)[0]?.object;
if (!pathNode) return { path: null, inverse: false };
if (pathNode.termType !== 'BlankNode') return { path: pathNode.value, inverse: false };
const inv = store.getQuads(pathNode, SH + 'inversePath', null)[0]?.object;
return inv ? { path: inv.value, inverse: true } : { path: null, inverse: false };
} }
function parseField(store, p) { function parseField(store, p) {
let path = null, inverse = false; const one = (pred) => store.getQuads(p, pred, null)[0]?.object?.value || null;
const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object;
if (pathQuad) {
if (pathQuad.termType === 'BlankNode') {
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
if (inv) { path = inv.value; inverse = true; }
} else {
path = pathQuad.value;
}
}
const one = (pred) => store.getQuads(p, pred, null)[0]?.object || null;
return { return {
path, inverse, ...readPath(store, p),
name: one(SH + 'name')?.value || null, name: one(SH + 'name'),
description: one(SH + 'description')?.value || null, datatype: one(SH + 'datatype'),
datatype: one(SH + 'datatype')?.value || null, class: one(SH + 'class'),
class: one(SH + 'class')?.value || null, node: one(SH + 'node'),
node: one(SH + 'node')?.value || null,
nodeKind: one(SH + 'nodeKind')?.value || null,
maxCount: one(SH + 'maxCount')?.value || null,
}; };
} }
@@ -89,16 +73,7 @@ function parseField(store, p) {
function parseDiscriminator(store, shape) { function parseDiscriminator(store, shape) {
const node = store.getQuads(shape, REP + 'discriminator', null)[0]?.object; const node = store.getQuads(shape, REP + 'discriminator', null)[0]?.object;
if (!node) return null; if (!node) return null;
let path = null, inverse = false; const { path, inverse } = readPath(store, node);
const pathQuad = store.getQuads(node, SH + 'path', null)[0]?.object;
if (pathQuad) {
if (pathQuad.termType === 'BlankNode') {
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
if (inv) { path = inv.value; inverse = true; }
} else {
path = pathQuad.value;
}
}
const one = (pred) => store.getQuads(node, pred, null)[0]?.object?.value || null; const one = (pred) => store.getQuads(node, pred, null)[0]?.object?.value || null;
const minCount = one(SH + 'minCount'); const minCount = one(SH + 'minCount');
const maxCount = one(SH + 'maxCount'); const maxCount = one(SH + 'maxCount');
@@ -110,11 +85,6 @@ function parseDiscriminator(store, shape) {
}; };
} }
export function entityForClass(profile, cls) {
const shape = profile.shapes.find(s => s.targetClass === cls);
return shape ? shape.uri : null;
}
// The shape a spec starts from. rep:entity wins; a targetClass only resolves // The shape a spec starts from. rep:entity wins; a targetClass only resolves
// when exactly one shape carries it, so a class two entities share is a // when exactly one shape carries it, so a class two entities share is a
// validator error, not a silent first match. // validator error, not a silent first match.
@@ -132,10 +102,7 @@ export function fieldsOf(profile, shapeUri) {
return shape(profile, shapeUri)?.fields || []; return shape(profile, shapeUri)?.fields || [];
} }
export function field(profile, shapeUri, predicate, inverse) { // The field a hop takes from a shape, or null.
return fieldsOf(profile, shapeUri).find(f => f.path === predicate && !!f.inverse === !!inverse) || null; export function field(profile, shapeUri, hop) {
return fieldsOf(profile, shapeUri).find(f => f.path === hop.predicate && !!f.inverse === !!hop.inverse) || null;
} }
export function prefixes(profile) {
return profile.prefixes;
}

63
src/runner/report.js
View File

@@ -1,63 +0,0 @@
import { randomUUID } from 'crypto';
import { statSync } from 'fs';
import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt,
sparqlEscapeDateTime } from '../db.js';
const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
// Register the CSV: a logical file (http://data.lblod.info/files/<uuid>) plus
// a physical share:// file pointing back with nie:dataSource. The logical
// URI is what you link to. When given, the file gets dct:type.
export async function registerFile(sessionUpdateFn, fileName, filePath, { type } = {}) {
const size = statSync(filePath).size;
const uuid = randomUUID();
const uuid2 = randomUUID();
const now = new Date();
const logicalUri = `http://data.lblod.info/files/${uuid}`;
const physicalUri = `share://${fileName}`;
await (sessionUpdateFn || sessionUpdate)(`
INSERT DATA {
${sparqlEscapeUri(logicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)}${type ? ` ;
${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} .
${sparqlEscapeUri(physicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid2)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated')} ${sparqlEscapeDateTime(now)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource')} ${sparqlEscapeUri(logicalUri)}${type ? ` ;
${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} .
}`);
return logicalUri;
}
// One report pointing at the logical file URI.
export async function registerReport(sessionUpdateFn, title, fileUri, extra = {}) {
const uuid = randomUUID();
const now = new Date();
const reportUri = `http://data.lblod.info/id/reports/${uuid}`;
const extraTriples = [];
if (extra.question) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/question')} ${sparqlEscapeString(extra.question)} .`);
if (extra.specFile) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/specFile')} ${sparqlEscapeString(extra.specFile)} .`);
if (extra.creator) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://purl.org/dc/terms/creator')} ${sparqlEscapeUri(extra.creator)} .`);
await (sessionUpdateFn || sessionUpdate)(`
INSERT DATA {
${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)} ;
${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} .
${extraTriples.join('\n ')}
}`);
return reportUri;
}

61
src/runner/run.js
View File

@@ -1,38 +1,49 @@
import { update, uuid, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
import { seed } from './seed.js'; import { seed } from './seed.js';
import { fetchColumns } from './columns.js'; import { fetchColumns } from './columns.js';
import { assemble } from './assemble.js'; import { assemble } from './assemble.js';
import { rowsToCsv, writeCsv, writeSpec } from './csv.js';
import { registerFile, registerReport } from './report.js';
import { startShape } from './profile.js'; import { startShape } from './profile.js';
// Ties the steps together: seed → columns → assemble → CSV → register. const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
// Returns { reportUri, fileUri, rowCount }. const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
export async function run(query, update, parsed, profile, title, extra = {}) { // run(spec, profile) → the CSV text. seed → columns → assemble → CSV.
const shape = startShape(profile, parsed); export async function run(spec, profile) {
const subjects = await seed(query, parsed, shape); const subjects = await seed(spec, startShape(profile, spec));
const values = await fetchColumns(subjects, spec);
const values = new Map(); const rows = assemble(subjects, values, spec);
await fetchColumns(query, subjects, parsed, values); return rows.map(row => row.map(csvCell).join(CSV_SEPARATOR)).join('\n');
}
const rows = assemble(subjects, values, parsed);
const fileName = extra.fileName || `${slug(title)}.csv`; // A cell is quoted only when it holds the separator, a quote or a newline;
const csv = rowsToCsv(rows); // a quote inside a cell is doubled, per RFC 4180.
const filePath = writeCsv(fileName, csv); function csvCell(cell) {
const specFile = extra.spec && writeSpec(fileName, extra.spec); const s = String(cell ?? '');
return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s)
const fileUri = await registerFile(update, fileName, filePath, { type: extra.fileType }); ? `"${s.replace(/"/g, '""')}"`
const reportUri = await registerReport(update, title, fileUri, : s;
specFile ? { ...extra, specFile } : extra); }
return { reportUri, fileUri, rowCount: rows.length - 1, filePath }; // One report pointing at the logical file URI of its CSV.
export async function registerReport(title, fileUri) {
const id = uuid();
const reportUri = `http://data.lblod.info/id/reports/${id}`;
await update(`
INSERT DATA {
${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(id)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(new Date())} ;
${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} .
}`);
return reportUri;
} }
// The file name must be known before the run starts, so the chat can show a
// pending card named after the report.
export function slug(title) { export function slug(title) {
return (title || 'report').toLowerCase() return (title || 'report').toLowerCase()
.replace(/[^a-z0-9]+/g, '-') .replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '') .replace(/^-+|-+$/g, '')
.slice(0, 80) || 'report'; .slice(0, 80) || 'report';
} }

30
src/runner/seed.js
View File

@@ -1,16 +1,15 @@
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from '../db.js'; import { query, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from 'mu';
import { sharedSteps } from './spec.js'; import { sharedSteps } from './spec.js';
// Subject selection. One paged query per page; stops at ROW_LIMIT. // Subject selection. One paged query per page; stops at ROW_LIMIT.
const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000); const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000);
const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000); const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000);
export async function seed(sessionQuery, spec, shape = null) { export async function seed(spec, shape) {
const subjects = []; const subjects = [];
let offset = 0; let offset = 0;
for (;;) { for (;;) {
const query = seedPageQuery(spec, PAGE_SIZE, offset, shape); const result = await query(seedPageQuery(spec, PAGE_SIZE, offset, shape));
const result = await sessionQuery(query);
const rows = result.results.bindings.map(b => b.s.value); const rows = result.results.bindings.map(b => b.s.value);
subjects.push(...rows); subjects.push(...rows);
if (subjects.length > ROW_LIMIT) { if (subjects.length > ROW_LIMIT) {
@@ -49,13 +48,13 @@ function discriminatorPattern(d) {
// Writes out every hop as its own triple pattern. // Writes out every hop as its own triple pattern.
// Forward: ?s <p1> ?f0_1 . ?f0_1 <p2> ?f0_v // Forward: ?s <p1> ?f0_1 . ?f0_1 <p2> ?f0_v
// Inverse: ?s ^<p1> → ?f0_1 <p1> ?s // Inverse: ?s ^<p1> → ?f0_1 <p1> ?s
export function hopsPattern(hops, varPrefix, startVar) { export function hopsPattern(hops, varPrefix, startVar, lastVar = `?${varPrefix}_v`) {
let prev = startVar; let prev = startVar;
const patterns = []; const patterns = [];
for (let i = 0; i < hops.length; i++) { for (let i = 0; i < hops.length; i++) {
const hop = hops[i]; const hop = hops[i];
const isLast = i === hops.length - 1; const isLast = i === hops.length - 1;
const v = isLast ? `?${varPrefix}_v` : `?${varPrefix}_${i + 1}`; const v = isLast ? lastVar : `?${varPrefix}_${i + 1}`;
if (hop.inverse) { if (hop.inverse) {
patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`); patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
} else { } else {
@@ -114,29 +113,14 @@ export function conditionPattern(c, varPrefix, hops, startVar) {
parts.push(`FILTER(REGEX(str(${v}), ${sparqlEscapeString(alternation)}${flags}))`); parts.push(`FILTER(REGEX(str(${v}), ${sparqlEscapeString(alternation)}${flags}))`);
} }
if (c.minCount === 2) { if (c.minCount === 2) {
// two distinct leaf values on the same path // two distinct leaf values: the same path again, to a second leaf
const w = `?${varPrefix}_w`; const w = `?${varPrefix}_w`;
parts.push(hopsPattern2(hops, varPrefix, startVar, w)); parts.push(hopsPattern(hops, `${varPrefix}_2`, startVar, w));
parts.push(`FILTER(${v} != ${w})`); parts.push(`FILTER(${v} != ${w})`);
} }
return parts.join('\n '); return parts.join('\n ');
} }
function hopsPattern2(hops, varPrefix, startVar, wVar) {
// repeat the path to a second leaf variable
let prev = startVar;
const patterns = [];
for (let i = 0; i < hops.length; i++) {
const hop = hops[i];
const isLast = i === hops.length - 1;
const v = isLast ? wVar : `?${varPrefix}_2_${i + 1}`;
if (hop.inverse) patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
else patterns.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`);
prev = v;
}
return patterns.join('\n ');
}
function escapeTerm(t) { function escapeTerm(t) {
if (t.type === 'uri') return sparqlEscapeUri(t.value); if (t.type === 'uri') return sparqlEscapeUri(t.value);
if (t.datatype === 'http://www.w3.org/2001/XMLSchema#dateTime') if (t.datatype === 'http://www.w3.org/2001/XMLSchema#dateTime')

45
src/runner/spec.js
View File

@@ -23,22 +23,21 @@ export function parseSpec(turtle) {
const profileUri = one(specNode, REP + 'profile')?.value || null; const profileUri = one(specNode, REP + 'profile')?.value || null;
const title = one(specNode, 'http://purl.org/dc/terms/title')?.value || null; const title = one(specNode, 'http://purl.org/dc/terms/title')?.value || null;
const filters = store.getQuads(specNode, SH + 'property', null).map(q => const filters = store.getQuads(specNode, SH + 'property', null).map(q => parseFilter(store, q.object));
parseFilter(store, q.object, { withLabel: true }));
const columnsQuad = one(specNode, REP + 'columns'); const columnsQuad = one(specNode, REP + 'columns');
const columns = columnsQuad ? parseColumnList(store, columnsQuad) : []; const columns = columnsQuad ? readRdfList(store, columnsQuad).map(col => parseColumn(store, col)) : [];
return { uri: specNode.value, title, profileUri, targetClass, entity, filters, columns }; return { uri: specNode.value, title, profileUri, targetClass, entity, filters, columns };
} }
function parseFilter(store, p, { withLabel = false } = {}) { function parseFilter(store, p) {
let path = []; let path = [];
const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object; const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object;
if (pathQuad) path = parsePath(store, pathQuad); if (pathQuad) path = parsePath(store, pathQuad);
return { return {
path, path,
constraints: readConstraints(store, p, { withLabel }), constraints: readConstraints(store, p, { withLabel: true }),
where: readWhere(store, p), where: readWhere(store, p),
}; };
} }
@@ -47,8 +46,7 @@ function parseFilter(store, p, { withLabel = false } = {}) {
// is written like a filter, from the row; check.js refuses one nested in // is written like a filter, from the row; check.js refuses one nested in
// another. // another.
function readWhere(store, node) { function readWhere(store, node) {
return store.getQuads(node, REP + 'where', null).map(q => return store.getQuads(node, REP + 'where', null).map(q => parseFilter(store, q.object));
parseFilter(store, q.object, { withLabel: true }));
} }
// The number of steps two paths have in common from the row. A rep:where // The number of steps two paths have in common from the row. A rep:where
@@ -60,18 +58,6 @@ export function sharedSteps(a, b) {
return k; return k;
} }
function parseColumnList(store, listNode) {
const columns = [];
let cur = listNode;
while (cur && cur.value !== RDF + 'nil') {
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object;
if (item) columns.push(parseColumn(store, item));
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
if (!cur) throw new Error('rep:columns list is broken (no rdf:rest terminator)');
}
return columns;
}
function parseColumn(store, col) { function parseColumn(store, col) {
let path = []; let path = [];
const pathQuad = store.getQuads(col, SH + 'path', null)[0]?.object; const pathQuad = store.getQuads(col, SH + 'path', null)[0]?.object;
@@ -107,21 +93,12 @@ function parsePath(store, pathQuad) {
return [{ predicate: pathQuad.value, inverse: false }]; return [{ predicate: pathQuad.value, inverse: false }];
} }
// RDF list of predicates // RDF list of predicates
const hops = []; return readRdfList(store, pathQuad).map(item => {
let cur = pathQuad; if (item.termType !== 'BlankNode') return { predicate: item.value, inverse: false };
while (cur && cur.value !== RDF + 'nil') { const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object; if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
if (!item) throw new Error('sh:path list is broken'); return { predicate: inv.value, inverse: true };
if (item.termType === 'BlankNode') { });
const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
hops.push({ predicate: inv.value, inverse: true });
} else {
hops.push({ predicate: item.value, inverse: false });
}
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
}
return hops;
} }
function readConstraints(store, p, { withLabel = false } = {}) { function readConstraints(store, p, { withLabel = false } = {}) {