felix.ruizdearcaute/natural-language-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup

Browse Source
This commit is contained in:
felix
2026-09-27 00:15:18 +02:00
parent ad278dd680
commit 9b0b8a85c9
25 changed files with 588 additions and 1959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@@ -44,6 +44,7 @@ Two documents drive everything:
A **profile** is a SHACL shapes graph: one `sh:NodeShape` per entity, fields
as `sh:property`, code lists via `sh:class`, links via `sh:node`. The service
ships no profile and knows no vocabulary; profiles live in the app repo.
Profiles load at boot; a broken profile stops the service.
A **spec** is `rep:ReportSpec` + `sh:NodeShape`: `sh:property` selects the
subjects, `rep:columns` says what the CSV holds. The two never interact. The
@@ -107,13 +108,11 @@ The scope URI is `SERVICE_SCOPE` (default
POST /assistant/conversations/:id/turns { content } → 202 { id }
```
This is the chat. The turn module in
`src/chat/` is generic — it imports nothing from `src/llm/` or `src/runner/`
and moves between services without edits — and `src/report-assistant.js` is
this service's one `answer` hook. It records the question, answers `202` with
the message id, then classifies the turn and refines or executes with the
conversation's history. The answer and its bijlagen are written when the turn
ends. The browser polls
This is the chat. The route is in `app.js`: it records the question, answers
`202` with the message id, then classifies the turn and refines or executes
(`src/report-assistant.js`) with the conversation's history. The
conversation, message and bijlage reads and writes are in `src/chat.js`. The
answer and its bijlagen are written when the turn ends. The browser polls
`/chat-conversations/:id?include=messages.attachments` and finds the answer
that way. Failure is a message, never a silent loader.

69
app.js
View File

@@ -1,25 +1,66 @@
import { app, errorHandler } from 'mu';
import bodyParser from 'body-parser';
import { loadProfiles } from './src/runner/profile.js';
import { mountChat } from './src/chat/index.js';
import { reportAssistant } from './src/report-assistant.js';
import { readConversation, readHistory, findAssistant, writeMessage } from './src/chat.js';
import { wantsExecution } from './src/llm/mode.js';
import { executeSpec, refineSpec } from './src/report-assistant.js';
const PROFILE_DIR = process.env.PROFILE_DIR || '/config/profiles';
const CHAT_ASSISTANT_URI = process.env.CHAT_ASSISTANT_URI;
const TITLE_LENGTH = 80;
const FAILED = 'Er ging iets mis. Probeer het opnieuw.';
// Profiles load at boot; handlers await the same promise. A failed load
// answers an error per request instead of crashing boot.
let profilesPromise = loadProfiles(PROFILE_DIR);
profilesPromise.catch(e => console.error('[profiles] loading failed:', e.message));
// Profiles load at boot. A broken profile stops the service.
const profiles = loadProfiles(PROFILE_DIR);
async function whenProfiles() {
// One chat turn. Records the question, answers 202, then writes the
// assistant's answer; the frontend polls the conversation for it. mu's
// query/update run as the caller for the whole turn, also after the 202.
// The template only parses application/vnd.api+json, the chat sends
// application/json.
app.post('/assistant/conversations/:id/turns', bodyParser.json(), async function(req, res) {
const content = String(req.body?.content ?? '').trim();
if (!content) return res.status(400).json({ error: 'content is required' });
let conversation, assistant, history, question;
try {
return await profilesPromise;
// the access check: the caller's session decides what is readable
conversation = await readConversation(req.params.id);
if (!conversation) return res.status(404).json({ error: 'no such conversation' });
assistant = CHAT_ASSISTANT_URI || await findAssistant();
history = await readHistory(conversation.uri, assistant);
question = await writeMessage({
conversationUri: conversation.uri,
content,
maker: conversation.creator,
title: conversation.title ? undefined : content.slice(0, TITLE_LENGTH),
});
} catch (e) {
throw new Error('profiles are not available: ' + e.message);
console.error('[chat] could not record the question:', e);
return res.status(500).json({ error: e.message });
}
}
// The chat. reportAssistant is the answer hook: it refines the spec with the
// LLM, or runs the agreed spec without it.
mountChat(app, { path: '/assistant', answer: reportAssistant(whenProfiles) });
res.status(202).json({ id: question.id });
app.use(errorHandler);
try {
const turns = [...history, { role: 'user', content }];
const answer = await wantsExecution(turns)
? await executeSpec(history, conversation.title, profiles)
: await refineSpec(turns, profiles);
await writeMessage({
conversationUri: conversation.uri,
content: answer.text,
maker: assistant,
attachments: answer.attachments,
});
} catch (e) {
console.error('[chat] turn failed:', e);
try {
await writeMessage({ conversationUri: conversation.uri, content: FAILED, maker: assistant });
} catch (e2) {
console.error('[chat] could not write the failure message:', e2);
}
}
});
app.use(errorHandler);

871
package-lock.json generated
View File

@@ -9,7 +9,6 @@
"version": "0.2.0",
"license": "MIT",
"dependencies": {
"express": "^5.2.1",
"n3": "^1.17.0"
}
},
@@ -25,19 +24,6 @@
"node": ">=6.5"
}
},
"node_modules/accepts": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz",
"integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==",
"license": "MIT",
"dependencies": {
"mime-types": "^3.0.0",
"negotiator": "^1.0.0"
},
"engines": {
"node": ">= 0.6"
}
},
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
@@ -58,43 +44,6 @@
],
"license": "MIT"
},
"node_modules/body-parser": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz",
"integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==",
"license": "MIT",
"dependencies": {
"bytes": "^3.1.2",
"content-type": "^2.0.0",
"debug": "^4.4.3",
"http-errors": "^2.0.1",
"iconv-lite": "^0.7.2",
"on-finished": "^2.4.1",
"qs": "^6.15.2",
"raw-body": "^3.0.2",
"type-is": "^2.1.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/body-parser/node_modules/content-type": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz",
"integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/buffer": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
@@ -119,184 +68,6 @@
"ieee754": "^1.2.1"
}
},
"node_modules/bytes": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
"integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/call-bind-apply-helpers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
"integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/call-bound": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz",
"integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"get-intrinsic": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/content-disposition": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz",
"integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/content-type": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz",
"integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie": {
"version": "0.7.2",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz",
"integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/cookie-signature": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz",
"integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==",
"license": "MIT",
"engines": {
"node": ">=6.6.0"
}
},
"node_modules/debug": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
"integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
"integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.1",
"es-errors": "^1.3.0",
"gopd": "^1.2.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==",
"license": "MIT"
},
"node_modules/encodeurl": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
"integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
"integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-errors": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
"integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/es-object-atoms": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz",
"integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
"license": "MIT"
},
"node_modules/etag": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
"integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/event-target-shim": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
@@ -315,206 +86,6 @@
"node": ">=0.8.x"
}
},
"node_modules/express": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz",
"integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==",
"license": "MIT",
"dependencies": {
"accepts": "^2.0.0",
"body-parser": "^2.2.1",
"content-disposition": "^1.0.0",
"content-type": "^1.0.5",
"cookie": "^0.7.1",
"cookie-signature": "^1.2.1",
"debug": "^4.4.0",
"depd": "^2.0.0",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"etag": "^1.8.1",
"finalhandler": "^2.1.0",
"fresh": "^2.0.0",
"http-errors": "^2.0.0",
"merge-descriptors": "^2.0.0",
"mime-types": "^3.0.0",
"on-finished": "^2.4.1",
"once": "^1.4.0",
"parseurl": "^1.3.3",
"proxy-addr": "^2.0.7",
"qs": "^6.14.0",
"range-parser": "^1.2.1",
"router": "^2.2.0",
"send": "^1.1.0",
"serve-static": "^2.2.0",
"statuses": "^2.0.1",
"type-is": "^2.0.1",
"vary": "^1.1.2"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/finalhandler": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz",
"integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==",
"license": "MIT",
"dependencies": {
"debug": "^4.4.0",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"on-finished": "^2.4.1",
"parseurl": "^1.3.3",
"statuses": "^2.0.1"
},
"engines": {
"node": ">= 18.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
"integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/fresh": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz",
"integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/function-bind": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
"integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-intrinsic": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
"integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
"license": "MIT",
"dependencies": {
"call-bind-apply-helpers": "^1.0.2",
"es-define-property": "^1.0.1",
"es-errors": "^1.3.0",
"es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
"get-proto": "^1.0.1",
"gopd": "^1.2.0",
"has-symbols": "^1.1.0",
"hasown": "^2.0.2",
"math-intrinsics": "^1.1.0"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/get-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
"integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
"license": "MIT",
"dependencies": {
"dunder-proto": "^1.0.1",
"es-object-atoms": "^1.0.0"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/gopd": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
"integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/has-symbols": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
"integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/hasown": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz",
"integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==",
"license": "MIT",
"dependencies": {
"function-bind": "^1.1.2"
},
"engines": {
"node": ">= 0.4"
}
},
"node_modules/http-errors": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
"integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
"license": "MIT",
"dependencies": {
"depd": "~2.0.0",
"inherits": "~2.0.4",
"setprototypeof": "~1.2.0",
"statuses": "~2.0.2",
"toidentifier": "~1.0.1"
},
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/iconv-lite": {
"version": "0.7.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz",
"integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==",
"license": "MIT",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
},
"engines": {
"node": ">=0.10.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
@@ -535,92 +106,6 @@
],
"license": "BSD-3-Clause"
},
"node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"license": "ISC"
},
"node_modules/ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
"integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==",
"license": "MIT",
"engines": {
"node": ">= 0.10"
}
},
"node_modules/is-promise": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz",
"integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==",
"license": "MIT"
},
"node_modules/math-intrinsics": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
}
},
"node_modules/media-typer": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz",
"integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/merge-descriptors": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz",
"integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/mime-db": {
"version": "1.54.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz",
"integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
}
},
"node_modules/mime-types": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz",
"integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==",
"license": "MIT",
"dependencies": {
"mime-db": "^1.54.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT"
},
"node_modules/n3": {
"version": "1.26.0",
"resolved": "https://registry.npmjs.org/n3/-/n3-1.26.0.tgz",
@@ -634,87 +119,6 @@
"node": ">=12.0"
}
},
"node_modules/negotiator": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz",
"integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==",
"license": "MIT",
"dependencies": {
"content-type": "^2.1.0"
},
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/negotiator/node_modules/content-type": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz",
"integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/object-inspect": {
"version": "1.13.4",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
"integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==",
"license": "MIT",
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/on-finished": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
"integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==",
"license": "MIT",
"dependencies": {
"ee-first": "1.1.1"
},
"engines": {
"node": ">= 0.8"
}
},
"node_modules/once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
"license": "ISC",
"dependencies": {
"wrappy": "1"
}
},
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/path-to-regexp": {
"version": "8.4.2",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz",
"integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==",
"license": "MIT",
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
@@ -724,63 +128,6 @@
"node": ">= 0.6.0"
}
},
"node_modules/proxy-addr": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
"integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==",
"license": "MIT",
"dependencies": {
"forwarded": "0.2.0",
"ipaddr.js": "1.9.1"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/qs": {
"version": "6.15.3",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz",
"integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==",
"license": "BSD-3-Clause",
"dependencies": {
"es-define-property": "^1.0.1",
"side-channel": "^1.1.1"
},
"engines": {
"node": ">=0.6"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/range-parser": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz",
"integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==",
"license": "MIT",
"engines": {
"node": ">= 0.6"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/raw-body": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz",
"integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==",
"license": "MIT",
"dependencies": {
"bytes": "~3.1.2",
"http-errors": "~2.0.1",
"iconv-lite": "~0.7.0",
"unpipe": "~1.0.0"
},
"engines": {
"node": ">= 0.10"
}
},
"node_modules/readable-stream": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz",
@@ -797,22 +144,6 @@
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
"node_modules/router": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz",
"integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==",
"license": "MIT",
"dependencies": {
"debug": "^4.4.0",
"depd": "^2.0.0",
"is-promise": "^4.0.0",
"parseurl": "^1.3.3",
"path-to-regexp": "^8.0.0"
},
"engines": {
"node": ">= 18"
}
},
"node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -833,144 +164,6 @@
],
"license": "MIT"
},
"node_modules/safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
"license": "MIT"
},
"node_modules/send": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz",
"integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==",
"license": "MIT",
"dependencies": {
"debug": "^4.4.3",
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"etag": "^1.8.1",
"fresh": "^2.0.0",
"http-errors": "^2.0.1",
"mime-types": "^3.0.2",
"ms": "^2.1.3",
"on-finished": "^2.4.1",
"range-parser": "^1.2.1",
"statuses": "^2.0.2"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/serve-static": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz",
"integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==",
"license": "MIT",
"dependencies": {
"encodeurl": "^2.0.0",
"escape-html": "^1.0.3",
"parseurl": "^1.3.3",
"send": "^1.2.0"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/setprototypeof": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
"license": "ISC"
},
"node_modules/side-channel": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz",
"integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"object-inspect": "^1.13.4",
"side-channel-list": "^1.0.1",
"side-channel-map": "^1.0.1",
"side-channel-weakmap": "^1.0.2"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-list": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz",
"integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==",
"license": "MIT",
"dependencies": {
"es-errors": "^1.3.0",
"object-inspect": "^1.13.4"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-map": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz",
"integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==",
"license": "MIT",
"dependencies": {
"call-bound": "^1.0.2",
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.5",
"object-inspect": "^1.13.3"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/side-channel-weakmap": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz",
"integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==",
"license": "MIT",
"dependencies": {
"call-bound": "^1.0.2",
"es-errors": "^1.3.0",
"get-intrinsic": "^1.2.5",
"object-inspect": "^1.13.3",
"side-channel-map": "^1.0.1"
},
"engines": {
"node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
@@ -979,70 +172,6 @@
"dependencies": {
"safe-buffer": "~5.2.0"
}
},
"node_modules/toidentifier": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
"integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==",
"license": "MIT",
"engines": {
"node": ">=0.6"
}
},
"node_modules/type-is": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz",
"integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==",
"license": "MIT",
"dependencies": {
"content-type": "^2.0.0",
"media-typer": "^1.1.0",
"mime-types": "^3.0.0"
},
"engines": {
"node": ">= 18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/type-is/node_modules/content-type": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz",
"integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
"integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
"license": "ISC"
}
}
}

1
package.json
View File

@@ -9,7 +9,6 @@
"start": "node app.js"
},
"dependencies": {
"express": "^5.2.1",
"n3": "^1.17.0"
}
}

122
src/chat/store.js → src/chat.js
View File

@@ -1,15 +1,14 @@
// The chat's reads and writes. Nothing here knows the service it lives in.
// The chat's reads and writes: conversations, messages and their bijlagen.
// Full URIs everywhere: the auth layer rejects a prefixed name with no
// PREFIX line with an opaque 500.
// PREFIX line with an opaque 500 (the xsd: types of mu's escape helpers are
// fine).
//
// A bijlage is a real file: a logical nfo:FileDataObject the file service
// serves at /files/<uuid>/download, over a physical share:// object with
// nie:dataSource back to the logical one. Every attachment carries the
// file's logical URI (see chat/attachment-file.js how those files are
// made); nothing here writes files.
import { randomUUID } from 'crypto';
import { query as muQuery, update as muUpdate,
sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
// nie:dataSource back to the logical one.
import { readFileSync, writeFileSync, statSync } from 'fs';
import { query, update, uuid,
sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt, sparqlEscapeDateTime } from 'mu';
const HISTORY_LIMIT = Number(process.env.CHAT_HISTORY_LIMIT || 20);
@@ -27,14 +26,28 @@ const T = {
maker: 'http://xmlns.com/foaf/0.1/maker',
attachment: 'https://www.w3.org/ns/activitystreams#attachment',
softwareAgent: 'http://www.w3.org/ns/prov#SoftwareAgent',
file: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject',
fileName: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName',
fileSize: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize',
fileCreated: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated',
format: 'http://purl.org/dc/terms/format',
fileType: 'http://purl.org/dc/terms/type',
dataSource: 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource',
};
const MESSAGE_BASE = 'http://data.lblod.info/id/chat-messages/';
const FILE_BASE = 'http://data.lblod.info/files/';
const SHARE_DIR = process.env.SHARE_DIR || '/share';
export const SPEC_MEDIA_TYPE = 'text/turtle';
// The bijlage types. Match the add-chat-attachment-types-codelist migration
// in the portal.
export const SPEC_TYPE = 'http://lblod.data.gift/concepts/6693014e-7b7d-448c-bb24-80c9758521f3';
export const RESULT_TYPE = 'http://lblod.data.gift/concepts/28d97698-e219-4a92-bdbd-7ab367a7524e';
// Every spec file is called like this; readSpecFile opens nothing else.
const SPEC_FILE_NAME = /^specificatie-[0-9a-f-]{36}\.ttl$/;
const u = sparqlEscapeUri;
const s = sparqlEscapeString;
// Full URIs, so spell the dateTime type out.
const dt = (value) => sparqlEscapeDateTime(value)
.replace('xsd:dateTime', '<http://www.w3.org/2001/XMLSchema#dateTime>');
const bindings = (json) => json.results.bindings;
@@ -42,7 +55,7 @@ const bindings = (json) => json.results.bindings;
// The conversation by uuid, null when the caller may not read it.
export async function readConversation(id) {
const json = await muQuery(`
const json = await query(`
SELECT ?conversation ?title ?creator WHERE {
?conversation ${u(T.type)} ${u(T.thread)} ;
${u(T.uuid)} ${s(id)} .
@@ -57,7 +70,7 @@ export async function readConversation(id) {
// The last HISTORY_LIMIT messages, oldest first, as
// { role, content, attachments: [{ uri, name, mediaType, type }] }.
export async function readHistory(conversationUri, assistantUri) {
const json = await muQuery(`
const json = await query(`
SELECT ?message ?content ?maker ?created ?doc ?docName ?docFormat ?docType WHERE {
?message ${u(T.hasContainer)} ${u(conversationUri)} ;
${u(T.content)} ?content ;
@@ -65,9 +78,9 @@ export async function readHistory(conversationUri, assistantUri) {
${u(T.created)} ?created .
OPTIONAL {
?message ${u(T.attachment)} ?doc .
OPTIONAL { ?doc ${u('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ?docName . }
OPTIONAL { ?doc ${u('http://purl.org/dc/terms/format')} ?docFormat . }
OPTIONAL { ?doc ${u('http://purl.org/dc/terms/type')} ?docType . }
OPTIONAL { ?doc ${u(T.fileName)} ?docName . }
OPTIONAL { ?doc ${u(T.format)} ?docFormat . }
OPTIONAL { ?doc ${u(T.fileType)} ?docType . }
}
} ORDER BY DESC(?created) LIMIT ${HISTORY_LIMIT * 4}`);
const messages = [];
@@ -98,7 +111,7 @@ export async function readHistory(conversationUri, assistantUri) {
// The one assistant the caller can read. Throws unless there is exactly one.
export async function findAssistant() {
const json = await muQuery(`
const json = await query(`
SELECT ?agent WHERE { ?agent ${u(T.type)} ${u(T.softwareAgent)} . } LIMIT 2`);
const rows = bindings(json);
if (rows.length !== 1) {
@@ -111,12 +124,12 @@ export async function findAssistant() {
// One message, both types, its bijlagen, and the conversation touched, in
// one update. `title` is only written when given (the first question names
// an unnamed conversation). Every attachment is the logical uri of a file
// that already exists.
// an unnamed conversation). attachments are the logical uris of files that
// already exist.
export async function writeMessage({ conversationUri, content, maker, attachments = [], title }) {
const id = randomUUID();
const id = uuid();
const uri = `${MESSAGE_BASE}${id}`;
const now = dt(new Date());
const now = sparqlEscapeDateTime(new Date());
// Predicate pairs joined with ';', never a trailing one: it breaks the Turtle.
const pairs = [
@@ -128,15 +141,14 @@ export async function writeMessage({ conversationUri, content, maker, attachment
`${u(T.created)} ${now}`,
];
if (maker) pairs.push(`${u(T.maker)} ${u(maker)}`);
for (const attachment of attachments) {
if (!attachment.uri) throw new Error('an attachment without a file uri');
pairs.push(`${u(T.attachment)} ${u(attachment.uri)}`);
for (const fileUri of attachments) {
pairs.push(`${u(T.attachment)} ${u(fileUri)}`);
}
const conversationPairs = [`${u(T.lastActivity)} ${now}`];
if (title) conversationPairs.push(`${u(T.title)} ${s(title)}`);
await muUpdate(`
await update(`
DELETE {
${u(conversationUri)} ${u(T.lastActivity)} ?old .
}
@@ -150,3 +162,63 @@ export async function writeMessage({ conversationUri, content, maker, attachment
return { uri, id };
}
// --- files --------------------------------------------------------------------
export function writeShareFile(fileName, content) {
writeFileSync(`${SHARE_DIR}/${fileName}`, content, 'utf8');
}
// storeSpecFile(spec) → the file uri of a new spec bijlage. A spec file
// keeps one name for the logical and the physical file: readSpecFile finds
// it by the name in the history.
export async function storeSpecFile(spec) {
const fileName = `specificatie-${uuid()}.ttl`;
writeShareFile(fileName, spec);
return registerFile(fileName, SPEC_MEDIA_TYPE, SPEC_TYPE);
}
// readSpecFile(fileName) → the Turtle of a spec bijlage, null when there is
// no such spec file.
export function readSpecFile(fileName) {
if (!SPEC_FILE_NAME.test(fileName || '')) return null;
try {
return readFileSync(`${SHARE_DIR}/${fileName}`, 'utf8');
} catch {
return null;
}
}
// registerFile(shareName, format, type, name) → the logical file uri. One
// logical file called `name` over the physical share file `shareName`, both
// nfo:FileDataObject, both typed with dct:type. A share name must be
// unique: two files on one share name overwrite each other.
export async function registerFile(shareName, format, type, name = shareName) {
const id = uuid();
const physicalId = uuid();
const logical = `${FILE_BASE}${id}`;
const physical = `share://${shareName}`;
const size = statSync(`${SHARE_DIR}/${shareName}`).size;
const now = sparqlEscapeDateTime(new Date());
await update(`
INSERT DATA {
${u(logical)} ${u(T.type)} ${u(T.file)} ;
${u(T.uuid)} ${s(id)} ;
${u(T.fileName)} ${s(name)} ;
${u(T.format)} ${s(format)} ;
${u(T.fileSize)} ${sparqlEscapeInt(size)} ;
${u(T.created)} ${now} ;
${u(T.fileType)} ${u(type)} .
${u(physical)} ${u(T.type)} ${u(T.file)} ;
${u(T.uuid)} ${s(physicalId)} ;
${u(T.fileName)} ${s(shareName)} ;
${u(T.format)} ${s(format)} ;
${u(T.fileSize)} ${sparqlEscapeInt(size)} ;
${u(T.fileCreated)} ${now} ;
${u(T.dataSource)} ${u(logical)} ;
${u(T.fileType)} ${u(type)} .
}`);
return logical;
}

81
src/chat/attachment-file.js
View File

@@ -1,81 +0,0 @@
// A file made to sit on a message: written to the share, registered as a
// logical nfo:FileDataObject over a physical share:// object, typed with
// dct:type. The file service serves /files/<uuid>/download for it.
import { writeFileSync } from 'fs';
import { randomUUID } from 'crypto';
import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString,
sparqlEscapeInt, sparqlEscapeDateTime } from '../db.js';
import { ATTACHMENT_TYPES, SPEC_MEDIA_TYPE } from './vocab.js';
const SHARE_DIR = process.env.SHARE_DIR || '/share';
const FILE_BASE = 'http://data.lblod.info/files/';
const SHARE_BASE = 'share://';
const UUID = 'http://mu.semte.ch/vocabularies/core/uuid';
const NFO = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#';
const FILE = `${NFO}FileDataObject`;
const FILE_NAME = `${NFO}fileName`;
const FILE_SIZE = `${NFO}fileSize`;
const FILE_CREATED = `${NFO}fileCreated`;
const FORMAT = 'http://purl.org/dc/terms/format';
const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type';
const FILE_TYPE = 'http://purl.org/dc/terms/type';
const DATA_SOURCE = 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource';
const file = sparqlEscapeUri;
const str = sparqlEscapeString;
// storeSpecFile(update, content) → { uri, name }. One spec, one unique file:
// spec bijlagen keep their name apart from the report's ttl-file. Every
// spec file starts with "specificatie-", the read_spec tool opens only
// those.
export async function storeSpecFile(update, content) {
const fileName = `specificatie-${randomUUID()}.ttl`;
writeFileSync(`${SHARE_DIR}/${fileName}`, content, 'utf8');
return registerFileObject(update, {
fileName,
format: SPEC_MEDIA_TYPE,
size: Buffer.byteLength(content),
type: ATTACHMENT_TYPES.spec,
});
}
// registerFileObject(update, { fileName, format, size, type }) → { uri, name }.
// One logical file over one physical share:// file, both nfo:FileDataObject,
// both carrying the dct:type when given.
export async function registerFileObject(update, { fileName, format, size, type }) {
const id = randomUUID();
const physicalId = randomUUID();
const logical = `${FILE_BASE}${id}`;
const physical = `${SHARE_BASE}${fileName}`;
const now = sparqlEscapeDateTime(new Date()) // full URI, xsd spelled out
.replace('xsd:dateTime', '<http://www.w3.org/2001/XMLSchema#dateTime>');
const logicalPairs = [
`${file(RDF_TYPE)} ${file(FILE)}`,
`${file(UUID)} ${str(id)}`,
`${file(FILE_NAME)} ${str(fileName)}`,
`${file(FORMAT)} ${str(format)}`,
`${file(FILE_SIZE)} ${sparqlEscapeInt(size)}`,
`${file('http://purl.org/dc/terms/created')} ${now}`,
...(type ? [`${file(FILE_TYPE)} ${file(type)}`] : []),
];
const physicalPairs = [
`${file(RDF_TYPE)} ${file(FILE)}`,
`${file(UUID)} ${str(physicalId)}`,
`${file(FILE_NAME)} ${str(fileName)}`,
`${file(FORMAT)} ${str(format)}`,
`${file(FILE_SIZE)} ${sparqlEscapeInt(size)}`,
`${file(FILE_CREATED)} ${now}`,
`${file(DATA_SOURCE)} ${file(logical)}`,
...(type ? [`${file(FILE_TYPE)} ${file(type)}`] : []),
];
await (update || sessionUpdate)(`
INSERT DATA {
${file(logical)} ${logicalPairs.join(' ;\n ')} .
${file(physical)} ${physicalPairs.join(' ;\n ')} .
}`);
return { uri: logical, name: fileName };
}

86
src/chat/index.js
View File

@@ -1,86 +0,0 @@
// The turn endpoint. Mount it with mountChat(app, { path, answer });
// answer(turn) is the only thing the mounting service writes.
import express from 'express';
import { query as muQuery, update as muUpdate } from 'mu';
import {
readConversation, readHistory, findAssistant,
writeMessage,
} from './store.js';
const TEXT = {
failed: 'Er ging iets mis. Probeer het opnieuw.',
};
const TITLE_LENGTH = 80;
export function mountChat(app, { path = '/assistant', answer }) {
if (typeof answer !== 'function') throw new Error('mountChat needs an answer(turn) function');
app.post(`${path}/conversations/:id/turns`, express.json(), async (req, res) => {
const content = String(req.body?.content ?? '').trim();
if (!content) return res.status(400).json({ error: 'content is required' });
// 1. the access check: the caller's session decides what is readable
let conversation;
try {
conversation = await readConversation(req.params.id);
} catch (e) {
console.error('[chat] conversation read failed:', e);
return res.status(500).json({ error: e.message });
}
if (!conversation) return res.status(404).json({ error: 'no such conversation' });
let assistant, history, userMessage;
try {
assistant = process.env.CHAT_ASSISTANT_URI || await findAssistant();
history = await readHistory(conversation.uri, assistant);
userMessage = await writeMessage({
conversationUri: conversation.uri,
content,
maker: conversation.creator,
title: conversation.title ? undefined : content.slice(0, TITLE_LENGTH),
});
} catch (e) {
console.error('[chat] could not record the question:', e);
return res.status(500).json({ error: e.message });
}
// 2. answer 202 now; the assistant's message comes when it comes. The
// continuation still runs as the caller.
res.status(202).json({ id: userMessage.id });
const turn = {
conversation: { uri: conversation.uri, id: conversation.id, title: conversation.title },
history,
content,
query: muQuery,
update: muUpdate,
finished: false,
say: (text, attachments = []) => writeMessage({
conversationUri: conversation.uri, content: text, maker: assistant, attachments,
}),
};
try {
// answer may return a string, or { text, attachments } for answers the
// assistant writes with an attachment, such as the report spec.
const result = await answer(turn);
const text = typeof result === 'string' ? result : result?.text;
const attachments
= typeof result === 'string' ? [] : (result?.attachments ?? []);
if (typeof text === 'string' && text.trim()) {
await turn.say(text.trim(), attachments);
}
} catch (e) {
console.error('[chat] turn failed:', e);
try {
await turn.say(TEXT.failed);
} catch (e2) {
console.error('[chat] could not write the failure message:', e2);
}
} finally {
turn.finished = true;
}
});
console.log(`[chat] turns at POST ${path}/conversations/:id/turns`);
}

8
src/chat/vocab.js
View File

@@ -1,8 +0,0 @@
// The bijlage types of the report assistant. Match the
// add-chat-attachment-types-codelist migration in the portal.
export const ATTACHMENT_TYPES = {
spec: 'http://lblod.data.gift/concepts/6693014e-7b7d-448c-bb24-80c9758521f3',
result: 'http://lblod.data.gift/concepts/28d97698-e219-4a92-bdbd-7ab367a7524e',
};
export const SPEC_BASE_NAME = 'specificatie';
export const SPEC_MEDIA_TYPE = 'text/turtle';

37
src/db.js
View File

@@ -1,37 +0,0 @@
// Thin re-export of mu's SPARQL helpers so the runner files share one import
// site. mu's query/update attach the caller's session from the request
// context, and mu-authorization decides graphs and visibility; the service
// never touches an auth header.
import {
query as muQuery,
update as muUpdate,
sparqlEscapeUri,
sparqlEscapeString,
sparqlEscapeInt,
sparqlEscapeDateTime,
sparqlEscapeDate,
sparqlEscapeBool,
} from 'mu';
export const sessionUpdate = muUpdate;
export {
sparqlEscapeUri,
sparqlEscapeString,
sparqlEscapeInt,
sparqlEscapeDateTime,
sparqlEscapeDate,
sparqlEscapeBool,
};
// The service's own scope. Reads the LLM drives during refinement (code
// lists, lookup_values) run under this scope, not the caller's session. The
// sparql-parser config grants this scope read access to the public graph
// only, so refinement reads only public data. Execution still runs as the
// caller. Set to match the with-scope grant in config.lisp.
export const SERVICE_SCOPE =
process.env.SERVICE_SCOPE || 'http://services.semantic.works/natural-language-report';
// Wrap a mu query/update so it carries the service scope. mu's query/update
// accept { scope } on mu-javascript-template >= 1.9.0.
export function scopedQuery(q) { return muQuery(q, { scope: SERVICE_SCOPE }); }
export function scopedUpdate(q) { return muUpdate(q, { scope: SERVICE_SCOPE }); }

70
src/llm/describe.js
View File

@@ -1,59 +1,9 @@
// A profile as a plain text menu: entities, fields, code lists. Code lists up
// to INLINE_VALUES_MAX are inlined and cached for VALUES_TTL, so the LLM
// never has to search the short ones.
// to INLINE_VALUES_MAX are inlined (and cached), so the LLM never has to
// search the short ones.
import { codeList, INLINE_VALUES_MAX } from './lookup.js';
import { fieldsOf, entityForClass } from '../runner/profile.js';
const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50);
const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000;
// In-memory cache: profile+field → { values, at }. Refreshed lazily when stale.
const cache = new Map();
// codeListValues(profile, queryFn) → map "shapeUri|predicate|inverse"
// → [{ uri, label }]. queryFn runs under the service scope (public graph), so
// code lists read the same regardless of the caller's rights.
export async function codeListValues(profile, queryFn) {
const out = {};
for (const shape of profile.shapes) {
for (const f of shape.fields) {
if (!f.class) continue;
const key = `${shape.uri}|${f.path}|${f.inverse ? 1 : 0}`;
out[key] = await loadCodeList(f, key, queryFn);
}
}
return out;
}
async function loadCodeList(f, key, queryFn) {
if (!f.class) return [];
const hit = cache.get(key);
if (hit && Date.now() - hit.at < VALUES_TTL) return hit.values;
let values = [];
try {
values = await queryCodeList(f.class, queryFn);
} catch (e) {
// the read failed; report nothing, the LLM will use rep:anyOf with words
values = [];
}
cache.set(key, { values, at: Date.now() });
return values;
}
async function queryCodeList(type, queryFn) {
const q = `SELECT ?uri ?label WHERE {
?uri a <${type}> .
OPTIONAL { ?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label . }
} ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`;
const r = await queryFn(q);
const bindings = r.results.bindings;
return bindings.slice(0, INLINE_VALUES_MAX).map(b => ({
uri: b.uri.value,
label: b.label?.value || b.uri.value,
}));
}
export function describeProfile(profile, codeLists = {}) {
export async function describeProfile(profile) {
const lines = [];
lines.push('A spec is Turtle, in exactly this shape (copy these prefixes):');
lines.push('@prefix rep: <http://mu.semte.ch/vocabularies/reporting/> .');
@@ -80,13 +30,12 @@ export function describeProfile(profile, codeLists = {}) {
for (const shape of profile.shapes) {
lines.push(`${shape.label || shape.targetClass} — sh:targetClass <${shape.targetClass}> ; rep:entity <${shape.uri}>`);
for (const f of shape.fields) {
const key = `${shape.uri}|${f.path}|${f.inverse ? 1 : 0}`;
const kind = f.datatype ? datatypeName(f.datatype)
: f.class ? `→ ${conceptLabel(f.class, codeLists[key])}`
: f.class ? `→ ${conceptLabel(f.class, await codeList(f.class))}`
: f.node ? `→ ${linkedLabel(profile, f.node)}`
: '?';
const dir = f.inverse ? ' (inverse)' : '';
const pathShown = typeof f.path === 'string' ? f.path : '?';
const pathShown = f.path || '?';
lines.push(` ${f.name || pathShown} sh:path <${pathShown}>${dir} ${kind}`);
}
lines.push('');
@@ -95,11 +44,10 @@ export function describeProfile(profile, codeLists = {}) {
}
function conceptLabel(type, values) {
const head = values.slice(0, INLINE_VALUES_MAX);
if (!head.length) return short(type);
const names = head.map(v => v.label).join(', ');
if (!values.length) return short(type);
const names = values.map(v => v.label).join(', ');
const more = values.length >= INLINE_VALUES_MAX ? ', …' : '';
return `${short(type)} — ${head.length} waarden: ${names}${more}`;
return `${short(type)} — ${values.length} waarden: ${names}${more}`;
}
function linkedLabel(profile, nodeUri) {

134
src/llm/lookup.js
View File

@@ -1,98 +1,118 @@
// Search a code list too long to inline: regex on str(?label), so
// language-tagged labels match. Returns the matches, the total, and an
// `exact` flag when a label equals the term (case aside).
// The LLM's own reads: code lists. They run under the service scope, not
// the caller's session; the sparql-parser config grants that scope read
// access to the public graph only, so they read the same for every caller.
// Execution still runs as the caller.
//
// This is a suggest helper, not a spec check. It reads candidate values the
// LLM can put in a filter; it never runs the spec and never sees report
// rows. queryFn runs under the service scope (public graph only), so the
// suggestions come from the same data set the profile describes, regardless
// of who calls.
import { sparqlEscapeString } from '../db.js';
// lookup_values is a suggest helper, not a spec check. It reads candidate
// values the LLM can put in a filter; it never runs the spec and never sees
// report rows.
import { query, sparqlEscapeString } from 'mu';
// Must match the with-scope grant in the app's config.lisp.
const SERVICE_SCOPE = process.env.SERVICE_SCOPE || 'http://services.semantic.works/natural-language-report';
export const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50);
const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000;
const LIMIT = 25;
// lookupValues(profile, term, fieldSpec, queryFn) →
// { total, exact, matches } or { error }.
// field is "shapeLabel.fieldLabel" as the LLM writes it, or a predicate URI.
// We resolve it to the field's sh:class to know what type to search.
export async function lookupValues(profile, term, fieldSpec, queryFn) {
const type = resolveClass(profile, fieldSpec);
if (!type) return { total: 0, exact: null, matches: [], error: `field "${fieldSpec}" not found in profile "${profile.title}"` };
// class → { values, at }. Refreshed lazily when stale.
const cache = new Map();
function publicQuery(q) {
return query(q, { scope: SERVICE_SCOPE });
}
// codeList(type) → up to INLINE_VALUES_MAX [{ uri, label }], cached for
// VALUES_TTL. Empty when the read fails; the LLM then uses rep:anyOf with
// words.
export async function codeList(type) {
const hit = cache.get(type);
if (hit && Date.now() - hit.at < VALUES_TTL) return hit.values;
let values = [];
try {
const r = await publicQuery(`SELECT ?uri ?label WHERE {
?uri a <${type}> .
OPTIONAL { ?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label . }
} ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`);
values = r.results.bindings.slice(0, INLINE_VALUES_MAX).map(b => ({
uri: b.uri.value,
label: b.label?.value || b.uri.value,
}));
} catch (e) {
values = [];
}
cache.set(type, { values, at: Date.now() });
return values;
}
// lookupValues(profile, field, term) → the matches as text for the LLM: up
// to LIMIT labels and URIs, an exact label (case aside) first. field is
// "shapeLabel.fieldLabel" as the LLM writes it; it resolves to the class to
// search.
export async function lookupValues(profile, field, term) {
const type = resolveClass(profile, field);
if (!type) return `field "${field}" not found in profile "${profile.title}"`;
// Search within the class first. A common word ("gemeente") also sits in
// hundreds of unrelated labels; an unscoped scan drowns the code.
let rows = await labelQuery(term, type, queryFn);
let rows = await labelQuery(term, type);
// Code lists sometimes type their entries differently than the profile
// says (MAR codes are ext:supervision/Nomenclature, the profile says
// skos:Concept). Fall back to a scan over everything.
if (!rows.length) rows = await labelQuery(term, null, queryFn);
// Order: an exact label (case aside) first, then the rest as the store
// ordered them. "Gent" must surface before "AGB Erfgoed Gent".
if (!rows.length) rows = await labelQuery(term, null);
// "Gent" must surface before "AGB Erfgoed Gent".
const lower = term.toLowerCase();
const exactRow = rows.find(b => b.label.toLowerCase() === lower);
const rest = rows.filter(b => b !== exactRow);
const ordered = exactRow ? [exactRow, ...rest] : rest;
let matches = ordered.slice(0, LIMIT);
if (!matches.length) {
const byCode = await codeQuery(profile, type, term, queryFn);
matches = byCode;
}
const exact = matches.find(m => m.label.toLowerCase() === term.toLowerCase());
return {
total: matches.length >= LIMIT ? `${LIMIT}+` : matches.length,
exact: exact ? exact.uri : null,
matches,
};
const exactRow = rows.find(r => r.label.toLowerCase() === lower);
let matches = (exactRow ? [exactRow, ...rows.filter(r => r !== exactRow)] : rows).slice(0, LIMIT);
if (!matches.length) matches = await codeQuery(term);
if (!matches.length) return `no value matches "${term}" in ${field}.`;
const lines = matches.map(m => `${m.label} <${m.uri}>`);
if (matches.length >= LIMIT) lines.push(`(${LIMIT}+ matches, narrow the term)`);
const exact = matches.find(m => m.label.toLowerCase() === lower);
if (exact) lines.push(`exact: <${exact.uri}>`);
return lines.join('\n');
}
async function labelQuery(term, type, queryFn) {
// Regex on str(?label), so language-tagged labels match.
async function labelQuery(term, type) {
const q = `SELECT DISTINCT ?uri ?label WHERE {
?uri a ${type ? `<${type}>` : '?anyType'} ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?label .
FILTER(REGEX(str(?label), ${sparqlEscapeString(escapeRegex(term))}, "i"))
} ORDER BY ?label LIMIT ${LIMIT * 8}`;
const r = await queryFn(q);
const r = await publicQuery(q);
return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value }));
}
// MAR codes: the label says "MAR7300 - …" but the searchable code sits on
// ext:supervision/nomenclatureCode. One extra query, only when the label
// search came up empty.
async function codeQuery(profile, type, term, queryFn) {
async function codeQuery(term) {
const q = `SELECT ?uri ?label WHERE {
?uri <http://mu.semte.ch/vocabularies/ext/supervision/nomenclatureCode> ?code ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?label .
FILTER(CONTAINS(LCASE(str(?code)), LCASE(${sparqlEscapeString(term)})))
} ORDER BY ?label LIMIT ${LIMIT}`;
try {
const r = await queryFn(q);
const r = await publicQuery(q);
return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value }));
} catch {
return [];
}
}
function resolveClass(profile, fieldSpec) {
if (fieldSpec.includes('/')) return null;
// "shapeLabel.fieldLabel" → find the field carrying sh:class
const dot = fieldSpec.lastIndexOf('.');
if (dot < 0) {
// bare predicate URI
for (const shape of profile.shapes) {
for (const f of shape.fields) {
if (f.path === fieldSpec && f.class) return f.class;
}
}
return null;
}
const shapeLabel = fieldSpec.slice(0, dot);
const fieldLabel = fieldSpec.slice(dot + 1);
// "shapeLabel.fieldLabel" → the class to search: the field's sh:class, or
// for a link (sh:node) the linked shape's sh:targetClass
// ("melding.bestuurseenheid" → besluit:Bestuurseenheid).
function resolveClass(profile, field) {
if (field.includes('/')) return null;
const dot = field.lastIndexOf('.');
if (dot < 0) return null;
const shapeLabel = field.slice(0, dot);
const fieldLabel = field.slice(dot + 1);
const shape = profile.shapes.find(s => s.label === shapeLabel);
if (!shape) return null;
const f = shape.fields.find(x => x.name === fieldLabel);
// sh:node marks a link, not a code list; the searchable type is the linked
// shape's sh:targetClass ("melding.bestuurseenheid" → besluit:Bestuurseenheid).
if (f?.class) return f.class;
if (f?.node) {
const linked = profile.shapes.find(s => s.uri === f.node);
@@ -114,4 +134,4 @@ function shortType(cls) {
function escapeRegex(s) {
return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
}

114
src/llm/loop.js
View File

@@ -8,7 +8,8 @@
// confirm before anything executes; execution runs the agreed spec without
// the LLM (../report-assistant.js).
import { buildTools } from './tools.js';
import { TOOLS, runTool } from './tools.js';
import { SPEC_MEDIA_TYPE } from '../chat.js';
const MAX_ROUNDS = Number(process.env.MAX_ROUNDS || 12);
const LLM_BASE_URL = process.env.LLM_BASE_URL;
@@ -77,70 +78,49 @@ what it says, up to three rounds. Then propose and stop.
Reply to the user in Dutch. Earlier turns are context only. Answer the last one.`;
// ask(questionOrMessages, profiles, session) → { text }
// A string is one question; an array is the conversation so far,
// [{ role, content }], newest last. The system prompt is always this file's
// own; a system message in the input is dropped.
export async function ask(questionOrMessages, profiles, session) {
const turns = typeof questionOrMessages === 'string'
? [{ role: 'user', content: questionOrMessages }]
: questionOrMessages.filter(m => m.role !== 'system');
// ask(turns, profiles) → { text, spec }. turns is the conversation so far,
// [{ role, content, attachments }], newest last. spec is the last spec
// validate_spec accepted in this turn, or null.
export async function ask(turns, profiles) {
const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns.map(withSpecHint)];
let spec = null;
return runLoop(withAttachmentSpecs(turns), profiles, session);
for (let round = 0; round < MAX_ROUNDS; round++) {
const { message } = await chat(messages, TOOLS);
messages.push(message);
const calls = message.tool_calls || [];
if (!calls.length) return { text: message.content || '', spec };
for (const call of calls) {
const args = parseArgs(call.function.arguments);
let result;
try {
result = await runTool(call.function.name, args, profiles);
} catch (e) {
result = `tool failed: ${String(e.message || e).split('\n')[0]}`;
}
// validate_spec answers "ok" (plus notes) when the spec checks out;
// that spec becomes the bijlage of the answer.
if (call.function.name === 'validate_spec' && /^ok(\n|$)/.test(result)) spec = args.spec;
messages.push({ role: 'tool', tool_call_id: call.id, content: result });
}
}
return { text: 'Sorry, dit rapport is te moeilijk. Contacteer de developers.', spec };
}
// The history carries the spec as a bijlage. Paste one hint line per spec
// under the message text, so the model knows a spec is there and opens it
// with read_spec; the chat itself still keeps the Turtle out of the text.
function withAttachmentSpecs(turns) {
return turns.map(t => {
const specs = (t.attachments || []).filter(a => a.mediaType === 'text/turtle' && a.uri);
if (!specs.length) return t;
return {
...t,
content: `${t.content}\n\n${specs.map(a =>
`Bijlage "${a.name}", a spec file (read it with read_spec): ${a.uri}`).join('\n\n')}`,
};
});
function withSpecHint(turn) {
const specs = (turn.attachments || []).filter(a => a.mediaType === SPEC_MEDIA_TYPE);
if (!specs.length) return turn;
const hints = specs.map(a => `Bijlage "${a.name}", a spec file (read it with read_spec).`);
return { ...turn, content: `${turn.content}\n\n${hints.join('\n\n')}` };
}
// runLoop drives the agent loop with the refinement prompt and tool set.
async function runLoop(turns, profiles, session) {
const { handlers } = buildTools(profiles, session);
const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns];
for (let round = 0; round < MAX_ROUNDS; round++) {
const reply = await chat(messages, { tools: toolDefs() });
messages.push(reply.message);
const calls = reply.message.tool_calls || [];
if (!calls.length) {
return { text: reply.message.content || '' };
}
for (const call of calls) {
const args = parseArgs(call.function.arguments);
const fn = handlers.get(call.function.name);
let resultText;
if (!fn) {
resultText = `unknown tool "${call.function.name}"`;
} else {
try {
const out = await fn(args);
resultText = out.content?.[0]?.text ?? JSON.stringify(out);
} catch (e) {
resultText = `tool failed: ${String(e.message || e).split('\n')[0]}`;
}
}
messages.push({ role: 'tool', tool_call_id: call.id, content: resultText });
}
}
return { text: '(loop did not finish)' };
}
// chat(messages, { tools, toolChoice }) → { message }
// The one place that talks to the provider. tools null (the default) means a
// plain completion (the mode classifier); runLoop passes the tool
// definitions; toolChoice defaults to 'auto'.
export async function chat(messages, { tools = null, toolChoice = 'auto' } = {}) {
// chat(messages, tools) → { message }. The one place that talks to the
// provider. Without tools it is a plain completion (the mode check).
export async function chat(messages, tools = null) {
// Send only the fields every provider accepts back.
const wire = messages.map(m => {
const out = { role: m.role, content: m.content ?? null };
@@ -154,13 +134,10 @@ export async function chat(messages, { tools = null, toolChoice = 'auto' } = {})
if (m.role === 'tool' && m.tool_call_id) out.tool_call_id = m.tool_call_id;
return out;
});
const body = {
model: LLM_MODEL,
messages: wire,
};
const body = { model: LLM_MODEL, messages: wire };
if (tools) {
body.tools = tools;
body.tool_choice = toolChoice === 'auto' ? 'auto' : toolChoice;
body.tool_choice = 'auto';
}
if (LOG_LLM) {
console.log(`[llm] POST ${LLM_BASE_URL}/chat/completions request:\n${
@@ -191,16 +168,3 @@ function parseArgs(s) {
if (!s) return {};
try { return JSON.parse(s); } catch { return {}; }
}
// The tool definitions sent to the model.
import { toolSchemas } from './schemas.js';
function toolDefs() {
return toolSchemas().map(t => ({
type: 'function',
function: {
name: t.name,
description: t.description,
parameters: t.inputSchema,
},
}));
}

43
src/llm/mode.js
View File

@@ -25,43 +25,18 @@ Rules:
Reply with exactly one token: true or false. No other words, no punctuation,
no markdown.`;
// classifyMode(turns) → boolean (true = execute now)
// turns: [{ role, content }], newest last. A leading system message is
// dropped; the assistant's own system prompt is never sent to the model.
export async function classifyMode(turns) {
const convo = (turns || [])
.filter(m => m.role !== 'system')
.map(m => ({ role: m.role === 'assistant' ? 'assistant' : 'user', content: m.content ?? '' }));
if (!convo.length) return false;
// Keep the last few turns; the decision is about the latest message.
const recent = convo.slice(-6);
const messages = [{ role: 'system', content: MODE_PROMPT }, ...recent];
let raw;
// wantsExecution(turns) → true when the user's last message says "run it
// now". turns: [{ role, content }], newest last; the last few are enough.
export async function wantsExecution(turns) {
let message;
try {
const reply = await chat(messages, { tools: null });
raw = firstText(reply.message);
({ message } = await chat([{ role: 'system', content: MODE_PROMPT }, ...turns.slice(-6)]));
} catch (e) {
// The classifier call failed (no LLM, provider down). Default to refine:
// safer than running a report on a guess, and the refine loop will
// surface the real error to the user.
// The call failed (no LLM, provider down). Refine: safer than running a
// report on a guess, and the refine loop will surface the real error.
return false;
}
const text = stripMarkdown(raw).trim().toLowerCase();
if (!text) return false;
if (text.startsWith('true')) return true;
if (text.startsWith('false')) return false;
// A JSON {"execute": ...} shape some models emit.
try {
const m = text.match(/\{[\s\S]*\}/);
if (m) {
const v = JSON.parse(m[0]);
if (typeof v === 'boolean') return v;
if (v && typeof v.execute === 'boolean') return v.execute;
if (v && typeof v.refine === 'boolean') return !v.refine;
}
} catch { /* ignore */ }
return false;
return stripMarkdown(firstText(message)).trim().toLowerCase().startsWith('true');
}
// Reasoning models may leave `content` empty and put the visible text in a
@@ -78,4 +53,4 @@ function firstText(message) {
function stripMarkdown(s) {
return String(s).replace(/[*_`>#]/g, ' ');
}
}

57
src/llm/schemas.js
View File

@@ -1,57 +0,0 @@
// The JSON schemas of the tools. The descriptions are what the LLM reads to
// decide which tool to call. Refinement only: list_profiles,
// describe_profile, validate_spec, read_spec and lookup_values.
const COMMON = [
{
name: 'list_profiles',
description: 'List the report profiles available, with their id and title.',
inputSchema: { type: 'object', properties: {}, required: [] },
},
{
name: 'describe_profile',
description: 'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.',
inputSchema: {
type: 'object',
properties: { profile_id: { type: 'string', description: 'The profile URI returned by list_profiles' } },
required: ['profile_id'],
},
},
{
name: 'validate_spec',
description: 'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.',
inputSchema: {
type: 'object',
properties: { spec: { type: 'string', description: 'The spec as Turtle text' } },
required: ['spec'],
},
},
];
const LOOKUP_VALUES = {
name: 'lookup_values',
description: 'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.',
inputSchema: {
type: 'object',
properties: {
profile_id: { type: 'string', description: 'The profile URI returned by list_profiles' },
field: { type: 'string', description: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"' },
term: { type: 'string', description: 'The Dutch word or name to search for' },
},
required: ['profile_id', 'field', 'term'],
},
};
const READ_SPEC = {
name: 'read_spec',
description: 'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.',
inputSchema: {
type: 'object',
properties: { file_uri: { type: 'string', description: 'The file URI of the spec bijlage, e.g. http://data.lblod.info/files/<uuid>' } },
required: ['file_uri'],
},
};
export function toolSchemas() {
return [...COMMON, READ_SPEC, LOOKUP_VALUES];
}

171
src/llm/tools.js
View File

@@ -1,94 +1,91 @@
// The tools the LLM calls, refinement only: list_profiles, describe_profile,
// validate_spec, lookup_values, read_spec. lookup_values searches a code list
// under the service scope (public graph) and returns candidate values, so the
// LLM can suggest concrete filter values. No report is created, no spec is
// evaluated.
//
// The shape every tool returns is { content: [{ type: 'text', text }] },
// which the loop passes straight back to the model.
import { checkSpec, whereNotes } from '../runner/check.js';
// The tools the LLM calls, refinement only: list_profiles,
// describe_profile, validate_spec, read_spec and lookup_values. Each returns
// plain text for the model. No report is created, no spec is evaluated;
// the code-list reads run under the service scope (public graph).
import { parseSpec } from '../runner/spec.js';
import { describeProfile, codeListValues } from './describe.js';
import { checkSpec, whereNotes, profileError } from '../runner/check.js';
import { describeProfile } from './describe.js';
import { lookupValues } from './lookup.js';
import { scopedQuery } from '../db.js';
import { readFileSync } from 'fs';
import { readSpecFile } from '../chat.js';
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
// The tool definitions sent to the model. The descriptions are what the LLM
// reads to decide which tool to call.
export const TOOLS = [
tool('list_profiles',
'List the report profiles available, with their id and title.',
{}),
tool('describe_profile',
'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.',
{ profile_id: 'The profile URI returned by list_profiles' }),
tool('validate_spec',
'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.',
{ spec: 'The spec as Turtle text' }),
tool('read_spec',
'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.',
{ file_name: 'The file name of the spec bijlage, e.g. specificatie-<uuid>.ttl' }),
tool('lookup_values',
'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.',
{
profile_id: 'The profile URI returned by list_profiles',
field: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"',
term: 'The Dutch word or name to search for',
}),
];
// buildTools(profiles, session) → { handlers }
export function buildTools(profiles, session) {
const profileList = () => [...profiles.values()].map(p => ({ id: p.uri, title: p.title }));
const findProfile = (id) => profiles.get(id);
async function describe_profile({ profile_id }) {
const p = findProfile(profile_id);
if (!p) return text(`no profile <${profile_id}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
const inlined = await codeListValues(p, scopedQuery);
return text(describeProfile(p, inlined));
function tool(name, description, params) {
const properties = {};
for (const [key, text] of Object.entries(params)) {
properties[key] = { type: 'string', description: text };
}
async function validate_spec({ spec }) {
let parsed;
try { parsed = parseSpec(spec); }
catch (e) { return text(`the spec did not parse: ${e.message}`); }
const p = findProfile(parsed.profileUri);
if (!p) return text(`no profile <${parsed.profileUri}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
const errors = checkSpec(parsed, p, MAX_PATH_DEPTH, profiles);
if (errors.length) return text(errors[0]);
// The spec that checks out becomes the attached proposal.
session.onSpecValidated?.(spec);
return text(['ok', ...whereNotes(parsed, p)].join('\n'));
}
async function list_profiles() {
return text(profileList().map(x => `${x.title} <${x.id}>`).join('\n'));
}
async function lookup_values({ profile_id, field, term }) {
const p = findProfile(profile_id);
if (!p) return text(`no profile <${profile_id}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
if (!term) return text('no term to search for.');
try {
const out = await lookupValues(p, term, field, scopedQuery);
if (out.error) return text(out.error);
if (!out.matches.length) return text(`no value matches "${term}" in ${field}.`);
const lines = out.matches.map(m => `${m.label} <${m.uri}>`);
if (out.total === '25+') lines.push('(25+ matches, narrow the term)');
if (out.exact) lines.push(`exact: <${out.exact}>`);
return text(lines.join('\n'));
} catch (e) {
return text(`the lookup failed: ${String(e.message || e).split('\n')[0]}`);
}
}
// A spec bijlage next to a message. The model opens it with read_spec:
// only files this service made are readable, hte spec files under share.
async function read_spec({ file_uri }) {
const fileName = String(file_uri || '').split('/').pop();
if (!fileName || !/^specificatie-[0-9a-f]{8}-[0-9a-f-]{27}\.ttl$/.test(fileName)) {
return text('no such spec file.');
}
const shareDir = process.env.SHARE_DIR || '/share';
try {
return text(readFileSync(`${shareDir}/${fileName}`, 'utf8'));
} catch (e) {
return text(`the spec failed to load: ${String(e.message || e).split('\n')[0]}`);
}
}
const handlers = new Map(
[
['list_profiles', list_profiles],
['describe_profile', describe_profile],
['validate_spec', validate_spec],
['read_spec', read_spec],
['lookup_values', lookup_values],
],
);
return { handlers };
return {
type: 'function',
function: {
name,
description,
parameters: { type: 'object', properties, required: Object.keys(params) },
},
};
}
function text(s) { return { content: [{ type: 'text', text: s }] }; }
export async function runTool(name, args, profiles) {
switch (name) {
case 'list_profiles': return listProfiles(profiles);
case 'describe_profile': return describe(args, profiles);
case 'validate_spec': return validateSpec(args, profiles);
case 'read_spec': return readSpec(args);
case 'lookup_values': return lookup(args, profiles);
default: return `unknown tool "${name}"`;
}
}
function listProfiles(profiles) {
return [...profiles.values()].map(p => `${p.title} <${p.uri}>`).join('\n');
}
async function describe({ profile_id }, profiles) {
return profileError(profiles, profile_id) || describeProfile(profiles.get(profile_id));
}
function validateSpec({ spec }, profiles) {
let parsed;
try {
parsed = parseSpec(spec);
} catch (e) {
return `the spec did not parse: ${e.message}`;
}
const error = checkSpec(parsed, profiles);
if (error) return error;
return ['ok', ...whereNotes(parsed, profiles.get(parsed.profileUri))].join('\n');
}
// Only spec files this service made are readable.
function readSpec({ file_name }) {
return readSpecFile(file_name) || 'no such spec file.';
}
async function lookup({ profile_id, field, term }, profiles) {
const error = profileError(profiles, profile_id);
if (error) return error;
if (!term) return 'no term to search for.';
return lookupValues(profiles.get(profile_id), field, term);
}

158
src/report-assistant.js
View File

@@ -1,129 +1,71 @@
// What this service answers a chat turn with. The service is synchronous
// per turn: when the answer is written, every bijlage already exists.
// What the assistant answers a chat turn with: { text, attachments }. When
// the answer is written, every bijlage already exists.
//
// Execute turns skip the LLM loop. The mode classifier says the user wants
// to run the report now; the agreed spec already sits in the conversation
// as a bijlage, so it is read from disk and fed straight to the runner.
// The answer is hard-coded: no extra prompt runs for it. When the run cannot
// happen or fails, the answer is a hard-coded failure message: nothing is
// retried and no spec is rewritten.
import { classifyMode } from './llm/mode.js';
// Refine: the LLM proposes a spec; the last spec that checked out goes on
// the answer as a bijlage. The prompt never carries the Turtle.
//
// Execute: no LLM runs. The last spec bijlage in the conversation is the
// agreed spec; it is read from the share and fed straight to the runner.
// The answer is fixed text. When the run cannot happen or fails, the answer
// is a fixed failure message: nothing is retried and no spec is rewritten.
import { uuid } from 'mu';
import { ask } from './llm/loop.js';
import { parseSpec } from './runner/spec.js';
import { checkSpec } from './runner/check.js';
import { run, slug } from './runner/run.js';
import { readFileSync } from 'fs';
import { storeSpecFile, registerFileObject } from './chat/attachment-file.js';
import { SPEC_MEDIA_TYPE, ATTACHMENT_TYPES } from './chat/vocab.js';
import { run, registerReport, slug } from './runner/run.js';
import { storeSpecFile, readSpecFile, writeShareFile, registerFile,
SPEC_MEDIA_TYPE, SPEC_TYPE, RESULT_TYPE } from './chat.js';
const TEXT = {
executed: 'Het rapport is uitgevoerd. De bijlagen staan erbij.',
noSpec: 'Er is nog geen voorstel om uit te voeren. Beschrijf eerst welk rapport je wilt.',
invalid: 'Het voorstel is niet meer geldig. Vraag een nieuw voorstel.',
failed: 'Het rapport kon niet uitgevoerd worden. Probeer het later opnieuw.',
noAnswer: 'Er kwam geen antwoord. Probeer het opnieuw.',
};
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
const SHARE_DIR = process.env.SHARE_DIR || '/share';
// The chat mounts before the profiles are in; whenProfiles waits for the load.
export function reportAssistant(whenProfiles) {
return async function answer(turn) {
const turns = [...turn.history, { role: 'user', content: turn.content }];
const profiles = await whenProfiles();
if (await classifyMode(turns)) {
try {
return await runStoredSpec(turn, profiles);
} catch (e) {
console.error('[reports] execution failed:', e);
return TEXT.failed;
}
}
return llmAnswer(turn, turns, profiles);
};
// An empty answer (a reasoning model can end with no text) gets a fixed
// text and no spec: only a spec the user saw explained may run.
export async function refineSpec(turns, profiles) {
const { text, spec } = await ask(turns, profiles);
if (!text.trim()) return { text: TEXT.noAnswer };
const attachments = spec ? [await storeSpecFile(spec)] : [];
return { text: text.trim(), attachments };
}
// Execution: the last spec bijlage in the conversation is the agreed spec.
// No prompt runs; the runner gets it straight away.
async function runStoredSpec(turn, profiles) {
const attachment = lastSpecAttachment(turn.history);
export async function executeSpec(history, conversationTitle, profiles) {
const attachment = lastSpecAttachment(history);
const content = attachment && readSpecFile(attachment.name);
if (!content) return TEXT.noSpec;
if (!content) return { text: TEXT.noSpec };
const parsed = parseSpec(content);
const profile = profiles.get(parsed.profileUri);
if (!profile) {
console.error(`[reports] no profile <${parsed.profileUri}>`);
return TEXT.invalid;
try {
const spec = parseSpec(content);
const error = checkSpec(spec, profiles);
if (error) {
console.error(`[reports] the agreed spec is not valid: ${error}`);
return { text: TEXT.invalid };
}
// The conversation's title (named by the first question) names the
// file; the share name is unique, so a second run never overwrites the
// first. The report itself keeps the dct:title of the spec.
const shareName = `${uuid()}.csv`;
writeShareFile(shareName, await run(spec, profiles.get(spec.profileUri)));
const csvFile = await registerFile(shareName, 'text/csv', RESULT_TYPE, `${slug(conversationTitle)}.csv`);
await registerReport(spec.title, csvFile);
// Same physical spec file, registered as a bijlage for this message.
const specFile = await registerFile(attachment.name, SPEC_MEDIA_TYPE, SPEC_TYPE);
return { text: TEXT.executed, attachments: [csvFile, specFile] };
} catch (e) {
console.error('[reports] execution failed:', e);
return { text: TEXT.failed };
}
const errors = checkSpec(parsed, profile, MAX_PATH_DEPTH, profiles);
if (errors.length) {
console.error(`[reports] the agreed spec is not valid: ${errors[0]}`);
return TEXT.invalid;
}
// The conversation's title (named by the first question) names the file.
// The report itself keeps the dct:title of the spec.
const { fileUri } = await run(turn.query, turn.update, parsed, profile,
parsed.title, {
spec: content,
fileType: ATTACHMENT_TYPES.result,
fileName: `${slug(turn.conversation.title)}.csv`,
});
// Same physical spec file, registered as a bijlage for this message.
const specFile = await registerFileObject(turn.update, {
fileName: attachment.name,
format: SPEC_MEDIA_TYPE,
size: Buffer.byteLength(content),
type: ATTACHMENT_TYPES.spec,
});
return {
text: TEXT.executed,
attachments: [
{ uri: fileUri },
{ ...specFile, mediaType: SPEC_MEDIA_TYPE },
],
};
}
function lastSpecAttachment(history) {
for (const message of [...(history || [])].reverse()) {
const specs = (message.attachments || []).filter(a =>
a.mediaType === SPEC_MEDIA_TYPE
&& /^specificatie-[0-9a-f-]+\.ttl$/.test(a.name || ''));
if (specs.length) return specs[specs.length - 1];
}
return null;
}
function readSpecFile(fileName) {
try {
return readFileSync(`${SHARE_DIR}/${fileName}`, 'utf8');
} catch {
return null;
}
}
// The LLM loop, refinement only. The spec that checked out goes on the
// answer as a bijlage. The prompt never carries the Turtle.
async function llmAnswer(turn, turns, profiles) {
// The last spec that checked out; stored as a file for this turn's
// final message.
let spec = null;
const session = {
onSpecValidated(validated) { spec = validated; },
};
const { text } = await ask(turns, profiles, session);
const attachments = [];
if (spec) {
const stored = await storeSpecFile(turn.update, spec);
attachments.push({ ...stored, mediaType: SPEC_MEDIA_TYPE });
}
return { text, attachments };
const specs = history.flatMap(message => message.attachments)
.filter(a => a.mediaType === SPEC_MEDIA_TYPE && a.name?.startsWith('specificatie-'));
return specs.at(-1) || null;
}

20
src/runner/assemble.js
View File

@@ -14,7 +14,6 @@ export const NUMERIC_DATATYPES = new Set([
'nonNegativeInteger', 'positiveInteger', 'nonPositiveInteger', 'negativeInteger',
'unsignedLong', 'unsignedInt', 'unsignedShort', 'unsignedByte',
].map(d => XSD + d));
export const TEMPORAL_DATATYPES = new Set([XSD + 'date', XSD + 'dateTime']);
export function assemble(subjects, values, spec) {
const sorted = [...subjects].sort();
@@ -57,19 +56,18 @@ function collect(terms, col) {
}
}
// min/max over values that share one datatype. Numbers compare as numbers,
// dates and times as instants. Mixed datatypes throw: check.js should have
// caught it, and a silently wrong cell is worse than a failed job.
// min/max. Numbers compare as numbers. Dates and times compare as text:
// xsd:date and xsd:dateTime are ISO strings, so text order is date order.
// Numbers mixed with other values throw: check.js should have caught it,
// and a silently wrong cell is worse than a failed job. The terms are SPARQL
// JSON bindings, so the datatype is a plain string.
function extreme(terms, col, direction) {
if (!terms.length) return '';
const datatypes = [...new Set(terms.map(t => t.datatype?.value || ''))];
if (datatypes.length > 1) {
throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but its values carry ${datatypes.length} datatypes (${datatypes.join(', ')}).`);
const numbers = terms.filter(t => NUMERIC_DATATYPES.has(t.datatype)).length;
if (numbers && numbers < terms.length) {
throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but mixes numbers with other values.`);
}
const dt = datatypes[0];
const key = NUMERIC_DATATYPES.has(dt) ? t => Number(t.value)
: TEMPORAL_DATATYPES.has(dt) ? t => new Date(t.value).getTime()
: t => t.value;
const key = numbers ? t => Number(t.value) : t => t.value;
const cmp = (a, b) => a < b ? -1 : a > b ? 1 : 0;
const best = terms.reduce((a, b) => (cmp(key(b), key(a)) * direction > 0 ? b : a));
return best.value;

137
src/runner/check.js
View File

@@ -1,4 +1,4 @@
import { fieldsOf } from './profile.js';
import { startShape, fieldsOf, field } from './profile.js';
import { NUMERIC_DATATYPES } from './assemble.js';
import { sharedSteps } from './spec.js';
@@ -9,23 +9,18 @@ const XSD_DATE = XSD + 'date';
const XSD_DATETIME = XSD + 'dateTime';
// The validator. checkSpec runs the checks in order and returns the first
// error as a one-element array (empty if ok). Every message names the thing
// that is wrong in the profile's own words and says what to write instead.
// No database call anywhere in this file.
// error, or null when the spec is fine. Every message names the thing that
// is wrong in the profile's own words and says what to write instead. No
// database call anywhere in this file.
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
const ANYOF_MAX_TERMS = 24;
const ANYOF_MAX_TERM_LENGTH = 100;
// The spec names a profile we have. knownProfiles is the profile Map loaded
// at boot; the check is skipped when it is omitted.
function profileNamed(spec, knownProfiles) {
if (!spec.profileUri) {
return `no rep:profile named. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'} .`;
}
if (knownProfiles && !knownProfiles.has(spec.profileUri)) {
return `no profile <${spec.profileUri}>. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'}.`;
}
return null;
// null when the profile exists, else what to write instead.
export function profileError(profiles, uri) {
if (profiles.has(uri)) return null;
return `no profile <${uri}>. Available: ${[...profiles.values()].map(p => p.title).join(', ')}.`;
}
// The spec names an entity we have: by rep:entity, or by a targetClass only
@@ -74,16 +69,6 @@ function conditions(spec) {
return out;
}
// The shape a spec starts from, after startEntity has approved it: rep:entity
// wins, otherwise the one shape carrying the targetClass.
function startShapeOf(spec, profile) {
if (spec.entity) {
return profile.shapes.find(s => s.uri === spec.entity)?.uri || null;
}
const matches = profile.shapes.filter(s => s.targetClass === spec.targetClass);
return matches.length === 1 ? matches[0].uri : null;
}
// Columns exist
function columnsPresent(spec) {
if (!spec.columns.length) return 'no columns. Add at least one rep:columns entry.';
@@ -100,26 +85,26 @@ function columnLabel(spec) {
// Labels are unique
function columnLabelsUnique(spec) {
const seen = new Map();
const seen = new Set();
for (const col of spec.columns) {
if (seen.has(col.label)) {
return `two columns are called "${col.label}". Labels must differ.`;
}
seen.set(col.label, true);
seen.add(col.label);
}
return null;
}
// Every path resolves, columns end on a value, depth capped
function pathsResolve(spec, profile, maxPathDepth) {
const startShapeUri = startShapeOf(spec, profile);
function pathsResolve(spec, profile) {
const startShapeUri = startShape(profile, spec).uri;
for (const col of spec.columns) {
const err = walk(profile, startShapeUri, col.path, maxPathDepth,
const err = walk(profile, startShapeUri, col.path,
`column "${col.label}"`, true, col.nodeKind === SH + 'IRI');
if (err) return err;
}
for (const [i, filter] of spec.filters.entries()) {
const err = walk(profile, startShapeUri, filter.path, maxPathDepth,
const err = walk(profile, startShapeUri, filter.path,
`filter ${i + 1}`, false);
if (err) return err;
}
@@ -128,7 +113,7 @@ function pathsResolve(spec, profile, maxPathDepth) {
// sh:min/sh:max only on numbers and dates
function collectMinMaxTyped(spec, profile) {
const startShapeUri = startShapeOf(spec, profile);
const startShapeUri = startShape(profile, spec).uri;
for (const col of spec.columns) {
if (col.collect !== SH + 'min' && col.collect !== SH + 'max') continue;
const name = col.collect === SH + 'min' ? 'sh:min' : 'sh:max';
@@ -136,8 +121,7 @@ function collectMinMaxTyped(spec, profile) {
const last = col.path[col.path.length - 1];
const shapeUri = shapeAfter(profile, startShapeUri, col.path);
if (!shapeUri) continue; // pathsResolve already reported the real error
const f = fieldsOf(profile, shapeUri)
.find(x => x.path === last.predicate && !!x.inverse === !!last.inverse);
const f = field(profile, shapeUri, last);
if (!f) continue;
const kind = f.datatype === XSD_DATETIME || f.datatype === XSD_DATE ? 'date'
: NUMERIC_DATATYPES.has(f.datatype) ? 'number'
@@ -205,10 +189,10 @@ function filterConstraint(spec) {
function maxCountZero(spec) {
for (const { cond } of conditions(spec)) {
const c = cond.constraints;
if (c.maxCount !== null && c.maxCount !== undefined && c.maxCount > 0) {
if (c.maxCount > 0) {
return `sh:maxCount ${c.maxCount} needs counting, which this service cannot do. Use sh:maxCount 0 for "has none", or drop it.`;
}
if (c.minCount !== null && c.minCount !== undefined && c.minCount > 2) {
if (c.minCount > 2) {
return `sh:minCount ${c.minCount} needs counting. Use sh:minCount 1 (at least one) or sh:minCount 2 (two distinct values); anything higher is beyond this service.`;
}
}
@@ -272,14 +256,14 @@ function literalsParse(spec) {
// rep:where: walks from the row through the profile, shares at least one
// step with its filter or column, and holds no rep:where of its own.
function whereConditions(spec, profile, maxPathDepth) {
const startShapeUri = startShapeOf(spec, profile);
function whereConditions(spec, profile) {
const startShapeUri = startShape(profile, spec).uri;
for (const { what, cond, host } of conditions(spec)) {
if (!host) continue;
if (cond.where.length) {
return `${what} holds a rep:where of its own. Put every rep:where directly on the filter or column.`;
}
const err = walk(profile, startShapeUri, cond.path, maxPathDepth, what, false);
const err = walk(profile, startShapeUri, cond.path, what, false);
if (err) return err;
const k = sharedSteps(host.path, cond.path);
if (!k) {
@@ -302,14 +286,13 @@ function whereConditions(spec, profile, maxPathDepth) {
// code) it compares a URI and never matches, so its path must end on a field
// with a datatype.
function rangeOnValue(spec, profile) {
const startShapeUri = startShapeOf(spec, profile);
const startShapeUri = startShape(profile, spec).uri;
for (const { what, cond } of conditions(spec)) {
const key = ['minInclusive', 'maxInclusive', 'minExclusive', 'maxExclusive']
.find(k => cond.constraints[k]);
if (!key || !cond.path.length) continue;
const last = cond.path[cond.path.length - 1];
const f = fieldsOf(profile, shapeAfter(profile, startShapeUri, cond.path))
.find(x => x.path === last.predicate && !!x.inverse === !!last.inverse);
const f = field(profile, shapeAfter(profile, startShapeUri, cond.path), last);
if (!f || f.datatype) continue; // pathsResolve reports a missing step
const target = f.node
? `"${profile.shapes.find(s => s.uri === f.node)?.label || f.node}"`
@@ -326,7 +309,7 @@ function rangeOnValue(spec, profile) {
// Where each rep:where applies, in the profile's words, so the proposal can
// say it. Profile only, no database.
export function whereNotes(spec, profile) {
const startShapeUri = startShapeOf(spec, profile);
const startShapeUri = startShape(profile, spec).uri;
return conditions(spec).filter(x => x.host).map(({ what, cond, host }) => {
const k = sharedSteps(host.path, cond.path);
return `${what} applies at ${nodeName(profile, startShapeUri, host.path.slice(0, k))}, after step ${k} of ${host.what}.`;
@@ -337,7 +320,7 @@ export function whereNotes(spec, profile) {
function nodeName(profile, startShapeUri, hops) {
let shapeUri = startShapeUri;
for (const [i, hop] of hops.entries()) {
const f = fieldsOf(profile, shapeUri).find(x => x.path === hop.predicate && !!x.inverse === !!hop.inverse);
const f = field(profile, shapeUri, hop);
if (!f) return `step ${hops.length}`;
if (!f.node) return i === hops.length - 1 ? `the value "${f.name}"` : `step ${hops.length}`;
shapeUri = f.node;
@@ -351,33 +334,37 @@ function parsesAsDate(value, withTime) {
return withTime ? value.includes('T') : /^\d{4}-\d{2}-\d{2}$/.test(value);
}
// In this order: startEntity comes first, the checks after it rely on a
// start shape.
const CHECKS = [
['no entity or unknown class', startEntity],
['no columns', columnsPresent],
['column without rdfs:label', columnLabel],
['two columns with one label', columnLabelsUnique],
['column carries constraints', columnPure],
['filter carries rdfs:label', filterLabel],
['filter without a condition', filterConstraint],
['sh:maxCount above 0', maxCountZero],
['rep:anyOf too big', anyOfLimits],
['sh:in mixes URIs and literals', inHomogeneous],
['typed literal does not parse', literalsParse],
['two rep:row columns', oneRowColumn],
['path does not resolve', pathsResolve],
['rep:where does not fit', whereConditions],
['date or number condition on a link', rangeOnValue],
['sh:min/sh:max on a non-number', collectMinMaxTyped],
startEntity,
columnsPresent,
columnLabel,
columnLabelsUnique,
columnPure,
filterLabel,
filterConstraint,
maxCountZero,
anyOfLimits,
inHomogeneous,
literalsParse,
oneRowColumn,
pathsResolve,
whereConditions,
rangeOnValue,
collectMinMaxTyped,
];
export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) {
const profileErr = profileNamed(spec, knownProfiles);
if (profileErr) return [profileErr];
for (const [name, check] of CHECKS) {
const err = check(spec, profile, maxPathDepth);
if (err) return [err];
// checkSpec(spec, profiles) → the first error, or null.
export function checkSpec(spec, profiles) {
const error = profileError(profiles, spec.profileUri);
if (error) return error;
const profile = profiles.get(spec.profileUri);
for (const check of CHECKS) {
const err = check(spec, profile);
if (err) return err;
}
return [];
return null;
}
// The shape a path lands in: follow sh:node through every hop but the
@@ -386,17 +373,16 @@ export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) {
function shapeAfter(profile, startShapeUri, hops) {
let shapeUri = startShapeUri;
for (let i = 0; i < hops.length - 1; i++) {
const fields = fieldsOf(profile, shapeUri);
const f = fields.find(x => x.path === hops[i].predicate && !!x.inverse === !!hops[i].inverse);
const f = field(profile, shapeUri, hops[i]);
if (!f || !f.node) return null;
shapeUri = f.node;
}
return shapeUri;
}
function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsIri = false) {
if (hops.length > maxPathDepth) {
return `${what} walks ${hops.length} hops. The limit is ${maxPathDepth}.`;
function walk(profile, startShapeUri, hops, what, isColumn, wantsIri = false) {
if (hops.length > MAX_PATH_DEPTH) {
return `${what} walks ${hops.length} hops. The limit is ${MAX_PATH_DEPTH}.`;
}
if (!hops.length) return null;
let shapeUri = startShapeUri;
@@ -416,20 +402,15 @@ function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsI
shapeUri = f.node;
} else if (isColumn) {
if (wantsIri && f.datatype) {
return `column "${colLabel(what)}" has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`;
return `${what} has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`;
}
if (!wantsIri && !f.datatype && !f.class) {
const valueFields = fieldsOf(profile, f.node)
.filter(x => x.datatype || x.class)
.map(x => `${x.name} (${x.path})`);
return `column "${colLabel(what)}" ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`;
return `${what} ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`;
}
}
}
return null;
}
function colLabel(what) {
const m = what.match(/"([^"]+)"/);
return m ? m[1] : what;
}

26
src/runner/columns.js
View File

@@ -1,5 +1,5 @@
import { sparqlEscapeUri } from '../db.js';
import { wherePatterns } from './seed.js';
import { query, sparqlEscapeUri } from 'mu';
import { hopsPattern, wherePatterns } from './seed.js';
// Fetch column values for known subjects. Columns are grouped by their
// shared non-leaf hops; two columns share a query only when every hop
@@ -8,12 +8,14 @@ import { wherePatterns } from './seed.js';
const SUBJECT_CHUNK_SIZE = Math.max(1, Number(process.env.SUBJECT_CHUNK_SIZE) || 100);
export async function fetchColumns(sessionQuery, subjects, spec, values = new Map()) {
// fetchColumns(subjects, spec) → subject → column index → [term]
export async function fetchColumns(subjects, spec) {
const values = new Map();
const groups = groupColumns(spec.columns);
for (const group of groups) {
for (let i = 0; i < subjects.length; i += SUBJECT_CHUNK_SIZE) {
const chunk = subjects.slice(i, i + SUBJECT_CHUNK_SIZE);
const result = await sessionQuery(groupQuery(group, chunk));
const result = await query(groupQuery(group, chunk));
collect(result, group, values);
}
}
@@ -41,21 +43,11 @@ export function groupColumns(columns) {
export function groupQuery(group, chunk) {
const subjects = chunk.map(sparqlEscapeUri).join(' ');
const lines = [];
let prev = '?s';
// every hop except the last gets its own variable; the leaf is ?v
group.hops.slice(0, -1).forEach((hop, i) => {
const v = `?x${i}`;
if (hop.inverse) lines.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
else lines.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`);
prev = v;
});
const leaf = group.hops[group.hops.length - 1];
if (leaf.inverse) lines.push(`?v ${sparqlEscapeUri(leaf.predicate)} ${prev} .`);
else lines.push(`${prev} ${sparqlEscapeUri(leaf.predicate)} ?v .`);
// rep:where hangs on the column's own nodes: ?x<k-1> after k hops, ?v at
const lines = [hopsPattern(group.hops, 'x', '?s', '?v')];
// rep:where hangs on the column's own nodes: ?x_<k> after k hops, ?v at
// the end.
const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x${k - 1}`);
const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x_${k}`);
lines.push(...wherePatterns(group.where, group.hops, nodeVar, 'c'));
return `SELECT ?s ?v WHERE {\n VALUES ?s { ${subjects} }\n ${lines.join('\n ')}\n}`;
}

35
src/runner/csv.js
View File

@@ -1,35 +0,0 @@
import { mkdirSync, writeFileSync } from 'fs';
// Rows to a CSV file. CSV_SEPARATOR is ';' by default. A cell is quoted
// only when it holds the separator, a quote or a newline; a quote inside a
// cell is doubled, per RFC 4180.
const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
const SHARE_DIR = process.env.SHARE_DIR || '/share';
export function rowsToCsv(rows) {
return rows.map(row => row.map(escapeCell).join(CSV_SEPARATOR)).join('\n');
}
function escapeCell(cell) {
const s = String(cell ?? '');
return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s)
? `"${s.replace(/"/g, '""')}"`
: s;
}
export function writeCsv(name, csv) {
mkdirSync(SHARE_DIR, { recursive: true });
const file = `${SHARE_DIR}/${name}`;
writeFileSync(file, csv, 'utf8');
return file;
}
// The spec of the run, next to its CSV (same base name, .ttl), so a report
// can be debugged afterwards. The store keeps only the pointer.
export function writeSpec(csvName, spec) {
mkdirSync(SHARE_DIR, { recursive: true });
const name = csvName.replace(/\.csv$/, '') + '.ttl';
writeFileSync(`${SHARE_DIR}/${name}`, spec, 'utf8');
return `share://${name}`;
}

95
src/runner/profile.js
View File

@@ -1,5 +1,5 @@
import N3 from 'n3';
import { readFileSync } from 'fs';
import { readFileSync, readdirSync } from 'fs';
// Parse a profile (a SHACL shapes graph) and answer questions about it.
@@ -8,13 +8,11 @@ const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
const OWL_ONTOLOGY = 'http://www.w3.org/2002/07/owl#Ontology';
const REP = 'http://mu.semte.ch/vocabularies/reporting/';
export async function loadProfiles(dir) {
const fs = await import('fs');
const path = await import('path');
const files = fs.readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
export function loadProfiles(dir) {
const files = readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
const profiles = new Map();
for (const file of files) {
const profile = await loadProfile(path.join(dir, file));
const profile = loadProfile(`${dir}/${file}`);
profiles.set(profile.uri, profile);
console.log(`[profile] "${profile.title}" (${file}): ${profile.shapes.length} entities, ` +
`${profile.shapes.reduce((n, s) => n + s.fields.length, 0)} fields`);
@@ -23,29 +21,19 @@ export async function loadProfiles(dir) {
return profiles;
}
export async function loadProfile(file) {
const parser = new N3.Parser();
const store = new N3.Store();
await new Promise((resolve, reject) => {
parser.parse(readFileSync(file, 'utf8'), (err, quad) => {
if (err) return reject(new Error(`${file}: ${err.message}`));
if (quad) store.addQuad(quad); else resolve();
});
});
function loadProfile(file) {
let store;
try {
store = new N3.Store(new N3.Parser().parse(readFileSync(file, 'utf8')));
} catch (e) {
throw new Error(`${file}: ${e.message}`);
}
const ontology = store.getQuads(null, RDF + 'type', OWL_ONTOLOGY)[0];
if (!ontology) throw new Error(`${file}: no owl:Ontology. A profile file must name itself.`);
const uri = ontology.subject.value;
const title = store.getQuads(ontology.subject, 'http://purl.org/dc/terms/title', null)[0]?.object.value || uri;
const prefixQuads = store.getQuads(null, SH + 'declare', null);
const prefixes = {};
for (const q of prefixQuads) {
const p = store.getQuads(q.object, SH + 'prefix', null)[0]?.object.value;
const ns = store.getQuads(q.object, SH + 'namespace', null)[0]?.object.value;
if (p && ns) prefixes[p] = ns;
}
const shapes = store.getQuads(null, RDF + 'type', SH + 'NodeShape').map(q => q.subject);
if (!shapes.length) throw new Error(`${file}: no sh:NodeShape. A profile without entities is useless.`);
@@ -57,30 +45,26 @@ export async function loadProfile(file) {
return { uri: shape.value, targetClass, label, fields, discriminator };
});
return { uri, title, prefixes, shapes: shapeList, store };
return { uri, title, shapes: shapeList };
}
// A field's sh:path: one predicate, or [ sh:inversePath <predicate> ].
function readPath(store, node) {
const pathNode = store.getQuads(node, SH + 'path', null)[0]?.object;
if (!pathNode) return { path: null, inverse: false };
if (pathNode.termType !== 'BlankNode') return { path: pathNode.value, inverse: false };
const inv = store.getQuads(pathNode, SH + 'inversePath', null)[0]?.object;
return inv ? { path: inv.value, inverse: true } : { path: null, inverse: false };
}
function parseField(store, p) {
let path = null, inverse = false;
const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object;
if (pathQuad) {
if (pathQuad.termType === 'BlankNode') {
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
if (inv) { path = inv.value; inverse = true; }
} else {
path = pathQuad.value;
}
}
const one = (pred) => store.getQuads(p, pred, null)[0]?.object || null;
const one = (pred) => store.getQuads(p, pred, null)[0]?.object?.value || null;
return {
path, inverse,
name: one(SH + 'name')?.value || null,
description: one(SH + 'description')?.value || null,
datatype: one(SH + 'datatype')?.value || null,
class: one(SH + 'class')?.value || null,
node: one(SH + 'node')?.value || null,
nodeKind: one(SH + 'nodeKind')?.value || null,
maxCount: one(SH + 'maxCount')?.value || null,
...readPath(store, p),
name: one(SH + 'name'),
datatype: one(SH + 'datatype'),
class: one(SH + 'class'),
node: one(SH + 'node'),
};
}
@@ -89,16 +73,7 @@ function parseField(store, p) {
function parseDiscriminator(store, shape) {
const node = store.getQuads(shape, REP + 'discriminator', null)[0]?.object;
if (!node) return null;
let path = null, inverse = false;
const pathQuad = store.getQuads(node, SH + 'path', null)[0]?.object;
if (pathQuad) {
if (pathQuad.termType === 'BlankNode') {
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
if (inv) { path = inv.value; inverse = true; }
} else {
path = pathQuad.value;
}
}
const { path, inverse } = readPath(store, node);
const one = (pred) => store.getQuads(node, pred, null)[0]?.object?.value || null;
const minCount = one(SH + 'minCount');
const maxCount = one(SH + 'maxCount');
@@ -110,11 +85,6 @@ function parseDiscriminator(store, shape) {
};
}
export function entityForClass(profile, cls) {
const shape = profile.shapes.find(s => s.targetClass === cls);
return shape ? shape.uri : null;
}
// The shape a spec starts from. rep:entity wins; a targetClass only resolves
// when exactly one shape carries it, so a class two entities share is a
// validator error, not a silent first match.
@@ -132,10 +102,7 @@ export function fieldsOf(profile, shapeUri) {
return shape(profile, shapeUri)?.fields || [];
}
export function field(profile, shapeUri, predicate, inverse) {
return fieldsOf(profile, shapeUri).find(f => f.path === predicate && !!f.inverse === !!inverse) || null;
// The field a hop takes from a shape, or null.
export function field(profile, shapeUri, hop) {
return fieldsOf(profile, shapeUri).find(f => f.path === hop.predicate && !!f.inverse === !!hop.inverse) || null;
}
export function prefixes(profile) {
return profile.prefixes;
}

63
src/runner/report.js
View File

@@ -1,63 +0,0 @@
import { randomUUID } from 'crypto';
import { statSync } from 'fs';
import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt,
sparqlEscapeDateTime } from '../db.js';
const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
// Register the CSV: a logical file (http://data.lblod.info/files/<uuid>) plus
// a physical share:// file pointing back with nie:dataSource. The logical
// URI is what you link to. When given, the file gets dct:type.
export async function registerFile(sessionUpdateFn, fileName, filePath, { type } = {}) {
const size = statSync(filePath).size;
const uuid = randomUUID();
const uuid2 = randomUUID();
const now = new Date();
const logicalUri = `http://data.lblod.info/files/${uuid}`;
const physicalUri = `share://${fileName}`;
await (sessionUpdateFn || sessionUpdate)(`
INSERT DATA {
${sparqlEscapeUri(logicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)}${type ? ` ;
${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} .
${sparqlEscapeUri(physicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid2)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated')} ${sparqlEscapeDateTime(now)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource')} ${sparqlEscapeUri(logicalUri)}${type ? ` ;
${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} .
}`);
return logicalUri;
}
// One report pointing at the logical file URI.
export async function registerReport(sessionUpdateFn, title, fileUri, extra = {}) {
const uuid = randomUUID();
const now = new Date();
const reportUri = `http://data.lblod.info/id/reports/${uuid}`;
const extraTriples = [];
if (extra.question) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/question')} ${sparqlEscapeString(extra.question)} .`);
if (extra.specFile) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/specFile')} ${sparqlEscapeString(extra.specFile)} .`);
if (extra.creator) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://purl.org/dc/terms/creator')} ${sparqlEscapeUri(extra.creator)} .`);
await (sessionUpdateFn || sessionUpdate)(`
INSERT DATA {
${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)} ;
${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} .
${extraTriples.join('\n ')}
}`);
return reportUri;
}

61
src/runner/run.js
View File

@@ -1,38 +1,49 @@
import { update, uuid, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
import { seed } from './seed.js';
import { fetchColumns } from './columns.js';
import { assemble } from './assemble.js';
import { rowsToCsv, writeCsv, writeSpec } from './csv.js';
import { registerFile, registerReport } from './report.js';
import { startShape } from './profile.js';
// Ties the steps together: seed → columns → assemble → CSV → register.
// Returns { reportUri, fileUri, rowCount }.
const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
export async function run(query, update, parsed, profile, title, extra = {}) {
const shape = startShape(profile, parsed);
const subjects = await seed(query, parsed, shape);
const values = new Map();
await fetchColumns(query, subjects, parsed, values);
const rows = assemble(subjects, values, parsed);
const fileName = extra.fileName || `${slug(title)}.csv`;
const csv = rowsToCsv(rows);
const filePath = writeCsv(fileName, csv);
const specFile = extra.spec && writeSpec(fileName, extra.spec);
const fileUri = await registerFile(update, fileName, filePath, { type: extra.fileType });
const reportUri = await registerReport(update, title, fileUri,
specFile ? { ...extra, specFile } : extra);
return { reportUri, fileUri, rowCount: rows.length - 1, filePath };
// run(spec, profile) → the CSV text. seed → columns → assemble → CSV.
export async function run(spec, profile) {
const subjects = await seed(spec, startShape(profile, spec));
const values = await fetchColumns(subjects, spec);
const rows = assemble(subjects, values, spec);
return rows.map(row => row.map(csvCell).join(CSV_SEPARATOR)).join('\n');
}
// A cell is quoted only when it holds the separator, a quote or a newline;
// a quote inside a cell is doubled, per RFC 4180.
function csvCell(cell) {
const s = String(cell ?? '');
return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s)
? `"${s.replace(/"/g, '""')}"`
: s;
}
// One report pointing at the logical file URI of its CSV.
export async function registerReport(title, fileUri) {
const id = uuid();
const reportUri = `http://data.lblod.info/id/reports/${id}`;
await update(`
INSERT DATA {
${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(id)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(new Date())} ;
${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} .
}`);
return reportUri;
}
// The file name must be known before the run starts, so the chat can show a
// pending card named after the report.
export function slug(title) {
return (title || 'report').toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80) || 'report';
}
}

30
src/runner/seed.js
View File

@@ -1,16 +1,15 @@
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from '../db.js';
import { query, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from 'mu';
import { sharedSteps } from './spec.js';
// Subject selection. One paged query per page; stops at ROW_LIMIT.
const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000);
const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000);
export async function seed(sessionQuery, spec, shape = null) {
export async function seed(spec, shape) {
const subjects = [];
let offset = 0;
for (;;) {
const query = seedPageQuery(spec, PAGE_SIZE, offset, shape);
const result = await sessionQuery(query);
const result = await query(seedPageQuery(spec, PAGE_SIZE, offset, shape));
const rows = result.results.bindings.map(b => b.s.value);
subjects.push(...rows);
if (subjects.length > ROW_LIMIT) {
@@ -49,13 +48,13 @@ function discriminatorPattern(d) {
// Writes out every hop as its own triple pattern.
// Forward: ?s <p1> ?f0_1 . ?f0_1 <p2> ?f0_v
// Inverse: ?s ^<p1> → ?f0_1 <p1> ?s
export function hopsPattern(hops, varPrefix, startVar) {
export function hopsPattern(hops, varPrefix, startVar, lastVar = `?${varPrefix}_v`) {
let prev = startVar;
const patterns = [];
for (let i = 0; i < hops.length; i++) {
const hop = hops[i];
const isLast = i === hops.length - 1;
const v = isLast ? `?${varPrefix}_v` : `?${varPrefix}_${i + 1}`;
const v = isLast ? lastVar : `?${varPrefix}_${i + 1}`;
if (hop.inverse) {
patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
} else {
@@ -114,29 +113,14 @@ export function conditionPattern(c, varPrefix, hops, startVar) {
parts.push(`FILTER(REGEX(str(${v}), ${sparqlEscapeString(alternation)}${flags}))`);
}
if (c.minCount === 2) {
// two distinct leaf values on the same path
// two distinct leaf values: the same path again, to a second leaf
const w = `?${varPrefix}_w`;
parts.push(hopsPattern2(hops, varPrefix, startVar, w));
parts.push(hopsPattern(hops, `${varPrefix}_2`, startVar, w));
parts.push(`FILTER(${v} != ${w})`);
}
return parts.join('\n ');
}
function hopsPattern2(hops, varPrefix, startVar, wVar) {
// repeat the path to a second leaf variable
let prev = startVar;
const patterns = [];
for (let i = 0; i < hops.length; i++) {
const hop = hops[i];
const isLast = i === hops.length - 1;
const v = isLast ? wVar : `?${varPrefix}_2_${i + 1}`;
if (hop.inverse) patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
else patterns.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`);
prev = v;
}
return patterns.join('\n ');
}
function escapeTerm(t) {
if (t.type === 'uri') return sparqlEscapeUri(t.value);
if (t.datatype === 'http://www.w3.org/2001/XMLSchema#dateTime')

45
src/runner/spec.js
View File

@@ -23,22 +23,21 @@ export function parseSpec(turtle) {
const profileUri = one(specNode, REP + 'profile')?.value || null;
const title = one(specNode, 'http://purl.org/dc/terms/title')?.value || null;
const filters = store.getQuads(specNode, SH + 'property', null).map(q =>
parseFilter(store, q.object, { withLabel: true }));
const filters = store.getQuads(specNode, SH + 'property', null).map(q => parseFilter(store, q.object));
const columnsQuad = one(specNode, REP + 'columns');
const columns = columnsQuad ? parseColumnList(store, columnsQuad) : [];
const columns = columnsQuad ? readRdfList(store, columnsQuad).map(col => parseColumn(store, col)) : [];
return { uri: specNode.value, title, profileUri, targetClass, entity, filters, columns };
}
function parseFilter(store, p, { withLabel = false } = {}) {
function parseFilter(store, p) {
let path = [];
const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object;
if (pathQuad) path = parsePath(store, pathQuad);
return {
path,
constraints: readConstraints(store, p, { withLabel }),
constraints: readConstraints(store, p, { withLabel: true }),
where: readWhere(store, p),
};
}
@@ -47,8 +46,7 @@ function parseFilter(store, p, { withLabel = false } = {}) {
// is written like a filter, from the row; check.js refuses one nested in
// another.
function readWhere(store, node) {
return store.getQuads(node, REP + 'where', null).map(q =>
parseFilter(store, q.object, { withLabel: true }));
return store.getQuads(node, REP + 'where', null).map(q => parseFilter(store, q.object));
}
// The number of steps two paths have in common from the row. A rep:where
@@ -60,18 +58,6 @@ export function sharedSteps(a, b) {
return k;
}
function parseColumnList(store, listNode) {
const columns = [];
let cur = listNode;
while (cur && cur.value !== RDF + 'nil') {
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object;
if (item) columns.push(parseColumn(store, item));
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
if (!cur) throw new Error('rep:columns list is broken (no rdf:rest terminator)');
}
return columns;
}
function parseColumn(store, col) {
let path = [];
const pathQuad = store.getQuads(col, SH + 'path', null)[0]?.object;
@@ -107,21 +93,12 @@ function parsePath(store, pathQuad) {
return [{ predicate: pathQuad.value, inverse: false }];
}
// RDF list of predicates
const hops = [];
let cur = pathQuad;
while (cur && cur.value !== RDF + 'nil') {
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object;
if (!item) throw new Error('sh:path list is broken');
if (item.termType === 'BlankNode') {
const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
hops.push({ predicate: inv.value, inverse: true });
} else {
hops.push({ predicate: item.value, inverse: false });
}
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
}
return hops;
return readRdfList(store, pathQuad).map(item => {
if (item.termType !== 'BlankNode') return { predicate: item.value, inverse: false };
const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
return { predicate: inv.value, inverse: true };
});
}
function readConstraints(store, p, { withLabel = false } = {}) {