diff --git a/README.md b/README.md index 11957e2..edcc757 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ Two documents drive everything: A **profile** is a SHACL shapes graph: one `sh:NodeShape` per entity, fields as `sh:property`, code lists via `sh:class`, links via `sh:node`. The service ships no profile and knows no vocabulary; profiles live in the app repo. +Profiles load at boot; a broken profile stops the service. A **spec** is `rep:ReportSpec` + `sh:NodeShape`: `sh:property` selects the subjects, `rep:columns` says what the CSV holds. The two never interact. The @@ -107,13 +108,11 @@ The scope URI is `SERVICE_SCOPE` (default POST /assistant/conversations/:id/turns { content } → 202 { id } ``` -This is the chat. The turn module in -`src/chat/` is generic — it imports nothing from `src/llm/` or `src/runner/` -and moves between services without edits — and `src/report-assistant.js` is -this service's one `answer` hook. It records the question, answers `202` with -the message id, then classifies the turn and refines or executes with the -conversation's history. The answer and its bijlagen are written when the turn -ends. The browser polls +This is the chat. The route is in `app.js`: it records the question, answers +`202` with the message id, then classifies the turn and refines or executes +(`src/report-assistant.js`) with the conversation's history. The +conversation, message and bijlage reads and writes are in `src/chat.js`. The +answer and its bijlagen are written when the turn ends. The browser polls `/chat-conversations/:id?include=messages.attachments` and finds the answer that way. Failure is a message, never a silent loader. diff --git a/app.js b/app.js index 0af1065..d8e5f86 100644 --- a/app.js +++ b/app.js @@ -1,25 +1,66 @@ import { app, errorHandler } from 'mu'; +import bodyParser from 'body-parser'; import { loadProfiles } from './src/runner/profile.js'; -import { mountChat } from './src/chat/index.js'; -import { reportAssistant } from './src/report-assistant.js'; +import { readConversation, readHistory, findAssistant, writeMessage } from './src/chat.js'; +import { wantsExecution } from './src/llm/mode.js'; +import { executeSpec, refineSpec } from './src/report-assistant.js'; const PROFILE_DIR = process.env.PROFILE_DIR || '/config/profiles'; +const CHAT_ASSISTANT_URI = process.env.CHAT_ASSISTANT_URI; +const TITLE_LENGTH = 80; +const FAILED = 'Er ging iets mis. Probeer het opnieuw.'; -// Profiles load at boot; handlers await the same promise. A failed load -// answers an error per request instead of crashing boot. -let profilesPromise = loadProfiles(PROFILE_DIR); -profilesPromise.catch(e => console.error('[profiles] loading failed:', e.message)); +// Profiles load at boot. A broken profile stops the service. +const profiles = loadProfiles(PROFILE_DIR); -async function whenProfiles() { +// One chat turn. Records the question, answers 202, then writes the +// assistant's answer; the frontend polls the conversation for it. mu's +// query/update run as the caller for the whole turn, also after the 202. +// The template only parses application/vnd.api+json, the chat sends +// application/json. +app.post('/assistant/conversations/:id/turns', bodyParser.json(), async function(req, res) { + const content = String(req.body?.content ?? '').trim(); + if (!content) return res.status(400).json({ error: 'content is required' }); + + let conversation, assistant, history, question; try { - return await profilesPromise; + // the access check: the caller's session decides what is readable + conversation = await readConversation(req.params.id); + if (!conversation) return res.status(404).json({ error: 'no such conversation' }); + assistant = CHAT_ASSISTANT_URI || await findAssistant(); + history = await readHistory(conversation.uri, assistant); + question = await writeMessage({ + conversationUri: conversation.uri, + content, + maker: conversation.creator, + title: conversation.title ? undefined : content.slice(0, TITLE_LENGTH), + }); } catch (e) { - throw new Error('profiles are not available: ' + e.message); + console.error('[chat] could not record the question:', e); + return res.status(500).json({ error: e.message }); } -} -// The chat. reportAssistant is the answer hook: it refines the spec with the -// LLM, or runs the agreed spec without it. -mountChat(app, { path: '/assistant', answer: reportAssistant(whenProfiles) }); + res.status(202).json({ id: question.id }); -app.use(errorHandler); \ No newline at end of file + try { + const turns = [...history, { role: 'user', content }]; + const answer = await wantsExecution(turns) + ? await executeSpec(history, conversation.title, profiles) + : await refineSpec(turns, profiles); + await writeMessage({ + conversationUri: conversation.uri, + content: answer.text, + maker: assistant, + attachments: answer.attachments, + }); + } catch (e) { + console.error('[chat] turn failed:', e); + try { + await writeMessage({ conversationUri: conversation.uri, content: FAILED, maker: assistant }); + } catch (e2) { + console.error('[chat] could not write the failure message:', e2); + } + } +}); + +app.use(errorHandler); diff --git a/package-lock.json b/package-lock.json index a58078d..612d169 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,6 @@ "version": "0.2.0", "license": "MIT", "dependencies": { - "express": "^5.2.1", "n3": "^1.17.0" } }, @@ -25,19 +24,6 @@ "node": ">=6.5" } }, - "node_modules/accepts": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", - "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", - "license": "MIT", - "dependencies": { - "mime-types": "^3.0.0", - "negotiator": "^1.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -58,43 +44,6 @@ ], "license": "MIT" }, - "node_modules/body-parser": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", - "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", - "license": "MIT", - "dependencies": { - "bytes": "^3.1.2", - "content-type": "^2.0.0", - "debug": "^4.4.3", - "http-errors": "^2.0.1", - "iconv-lite": "^0.7.2", - "on-finished": "^2.4.1", - "qs": "^6.15.2", - "raw-body": "^3.0.2", - "type-is": "^2.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/body-parser/node_modules/content-type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", - "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/buffer": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", @@ -119,184 +68,6 @@ "ieee754": "^1.2.1" } }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/call-bind-apply-helpers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", - "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/content-disposition": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", - "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", - "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", - "license": "MIT", - "engines": { - "node": ">=6.6.0" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/dunder-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", - "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.1", - "es-errors": "^1.3.0", - "gopd": "^1.2.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "license": "MIT" - }, - "node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/es-define-property": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", - "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/es-object-atoms": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", - "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "license": "MIT" - }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -315,206 +86,6 @@ "node": ">=0.8.x" } }, - "node_modules/express": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", - "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", - "license": "MIT", - "dependencies": { - "accepts": "^2.0.0", - "body-parser": "^2.2.1", - "content-disposition": "^1.0.0", - "content-type": "^1.0.5", - "cookie": "^0.7.1", - "cookie-signature": "^1.2.1", - "debug": "^4.4.0", - "depd": "^2.0.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "finalhandler": "^2.1.0", - "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "merge-descriptors": "^2.0.0", - "mime-types": "^3.0.0", - "on-finished": "^2.4.1", - "once": "^1.4.0", - "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", - "qs": "^6.14.0", - "range-parser": "^1.2.1", - "router": "^2.2.0", - "send": "^1.1.0", - "serve-static": "^2.2.0", - "statuses": "^2.0.1", - "type-is": "^2.0.1", - "vary": "^1.1.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/finalhandler": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", - "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "on-finished": "^2.4.1", - "parseurl": "^1.3.3", - "statuses": "^2.0.1" - }, - "engines": { - "node": ">= 18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fresh": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", - "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-intrinsic": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", - "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "es-define-property": "^1.0.1", - "es-errors": "^1.3.0", - "es-object-atoms": "^1.1.1", - "function-bind": "^1.1.2", - "get-proto": "^1.0.1", - "gopd": "^1.2.0", - "has-symbols": "^1.1.0", - "hasown": "^2.0.2", - "math-intrinsics": "^1.1.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-proto": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", - "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "license": "MIT", - "dependencies": { - "dunder-proto": "^1.0.1", - "es-object-atoms": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/gopd": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", - "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-symbols": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", - "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/iconv-lite": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", - "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -535,92 +106,6 @@ ], "license": "BSD-3-Clause" }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "license": "ISC" - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-promise": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", - "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", - "license": "MIT" - }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/media-typer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", - "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/merge-descriptors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", - "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, "node_modules/n3": { "version": "1.26.0", "resolved": "https://registry.npmjs.org/n3/-/n3-1.26.0.tgz", @@ -634,87 +119,6 @@ "node": ">=12.0" } }, - "node_modules/negotiator": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", - "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", - "license": "MIT", - "dependencies": { - "content-type": "^2.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/negotiator/node_modules/content-type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", - "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-to-regexp": { - "version": "8.4.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", - "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -724,63 +128,6 @@ "node": ">= 0.6.0" } }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/qs": { - "version": "6.15.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.3.tgz", - "integrity": "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A==", - "license": "BSD-3-Clause", - "dependencies": { - "es-define-property": "^1.0.1", - "side-channel": "^1.1.1" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/range-parser": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", - "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/raw-body": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", - "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.7.0", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.10" - } - }, "node_modules/readable-stream": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", @@ -797,22 +144,6 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, - "node_modules/router": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", - "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "depd": "^2.0.0", - "is-promise": "^4.0.0", - "parseurl": "^1.3.3", - "path-to-regexp": "^8.0.0" - }, - "engines": { - "node": ">= 18" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -833,144 +164,6 @@ ], "license": "MIT" }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/send": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", - "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "fresh": "^2.0.0", - "http-errors": "^2.0.1", - "mime-types": "^3.0.2", - "ms": "^2.1.3", - "on-finished": "^2.4.1", - "range-parser": "^1.2.1", - "statuses": "^2.0.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/serve-static": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", - "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", - "license": "MIT", - "dependencies": { - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "parseurl": "^1.3.3", - "send": "^1.2.0" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "license": "ISC" - }, - "node_modules/side-channel": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", - "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4", - "side-channel-list": "^1.0.1", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", - "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -979,70 +172,6 @@ "dependencies": { "safe-buffer": "~5.2.0" } - }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/type-is": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", - "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", - "license": "MIT", - "dependencies": { - "content-type": "^2.0.0", - "media-typer": "^1.1.0", - "mime-types": "^3.0.0" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/type-is/node_modules/content-type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", - "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "license": "ISC" } } } diff --git a/package.json b/package.json index 2b87e5b..d1c0641 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,6 @@ "start": "node app.js" }, "dependencies": { - "express": "^5.2.1", "n3": "^1.17.0" } } diff --git a/src/chat/store.js b/src/chat.js similarity index 52% rename from src/chat/store.js rename to src/chat.js index 922afd5..33bb989 100644 --- a/src/chat/store.js +++ b/src/chat.js @@ -1,15 +1,14 @@ -// The chat's reads and writes. Nothing here knows the service it lives in. +// The chat's reads and writes: conversations, messages and their bijlagen. // Full URIs everywhere: the auth layer rejects a prefixed name with no -// PREFIX line with an opaque 500. +// PREFIX line with an opaque 500 (the xsd: types of mu's escape helpers are +// fine). // // A bijlage is a real file: a logical nfo:FileDataObject the file service // serves at /files//download, over a physical share:// object with -// nie:dataSource back to the logical one. Every attachment carries the -// file's logical URI (see chat/attachment-file.js how those files are -// made); nothing here writes files. -import { randomUUID } from 'crypto'; -import { query as muQuery, update as muUpdate, - sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu'; +// nie:dataSource back to the logical one. +import { readFileSync, writeFileSync, statSync } from 'fs'; +import { query, update, uuid, + sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt, sparqlEscapeDateTime } from 'mu'; const HISTORY_LIMIT = Number(process.env.CHAT_HISTORY_LIMIT || 20); @@ -27,14 +26,28 @@ const T = { maker: 'http://xmlns.com/foaf/0.1/maker', attachment: 'https://www.w3.org/ns/activitystreams#attachment', softwareAgent: 'http://www.w3.org/ns/prov#SoftwareAgent', + file: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject', + fileName: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName', + fileSize: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize', + fileCreated: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated', + format: 'http://purl.org/dc/terms/format', + fileType: 'http://purl.org/dc/terms/type', + dataSource: 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource', }; const MESSAGE_BASE = 'http://data.lblod.info/id/chat-messages/'; +const FILE_BASE = 'http://data.lblod.info/files/'; +const SHARE_DIR = process.env.SHARE_DIR || '/share'; + +export const SPEC_MEDIA_TYPE = 'text/turtle'; +// The bijlage types. Match the add-chat-attachment-types-codelist migration +// in the portal. +export const SPEC_TYPE = 'http://lblod.data.gift/concepts/6693014e-7b7d-448c-bb24-80c9758521f3'; +export const RESULT_TYPE = 'http://lblod.data.gift/concepts/28d97698-e219-4a92-bdbd-7ab367a7524e'; +// Every spec file is called like this; readSpecFile opens nothing else. +const SPEC_FILE_NAME = /^specificatie-[0-9a-f-]{36}\.ttl$/; const u = sparqlEscapeUri; const s = sparqlEscapeString; -// Full URIs, so spell the dateTime type out. -const dt = (value) => sparqlEscapeDateTime(value) - .replace('xsd:dateTime', ''); const bindings = (json) => json.results.bindings; @@ -42,7 +55,7 @@ const bindings = (json) => json.results.bindings; // The conversation by uuid, null when the caller may not read it. export async function readConversation(id) { - const json = await muQuery(` + const json = await query(` SELECT ?conversation ?title ?creator WHERE { ?conversation ${u(T.type)} ${u(T.thread)} ; ${u(T.uuid)} ${s(id)} . @@ -57,7 +70,7 @@ export async function readConversation(id) { // The last HISTORY_LIMIT messages, oldest first, as // { role, content, attachments: [{ uri, name, mediaType, type }] }. export async function readHistory(conversationUri, assistantUri) { - const json = await muQuery(` + const json = await query(` SELECT ?message ?content ?maker ?created ?doc ?docName ?docFormat ?docType WHERE { ?message ${u(T.hasContainer)} ${u(conversationUri)} ; ${u(T.content)} ?content ; @@ -65,9 +78,9 @@ export async function readHistory(conversationUri, assistantUri) { ${u(T.created)} ?created . OPTIONAL { ?message ${u(T.attachment)} ?doc . - OPTIONAL { ?doc ${u('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ?docName . } - OPTIONAL { ?doc ${u('http://purl.org/dc/terms/format')} ?docFormat . } - OPTIONAL { ?doc ${u('http://purl.org/dc/terms/type')} ?docType . } + OPTIONAL { ?doc ${u(T.fileName)} ?docName . } + OPTIONAL { ?doc ${u(T.format)} ?docFormat . } + OPTIONAL { ?doc ${u(T.fileType)} ?docType . } } } ORDER BY DESC(?created) LIMIT ${HISTORY_LIMIT * 4}`); const messages = []; @@ -98,7 +111,7 @@ export async function readHistory(conversationUri, assistantUri) { // The one assistant the caller can read. Throws unless there is exactly one. export async function findAssistant() { - const json = await muQuery(` + const json = await query(` SELECT ?agent WHERE { ?agent ${u(T.type)} ${u(T.softwareAgent)} . } LIMIT 2`); const rows = bindings(json); if (rows.length !== 1) { @@ -111,12 +124,12 @@ export async function findAssistant() { // One message, both types, its bijlagen, and the conversation touched, in // one update. `title` is only written when given (the first question names -// an unnamed conversation). Every attachment is the logical uri of a file -// that already exists. +// an unnamed conversation). attachments are the logical uris of files that +// already exist. export async function writeMessage({ conversationUri, content, maker, attachments = [], title }) { - const id = randomUUID(); + const id = uuid(); const uri = `${MESSAGE_BASE}${id}`; - const now = dt(new Date()); + const now = sparqlEscapeDateTime(new Date()); // Predicate pairs joined with ';', never a trailing one: it breaks the Turtle. const pairs = [ @@ -128,15 +141,14 @@ export async function writeMessage({ conversationUri, content, maker, attachment `${u(T.created)} ${now}`, ]; if (maker) pairs.push(`${u(T.maker)} ${u(maker)}`); - for (const attachment of attachments) { - if (!attachment.uri) throw new Error('an attachment without a file uri'); - pairs.push(`${u(T.attachment)} ${u(attachment.uri)}`); + for (const fileUri of attachments) { + pairs.push(`${u(T.attachment)} ${u(fileUri)}`); } const conversationPairs = [`${u(T.lastActivity)} ${now}`]; if (title) conversationPairs.push(`${u(T.title)} ${s(title)}`); - await muUpdate(` + await update(` DELETE { ${u(conversationUri)} ${u(T.lastActivity)} ?old . } @@ -150,3 +162,63 @@ export async function writeMessage({ conversationUri, content, maker, attachment return { uri, id }; } + +// --- files -------------------------------------------------------------------- + +export function writeShareFile(fileName, content) { + writeFileSync(`${SHARE_DIR}/${fileName}`, content, 'utf8'); +} + +// storeSpecFile(spec) → the file uri of a new spec bijlage. A spec file +// keeps one name for the logical and the physical file: readSpecFile finds +// it by the name in the history. +export async function storeSpecFile(spec) { + const fileName = `specificatie-${uuid()}.ttl`; + writeShareFile(fileName, spec); + return registerFile(fileName, SPEC_MEDIA_TYPE, SPEC_TYPE); +} + +// readSpecFile(fileName) → the Turtle of a spec bijlage, null when there is +// no such spec file. +export function readSpecFile(fileName) { + if (!SPEC_FILE_NAME.test(fileName || '')) return null; + try { + return readFileSync(`${SHARE_DIR}/${fileName}`, 'utf8'); + } catch { + return null; + } +} + +// registerFile(shareName, format, type, name) → the logical file uri. One +// logical file called `name` over the physical share file `shareName`, both +// nfo:FileDataObject, both typed with dct:type. A share name must be +// unique: two files on one share name overwrite each other. +export async function registerFile(shareName, format, type, name = shareName) { + const id = uuid(); + const physicalId = uuid(); + const logical = `${FILE_BASE}${id}`; + const physical = `share://${shareName}`; + const size = statSync(`${SHARE_DIR}/${shareName}`).size; + const now = sparqlEscapeDateTime(new Date()); + + await update(` + INSERT DATA { + ${u(logical)} ${u(T.type)} ${u(T.file)} ; + ${u(T.uuid)} ${s(id)} ; + ${u(T.fileName)} ${s(name)} ; + ${u(T.format)} ${s(format)} ; + ${u(T.fileSize)} ${sparqlEscapeInt(size)} ; + ${u(T.created)} ${now} ; + ${u(T.fileType)} ${u(type)} . + ${u(physical)} ${u(T.type)} ${u(T.file)} ; + ${u(T.uuid)} ${s(physicalId)} ; + ${u(T.fileName)} ${s(shareName)} ; + ${u(T.format)} ${s(format)} ; + ${u(T.fileSize)} ${sparqlEscapeInt(size)} ; + ${u(T.fileCreated)} ${now} ; + ${u(T.dataSource)} ${u(logical)} ; + ${u(T.fileType)} ${u(type)} . + }`); + + return logical; +} diff --git a/src/chat/attachment-file.js b/src/chat/attachment-file.js deleted file mode 100644 index dbdf284..0000000 --- a/src/chat/attachment-file.js +++ /dev/null @@ -1,81 +0,0 @@ -// A file made to sit on a message: written to the share, registered as a -// logical nfo:FileDataObject over a physical share:// object, typed with -// dct:type. The file service serves /files//download for it. -import { writeFileSync } from 'fs'; -import { randomUUID } from 'crypto'; -import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString, - sparqlEscapeInt, sparqlEscapeDateTime } from '../db.js'; -import { ATTACHMENT_TYPES, SPEC_MEDIA_TYPE } from './vocab.js'; - -const SHARE_DIR = process.env.SHARE_DIR || '/share'; -const FILE_BASE = 'http://data.lblod.info/files/'; -const SHARE_BASE = 'share://'; -const UUID = 'http://mu.semte.ch/vocabularies/core/uuid'; - -const NFO = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#'; -const FILE = `${NFO}FileDataObject`; -const FILE_NAME = `${NFO}fileName`; -const FILE_SIZE = `${NFO}fileSize`; -const FILE_CREATED = `${NFO}fileCreated`; -const FORMAT = 'http://purl.org/dc/terms/format'; -const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; -const FILE_TYPE = 'http://purl.org/dc/terms/type'; -const DATA_SOURCE = 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource'; - -const file = sparqlEscapeUri; -const str = sparqlEscapeString; - -// storeSpecFile(update, content) → { uri, name }. One spec, one unique file: -// spec bijlagen keep their name apart from the report's ttl-file. Every -// spec file starts with "specificatie-", the read_spec tool opens only -// those. -export async function storeSpecFile(update, content) { - const fileName = `specificatie-${randomUUID()}.ttl`; - writeFileSync(`${SHARE_DIR}/${fileName}`, content, 'utf8'); - return registerFileObject(update, { - fileName, - format: SPEC_MEDIA_TYPE, - size: Buffer.byteLength(content), - type: ATTACHMENT_TYPES.spec, - }); -} - -// registerFileObject(update, { fileName, format, size, type }) → { uri, name }. -// One logical file over one physical share:// file, both nfo:FileDataObject, -// both carrying the dct:type when given. -export async function registerFileObject(update, { fileName, format, size, type }) { - const id = randomUUID(); - const physicalId = randomUUID(); - const logical = `${FILE_BASE}${id}`; - const physical = `${SHARE_BASE}${fileName}`; - const now = sparqlEscapeDateTime(new Date()) // full URI, xsd spelled out - .replace('xsd:dateTime', ''); - - const logicalPairs = [ - `${file(RDF_TYPE)} ${file(FILE)}`, - `${file(UUID)} ${str(id)}`, - `${file(FILE_NAME)} ${str(fileName)}`, - `${file(FORMAT)} ${str(format)}`, - `${file(FILE_SIZE)} ${sparqlEscapeInt(size)}`, - `${file('http://purl.org/dc/terms/created')} ${now}`, - ...(type ? [`${file(FILE_TYPE)} ${file(type)}`] : []), - ]; - const physicalPairs = [ - `${file(RDF_TYPE)} ${file(FILE)}`, - `${file(UUID)} ${str(physicalId)}`, - `${file(FILE_NAME)} ${str(fileName)}`, - `${file(FORMAT)} ${str(format)}`, - `${file(FILE_SIZE)} ${sparqlEscapeInt(size)}`, - `${file(FILE_CREATED)} ${now}`, - `${file(DATA_SOURCE)} ${file(logical)}`, - ...(type ? [`${file(FILE_TYPE)} ${file(type)}`] : []), - ]; - - await (update || sessionUpdate)(` - INSERT DATA { - ${file(logical)} ${logicalPairs.join(' ;\n ')} . - ${file(physical)} ${physicalPairs.join(' ;\n ')} . - }`); - - return { uri: logical, name: fileName }; -} diff --git a/src/chat/index.js b/src/chat/index.js deleted file mode 100644 index 553bc13..0000000 --- a/src/chat/index.js +++ /dev/null @@ -1,86 +0,0 @@ -// The turn endpoint. Mount it with mountChat(app, { path, answer }); -// answer(turn) is the only thing the mounting service writes. -import express from 'express'; -import { query as muQuery, update as muUpdate } from 'mu'; -import { - readConversation, readHistory, findAssistant, - writeMessage, -} from './store.js'; - -const TEXT = { - failed: 'Er ging iets mis. Probeer het opnieuw.', -}; -const TITLE_LENGTH = 80; - -export function mountChat(app, { path = '/assistant', answer }) { - if (typeof answer !== 'function') throw new Error('mountChat needs an answer(turn) function'); - - app.post(`${path}/conversations/:id/turns`, express.json(), async (req, res) => { - const content = String(req.body?.content ?? '').trim(); - if (!content) return res.status(400).json({ error: 'content is required' }); - - // 1. the access check: the caller's session decides what is readable - let conversation; - try { - conversation = await readConversation(req.params.id); - } catch (e) { - console.error('[chat] conversation read failed:', e); - return res.status(500).json({ error: e.message }); - } - if (!conversation) return res.status(404).json({ error: 'no such conversation' }); - - let assistant, history, userMessage; - try { - assistant = process.env.CHAT_ASSISTANT_URI || await findAssistant(); - history = await readHistory(conversation.uri, assistant); - userMessage = await writeMessage({ - conversationUri: conversation.uri, - content, - maker: conversation.creator, - title: conversation.title ? undefined : content.slice(0, TITLE_LENGTH), - }); - } catch (e) { - console.error('[chat] could not record the question:', e); - return res.status(500).json({ error: e.message }); - } - - // 2. answer 202 now; the assistant's message comes when it comes. The - // continuation still runs as the caller. - res.status(202).json({ id: userMessage.id }); - - const turn = { - conversation: { uri: conversation.uri, id: conversation.id, title: conversation.title }, - history, - content, - query: muQuery, - update: muUpdate, - finished: false, - say: (text, attachments = []) => writeMessage({ - conversationUri: conversation.uri, content: text, maker: assistant, attachments, - }), - }; - - try { - // answer may return a string, or { text, attachments } for answers the - // assistant writes with an attachment, such as the report spec. - const result = await answer(turn); - const text = typeof result === 'string' ? result : result?.text; - const attachments - = typeof result === 'string' ? [] : (result?.attachments ?? []); - if (typeof text === 'string' && text.trim()) { - await turn.say(text.trim(), attachments); - } - } catch (e) { - console.error('[chat] turn failed:', e); - try { - await turn.say(TEXT.failed); - } catch (e2) { - console.error('[chat] could not write the failure message:', e2); - } - } finally { - turn.finished = true; - } - }); - - console.log(`[chat] turns at POST ${path}/conversations/:id/turns`); -} diff --git a/src/chat/vocab.js b/src/chat/vocab.js deleted file mode 100644 index c383670..0000000 --- a/src/chat/vocab.js +++ /dev/null @@ -1,8 +0,0 @@ -// The bijlage types of the report assistant. Match the -// add-chat-attachment-types-codelist migration in the portal. -export const ATTACHMENT_TYPES = { - spec: 'http://lblod.data.gift/concepts/6693014e-7b7d-448c-bb24-80c9758521f3', - result: 'http://lblod.data.gift/concepts/28d97698-e219-4a92-bdbd-7ab367a7524e', -}; -export const SPEC_BASE_NAME = 'specificatie'; -export const SPEC_MEDIA_TYPE = 'text/turtle'; diff --git a/src/db.js b/src/db.js deleted file mode 100644 index ccb2522..0000000 --- a/src/db.js +++ /dev/null @@ -1,37 +0,0 @@ -// Thin re-export of mu's SPARQL helpers so the runner files share one import -// site. mu's query/update attach the caller's session from the request -// context, and mu-authorization decides graphs and visibility; the service -// never touches an auth header. -import { - query as muQuery, - update as muUpdate, - sparqlEscapeUri, - sparqlEscapeString, - sparqlEscapeInt, - sparqlEscapeDateTime, - sparqlEscapeDate, - sparqlEscapeBool, -} from 'mu'; - -export const sessionUpdate = muUpdate; -export { - sparqlEscapeUri, - sparqlEscapeString, - sparqlEscapeInt, - sparqlEscapeDateTime, - sparqlEscapeDate, - sparqlEscapeBool, -}; - -// The service's own scope. Reads the LLM drives during refinement (code -// lists, lookup_values) run under this scope, not the caller's session. The -// sparql-parser config grants this scope read access to the public graph -// only, so refinement reads only public data. Execution still runs as the -// caller. Set to match the with-scope grant in config.lisp. -export const SERVICE_SCOPE = - process.env.SERVICE_SCOPE || 'http://services.semantic.works/natural-language-report'; - -// Wrap a mu query/update so it carries the service scope. mu's query/update -// accept { scope } on mu-javascript-template >= 1.9.0. -export function scopedQuery(q) { return muQuery(q, { scope: SERVICE_SCOPE }); } -export function scopedUpdate(q) { return muUpdate(q, { scope: SERVICE_SCOPE }); } \ No newline at end of file diff --git a/src/llm/describe.js b/src/llm/describe.js index 5d09847..4cf661c 100644 --- a/src/llm/describe.js +++ b/src/llm/describe.js @@ -1,59 +1,9 @@ // A profile as a plain text menu: entities, fields, code lists. Code lists up -// to INLINE_VALUES_MAX are inlined and cached for VALUES_TTL, so the LLM -// never has to search the short ones. +// to INLINE_VALUES_MAX are inlined (and cached), so the LLM never has to +// search the short ones. +import { codeList, INLINE_VALUES_MAX } from './lookup.js'; -import { fieldsOf, entityForClass } from '../runner/profile.js'; - -const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50); -const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000; - -// In-memory cache: profile+field → { values, at }. Refreshed lazily when stale. -const cache = new Map(); - -// codeListValues(profile, queryFn) → map "shapeUri|predicate|inverse" -// → [{ uri, label }]. queryFn runs under the service scope (public graph), so -// code lists read the same regardless of the caller's rights. -export async function codeListValues(profile, queryFn) { - const out = {}; - for (const shape of profile.shapes) { - for (const f of shape.fields) { - if (!f.class) continue; - const key = `${shape.uri}|${f.path}|${f.inverse ? 1 : 0}`; - out[key] = await loadCodeList(f, key, queryFn); - } - } - return out; -} - -async function loadCodeList(f, key, queryFn) { - if (!f.class) return []; - const hit = cache.get(key); - if (hit && Date.now() - hit.at < VALUES_TTL) return hit.values; - let values = []; - try { - values = await queryCodeList(f.class, queryFn); - } catch (e) { - // the read failed; report nothing, the LLM will use rep:anyOf with words - values = []; - } - cache.set(key, { values, at: Date.now() }); - return values; -} - -async function queryCodeList(type, queryFn) { - const q = `SELECT ?uri ?label WHERE { - ?uri a <${type}> . - OPTIONAL { ?uri ?label . } - } ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`; - const r = await queryFn(q); - const bindings = r.results.bindings; - return bindings.slice(0, INLINE_VALUES_MAX).map(b => ({ - uri: b.uri.value, - label: b.label?.value || b.uri.value, - })); -} - -export function describeProfile(profile, codeLists = {}) { +export async function describeProfile(profile) { const lines = []; lines.push('A spec is Turtle, in exactly this shape (copy these prefixes):'); lines.push('@prefix rep: .'); @@ -80,13 +30,12 @@ export function describeProfile(profile, codeLists = {}) { for (const shape of profile.shapes) { lines.push(`${shape.label || shape.targetClass} — sh:targetClass <${shape.targetClass}> ; rep:entity <${shape.uri}>`); for (const f of shape.fields) { - const key = `${shape.uri}|${f.path}|${f.inverse ? 1 : 0}`; const kind = f.datatype ? datatypeName(f.datatype) - : f.class ? `→ ${conceptLabel(f.class, codeLists[key])}` + : f.class ? `→ ${conceptLabel(f.class, await codeList(f.class))}` : f.node ? `→ ${linkedLabel(profile, f.node)}` : '?'; const dir = f.inverse ? ' (inverse)' : ''; - const pathShown = typeof f.path === 'string' ? f.path : '?'; + const pathShown = f.path || '?'; lines.push(` ${f.name || pathShown} sh:path <${pathShown}>${dir} ${kind}`); } lines.push(''); @@ -95,11 +44,10 @@ export function describeProfile(profile, codeLists = {}) { } function conceptLabel(type, values) { - const head = values.slice(0, INLINE_VALUES_MAX); - if (!head.length) return short(type); - const names = head.map(v => v.label).join(', '); + if (!values.length) return short(type); + const names = values.map(v => v.label).join(', '); const more = values.length >= INLINE_VALUES_MAX ? ', …' : ''; - return `${short(type)} — ${head.length} waarden: ${names}${more}`; + return `${short(type)} — ${values.length} waarden: ${names}${more}`; } function linkedLabel(profile, nodeUri) { diff --git a/src/llm/lookup.js b/src/llm/lookup.js index de1764c..9793ee5 100644 --- a/src/llm/lookup.js +++ b/src/llm/lookup.js @@ -1,98 +1,118 @@ -// Search a code list too long to inline: regex on str(?label), so -// language-tagged labels match. Returns the matches, the total, and an -// `exact` flag when a label equals the term (case aside). +// The LLM's own reads: code lists. They run under the service scope, not +// the caller's session; the sparql-parser config grants that scope read +// access to the public graph only, so they read the same for every caller. +// Execution still runs as the caller. // -// This is a suggest helper, not a spec check. It reads candidate values the -// LLM can put in a filter; it never runs the spec and never sees report -// rows. queryFn runs under the service scope (public graph only), so the -// suggestions come from the same data set the profile describes, regardless -// of who calls. - -import { sparqlEscapeString } from '../db.js'; +// lookup_values is a suggest helper, not a spec check. It reads candidate +// values the LLM can put in a filter; it never runs the spec and never sees +// report rows. +import { query, sparqlEscapeString } from 'mu'; +// Must match the with-scope grant in the app's config.lisp. +const SERVICE_SCOPE = process.env.SERVICE_SCOPE || 'http://services.semantic.works/natural-language-report'; +export const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50); +const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000; const LIMIT = 25; -// lookupValues(profile, term, fieldSpec, queryFn) → -// { total, exact, matches } or { error }. -// field is "shapeLabel.fieldLabel" as the LLM writes it, or a predicate URI. -// We resolve it to the field's sh:class to know what type to search. -export async function lookupValues(profile, term, fieldSpec, queryFn) { - const type = resolveClass(profile, fieldSpec); - if (!type) return { total: 0, exact: null, matches: [], error: `field "${fieldSpec}" not found in profile "${profile.title}"` }; +// class → { values, at }. Refreshed lazily when stale. +const cache = new Map(); + +function publicQuery(q) { + return query(q, { scope: SERVICE_SCOPE }); +} + +// codeList(type) → up to INLINE_VALUES_MAX [{ uri, label }], cached for +// VALUES_TTL. Empty when the read fails; the LLM then uses rep:anyOf with +// words. +export async function codeList(type) { + const hit = cache.get(type); + if (hit && Date.now() - hit.at < VALUES_TTL) return hit.values; + let values = []; + try { + const r = await publicQuery(`SELECT ?uri ?label WHERE { + ?uri a <${type}> . + OPTIONAL { ?uri ?label . } + } ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`); + values = r.results.bindings.slice(0, INLINE_VALUES_MAX).map(b => ({ + uri: b.uri.value, + label: b.label?.value || b.uri.value, + })); + } catch (e) { + values = []; + } + cache.set(type, { values, at: Date.now() }); + return values; +} + +// lookupValues(profile, field, term) → the matches as text for the LLM: up +// to LIMIT labels and URIs, an exact label (case aside) first. field is +// "shapeLabel.fieldLabel" as the LLM writes it; it resolves to the class to +// search. +export async function lookupValues(profile, field, term) { + const type = resolveClass(profile, field); + if (!type) return `field "${field}" not found in profile "${profile.title}"`; // Search within the class first. A common word ("gemeente") also sits in // hundreds of unrelated labels; an unscoped scan drowns the code. - let rows = await labelQuery(term, type, queryFn); + let rows = await labelQuery(term, type); // Code lists sometimes type their entries differently than the profile // says (MAR codes are ext:supervision/Nomenclature, the profile says // skos:Concept). Fall back to a scan over everything. - if (!rows.length) rows = await labelQuery(term, null, queryFn); - // Order: an exact label (case aside) first, then the rest as the store - // ordered them. "Gent" must surface before "AGB Erfgoed Gent". + if (!rows.length) rows = await labelQuery(term, null); + // "Gent" must surface before "AGB Erfgoed Gent". const lower = term.toLowerCase(); - const exactRow = rows.find(b => b.label.toLowerCase() === lower); - const rest = rows.filter(b => b !== exactRow); - const ordered = exactRow ? [exactRow, ...rest] : rest; - let matches = ordered.slice(0, LIMIT); - if (!matches.length) { - const byCode = await codeQuery(profile, type, term, queryFn); - matches = byCode; - } - const exact = matches.find(m => m.label.toLowerCase() === term.toLowerCase()); - return { - total: matches.length >= LIMIT ? `${LIMIT}+` : matches.length, - exact: exact ? exact.uri : null, - matches, - }; + const exactRow = rows.find(r => r.label.toLowerCase() === lower); + let matches = (exactRow ? [exactRow, ...rows.filter(r => r !== exactRow)] : rows).slice(0, LIMIT); + if (!matches.length) matches = await codeQuery(term); + if (!matches.length) return `no value matches "${term}" in ${field}.`; + + const lines = matches.map(m => `${m.label} <${m.uri}>`); + if (matches.length >= LIMIT) lines.push(`(${LIMIT}+ matches, narrow the term)`); + const exact = matches.find(m => m.label.toLowerCase() === lower); + if (exact) lines.push(`exact: <${exact.uri}>`); + return lines.join('\n'); } -async function labelQuery(term, type, queryFn) { +// Regex on str(?label), so language-tagged labels match. +async function labelQuery(term, type) { const q = `SELECT DISTINCT ?uri ?label WHERE { ?uri a ${type ? `<${type}>` : '?anyType'} ; ?label . FILTER(REGEX(str(?label), ${sparqlEscapeString(escapeRegex(term))}, "i")) } ORDER BY ?label LIMIT ${LIMIT * 8}`; - const r = await queryFn(q); + const r = await publicQuery(q); return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value })); } // MAR codes: the label says "MAR7300 - …" but the searchable code sits on // ext:supervision/nomenclatureCode. One extra query, only when the label // search came up empty. -async function codeQuery(profile, type, term, queryFn) { +async function codeQuery(term) { const q = `SELECT ?uri ?label WHERE { ?uri ?code ; ?label . FILTER(CONTAINS(LCASE(str(?code)), LCASE(${sparqlEscapeString(term)}))) } ORDER BY ?label LIMIT ${LIMIT}`; try { - const r = await queryFn(q); + const r = await publicQuery(q); return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value })); } catch { return []; } } -function resolveClass(profile, fieldSpec) { - if (fieldSpec.includes('/')) return null; - // "shapeLabel.fieldLabel" → find the field carrying sh:class - const dot = fieldSpec.lastIndexOf('.'); - if (dot < 0) { - // bare predicate URI - for (const shape of profile.shapes) { - for (const f of shape.fields) { - if (f.path === fieldSpec && f.class) return f.class; - } - } - return null; - } - const shapeLabel = fieldSpec.slice(0, dot); - const fieldLabel = fieldSpec.slice(dot + 1); +// "shapeLabel.fieldLabel" → the class to search: the field's sh:class, or +// for a link (sh:node) the linked shape's sh:targetClass +// ("melding.bestuurseenheid" → besluit:Bestuurseenheid). +function resolveClass(profile, field) { + if (field.includes('/')) return null; + const dot = field.lastIndexOf('.'); + if (dot < 0) return null; + const shapeLabel = field.slice(0, dot); + const fieldLabel = field.slice(dot + 1); const shape = profile.shapes.find(s => s.label === shapeLabel); if (!shape) return null; const f = shape.fields.find(x => x.name === fieldLabel); - // sh:node marks a link, not a code list; the searchable type is the linked - // shape's sh:targetClass ("melding.bestuurseenheid" → besluit:Bestuurseenheid). if (f?.class) return f.class; if (f?.node) { const linked = profile.shapes.find(s => s.uri === f.node); @@ -114,4 +134,4 @@ function shortType(cls) { function escapeRegex(s) { return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} \ No newline at end of file +} diff --git a/src/llm/loop.js b/src/llm/loop.js index 4a35cb5..2ce38a7 100644 --- a/src/llm/loop.js +++ b/src/llm/loop.js @@ -8,7 +8,8 @@ // confirm before anything executes; execution runs the agreed spec without // the LLM (../report-assistant.js). -import { buildTools } from './tools.js'; +import { TOOLS, runTool } from './tools.js'; +import { SPEC_MEDIA_TYPE } from '../chat.js'; const MAX_ROUNDS = Number(process.env.MAX_ROUNDS || 12); const LLM_BASE_URL = process.env.LLM_BASE_URL; @@ -77,70 +78,49 @@ what it says, up to three rounds. Then propose and stop. Reply to the user in Dutch. Earlier turns are context only. Answer the last one.`; -// ask(questionOrMessages, profiles, session) → { text } -// A string is one question; an array is the conversation so far, -// [{ role, content }], newest last. The system prompt is always this file's -// own; a system message in the input is dropped. -export async function ask(questionOrMessages, profiles, session) { - const turns = typeof questionOrMessages === 'string' - ? [{ role: 'user', content: questionOrMessages }] - : questionOrMessages.filter(m => m.role !== 'system'); +// ask(turns, profiles) → { text, spec }. turns is the conversation so far, +// [{ role, content, attachments }], newest last. spec is the last spec +// validate_spec accepted in this turn, or null. +export async function ask(turns, profiles) { + const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns.map(withSpecHint)]; + let spec = null; - return runLoop(withAttachmentSpecs(turns), profiles, session); + for (let round = 0; round < MAX_ROUNDS; round++) { + const { message } = await chat(messages, TOOLS); + messages.push(message); + const calls = message.tool_calls || []; + if (!calls.length) return { text: message.content || '', spec }; + + for (const call of calls) { + const args = parseArgs(call.function.arguments); + let result; + try { + result = await runTool(call.function.name, args, profiles); + } catch (e) { + result = `tool failed: ${String(e.message || e).split('\n')[0]}`; + } + // validate_spec answers "ok" (plus notes) when the spec checks out; + // that spec becomes the bijlage of the answer. + if (call.function.name === 'validate_spec' && /^ok(\n|$)/.test(result)) spec = args.spec; + messages.push({ role: 'tool', tool_call_id: call.id, content: result }); + } + } + return { text: 'Sorry, dit rapport is te moeilijk. Contacteer de developers.', spec }; } // The history carries the spec as a bijlage. Paste one hint line per spec // under the message text, so the model knows a spec is there and opens it // with read_spec; the chat itself still keeps the Turtle out of the text. -function withAttachmentSpecs(turns) { - return turns.map(t => { - const specs = (t.attachments || []).filter(a => a.mediaType === 'text/turtle' && a.uri); - if (!specs.length) return t; - return { - ...t, - content: `${t.content}\n\n${specs.map(a => - `Bijlage "${a.name}", a spec file (read it with read_spec): ${a.uri}`).join('\n\n')}`, - }; - }); +function withSpecHint(turn) { + const specs = (turn.attachments || []).filter(a => a.mediaType === SPEC_MEDIA_TYPE); + if (!specs.length) return turn; + const hints = specs.map(a => `Bijlage "${a.name}", a spec file (read it with read_spec).`); + return { ...turn, content: `${turn.content}\n\n${hints.join('\n\n')}` }; } -// runLoop drives the agent loop with the refinement prompt and tool set. -async function runLoop(turns, profiles, session) { - const { handlers } = buildTools(profiles, session); - const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns]; - - for (let round = 0; round < MAX_ROUNDS; round++) { - const reply = await chat(messages, { tools: toolDefs() }); - messages.push(reply.message); - const calls = reply.message.tool_calls || []; - if (!calls.length) { - return { text: reply.message.content || '' }; - } - for (const call of calls) { - const args = parseArgs(call.function.arguments); - const fn = handlers.get(call.function.name); - let resultText; - if (!fn) { - resultText = `unknown tool "${call.function.name}"`; - } else { - try { - const out = await fn(args); - resultText = out.content?.[0]?.text ?? JSON.stringify(out); - } catch (e) { - resultText = `tool failed: ${String(e.message || e).split('\n')[0]}`; - } - } - messages.push({ role: 'tool', tool_call_id: call.id, content: resultText }); - } - } - return { text: '(loop did not finish)' }; -} - -// chat(messages, { tools, toolChoice }) → { message } -// The one place that talks to the provider. tools null (the default) means a -// plain completion (the mode classifier); runLoop passes the tool -// definitions; toolChoice defaults to 'auto'. -export async function chat(messages, { tools = null, toolChoice = 'auto' } = {}) { +// chat(messages, tools) → { message }. The one place that talks to the +// provider. Without tools it is a plain completion (the mode check). +export async function chat(messages, tools = null) { // Send only the fields every provider accepts back. const wire = messages.map(m => { const out = { role: m.role, content: m.content ?? null }; @@ -154,13 +134,10 @@ export async function chat(messages, { tools = null, toolChoice = 'auto' } = {}) if (m.role === 'tool' && m.tool_call_id) out.tool_call_id = m.tool_call_id; return out; }); - const body = { - model: LLM_MODEL, - messages: wire, - }; + const body = { model: LLM_MODEL, messages: wire }; if (tools) { body.tools = tools; - body.tool_choice = toolChoice === 'auto' ? 'auto' : toolChoice; + body.tool_choice = 'auto'; } if (LOG_LLM) { console.log(`[llm] POST ${LLM_BASE_URL}/chat/completions request:\n${ @@ -191,16 +168,3 @@ function parseArgs(s) { if (!s) return {}; try { return JSON.parse(s); } catch { return {}; } } - -// The tool definitions sent to the model. -import { toolSchemas } from './schemas.js'; -function toolDefs() { - return toolSchemas().map(t => ({ - type: 'function', - function: { - name: t.name, - description: t.description, - parameters: t.inputSchema, - }, - })); -} \ No newline at end of file diff --git a/src/llm/mode.js b/src/llm/mode.js index 2cf3000..68351a0 100644 --- a/src/llm/mode.js +++ b/src/llm/mode.js @@ -25,43 +25,18 @@ Rules: Reply with exactly one token: true or false. No other words, no punctuation, no markdown.`; -// classifyMode(turns) → boolean (true = execute now) -// turns: [{ role, content }], newest last. A leading system message is -// dropped; the assistant's own system prompt is never sent to the model. -export async function classifyMode(turns) { - const convo = (turns || []) - .filter(m => m.role !== 'system') - .map(m => ({ role: m.role === 'assistant' ? 'assistant' : 'user', content: m.content ?? '' })); - if (!convo.length) return false; - - // Keep the last few turns; the decision is about the latest message. - const recent = convo.slice(-6); - const messages = [{ role: 'system', content: MODE_PROMPT }, ...recent]; - let raw; +// wantsExecution(turns) → true when the user's last message says "run it +// now". turns: [{ role, content }], newest last; the last few are enough. +export async function wantsExecution(turns) { + let message; try { - const reply = await chat(messages, { tools: null }); - raw = firstText(reply.message); + ({ message } = await chat([{ role: 'system', content: MODE_PROMPT }, ...turns.slice(-6)])); } catch (e) { - // The classifier call failed (no LLM, provider down). Default to refine: - // safer than running a report on a guess, and the refine loop will - // surface the real error to the user. + // The call failed (no LLM, provider down). Refine: safer than running a + // report on a guess, and the refine loop will surface the real error. return false; } - const text = stripMarkdown(raw).trim().toLowerCase(); - if (!text) return false; - if (text.startsWith('true')) return true; - if (text.startsWith('false')) return false; - // A JSON {"execute": ...} shape some models emit. - try { - const m = text.match(/\{[\s\S]*\}/); - if (m) { - const v = JSON.parse(m[0]); - if (typeof v === 'boolean') return v; - if (v && typeof v.execute === 'boolean') return v.execute; - if (v && typeof v.refine === 'boolean') return !v.refine; - } - } catch { /* ignore */ } - return false; + return stripMarkdown(firstText(message)).trim().toLowerCase().startsWith('true'); } // Reasoning models may leave `content` empty and put the visible text in a @@ -78,4 +53,4 @@ function firstText(message) { function stripMarkdown(s) { return String(s).replace(/[*_`>#]/g, ' '); -} \ No newline at end of file +} diff --git a/src/llm/schemas.js b/src/llm/schemas.js deleted file mode 100644 index 4718501..0000000 --- a/src/llm/schemas.js +++ /dev/null @@ -1,57 +0,0 @@ -// The JSON schemas of the tools. The descriptions are what the LLM reads to -// decide which tool to call. Refinement only: list_profiles, -// describe_profile, validate_spec, read_spec and lookup_values. - -const COMMON = [ - { - name: 'list_profiles', - description: 'List the report profiles available, with their id and title.', - inputSchema: { type: 'object', properties: {}, required: [] }, - }, - { - name: 'describe_profile', - description: 'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.', - inputSchema: { - type: 'object', - properties: { profile_id: { type: 'string', description: 'The profile URI returned by list_profiles' } }, - required: ['profile_id'], - }, - }, - { - name: 'validate_spec', - description: 'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.', - inputSchema: { - type: 'object', - properties: { spec: { type: 'string', description: 'The spec as Turtle text' } }, - required: ['spec'], - }, - }, -]; - -const LOOKUP_VALUES = { - name: 'lookup_values', - description: 'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.', - inputSchema: { - type: 'object', - properties: { - profile_id: { type: 'string', description: 'The profile URI returned by list_profiles' }, - field: { type: 'string', description: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"' }, - term: { type: 'string', description: 'The Dutch word or name to search for' }, - }, - required: ['profile_id', 'field', 'term'], - }, -}; - -const READ_SPEC = { - name: 'read_spec', - description: 'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.', - inputSchema: { - type: 'object', - properties: { file_uri: { type: 'string', description: 'The file URI of the spec bijlage, e.g. http://data.lblod.info/files/' } }, - required: ['file_uri'], - }, -}; - -export function toolSchemas() { - return [...COMMON, READ_SPEC, LOOKUP_VALUES]; -} \ No newline at end of file diff --git a/src/llm/tools.js b/src/llm/tools.js index abb3e3b..55abf1c 100644 --- a/src/llm/tools.js +++ b/src/llm/tools.js @@ -1,94 +1,91 @@ -// The tools the LLM calls, refinement only: list_profiles, describe_profile, -// validate_spec, lookup_values, read_spec. lookup_values searches a code list -// under the service scope (public graph) and returns candidate values, so the -// LLM can suggest concrete filter values. No report is created, no spec is -// evaluated. -// -// The shape every tool returns is { content: [{ type: 'text', text }] }, -// which the loop passes straight back to the model. - -import { checkSpec, whereNotes } from '../runner/check.js'; +// The tools the LLM calls, refinement only: list_profiles, +// describe_profile, validate_spec, read_spec and lookup_values. Each returns +// plain text for the model. No report is created, no spec is evaluated; +// the code-list reads run under the service scope (public graph). import { parseSpec } from '../runner/spec.js'; -import { describeProfile, codeListValues } from './describe.js'; +import { checkSpec, whereNotes, profileError } from '../runner/check.js'; +import { describeProfile } from './describe.js'; import { lookupValues } from './lookup.js'; -import { scopedQuery } from '../db.js'; -import { readFileSync } from 'fs'; +import { readSpecFile } from '../chat.js'; -const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8); +// The tool definitions sent to the model. The descriptions are what the LLM +// reads to decide which tool to call. +export const TOOLS = [ + tool('list_profiles', + 'List the report profiles available, with their id and title.', + {}), + tool('describe_profile', + 'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.', + { profile_id: 'The profile URI returned by list_profiles' }), + tool('validate_spec', + 'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.', + { spec: 'The spec as Turtle text' }), + tool('read_spec', + 'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.', + { file_name: 'The file name of the spec bijlage, e.g. specificatie-.ttl' }), + tool('lookup_values', + 'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.', + { + profile_id: 'The profile URI returned by list_profiles', + field: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"', + term: 'The Dutch word or name to search for', + }), +]; -// buildTools(profiles, session) → { handlers } -export function buildTools(profiles, session) { - const profileList = () => [...profiles.values()].map(p => ({ id: p.uri, title: p.title })); - - const findProfile = (id) => profiles.get(id); - - async function describe_profile({ profile_id }) { - const p = findProfile(profile_id); - if (!p) return text(`no profile <${profile_id}>. Available: ${profileList().map(x => x.title).join(', ')}.`); - const inlined = await codeListValues(p, scopedQuery); - return text(describeProfile(p, inlined)); +function tool(name, description, params) { + const properties = {}; + for (const [key, text] of Object.entries(params)) { + properties[key] = { type: 'string', description: text }; } - - async function validate_spec({ spec }) { - let parsed; - try { parsed = parseSpec(spec); } - catch (e) { return text(`the spec did not parse: ${e.message}`); } - const p = findProfile(parsed.profileUri); - if (!p) return text(`no profile <${parsed.profileUri}>. Available: ${profileList().map(x => x.title).join(', ')}.`); - const errors = checkSpec(parsed, p, MAX_PATH_DEPTH, profiles); - if (errors.length) return text(errors[0]); - // The spec that checks out becomes the attached proposal. - session.onSpecValidated?.(spec); - return text(['ok', ...whereNotes(parsed, p)].join('\n')); - } - - async function list_profiles() { - return text(profileList().map(x => `${x.title} <${x.id}>`).join('\n')); - } - - async function lookup_values({ profile_id, field, term }) { - const p = findProfile(profile_id); - if (!p) return text(`no profile <${profile_id}>. Available: ${profileList().map(x => x.title).join(', ')}.`); - if (!term) return text('no term to search for.'); - try { - const out = await lookupValues(p, term, field, scopedQuery); - if (out.error) return text(out.error); - if (!out.matches.length) return text(`no value matches "${term}" in ${field}.`); - const lines = out.matches.map(m => `${m.label} <${m.uri}>`); - if (out.total === '25+') lines.push('(25+ matches, narrow the term)'); - if (out.exact) lines.push(`exact: <${out.exact}>`); - return text(lines.join('\n')); - } catch (e) { - return text(`the lookup failed: ${String(e.message || e).split('\n')[0]}`); - } - } - - // A spec bijlage next to a message. The model opens it with read_spec: - // only files this service made are readable, hte spec files under share. - async function read_spec({ file_uri }) { - const fileName = String(file_uri || '').split('/').pop(); - if (!fileName || !/^specificatie-[0-9a-f]{8}-[0-9a-f-]{27}\.ttl$/.test(fileName)) { - return text('no such spec file.'); - } - const shareDir = process.env.SHARE_DIR || '/share'; - try { - return text(readFileSync(`${shareDir}/${fileName}`, 'utf8')); - } catch (e) { - return text(`the spec failed to load: ${String(e.message || e).split('\n')[0]}`); - } - } - - const handlers = new Map( - [ - ['list_profiles', list_profiles], - ['describe_profile', describe_profile], - ['validate_spec', validate_spec], - ['read_spec', read_spec], - ['lookup_values', lookup_values], - ], - ); - - return { handlers }; + return { + type: 'function', + function: { + name, + description, + parameters: { type: 'object', properties, required: Object.keys(params) }, + }, + }; } -function text(s) { return { content: [{ type: 'text', text: s }] }; } +export async function runTool(name, args, profiles) { + switch (name) { + case 'list_profiles': return listProfiles(profiles); + case 'describe_profile': return describe(args, profiles); + case 'validate_spec': return validateSpec(args, profiles); + case 'read_spec': return readSpec(args); + case 'lookup_values': return lookup(args, profiles); + default: return `unknown tool "${name}"`; + } +} + +function listProfiles(profiles) { + return [...profiles.values()].map(p => `${p.title} <${p.uri}>`).join('\n'); +} + +async function describe({ profile_id }, profiles) { + return profileError(profiles, profile_id) || describeProfile(profiles.get(profile_id)); +} + +function validateSpec({ spec }, profiles) { + let parsed; + try { + parsed = parseSpec(spec); + } catch (e) { + return `the spec did not parse: ${e.message}`; + } + const error = checkSpec(parsed, profiles); + if (error) return error; + return ['ok', ...whereNotes(parsed, profiles.get(parsed.profileUri))].join('\n'); +} + +// Only spec files this service made are readable. +function readSpec({ file_name }) { + return readSpecFile(file_name) || 'no such spec file.'; +} + +async function lookup({ profile_id, field, term }, profiles) { + const error = profileError(profiles, profile_id); + if (error) return error; + if (!term) return 'no term to search for.'; + return lookupValues(profiles.get(profile_id), field, term); +} diff --git a/src/report-assistant.js b/src/report-assistant.js index 8f1f786..3c656ef 100644 --- a/src/report-assistant.js +++ b/src/report-assistant.js @@ -1,129 +1,71 @@ -// What this service answers a chat turn with. The service is synchronous -// per turn: when the answer is written, every bijlage already exists. +// What the assistant answers a chat turn with: { text, attachments }. When +// the answer is written, every bijlage already exists. // -// Execute turns skip the LLM loop. The mode classifier says the user wants -// to run the report now; the agreed spec already sits in the conversation -// as a bijlage, so it is read from disk and fed straight to the runner. -// The answer is hard-coded: no extra prompt runs for it. When the run cannot -// happen or fails, the answer is a hard-coded failure message: nothing is -// retried and no spec is rewritten. -import { classifyMode } from './llm/mode.js'; +// Refine: the LLM proposes a spec; the last spec that checked out goes on +// the answer as a bijlage. The prompt never carries the Turtle. +// +// Execute: no LLM runs. The last spec bijlage in the conversation is the +// agreed spec; it is read from the share and fed straight to the runner. +// The answer is fixed text. When the run cannot happen or fails, the answer +// is a fixed failure message: nothing is retried and no spec is rewritten. +import { uuid } from 'mu'; import { ask } from './llm/loop.js'; import { parseSpec } from './runner/spec.js'; import { checkSpec } from './runner/check.js'; -import { run, slug } from './runner/run.js'; -import { readFileSync } from 'fs'; -import { storeSpecFile, registerFileObject } from './chat/attachment-file.js'; -import { SPEC_MEDIA_TYPE, ATTACHMENT_TYPES } from './chat/vocab.js'; +import { run, registerReport, slug } from './runner/run.js'; +import { storeSpecFile, readSpecFile, writeShareFile, registerFile, + SPEC_MEDIA_TYPE, SPEC_TYPE, RESULT_TYPE } from './chat.js'; const TEXT = { executed: 'Het rapport is uitgevoerd. De bijlagen staan erbij.', noSpec: 'Er is nog geen voorstel om uit te voeren. Beschrijf eerst welk rapport je wilt.', invalid: 'Het voorstel is niet meer geldig. Vraag een nieuw voorstel.', failed: 'Het rapport kon niet uitgevoerd worden. Probeer het later opnieuw.', + noAnswer: 'Er kwam geen antwoord. Probeer het opnieuw.', }; -const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8); -const SHARE_DIR = process.env.SHARE_DIR || '/share'; - -// The chat mounts before the profiles are in; whenProfiles waits for the load. -export function reportAssistant(whenProfiles) { - return async function answer(turn) { - const turns = [...turn.history, { role: 'user', content: turn.content }]; - const profiles = await whenProfiles(); - - if (await classifyMode(turns)) { - try { - return await runStoredSpec(turn, profiles); - } catch (e) { - console.error('[reports] execution failed:', e); - return TEXT.failed; - } - } - return llmAnswer(turn, turns, profiles); - }; +// An empty answer (a reasoning model can end with no text) gets a fixed +// text and no spec: only a spec the user saw explained may run. +export async function refineSpec(turns, profiles) { + const { text, spec } = await ask(turns, profiles); + if (!text.trim()) return { text: TEXT.noAnswer }; + const attachments = spec ? [await storeSpecFile(spec)] : []; + return { text: text.trim(), attachments }; } -// Execution: the last spec bijlage in the conversation is the agreed spec. -// No prompt runs; the runner gets it straight away. -async function runStoredSpec(turn, profiles) { - const attachment = lastSpecAttachment(turn.history); +export async function executeSpec(history, conversationTitle, profiles) { + const attachment = lastSpecAttachment(history); const content = attachment && readSpecFile(attachment.name); - if (!content) return TEXT.noSpec; + if (!content) return { text: TEXT.noSpec }; - const parsed = parseSpec(content); - const profile = profiles.get(parsed.profileUri); - if (!profile) { - console.error(`[reports] no profile <${parsed.profileUri}>`); - return TEXT.invalid; + try { + const spec = parseSpec(content); + const error = checkSpec(spec, profiles); + if (error) { + console.error(`[reports] the agreed spec is not valid: ${error}`); + return { text: TEXT.invalid }; + } + + // The conversation's title (named by the first question) names the + // file; the share name is unique, so a second run never overwrites the + // first. The report itself keeps the dct:title of the spec. + const shareName = `${uuid()}.csv`; + writeShareFile(shareName, await run(spec, profiles.get(spec.profileUri))); + const csvFile = await registerFile(shareName, 'text/csv', RESULT_TYPE, `${slug(conversationTitle)}.csv`); + await registerReport(spec.title, csvFile); + + // Same physical spec file, registered as a bijlage for this message. + const specFile = await registerFile(attachment.name, SPEC_MEDIA_TYPE, SPEC_TYPE); + + return { text: TEXT.executed, attachments: [csvFile, specFile] }; + } catch (e) { + console.error('[reports] execution failed:', e); + return { text: TEXT.failed }; } - const errors = checkSpec(parsed, profile, MAX_PATH_DEPTH, profiles); - if (errors.length) { - console.error(`[reports] the agreed spec is not valid: ${errors[0]}`); - return TEXT.invalid; - } - - // The conversation's title (named by the first question) names the file. - // The report itself keeps the dct:title of the spec. - const { fileUri } = await run(turn.query, turn.update, parsed, profile, - parsed.title, { - spec: content, - fileType: ATTACHMENT_TYPES.result, - fileName: `${slug(turn.conversation.title)}.csv`, - }); - - // Same physical spec file, registered as a bijlage for this message. - const specFile = await registerFileObject(turn.update, { - fileName: attachment.name, - format: SPEC_MEDIA_TYPE, - size: Buffer.byteLength(content), - type: ATTACHMENT_TYPES.spec, - }); - - return { - text: TEXT.executed, - attachments: [ - { uri: fileUri }, - { ...specFile, mediaType: SPEC_MEDIA_TYPE }, - ], - }; } function lastSpecAttachment(history) { - for (const message of [...(history || [])].reverse()) { - const specs = (message.attachments || []).filter(a => - a.mediaType === SPEC_MEDIA_TYPE - && /^specificatie-[0-9a-f-]+\.ttl$/.test(a.name || '')); - if (specs.length) return specs[specs.length - 1]; - } - return null; -} - -function readSpecFile(fileName) { - try { - return readFileSync(`${SHARE_DIR}/${fileName}`, 'utf8'); - } catch { - return null; - } -} - -// The LLM loop, refinement only. The spec that checked out goes on the -// answer as a bijlage. The prompt never carries the Turtle. -async function llmAnswer(turn, turns, profiles) { - // The last spec that checked out; stored as a file for this turn's - // final message. - let spec = null; - - const session = { - onSpecValidated(validated) { spec = validated; }, - }; - - const { text } = await ask(turns, profiles, session); - - const attachments = []; - if (spec) { - const stored = await storeSpecFile(turn.update, spec); - attachments.push({ ...stored, mediaType: SPEC_MEDIA_TYPE }); - } - return { text, attachments }; + const specs = history.flatMap(message => message.attachments) + .filter(a => a.mediaType === SPEC_MEDIA_TYPE && a.name?.startsWith('specificatie-')); + return specs.at(-1) || null; } diff --git a/src/runner/assemble.js b/src/runner/assemble.js index bc9c3d9..e0cc153 100644 --- a/src/runner/assemble.js +++ b/src/runner/assemble.js @@ -14,7 +14,6 @@ export const NUMERIC_DATATYPES = new Set([ 'nonNegativeInteger', 'positiveInteger', 'nonPositiveInteger', 'negativeInteger', 'unsignedLong', 'unsignedInt', 'unsignedShort', 'unsignedByte', ].map(d => XSD + d)); -export const TEMPORAL_DATATYPES = new Set([XSD + 'date', XSD + 'dateTime']); export function assemble(subjects, values, spec) { const sorted = [...subjects].sort(); @@ -57,19 +56,18 @@ function collect(terms, col) { } } -// min/max over values that share one datatype. Numbers compare as numbers, -// dates and times as instants. Mixed datatypes throw: check.js should have -// caught it, and a silently wrong cell is worse than a failed job. +// min/max. Numbers compare as numbers. Dates and times compare as text: +// xsd:date and xsd:dateTime are ISO strings, so text order is date order. +// Numbers mixed with other values throw: check.js should have caught it, +// and a silently wrong cell is worse than a failed job. The terms are SPARQL +// JSON bindings, so the datatype is a plain string. function extreme(terms, col, direction) { if (!terms.length) return ''; - const datatypes = [...new Set(terms.map(t => t.datatype?.value || ''))]; - if (datatypes.length > 1) { - throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but its values carry ${datatypes.length} datatypes (${datatypes.join(', ')}).`); + const numbers = terms.filter(t => NUMERIC_DATATYPES.has(t.datatype)).length; + if (numbers && numbers < terms.length) { + throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but mixes numbers with other values.`); } - const dt = datatypes[0]; - const key = NUMERIC_DATATYPES.has(dt) ? t => Number(t.value) - : TEMPORAL_DATATYPES.has(dt) ? t => new Date(t.value).getTime() - : t => t.value; + const key = numbers ? t => Number(t.value) : t => t.value; const cmp = (a, b) => a < b ? -1 : a > b ? 1 : 0; const best = terms.reduce((a, b) => (cmp(key(b), key(a)) * direction > 0 ? b : a)); return best.value; diff --git a/src/runner/check.js b/src/runner/check.js index 133f066..1a44598 100644 --- a/src/runner/check.js +++ b/src/runner/check.js @@ -1,4 +1,4 @@ -import { fieldsOf } from './profile.js'; +import { startShape, fieldsOf, field } from './profile.js'; import { NUMERIC_DATATYPES } from './assemble.js'; import { sharedSteps } from './spec.js'; @@ -9,23 +9,18 @@ const XSD_DATE = XSD + 'date'; const XSD_DATETIME = XSD + 'dateTime'; // The validator. checkSpec runs the checks in order and returns the first -// error as a one-element array (empty if ok). Every message names the thing -// that is wrong in the profile's own words and says what to write instead. -// No database call anywhere in this file. +// error, or null when the spec is fine. Every message names the thing that +// is wrong in the profile's own words and says what to write instead. No +// database call anywhere in this file. +const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8); const ANYOF_MAX_TERMS = 24; const ANYOF_MAX_TERM_LENGTH = 100; -// The spec names a profile we have. knownProfiles is the profile Map loaded -// at boot; the check is skipped when it is omitted. -function profileNamed(spec, knownProfiles) { - if (!spec.profileUri) { - return `no rep:profile named. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'} .`; - } - if (knownProfiles && !knownProfiles.has(spec.profileUri)) { - return `no profile <${spec.profileUri}>. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'}.`; - } - return null; +// null when the profile exists, else what to write instead. +export function profileError(profiles, uri) { + if (profiles.has(uri)) return null; + return `no profile <${uri}>. Available: ${[...profiles.values()].map(p => p.title).join(', ')}.`; } // The spec names an entity we have: by rep:entity, or by a targetClass only @@ -74,16 +69,6 @@ function conditions(spec) { return out; } -// The shape a spec starts from, after startEntity has approved it: rep:entity -// wins, otherwise the one shape carrying the targetClass. -function startShapeOf(spec, profile) { - if (spec.entity) { - return profile.shapes.find(s => s.uri === spec.entity)?.uri || null; - } - const matches = profile.shapes.filter(s => s.targetClass === spec.targetClass); - return matches.length === 1 ? matches[0].uri : null; -} - // Columns exist function columnsPresent(spec) { if (!spec.columns.length) return 'no columns. Add at least one rep:columns entry.'; @@ -100,26 +85,26 @@ function columnLabel(spec) { // Labels are unique function columnLabelsUnique(spec) { - const seen = new Map(); + const seen = new Set(); for (const col of spec.columns) { if (seen.has(col.label)) { return `two columns are called "${col.label}". Labels must differ.`; } - seen.set(col.label, true); + seen.add(col.label); } return null; } // Every path resolves, columns end on a value, depth capped -function pathsResolve(spec, profile, maxPathDepth) { - const startShapeUri = startShapeOf(spec, profile); +function pathsResolve(spec, profile) { + const startShapeUri = startShape(profile, spec).uri; for (const col of spec.columns) { - const err = walk(profile, startShapeUri, col.path, maxPathDepth, + const err = walk(profile, startShapeUri, col.path, `column "${col.label}"`, true, col.nodeKind === SH + 'IRI'); if (err) return err; } for (const [i, filter] of spec.filters.entries()) { - const err = walk(profile, startShapeUri, filter.path, maxPathDepth, + const err = walk(profile, startShapeUri, filter.path, `filter ${i + 1}`, false); if (err) return err; } @@ -128,7 +113,7 @@ function pathsResolve(spec, profile, maxPathDepth) { // sh:min/sh:max only on numbers and dates function collectMinMaxTyped(spec, profile) { - const startShapeUri = startShapeOf(spec, profile); + const startShapeUri = startShape(profile, spec).uri; for (const col of spec.columns) { if (col.collect !== SH + 'min' && col.collect !== SH + 'max') continue; const name = col.collect === SH + 'min' ? 'sh:min' : 'sh:max'; @@ -136,8 +121,7 @@ function collectMinMaxTyped(spec, profile) { const last = col.path[col.path.length - 1]; const shapeUri = shapeAfter(profile, startShapeUri, col.path); if (!shapeUri) continue; // pathsResolve already reported the real error - const f = fieldsOf(profile, shapeUri) - .find(x => x.path === last.predicate && !!x.inverse === !!last.inverse); + const f = field(profile, shapeUri, last); if (!f) continue; const kind = f.datatype === XSD_DATETIME || f.datatype === XSD_DATE ? 'date' : NUMERIC_DATATYPES.has(f.datatype) ? 'number' @@ -205,10 +189,10 @@ function filterConstraint(spec) { function maxCountZero(spec) { for (const { cond } of conditions(spec)) { const c = cond.constraints; - if (c.maxCount !== null && c.maxCount !== undefined && c.maxCount > 0) { + if (c.maxCount > 0) { return `sh:maxCount ${c.maxCount} needs counting, which this service cannot do. Use sh:maxCount 0 for "has none", or drop it.`; } - if (c.minCount !== null && c.minCount !== undefined && c.minCount > 2) { + if (c.minCount > 2) { return `sh:minCount ${c.minCount} needs counting. Use sh:minCount 1 (at least one) or sh:minCount 2 (two distinct values); anything higher is beyond this service.`; } } @@ -272,14 +256,14 @@ function literalsParse(spec) { // rep:where: walks from the row through the profile, shares at least one // step with its filter or column, and holds no rep:where of its own. -function whereConditions(spec, profile, maxPathDepth) { - const startShapeUri = startShapeOf(spec, profile); +function whereConditions(spec, profile) { + const startShapeUri = startShape(profile, spec).uri; for (const { what, cond, host } of conditions(spec)) { if (!host) continue; if (cond.where.length) { return `${what} holds a rep:where of its own. Put every rep:where directly on the filter or column.`; } - const err = walk(profile, startShapeUri, cond.path, maxPathDepth, what, false); + const err = walk(profile, startShapeUri, cond.path, what, false); if (err) return err; const k = sharedSteps(host.path, cond.path); if (!k) { @@ -302,14 +286,13 @@ function whereConditions(spec, profile, maxPathDepth) { // code) it compares a URI and never matches, so its path must end on a field // with a datatype. function rangeOnValue(spec, profile) { - const startShapeUri = startShapeOf(spec, profile); + const startShapeUri = startShape(profile, spec).uri; for (const { what, cond } of conditions(spec)) { const key = ['minInclusive', 'maxInclusive', 'minExclusive', 'maxExclusive'] .find(k => cond.constraints[k]); if (!key || !cond.path.length) continue; const last = cond.path[cond.path.length - 1]; - const f = fieldsOf(profile, shapeAfter(profile, startShapeUri, cond.path)) - .find(x => x.path === last.predicate && !!x.inverse === !!last.inverse); + const f = field(profile, shapeAfter(profile, startShapeUri, cond.path), last); if (!f || f.datatype) continue; // pathsResolve reports a missing step const target = f.node ? `"${profile.shapes.find(s => s.uri === f.node)?.label || f.node}"` @@ -326,7 +309,7 @@ function rangeOnValue(spec, profile) { // Where each rep:where applies, in the profile's words, so the proposal can // say it. Profile only, no database. export function whereNotes(spec, profile) { - const startShapeUri = startShapeOf(spec, profile); + const startShapeUri = startShape(profile, spec).uri; return conditions(spec).filter(x => x.host).map(({ what, cond, host }) => { const k = sharedSteps(host.path, cond.path); return `${what} applies at ${nodeName(profile, startShapeUri, host.path.slice(0, k))}, after step ${k} of ${host.what}.`; @@ -337,7 +320,7 @@ export function whereNotes(spec, profile) { function nodeName(profile, startShapeUri, hops) { let shapeUri = startShapeUri; for (const [i, hop] of hops.entries()) { - const f = fieldsOf(profile, shapeUri).find(x => x.path === hop.predicate && !!x.inverse === !!hop.inverse); + const f = field(profile, shapeUri, hop); if (!f) return `step ${hops.length}`; if (!f.node) return i === hops.length - 1 ? `the value "${f.name}"` : `step ${hops.length}`; shapeUri = f.node; @@ -351,33 +334,37 @@ function parsesAsDate(value, withTime) { return withTime ? value.includes('T') : /^\d{4}-\d{2}-\d{2}$/.test(value); } +// In this order: startEntity comes first, the checks after it rely on a +// start shape. const CHECKS = [ - ['no entity or unknown class', startEntity], - ['no columns', columnsPresent], - ['column without rdfs:label', columnLabel], - ['two columns with one label', columnLabelsUnique], - ['column carries constraints', columnPure], - ['filter carries rdfs:label', filterLabel], - ['filter without a condition', filterConstraint], - ['sh:maxCount above 0', maxCountZero], - ['rep:anyOf too big', anyOfLimits], - ['sh:in mixes URIs and literals', inHomogeneous], - ['typed literal does not parse', literalsParse], - ['two rep:row columns', oneRowColumn], - ['path does not resolve', pathsResolve], - ['rep:where does not fit', whereConditions], - ['date or number condition on a link', rangeOnValue], - ['sh:min/sh:max on a non-number', collectMinMaxTyped], + startEntity, + columnsPresent, + columnLabel, + columnLabelsUnique, + columnPure, + filterLabel, + filterConstraint, + maxCountZero, + anyOfLimits, + inHomogeneous, + literalsParse, + oneRowColumn, + pathsResolve, + whereConditions, + rangeOnValue, + collectMinMaxTyped, ]; -export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) { - const profileErr = profileNamed(spec, knownProfiles); - if (profileErr) return [profileErr]; - for (const [name, check] of CHECKS) { - const err = check(spec, profile, maxPathDepth); - if (err) return [err]; +// checkSpec(spec, profiles) → the first error, or null. +export function checkSpec(spec, profiles) { + const error = profileError(profiles, spec.profileUri); + if (error) return error; + const profile = profiles.get(spec.profileUri); + for (const check of CHECKS) { + const err = check(spec, profile); + if (err) return err; } - return []; + return null; } // The shape a path lands in: follow sh:node through every hop but the @@ -386,17 +373,16 @@ export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) { function shapeAfter(profile, startShapeUri, hops) { let shapeUri = startShapeUri; for (let i = 0; i < hops.length - 1; i++) { - const fields = fieldsOf(profile, shapeUri); - const f = fields.find(x => x.path === hops[i].predicate && !!x.inverse === !!hops[i].inverse); + const f = field(profile, shapeUri, hops[i]); if (!f || !f.node) return null; shapeUri = f.node; } return shapeUri; } -function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsIri = false) { - if (hops.length > maxPathDepth) { - return `${what} walks ${hops.length} hops. The limit is ${maxPathDepth}.`; +function walk(profile, startShapeUri, hops, what, isColumn, wantsIri = false) { + if (hops.length > MAX_PATH_DEPTH) { + return `${what} walks ${hops.length} hops. The limit is ${MAX_PATH_DEPTH}.`; } if (!hops.length) return null; let shapeUri = startShapeUri; @@ -416,20 +402,15 @@ function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsI shapeUri = f.node; } else if (isColumn) { if (wantsIri && f.datatype) { - return `column "${colLabel(what)}" has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`; + return `${what} has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`; } if (!wantsIri && !f.datatype && !f.class) { const valueFields = fieldsOf(profile, f.node) .filter(x => x.datatype || x.class) .map(x => `${x.name} (${x.path})`); - return `column "${colLabel(what)}" ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`; + return `${what} ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`; } } } return null; -} - -function colLabel(what) { - const m = what.match(/"([^"]+)"/); - return m ? m[1] : what; } \ No newline at end of file diff --git a/src/runner/columns.js b/src/runner/columns.js index 73a3686..d916199 100644 --- a/src/runner/columns.js +++ b/src/runner/columns.js @@ -1,5 +1,5 @@ -import { sparqlEscapeUri } from '../db.js'; -import { wherePatterns } from './seed.js'; +import { query, sparqlEscapeUri } from 'mu'; +import { hopsPattern, wherePatterns } from './seed.js'; // Fetch column values for known subjects. Columns are grouped by their // shared non-leaf hops; two columns share a query only when every hop @@ -8,12 +8,14 @@ import { wherePatterns } from './seed.js'; const SUBJECT_CHUNK_SIZE = Math.max(1, Number(process.env.SUBJECT_CHUNK_SIZE) || 100); -export async function fetchColumns(sessionQuery, subjects, spec, values = new Map()) { +// fetchColumns(subjects, spec) → subject → column index → [term] +export async function fetchColumns(subjects, spec) { + const values = new Map(); const groups = groupColumns(spec.columns); for (const group of groups) { for (let i = 0; i < subjects.length; i += SUBJECT_CHUNK_SIZE) { const chunk = subjects.slice(i, i + SUBJECT_CHUNK_SIZE); - const result = await sessionQuery(groupQuery(group, chunk)); + const result = await query(groupQuery(group, chunk)); collect(result, group, values); } } @@ -41,21 +43,11 @@ export function groupColumns(columns) { export function groupQuery(group, chunk) { const subjects = chunk.map(sparqlEscapeUri).join(' '); - const lines = []; - let prev = '?s'; // every hop except the last gets its own variable; the leaf is ?v - group.hops.slice(0, -1).forEach((hop, i) => { - const v = `?x${i}`; - if (hop.inverse) lines.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`); - else lines.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`); - prev = v; - }); - const leaf = group.hops[group.hops.length - 1]; - if (leaf.inverse) lines.push(`?v ${sparqlEscapeUri(leaf.predicate)} ${prev} .`); - else lines.push(`${prev} ${sparqlEscapeUri(leaf.predicate)} ?v .`); - // rep:where hangs on the column's own nodes: ?x after k hops, ?v at + const lines = [hopsPattern(group.hops, 'x', '?s', '?v')]; + // rep:where hangs on the column's own nodes: ?x_ after k hops, ?v at // the end. - const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x${k - 1}`); + const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x_${k}`); lines.push(...wherePatterns(group.where, group.hops, nodeVar, 'c')); return `SELECT ?s ?v WHERE {\n VALUES ?s { ${subjects} }\n ${lines.join('\n ')}\n}`; } diff --git a/src/runner/csv.js b/src/runner/csv.js deleted file mode 100644 index b03ffd0..0000000 --- a/src/runner/csv.js +++ /dev/null @@ -1,35 +0,0 @@ -import { mkdirSync, writeFileSync } from 'fs'; - -// Rows to a CSV file. CSV_SEPARATOR is ';' by default. A cell is quoted -// only when it holds the separator, a quote or a newline; a quote inside a -// cell is doubled, per RFC 4180. - -const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';'; -const SHARE_DIR = process.env.SHARE_DIR || '/share'; - -export function rowsToCsv(rows) { - return rows.map(row => row.map(escapeCell).join(CSV_SEPARATOR)).join('\n'); -} - -function escapeCell(cell) { - const s = String(cell ?? ''); - return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s) - ? `"${s.replace(/"/g, '""')}"` - : s; -} - -export function writeCsv(name, csv) { - mkdirSync(SHARE_DIR, { recursive: true }); - const file = `${SHARE_DIR}/${name}`; - writeFileSync(file, csv, 'utf8'); - return file; -} - -// The spec of the run, next to its CSV (same base name, .ttl), so a report -// can be debugged afterwards. The store keeps only the pointer. -export function writeSpec(csvName, spec) { - mkdirSync(SHARE_DIR, { recursive: true }); - const name = csvName.replace(/\.csv$/, '') + '.ttl'; - writeFileSync(`${SHARE_DIR}/${name}`, spec, 'utf8'); - return `share://${name}`; -} \ No newline at end of file diff --git a/src/runner/profile.js b/src/runner/profile.js index b926fb2..04ae572 100644 --- a/src/runner/profile.js +++ b/src/runner/profile.js @@ -1,5 +1,5 @@ import N3 from 'n3'; -import { readFileSync } from 'fs'; +import { readFileSync, readdirSync } from 'fs'; // Parse a profile (a SHACL shapes graph) and answer questions about it. @@ -8,13 +8,11 @@ const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'; const OWL_ONTOLOGY = 'http://www.w3.org/2002/07/owl#Ontology'; const REP = 'http://mu.semte.ch/vocabularies/reporting/'; -export async function loadProfiles(dir) { - const fs = await import('fs'); - const path = await import('path'); - const files = fs.readdirSync(dir).filter(f => f.endsWith('.ttl')).sort(); +export function loadProfiles(dir) { + const files = readdirSync(dir).filter(f => f.endsWith('.ttl')).sort(); const profiles = new Map(); for (const file of files) { - const profile = await loadProfile(path.join(dir, file)); + const profile = loadProfile(`${dir}/${file}`); profiles.set(profile.uri, profile); console.log(`[profile] "${profile.title}" (${file}): ${profile.shapes.length} entities, ` + `${profile.shapes.reduce((n, s) => n + s.fields.length, 0)} fields`); @@ -23,29 +21,19 @@ export async function loadProfiles(dir) { return profiles; } -export async function loadProfile(file) { - const parser = new N3.Parser(); - const store = new N3.Store(); - await new Promise((resolve, reject) => { - parser.parse(readFileSync(file, 'utf8'), (err, quad) => { - if (err) return reject(new Error(`${file}: ${err.message}`)); - if (quad) store.addQuad(quad); else resolve(); - }); - }); +function loadProfile(file) { + let store; + try { + store = new N3.Store(new N3.Parser().parse(readFileSync(file, 'utf8'))); + } catch (e) { + throw new Error(`${file}: ${e.message}`); + } const ontology = store.getQuads(null, RDF + 'type', OWL_ONTOLOGY)[0]; if (!ontology) throw new Error(`${file}: no owl:Ontology. A profile file must name itself.`); const uri = ontology.subject.value; const title = store.getQuads(ontology.subject, 'http://purl.org/dc/terms/title', null)[0]?.object.value || uri; - const prefixQuads = store.getQuads(null, SH + 'declare', null); - const prefixes = {}; - for (const q of prefixQuads) { - const p = store.getQuads(q.object, SH + 'prefix', null)[0]?.object.value; - const ns = store.getQuads(q.object, SH + 'namespace', null)[0]?.object.value; - if (p && ns) prefixes[p] = ns; - } - const shapes = store.getQuads(null, RDF + 'type', SH + 'NodeShape').map(q => q.subject); if (!shapes.length) throw new Error(`${file}: no sh:NodeShape. A profile without entities is useless.`); @@ -57,30 +45,26 @@ export async function loadProfile(file) { return { uri: shape.value, targetClass, label, fields, discriminator }; }); - return { uri, title, prefixes, shapes: shapeList, store }; + return { uri, title, shapes: shapeList }; +} + +// A field's sh:path: one predicate, or [ sh:inversePath ]. +function readPath(store, node) { + const pathNode = store.getQuads(node, SH + 'path', null)[0]?.object; + if (!pathNode) return { path: null, inverse: false }; + if (pathNode.termType !== 'BlankNode') return { path: pathNode.value, inverse: false }; + const inv = store.getQuads(pathNode, SH + 'inversePath', null)[0]?.object; + return inv ? { path: inv.value, inverse: true } : { path: null, inverse: false }; } function parseField(store, p) { - let path = null, inverse = false; - const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object; - if (pathQuad) { - if (pathQuad.termType === 'BlankNode') { - const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object; - if (inv) { path = inv.value; inverse = true; } - } else { - path = pathQuad.value; - } - } - const one = (pred) => store.getQuads(p, pred, null)[0]?.object || null; + const one = (pred) => store.getQuads(p, pred, null)[0]?.object?.value || null; return { - path, inverse, - name: one(SH + 'name')?.value || null, - description: one(SH + 'description')?.value || null, - datatype: one(SH + 'datatype')?.value || null, - class: one(SH + 'class')?.value || null, - node: one(SH + 'node')?.value || null, - nodeKind: one(SH + 'nodeKind')?.value || null, - maxCount: one(SH + 'maxCount')?.value || null, + ...readPath(store, p), + name: one(SH + 'name'), + datatype: one(SH + 'datatype'), + class: one(SH + 'class'), + node: one(SH + 'node'), }; } @@ -89,16 +73,7 @@ function parseField(store, p) { function parseDiscriminator(store, shape) { const node = store.getQuads(shape, REP + 'discriminator', null)[0]?.object; if (!node) return null; - let path = null, inverse = false; - const pathQuad = store.getQuads(node, SH + 'path', null)[0]?.object; - if (pathQuad) { - if (pathQuad.termType === 'BlankNode') { - const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object; - if (inv) { path = inv.value; inverse = true; } - } else { - path = pathQuad.value; - } - } + const { path, inverse } = readPath(store, node); const one = (pred) => store.getQuads(node, pred, null)[0]?.object?.value || null; const minCount = one(SH + 'minCount'); const maxCount = one(SH + 'maxCount'); @@ -110,11 +85,6 @@ function parseDiscriminator(store, shape) { }; } -export function entityForClass(profile, cls) { - const shape = profile.shapes.find(s => s.targetClass === cls); - return shape ? shape.uri : null; -} - // The shape a spec starts from. rep:entity wins; a targetClass only resolves // when exactly one shape carries it, so a class two entities share is a // validator error, not a silent first match. @@ -132,10 +102,7 @@ export function fieldsOf(profile, shapeUri) { return shape(profile, shapeUri)?.fields || []; } -export function field(profile, shapeUri, predicate, inverse) { - return fieldsOf(profile, shapeUri).find(f => f.path === predicate && !!f.inverse === !!inverse) || null; +// The field a hop takes from a shape, or null. +export function field(profile, shapeUri, hop) { + return fieldsOf(profile, shapeUri).find(f => f.path === hop.predicate && !!f.inverse === !!hop.inverse) || null; } - -export function prefixes(profile) { - return profile.prefixes; -} \ No newline at end of file diff --git a/src/runner/report.js b/src/runner/report.js deleted file mode 100644 index cd8e99f..0000000 --- a/src/runner/report.js +++ /dev/null @@ -1,63 +0,0 @@ -import { randomUUID } from 'crypto'; -import { statSync } from 'fs'; -import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt, - sparqlEscapeDateTime } from '../db.js'; - -const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report'; - -// Register the CSV: a logical file (http://data.lblod.info/files/) plus -// a physical share:// file pointing back with nie:dataSource. The logical -// URI is what you link to. When given, the file gets dct:type. -export async function registerFile(sessionUpdateFn, fileName, filePath, { type } = {}) { - const size = statSync(filePath).size; - const uuid = randomUUID(); - const uuid2 = randomUUID(); - const now = new Date(); - const logicalUri = `http://data.lblod.info/files/${uuid}`; - const physicalUri = `share://${fileName}`; - - await (sessionUpdateFn || sessionUpdate)(` - INSERT DATA { - ${sparqlEscapeUri(logicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ; - ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ; - ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ; - ${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ; - ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ; - ${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)}${type ? ` ; - ${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} . - ${sparqlEscapeUri(physicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ; - ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid2)} ; - ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ; - ${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ; - ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ; - ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated')} ${sparqlEscapeDateTime(now)} ; - ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource')} ${sparqlEscapeUri(logicalUri)}${type ? ` ; - ${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} . - }`); - - return logicalUri; -} - -// One report pointing at the logical file URI. -export async function registerReport(sessionUpdateFn, title, fileUri, extra = {}) { - const uuid = randomUUID(); - const now = new Date(); - const reportUri = `http://data.lblod.info/id/reports/${uuid}`; - - const extraTriples = []; - if (extra.question) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/question')} ${sparqlEscapeString(extra.question)} .`); - if (extra.specFile) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/specFile')} ${sparqlEscapeString(extra.specFile)} .`); - if (extra.creator) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://purl.org/dc/terms/creator')} ${sparqlEscapeUri(extra.creator)} .`); - - await (sessionUpdateFn || sessionUpdate)(` - INSERT DATA { - ${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ; - ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ; - ${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ; - ${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)} ; - ${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} . - ${extraTriples.join('\n ')} - }`); - - return reportUri; -} \ No newline at end of file diff --git a/src/runner/run.js b/src/runner/run.js index e8c3a6c..e30db47 100644 --- a/src/runner/run.js +++ b/src/runner/run.js @@ -1,38 +1,49 @@ +import { update, uuid, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu'; import { seed } from './seed.js'; import { fetchColumns } from './columns.js'; import { assemble } from './assemble.js'; -import { rowsToCsv, writeCsv, writeSpec } from './csv.js'; -import { registerFile, registerReport } from './report.js'; import { startShape } from './profile.js'; -// Ties the steps together: seed → columns → assemble → CSV → register. -// Returns { reportUri, fileUri, rowCount }. +const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report'; +const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';'; -export async function run(query, update, parsed, profile, title, extra = {}) { - const shape = startShape(profile, parsed); - const subjects = await seed(query, parsed, shape); - - const values = new Map(); - await fetchColumns(query, subjects, parsed, values); - - const rows = assemble(subjects, values, parsed); - const fileName = extra.fileName || `${slug(title)}.csv`; - const csv = rowsToCsv(rows); - const filePath = writeCsv(fileName, csv); - const specFile = extra.spec && writeSpec(fileName, extra.spec); - - const fileUri = await registerFile(update, fileName, filePath, { type: extra.fileType }); - const reportUri = await registerReport(update, title, fileUri, - specFile ? { ...extra, specFile } : extra); - - return { reportUri, fileUri, rowCount: rows.length - 1, filePath }; +// run(spec, profile) → the CSV text. seed → columns → assemble → CSV. +export async function run(spec, profile) { + const subjects = await seed(spec, startShape(profile, spec)); + const values = await fetchColumns(subjects, spec); + const rows = assemble(subjects, values, spec); + return rows.map(row => row.map(csvCell).join(CSV_SEPARATOR)).join('\n'); +} + +// A cell is quoted only when it holds the separator, a quote or a newline; +// a quote inside a cell is doubled, per RFC 4180. +function csvCell(cell) { + const s = String(cell ?? ''); + return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s) + ? `"${s.replace(/"/g, '""')}"` + : s; +} + +// One report pointing at the logical file URI of its CSV. +export async function registerReport(title, fileUri) { + const id = uuid(); + const reportUri = `http://data.lblod.info/id/reports/${id}`; + + await update(` + INSERT DATA { + ${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ; + ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(id)} ; + ${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ; + ${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(new Date())} ; + ${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} . + }`); + + return reportUri; } -// The file name must be known before the run starts, so the chat can show a -// pending card named after the report. export function slug(title) { return (title || 'report').toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') .slice(0, 80) || 'report'; -} \ No newline at end of file +} diff --git a/src/runner/seed.js b/src/runner/seed.js index fbd6154..4739394 100644 --- a/src/runner/seed.js +++ b/src/runner/seed.js @@ -1,16 +1,15 @@ -import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from '../db.js'; +import { query, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from 'mu'; import { sharedSteps } from './spec.js'; // Subject selection. One paged query per page; stops at ROW_LIMIT. const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000); const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000); -export async function seed(sessionQuery, spec, shape = null) { +export async function seed(spec, shape) { const subjects = []; let offset = 0; for (;;) { - const query = seedPageQuery(spec, PAGE_SIZE, offset, shape); - const result = await sessionQuery(query); + const result = await query(seedPageQuery(spec, PAGE_SIZE, offset, shape)); const rows = result.results.bindings.map(b => b.s.value); subjects.push(...rows); if (subjects.length > ROW_LIMIT) { @@ -49,13 +48,13 @@ function discriminatorPattern(d) { // Writes out every hop as its own triple pattern. // Forward: ?s ?f0_1 . ?f0_1 ?f0_v // Inverse: ?s ^ → ?f0_1 ?s -export function hopsPattern(hops, varPrefix, startVar) { +export function hopsPattern(hops, varPrefix, startVar, lastVar = `?${varPrefix}_v`) { let prev = startVar; const patterns = []; for (let i = 0; i < hops.length; i++) { const hop = hops[i]; const isLast = i === hops.length - 1; - const v = isLast ? `?${varPrefix}_v` : `?${varPrefix}_${i + 1}`; + const v = isLast ? lastVar : `?${varPrefix}_${i + 1}`; if (hop.inverse) { patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`); } else { @@ -114,29 +113,14 @@ export function conditionPattern(c, varPrefix, hops, startVar) { parts.push(`FILTER(REGEX(str(${v}), ${sparqlEscapeString(alternation)}${flags}))`); } if (c.minCount === 2) { - // two distinct leaf values on the same path + // two distinct leaf values: the same path again, to a second leaf const w = `?${varPrefix}_w`; - parts.push(hopsPattern2(hops, varPrefix, startVar, w)); + parts.push(hopsPattern(hops, `${varPrefix}_2`, startVar, w)); parts.push(`FILTER(${v} != ${w})`); } return parts.join('\n '); } -function hopsPattern2(hops, varPrefix, startVar, wVar) { - // repeat the path to a second leaf variable - let prev = startVar; - const patterns = []; - for (let i = 0; i < hops.length; i++) { - const hop = hops[i]; - const isLast = i === hops.length - 1; - const v = isLast ? wVar : `?${varPrefix}_2_${i + 1}`; - if (hop.inverse) patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`); - else patterns.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`); - prev = v; - } - return patterns.join('\n '); -} - function escapeTerm(t) { if (t.type === 'uri') return sparqlEscapeUri(t.value); if (t.datatype === 'http://www.w3.org/2001/XMLSchema#dateTime') diff --git a/src/runner/spec.js b/src/runner/spec.js index 178f8d4..801fe57 100644 --- a/src/runner/spec.js +++ b/src/runner/spec.js @@ -23,22 +23,21 @@ export function parseSpec(turtle) { const profileUri = one(specNode, REP + 'profile')?.value || null; const title = one(specNode, 'http://purl.org/dc/terms/title')?.value || null; - const filters = store.getQuads(specNode, SH + 'property', null).map(q => - parseFilter(store, q.object, { withLabel: true })); + const filters = store.getQuads(specNode, SH + 'property', null).map(q => parseFilter(store, q.object)); const columnsQuad = one(specNode, REP + 'columns'); - const columns = columnsQuad ? parseColumnList(store, columnsQuad) : []; + const columns = columnsQuad ? readRdfList(store, columnsQuad).map(col => parseColumn(store, col)) : []; return { uri: specNode.value, title, profileUri, targetClass, entity, filters, columns }; } -function parseFilter(store, p, { withLabel = false } = {}) { +function parseFilter(store, p) { let path = []; const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object; if (pathQuad) path = parsePath(store, pathQuad); return { path, - constraints: readConstraints(store, p, { withLabel }), + constraints: readConstraints(store, p, { withLabel: true }), where: readWhere(store, p), }; } @@ -47,8 +46,7 @@ function parseFilter(store, p, { withLabel = false } = {}) { // is written like a filter, from the row; check.js refuses one nested in // another. function readWhere(store, node) { - return store.getQuads(node, REP + 'where', null).map(q => - parseFilter(store, q.object, { withLabel: true })); + return store.getQuads(node, REP + 'where', null).map(q => parseFilter(store, q.object)); } // The number of steps two paths have in common from the row. A rep:where @@ -60,18 +58,6 @@ export function sharedSteps(a, b) { return k; } -function parseColumnList(store, listNode) { - const columns = []; - let cur = listNode; - while (cur && cur.value !== RDF + 'nil') { - const item = store.getQuads(cur, RDF + 'first', null)[0]?.object; - if (item) columns.push(parseColumn(store, item)); - cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object; - if (!cur) throw new Error('rep:columns list is broken (no rdf:rest terminator)'); - } - return columns; -} - function parseColumn(store, col) { let path = []; const pathQuad = store.getQuads(col, SH + 'path', null)[0]?.object; @@ -107,21 +93,12 @@ function parsePath(store, pathQuad) { return [{ predicate: pathQuad.value, inverse: false }]; } // RDF list of predicates - const hops = []; - let cur = pathQuad; - while (cur && cur.value !== RDF + 'nil') { - const item = store.getQuads(cur, RDF + 'first', null)[0]?.object; - if (!item) throw new Error('sh:path list is broken'); - if (item.termType === 'BlankNode') { - const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object; - if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath ]'); - hops.push({ predicate: inv.value, inverse: true }); - } else { - hops.push({ predicate: item.value, inverse: false }); - } - cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object; - } - return hops; + return readRdfList(store, pathQuad).map(item => { + if (item.termType !== 'BlankNode') return { predicate: item.value, inverse: false }; + const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object; + if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath ]'); + return { predicate: inv.value, inverse: true }; + }); } function readConstraints(store, p, { withLabel = false } = {}) {