This commit is contained in:
felix
2026-09-14 16:34:17 +02:00
parent 789a676425
commit f128255226
27 changed files with 108 additions and 588 deletions

View File

@@ -9,7 +9,7 @@ Ad-hoc reports only. The scheduled reports stay in
## How it works
Two documents drive everything (plan §1, §2, §3):
Two documents drive everything:
| | Who writes it | When | Where |
|---|---|---|---|
@@ -32,7 +32,7 @@ with `mu-call-scope-id` set so jobs-controller ignores it.
## One identity: mu-authorization's
All SPARQL goes through the template's own helpers (`mu`'s `query` and
`update`), like ipdc-bookmarks-service. They attach the caller's session from
`update`). They attach the caller's session from
the request context, and mu-authorization decides graphs and visibility; the
service never touches an auth header. The context lives for the whole run,
also past the `202` the chat turn answers with, so background writes stay
@@ -54,7 +54,7 @@ Dutch comes back with a trace of the tool calls. The CSV lands in
`data/files/`. `history` carries the earlier turns for follow-ups; the
conversation lives on the caller's side, the service stores nothing.
`/assistant` is the chat (plan-fable-5-1, steps 1 and 3). The turn module in
`/assistant` is the chat. The turn module in
`src/chat/` is generic — it imports nothing from `src/llm/` or `src/runner/`
and moves between services without edits — and `src/report-assistant.js` is
this service's one `answer` hook. It records the question, answers `202` with
@@ -134,7 +134,7 @@ alone. A toezicht user can run the seed and read the CSV data, but the
may not write that type), so the report URI the tools return is dangling for
non-admin callers. Opening this up means adding the type to their group first.
## Rules the code holds to (plan §9)
## Rules the code holds to
- Query `http://database:8890/sparql`. Never virtuoso. Never sudo.
- No `GRAPH` clauses, no property paths, no aggregates, no inline comments:
@@ -142,15 +142,13 @@ non-admin callers. Opening this up means adding the type to their group first.
- Generated queries use full URIs, no prefixes (a bare `mu:uuid` prefixed
name without its PREFIX line is an opaque 500).
## Libraries (plan §0)
## Libraries
- `n3` — Turtle parsing (profiles, specs)
- hand-written: CSV writer (quotes only when needed, like today's reports),
the query builders, the agent loop (bare `fetch`, no provider SDK, no MCP
SDK — the tools are internal handlers, not an HTTP protocol)
- hand-written: CSV writer (quotes only when needed), the query builders,
the agent loop (bare `fetch`, no provider SDK)
## What it will not do
No counting, no sorting, no top-N, no comparing two subjects, one class per
report, suggestions stop at the public graph. §10 of the plan has the full
list and why.
report, suggestions stop at the public graph.

25
app.js
View File

@@ -8,10 +8,8 @@ import { reportAssistant } from './src/report-assistant.js';
const PROFILE_DIR = process.env.PROFILE_DIR || '/config/profiles';
// Profiles load fire-and-forget, like waitForDatabase in the bbcdr services:
// routes are registered synchronously, handlers await the same promise, and a
// call that lands before the load finishes just waits it out (local file
// reads). If loading failed, calls answer an error instead of crashing boot.
// Profiles load at boot; handlers await the same promise. A failed load
// answers an error per request instead of crashing boot.
let profilesPromise = loadProfiles(PROFILE_DIR);
profilesPromise.catch(e => console.error('[profiles] loading failed:', e.message));
@@ -23,20 +21,15 @@ async function whenProfiles() {
}
}
// The chat. The turn module is generic (src/chat/ moves between services
// without edits); reportAssistant is this service's one answer hook: it runs
// the loop below with the conversation so far and turns a report run into an
// interim message with a pending file. Step 1's echo hook, kept for tests:
// answer: async (turn) => { await sleep(5000); return `Je schreef: ${turn.content}`; }
// The chat. reportAssistant is the answer hook: it runs the LLM loop with the
// conversation so far and turns a report run into an interim message with a
// pending file.
mountChat(app, { path: '/assistant', answer: reportAssistant(whenProfiles) });
// The endpoint. One conversation turn in Dutch, the agent loop runs inside
// (read the profiles, look up values on the public graph, write a spec,
// validate and repair it, run the report) and the answer in Dutch comes back
// with a trace of the tool calls. Follow-ups carry the earlier turns in
// `history`; the conversation lives on the caller's side.
// The template parses application/vnd.api+json only; the question comes as
// plain JSON, so the route parses it itself, like the chat route does.
// POST /ask: one question in, the answer in Dutch and a trace of the tool
// calls back. Follow-ups carry the earlier turns in `history`.
// The template parses application/vnd.api+json only; this route parses the
// plain JSON body itself.
app.post('/ask', express.json(), async (req, res) => {
const { question, history } = req.body || {};
if (!question) return res.status(400).json({ error: 'no question in the request body' });

View File

@@ -1,8 +0,0 @@
// Maps the 'mu' import to dev-stubs/mu-stub.mjs, so tests run without the
// template container.
export async function resolve(specifier, context, nextResolve) {
if (specifier === 'mu') {
return { shortCircuit: true, url: new URL('./mu-stub.mjs', import.meta.url).href };
}
return nextResolve(specifier, context);
}

View File

@@ -1,51 +0,0 @@
// The template's helpers, shaped for the tests: query/update record every
// SPARQL string, and a test steers the answers with setQueryHandler.
// Registered for the 'mu' module by dev-stubs/register.js during tests only.
import express from 'express';
// The template's own app, stripped to what the tests mount routes on.
export const app = express();
export const state = { queries: [], updates: [], queryHandler: null };
// Steer the stub's answers: setQueryHandler(sparql => bindings[])
export function setQueryHandler(fn) { state.queryHandler = fn; }
// Clear the recordings only; the handler stays.
export function resetMu() {
state.queries.length = 0;
state.updates.length = 0;
}
export async function query(sparql) {
state.queries.push(sparql);
return { results: { bindings: state.queryHandler ? state.queryHandler(sparql) : [] }, head: { vars: [] } };
}
export async function update(sparql) {
state.updates.push(sparql);
return {};
}
export function sparqlEscapeString(value) {
return '"""' + value.replace(/[\\"]/g, (m) => '\\' + m) + '"""';
}
export function sparqlEscapeUri(value) {
return '<' + value.replace(/[\\"<>]/g, function (match) { return '\\' + match; }) + '>';
}
export function sparqlEscapeDateTime(value) {
return '"' + new Date(value).toISOString() + '"^^xsd:dateTime';
}
// The remaining template helpers db.js imports, copied from
// helpers/mu/sparql.js in mu-javascript-template.
export function sparqlEscapeInt(value) {
return '"' + Number.parseInt(value) + '"^^xsd:integer';
}
export function sparqlEscapeDate(value) {
return '"' + new Date(value).toISOString().substring(0, 10) + '"^^xsd:date';
}
export function sparqlEscapeBool(value) {
return value ? '"true"^^xsd:boolean' : '"false"^^xsd:boolean';
}

View File

@@ -1,5 +0,0 @@
// node --import ./dev-stubs/register.js <test file>
// Registers the 'mu' resolve hook (mu-hooks.mjs -> mu-stub.mjs).
import { register } from 'node:module';
register(new URL('./mu-hooks.mjs', import.meta.url).href);

57
package-lock.json generated
View File

@@ -11,29 +11,6 @@
"dependencies": {
"express": "^5.2.1",
"n3": "^1.17.0"
},
"devDependencies": {
"sparqljs": "^3.7.4"
}
},
"node_modules/@rdfjs/types": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@rdfjs/types/-/types-1.1.2.tgz",
"integrity": "sha512-wqpOJK1QCbmsGNtyzYnojPU8gRDPid2JO0Q0kMtb4j65xhCK880cnKAfEOwC+dX85VJcCByQx5zOwyyfCjDJsg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/node": {
"version": "26.5.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.1.tgz",
"integrity": "sha512-CzNm2FezW4VR/LjG6yUdiEgLE/rAQ9Slj5gCu/C2VrdcW7I0ahNZ8DRbHT7zOZ6r3ONgd/bsQIeSaoDGrd1C6g==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~8.9.0"
}
},
"node_modules/abort-controller": {
@@ -804,16 +781,6 @@
"node": ">= 0.10"
}
},
"node_modules/rdf-data-factory": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/rdf-data-factory/-/rdf-data-factory-1.1.3.tgz",
"integrity": "sha512-ny6CI7m2bq4lfQQmDYvcb2l1F9KtGwz9chipX4oWu2aAtVoXjb7k3d8J1EsgAsEbMXnBipB/iuRen5H2fwRWWQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@rdfjs/types": "^1.0.0"
}
},
"node_modules/readable-stream": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz",
@@ -995,23 +962,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/sparqljs": {
"version": "3.7.4",
"resolved": "https://registry.npmjs.org/sparqljs/-/sparqljs-3.7.4.tgz",
"integrity": "sha512-hb4C84gf7KM7vz+iG595mPwvqxOvBJCm9L3dCxtV2zZDht6ZMmMG0tHeeqFeqwb9875yU9U4lhYe4LYRvWj0BQ==",
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
"dev": true,
"license": "MIT",
"dependencies": {
"rdf-data-factory": "^1.1.2"
},
"bin": {
"sparqljs": "bin/sparql-to-json"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
"node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
@@ -1070,13 +1020,6 @@
"url": "https://opencollective.com/express"
}
},
"node_modules/undici-types": {
"version": "8.9.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
"integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
"dev": true,
"license": "MIT"
},
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

View File

@@ -6,14 +6,10 @@
"type": "module",
"main": "app.js",
"scripts": {
"start": "node app.js",
"test": "node --import ./dev-stubs/register.js test/chat-store.mjs && node --import ./dev-stubs/register.js test/chat-turn.mjs && node --import ./dev-stubs/register.js test/ask-parse.mjs && node test/import-boundaries.mjs"
"start": "node app.js"
},
"dependencies": {
"express": "^5.2.1",
"n3": "^1.17.0"
},
"devDependencies": {
"sparqljs": "^3.7.4"
}
}

View File

@@ -1,9 +1,5 @@
// The turn endpoint. Mount it on the service that answers:
//
// mountChat(app, { path: '/assistant', answer });
//
// answer(turn) is the only thing the service writes. See the plan, §3, for
// the turn object. This file knows nothing about what the answer is.
// The turn endpoint. Mount it with mountChat(app, { path, answer });
// answer(turn) is the only thing the mounting service writes.
import express from 'express';
import { query as muQuery, update as muUpdate } from 'mu';
import {
@@ -23,8 +19,7 @@ export function mountChat(app, { path = '/assistant', answer }) {
const content = String(req.body?.content ?? '').trim();
if (!content) return res.status(400).json({ error: 'content is required' });
// 1. the access check: the query carries the caller's session, and the
// auth layer answers with what they may read
// 1. the access check: the caller's session decides what is readable
let conversation;
try {
conversation = await readConversation(req.params.id);
@@ -49,10 +44,8 @@ export function mountChat(app, { path = '/assistant', answer }) {
return res.status(500).json({ error: e.message });
}
// 2. answer now; the assistant's message comes when it comes. The
// template's helpers keep attaching this request's session for the
// whole continuation, so the writes below stay the caller's;
// mu-authorization is in charge of what lands where.
// 2. answer 202 now; the assistant's message comes when it comes. The
// continuation still runs as the caller.
res.status(202).json({ id: userMessage.id });
const turn = {

View File

@@ -1,6 +1,6 @@
// The chat's reads and writes. Copy the folder as is; nothing here knows the
// service it lives in. Full URIs everywhere: the auth layer rejects a prefixed
// name with no PREFIX line with an opaque 500.
// The chat's reads and writes. Nothing here knows the service it lives in.
// Full URIs everywhere: the auth layer rejects a prefixed name with no
// PREFIX line with an opaque 500.
import { randomUUID } from 'crypto';
import { query as muQuery, update as muUpdate,
sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
@@ -31,8 +31,7 @@ const DOCUMENT_BASE = 'http://data.lblod.info/id/chat-documents/';
const u = sparqlEscapeUri;
const s = sparqlEscapeString;
// The template helper writes ^^xsd:dateTime; the auth layer happens to bind
// xsd, but the rule here is full URIs, so spell the type out.
// Full URIs, so spell the dateTime type out.
const dt = (value) => sparqlEscapeDateTime(value)
.replace('xsd:dateTime', '<http://www.w3.org/2001/XMLSchema#dateTime>');
@@ -40,9 +39,7 @@ const bindings = (json) => json.results.bindings;
// --- reads --------------------------------------------------------------------
// The conversation by uuid, as the caller sees it (the query carries the
// caller's session; the auth layer answers with what they may read).
// null when not readable.
// The conversation by uuid, null when the caller may not read it.
export async function readConversation(id) {
const json = await muQuery(`
SELECT ?conversation ?title ?creator WHERE {
@@ -96,7 +93,7 @@ export async function writeMessage({ conversationUri, content, maker, attachment
return { ...a, id, uri: `${DOCUMENT_BASE}${id}` };
});
// Predicate pairs are joined with a semicolon, so no list ever ends on one.
// Predicate pairs joined with ';', never a trailing one: it breaks the Turtle.
const pairs = [
`${u(T.type)} ${u(T.post)}`,
`${u(T.type)} ${u(T.instantMessage)}`,
@@ -152,7 +149,7 @@ export async function setDocumentUrl(documentUri, url, conversationUri) {
}`);
}
// The file failed. The card disappears; the caller says why in a message.
// The file failed; the card disappears, the caller says why in a message.
export async function dropDocument(documentUri) {
await muUpdate(`
DELETE {

View File

@@ -5,11 +5,6 @@ import { query as muQuery, update as muUpdate,
export { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt,
sparqlEscapeDateTime, sparqlEscapeDate, sparqlEscapeBool };
// All SPARQL goes through the template's own helpers, like
// ipdc-bookmarks-service. They attach the caller's session
// (mu-session-id, mu-call-id) from the request context, and
// mu-authorization decides graphs and visibility. The service never
// touches an auth header itself.
export async function sessionQuery(sparql) { return muQuery(sparql); }
export async function sessionUpdate(sparql) { return muUpdate(sparql); }
@@ -18,10 +13,8 @@ export function stripStoreError(text) {
return String(text).split('\n')[0].slice(0, 300);
}
// The per-request bundle the tools use: the template's query and update.
// The request context lives for the whole run, also past the 202 the chat
// turn answers with, so mu-authorization keeps rewriting with the caller's
// session. We add nothing to it.
// The request context lives for the whole run, also past the 202 a chat turn
// answers with, so every write stays the caller's.
export async function buildSession() {
return { query: sessionQuery, update: sessionUpdate };
}

View File

@@ -1,8 +1,6 @@
// Turn a shapes graph into the flat list an LLM reads well (plan §7.3). Not
// Turtle: a plain text menu of entities, their fields and the code lists.
// Code lists under INLINE_VALUES_MAX are inlined at request time
// (sessionQuery; mu-authorization decides what is visible) and refreshed
// every VALUES_TTL, so the LLM never has to search the short ones.
// A profile as a plain text menu: entities, fields, code lists. Code lists up
// to INLINE_VALUES_MAX are inlined and cached for VALUES_TTL, so the LLM
// never has to search the short ones.
import { fieldsOf, entityForClass } from '../runner/profile.js';
import { sessionQuery } from '../db.js';

View File

@@ -1,8 +1,6 @@
// lookup_values: search a code list too long to inline. One query through
// the template helper, regex on str(?label) so language-tagged labels still
// match. Returns the matches, the total, and an `exact` flag when a label
// equals the term (case aside) so the LLM can settle disambiguation on its
// own.
// Search a code list too long to inline: regex on str(?label), so
// language-tagged labels match. Returns the matches, the total, and an
// `exact` flag when a label equals the term (case aside).
import { sessionQuery, sparqlEscapeString } from '../db.js';
import { fieldsOf, entityForClass } from '../runner/profile.js';
@@ -24,7 +22,7 @@ export async function lookupValues(profile, term, fieldSpec) {
// The profile's sh:class is the type the data should carry, but code lists
// sometimes type their entries differently (MAR codes are
// ext:supervision/Nomenclature, the profile says skos:Concept). Prefer rows
// matching the asked class; fall back to all rows. The query reads more
// matching the asked class, fall back to all rows. The query reads more
// rows than LIMIT so an exact hit deeper in the alphabet still surfaces.
const wanted = bindings.filter(b => type && b.type?.value === type);
const rows = wanted.length ? wanted : bindings;
@@ -92,9 +90,8 @@ function resolveClass(profile, fieldSpec) {
const linked = profile.shapes.find(s => s.uri === f.node);
return linked?.targetClass || null;
}
// "bestuurseenheid.naam": the model chained link→value (shape.field on the
// linked shape). Resolve it: find the shape whose *name or label* matches,
// and whose rdf:type is in the data, then search that type.
// The model sometimes writes shape.field on a linked shape
// ("bestuurseenheid.naam"); resolve it to that shape's target class.
for (const s of profile.shapes) {
if (s.label === shapeLabel || shortType(s.targetClass) === shapeLabel) {
return s.targetClass;

View File

@@ -1,16 +1,6 @@
// The agent loop. One code path, one provider: POST /v1/chat/completions
// with tool calls, OpenAI-compatible (OpenAI, Azure, Ollama, LiteLLM, vLLM,
// Mistral). No provider SDKs. Three variables: LLM_BASE_URL, LLM_MODEL,
// LLM_API_KEY.
//
// The loop is the only stateful party: it keeps the message history, calls the
// MCP tools through the same handlers a real MCP client would, and stops when
// the model writes no more tool calls. validate_spec is driven up to
// MAX_REPAIR_ROUNDS, then run_report.
//
// This module is what the ten-question check runs against. It does not import
// from 'mu' or sessionQuery; it receives the tool handlers and a session
// bundle from the caller, so it works against any profile set and any session.
// The agent loop: POST /v1/chat/completions with tool calls,
// OpenAI-compatible (LLM_BASE_URL, LLM_MODEL, LLM_API_KEY). No provider
// SDKs. Stops when the model answers without tool calls.
import { buildTools } from './tools.js';
@@ -44,13 +34,9 @@ up to three rounds. Then call run_report.
Earlier turns are context only. Answer the last one.`;
// ask(questionOrMessages, profiles, session) → { text, trace }
// A string is one question, as before. An array is the conversation so far,
// [{ role: 'user' | 'assistant', content }], newest last. The system prompt
// is always this file's own; a system message in the input is dropped, so a
// caller cannot widen it. The loop appends every model reply and tool result
// to the messages, so a caller can keep one conversation across many reports.
// Returns the model's final text plus a trace of every tool call it made (for
// the leak test and for debugging).
// A string is one question; an array is the conversation so far,
// [{ role, content }], newest last. The system prompt is always this file's
// own; a system message in the input is dropped.
export async function ask(questionOrMessages, profiles, session) {
const { handlers } = buildTools(profiles, session);
const trace = [];
@@ -90,8 +76,7 @@ export async function ask(questionOrMessages, profiles, session) {
}
async function chat(messages) {
// Providers echo extra fields (litellm adds provider_specific_fields, Mistral
// rejects them on the way back). Send only the portable wire shape.
// Send only the fields every provider accepts back.
const wire = messages.map(m => {
const out = { role: m.role, content: m.content ?? null };
if (m.tool_calls) {
@@ -139,8 +124,7 @@ function parseArgs(s) {
}
// messagesFor(question) — a fresh conversation: system prompt plus the user
// question. Turn this into your own history array to follow up in the same
// conversation.
// question.
export function messagesFor(question) {
return [
{ role: 'system', content: SYSTEM_PROMPT },
@@ -148,8 +132,7 @@ export function messagesFor(question) {
];
}
// The tool definitions sent to the model. Same names and args as the MCP
// schemas, trimmed to what the model needs.
// The tool definitions sent to the model.
import { toolSchemas } from './schemas.js';
function toolDefs() {
return toolSchemas().map(t => ({

View File

@@ -1,6 +1,5 @@
// The JSON schemas for the eight tools, in the shape the MCP ListTools call
// returns. Keep these short and Dutch-aware: the descriptions are what the
// LLM reads to decide which tool to call.
// The JSON schemas of the eight tools. The descriptions are what the LLM
// reads to decide which tool to call.
export function toolSchemas() {
return [

View File

@@ -1,10 +1,7 @@
// The eight MCP tools the LLM calls. Each maps to a runner/ function or a
// profile/db read. What they return is always a job URI, a status, a count,
// a link, column names or an error we wrote, never a cell value (the §8
// guarantee).
//
// The shape every tool returns is { content: [{ type: 'text', text }] } so the
// loop can pass it straight back to the model.
// The eight tools the LLM calls. They return a job URI, a status, a count, a
// link, column names or an error we wrote — never a cell value. The shape
// every tool returns is { content: [{ type: 'text', text }] }, which the loop
// passes straight back to the model.
import { checkSpec } from '../runner/check.js';
import { parseSpec } from '../runner/spec.js';
@@ -16,12 +13,8 @@ import { lookupValues } from './lookup.js';
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
const RUN_TIMEOUT = Number(process.env.RUN_TIMEOUT || 60) * 1000;
// buildTools(profiles, session) → { tools: Tool[], handlers: Map<name, fn> }
// `session` is the request's { query, update } bundle: the template's own
// helpers, which attach the caller's session from the request context.
// run_report starts the run and waits up to RUN_TIMEOUT so the LLM gets the
// result in one turn when it is quick.
// buildTools(profiles, session) → { handlers }
// run_report waits up to RUN_TIMEOUT so a quick run returns in one turn.
export function buildTools(profiles, session) {
const profileList = () => [...profiles.values()].map(p => ({ id: p.uri, title: p.title }));
@@ -75,13 +68,13 @@ export function buildTools(profiles, session) {
const errors = checkSpec(parsed, p, MAX_PATH_DEPTH, profiles);
if (errors.length) return text(`the spec is not valid: ${errors[0]}`);
try {
// The spec's own dct:title names the report; there is no other input.
// The spec's dct:title names the report.
const name = parsed.title || 'report';
await session.onReportStart?.({ title: name, fileName: `${slug(name)}.csv` });
const running = run(session.query, session.update, parsed, p, name);
// The end callback fires even when withTimeout below has given up: the
// run is not cancelled, and the file arrives when it arrives. The
// rejection handler also keeps a late failure from crashing Node 20.
// The end callback also fires after the timeout gave up: the run is not
// cancelled, the file arrives when it arrives. The .catch keeps a late
// failure from crashing the process.
running
.then((r) => session.onReportEnd?.(null, r), (e) => session.onReportEnd?.(e))
.catch((e) => console.error('[reports] end callback failed:', e));

View File

@@ -1,6 +1,6 @@
// The report assistant: what the chat says when this service answers a turn.
// Runs the iteration-1 loop with the conversation so far, and turns a report
// run into an interim message with a pending file.
// What this service answers a chat turn with: the LLM loop with the
// conversation so far, and a report run becomes an interim message with a
// pending file.
import { ask } from './llm/loop.js';
const TEXT = {
@@ -8,13 +8,11 @@ const TEXT = {
failed: 'Het rapport is mislukt. Stel de vraag opnieuw, of anders geformuleerd.',
};
// whenProfiles() resolves once the SHACL profiles are loaded (app.js keeps
// the load promise), so the chat mounts before the profiles are in.
// The chat mounts before the profiles are in; whenProfiles waits for the load.
export function reportAssistant(whenProfiles) {
return async function answer(turn) {
// the pending file of the current run; the end callback needs the one
// the start callback made. One report per turn: a second run_report in
// the same turn overwrites this before the first end callback fires.
// One pending file per turn: a second run_report overwrites it before the
// first end callback fires.
let pendingDocument = null;
const session = {
@@ -43,8 +41,8 @@ export function reportAssistant(whenProfiles) {
};
}
// The file service serves /files/<uuid>/download, and the logical file URI
// that registerFile returns ends in that uuid.
// The file service serves /files/<uuid>/download; the logical file URI ends
// in that uuid.
function downloadHref(fileUri) {
return `/files/${fileUri.split('/').pop()}/download`;
}

View File

@@ -7,17 +7,16 @@ const XSD = 'http://www.w3.org/2001/XMLSchema#';
const XSD_DATE = XSD + 'date';
const XSD_DATETIME = XSD + 'dateTime';
// The validator. checkSpec(spec, profile, maxPathDepth) runs the named checks
// in order and returns the FIRST error as a one-element array (empty if ok).
// Every message names the thing that is wrong in the profile's own words and
// says what to write instead. No database call anywhere in this file — that is
// what lets an LLM iterate for free.
// The validator. checkSpec runs the checks in order and returns the first
// error as a one-element array (empty if ok). Every message names the thing
// that is wrong in the profile's own words and says what to write instead.
// No database call anywhere in this file.
const ANYOF_MAX_TERMS = 24;
const ANYOF_MAX_TERM_LENGTH = 100;
// 1 + 2. the profile names itself and is one we have. knownProfiles is the
// Map app.js loaded at boot; omit it and the check is skipped (fixtures).
// The spec names a profile we have. knownProfiles is the profile Map loaded
// at boot; the check is skipped when it is omitted.
function profileNamed(spec, knownProfiles) {
if (!spec.profileUri) {
return `no rep:profile named. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'} .`;
@@ -28,7 +27,7 @@ function profileNamed(spec, knownProfiles) {
return null;
}
// 3 + 4. the target class is in the profile
// The target class is in the profile
function targetClass(spec, profile) {
if (!spec.targetClass) {
return `no sh:targetClass. In "${profile.title}" you can list: ${shapeList(profile)}.`;
@@ -43,13 +42,13 @@ function shapeList(profile) {
return profile.shapes.map(s => s.label || s.targetClass).join(', ');
}
// 5. columns exist
// Columns exist
function columnsPresent(spec) {
if (!spec.columns.length) return 'no columns. Add at least one rep:columns entry.';
return null;
}
// 6. every column has a label
// Every column has a label
function columnLabel(spec) {
for (const col of spec.columns) {
if (!col.label) return 'a column has no rdfs:label.';
@@ -57,7 +56,7 @@ function columnLabel(spec) {
return null;
}
// 7. labels unique
// Labels are unique
function columnLabelsUnique(spec) {
const seen = new Map();
for (const col of spec.columns) {
@@ -69,7 +68,7 @@ function columnLabelsUnique(spec) {
return null;
}
// 8 + 9 + 10. every path resolves, columns end on a value, depth capped
// Every path resolves, columns end on a value, depth capped
function pathsResolve(spec, profile, maxPathDepth) {
const startShapeUri = entityForClass(profile, spec.targetClass);
for (const col of spec.columns) {
@@ -85,7 +84,7 @@ function pathsResolve(spec, profile, maxPathDepth) {
return null;
}
// 11. sh:min/sh:max only on numbers and dates
// sh:min/sh:max only on numbers and dates
function collectMinMaxTyped(spec, profile) {
const startShapeUri = entityForClass(profile, spec.targetClass);
for (const col of spec.columns) {
@@ -108,7 +107,7 @@ function collectMinMaxTyped(spec, profile) {
return null;
}
// 13. at most one rep:row column
// At most one rep:row column
function oneRowColumn(spec) {
const rowCols = spec.columns.filter(c => c.collect === REP + 'row');
if (rowCols.length > 1) {
@@ -118,7 +117,7 @@ function oneRowColumn(spec) {
return null;
}
// 12 + 14. columns carry no constraints; known collect mode
// Columns carry no constraints; known collect mode
function columnPure(spec) {
for (const col of spec.columns) {
if (Object.keys(col.constraints || {}).length) {
@@ -132,7 +131,7 @@ function columnPure(spec) {
return null;
}
// 15. no label inside a filter
// No label inside a filter
function filterLabel(spec) {
for (const filter of spec.filters) {
if ((filter.constraints || {}).label) {
@@ -142,7 +141,7 @@ function filterLabel(spec) {
return null;
}
// 15b. a filter with a path but no constraint selects everything; the model
// A filter with a path but no constraint selects everything; the model
// almost certainly meant to filter. Refuse and say what to add.
function filterConstraint(spec) {
const MEANINGFUL = ['minCount', 'maxCount', 'hasValue', 'in', 'minInclusive',
@@ -157,7 +156,7 @@ function filterConstraint(spec) {
return null;
}
// 16. sh:maxCount above 0 needs counting
// sh:maxCount above 0 needs counting
function maxCountZero(spec) {
for (const [i, filter] of spec.filters.entries()) {
const c = filter.constraints;
@@ -171,7 +170,7 @@ function maxCountZero(spec) {
return null;
}
// 17 + 18. rep:anyOf size limits
// rep:anyOf size limits
function anyOfLimits(spec) {
for (const [i, filter] of spec.filters.entries()) {
const c = filter.constraints;
@@ -191,7 +190,7 @@ function anyOfLimits(spec) {
return null;
}
// 19. sh:in mixes URIs and literals
// sh:in must not mix URIs and literals
function inHomogeneous(spec) {
for (const filter of spec.filters) {
const c = filter.constraints;
@@ -204,7 +203,7 @@ function inHomogeneous(spec) {
return null;
}
// 20. typed literals that will not parse
// Typed literals must parse
function literalsParse(spec) {
const constraintTerms = [];
for (const filter of spec.filters) {
@@ -259,8 +258,8 @@ export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) {
return [];
}
// The shape a column's path lands in: follow sh:node through every hop but
// the last. walk() has already validated the path, so any gap means the walk
// The shape a path lands in: follow sh:node through every hop but the
// last. walk() has already validated the path, so a gap here means the walk
// reported the real error and this can bail.
function shapeAfter(profile, startShapeUri, hops) {
let shapeUri = startShapeUri;

View File

@@ -1,9 +1,8 @@
import { mkdirSync, writeFileSync } from 'fs';
// Rows to a CSV file. CSV_SEPARATOR is ';' by default.
// A cell is quoted only when it holds the separator, a quote or a newline —
// this matches the shape of today's loket-report-generation-service output.
// A quote inside a cell is doubled, per RFC 4180.
// Rows to a CSV file. CSV_SEPARATOR is ';' by default. A cell is quoted
// only when it holds the separator, a quote or a newline; a quote inside a
// cell is doubled, per RFC 4180.
const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
const SHARE_DIR = process.env.SHARE_DIR || '/share';

View File

@@ -1,13 +1,9 @@
import { randomUUID } from 'crypto';
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from '../db.js';
// cogs:Job + task:Task, written by us. No other service is involved.
// Triples follow §6 of plan-opus-5.iteration-1-4.md; the shape follows
// config/resources/master-job-domain.lisp.
// Every write carries mu-call-scope-id (REPORT_SCOPE) so the delta notifier
// lets jobs-controller ignore our statuses — it throws on unknown operations.
// All functions take the update function to use: scopedUpdate(sparql, extra),
// which sends the captured groups plus this scope header.
// cogs:Job + task:Task, written by us; no other service is involved. Every
// write carries mu-call-scope-id (REPORT_SCOPE) so the delta notifier lets
// jobs-controller ignore our statuses — it throws on unknown operations.
const REPORT_SCOPE = 'http://redpencil.data.gift/id/concept/muScope/report-generation';
const JOB_BASE = process.env.JOB_URI_BASE || 'http://redpencil.data.gift/id/job/';
@@ -32,9 +28,7 @@ const TASK = 'http://redpencil.data.gift/vocabularies/tasks/';
const OSLC_MESSAGE = 'http://open-services.net/ns/core#message';
const OSLC_ERROR_URI = 'http://open-services.net/ns/core#Error';
// updateFn(sparql, extraHeaders) — app.js passes scopedUpdate, which sends
// the captured groups; the scope header is added here so every job write is
// opt-out-able for the delta notifier.
// Adds the scope header to every job write.
function withScope(updateFn) {
return async sparql => updateFn(sparql, { 'mu-call-scope-id': REPORT_SCOPE });
}

View File

@@ -8,9 +8,7 @@ const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
const OWL_ONTOLOGY = 'http://www.w3.org/2002/07/owl#Ontology';
export async function loadProfiles(dir) {
// eslint-disable-next-line no-undef
const fs = await import('fs');
// eslint-disable-next-line no-undef
const path = await import('path');
const files = fs.readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
const profiles = new Map();

View File

@@ -5,9 +5,9 @@ import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt,
const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
// Write the file to /share, then register it once: a logical file
// (http://data.lblod.info/files/<uuid>) plus a physical share:// file
// pointing back with nie:dataSource. The logical URI is what you link to.
// Register the CSV: a logical file (http://data.lblod.info/files/<uuid>) plus
// a physical share:// file pointing back with nie:dataSource. The logical
// URI is what you link to.
export async function registerFile(sessionUpdateFn, fileName, filePath) {
const size = statSync(filePath).size;
const uuid = randomUUID();
@@ -36,8 +36,7 @@ export async function registerFile(sessionUpdateFn, fileName, filePath) {
return logicalUri;
}
// One report pointing at the logical file URI. Insert without naming a graph;
// the auth layer places the triples from the caller's session.
// One report pointing at the logical file URI.
export async function registerReport(sessionUpdateFn, title, fileUri, extra = {}) {
const uuid = randomUUID();
const now = new Date();

View File

@@ -5,11 +5,9 @@ import { rowsToCsv, writeCsv } from './csv.js';
import { registerFile, registerReport } from './report.js';
import { touch } from './job.js';
// Ties the six steps together. `query` and `update` are the template's own
// helpers: they attach the caller's session from the request context, which
// lives for the whole run (also past the 202), so mu-authorization keeps
// deciding graphs and visibility. Returns { reportUri, fileUri, rowCount }.
// The task is touched once per column group so the dashboard shows movement.
// Ties the steps together: seed → columns → assemble → CSV → register.
// Returns { reportUri, fileUri, rowCount }. The task is touched once per
// column group so the dashboard shows movement.
export async function run(query, update, parsed, profile, title, taskUri, extra = {}) {
const subjects = await seed(query, parsed);
@@ -29,8 +27,8 @@ export async function run(query, update, parsed, profile, title, taskUri, extra
return { reportUri, fileUri, rowCount: rows.length - 1, filePath };
}
// Exported for run_report: the file name must be known before the run starts,
// so the chat can show a pending card named after the report.
// The file name must be known before the run starts, so the chat can show a
// pending card named after the report.
export function slug(title) {
return (title || 'report').toLowerCase()
.replace(/[^a-z0-9]+/g, '-')

View File

@@ -1,8 +1,6 @@
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from '../db.js';
// Subject selection. One paged query per page; stops at ROW_LIMIT.
// No GRAPH, no property paths, every hop its own variable.
const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000);
const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000);

View File

@@ -1,45 +0,0 @@
// POST /ask reads req.body, but the mu template only parses
// application/vnd.api+json; the route mounts express.json() for plain
// application/json, like the chat route does. Without an LLM configured a
// well-formed question must answer 503, not the "no question" 400.
//
// npm test (or: node --import ./dev-stubs/register.js test/ask-parse.mjs)
import assert from 'node:assert/strict';
delete process.env.LLM_BASE_URL;
let failures = 0;
async function check(name, fn) {
try {
await fn();
console.log(`ok - ${name}`);
} catch (e) {
failures++;
console.error(`FAIL - ${name}\n ${e.message}`);
}
}
const { app } = await import('mu');
await import('../app.js');
const server = app.listen(0);
const base = `http://localhost:${server.address().port}`;
const post = (path, body) => fetch(`${base}${path}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
await check('a plain JSON question parses and answers 503 without an LLM', async () => {
const res = await post('/ask', { question: 'meldingen met MAR-code 7300' });
assert.equal(res.status, 503);
});
await check('an empty body is still the 400', async () => {
const res = await post('/ask', {});
assert.equal(res.status, 400);
});
server.close();
process.exit(failures ? 1 : 0);

View File

@@ -1,94 +0,0 @@
// The chat store's SPARQL, asserted on the strings the mu stub records and
// parsed with sparqljs, so a syntax slip fails in the test and not against
// the auth layer.
//
// npm test (or: node --import ./dev-stubs/register.js test/chat-store.mjs)
import assert from 'node:assert/strict';
import { resetMu, state } from 'mu';
import { writeMessage, setDocumentUrl, dropDocument } from '../src/chat/store.js';
import { Parser } from 'sparqljs';
const CONVERSATION = 'http://data.lblod.info/id/chat-conversations/9ab4';
let failures = 0;
async function check(name, fn) {
try {
await fn();
console.log(`ok - ${name}`);
} catch (e) {
failures++;
console.error(`FAIL - ${name}\n ${e.message}`);
}
}
await check('writeMessage: both types, the container, the uuid', async () => {
resetMu();
const { id } = await writeMessage({ conversationUri: CONVERSATION, content: 'Hallo?' });
const q = state.updates[0];
assert.ok(q.includes('<http://rdfs.org/sioc/ns#Post>'), 'sioc:Post');
assert.ok(q.includes('<http://rdfs.org/sioc/types#InstantMessage>'), 'sioct:InstantMessage');
assert.ok(q.includes(`<${CONVERSATION}>`), 'the container');
assert.ok(q.includes(id), 'the uuid');
assert.ok(q.includes('Hallo?'), 'the content');
});
await check('writeMessage: the title on the first question', async () => {
resetMu();
await writeMessage({ conversationUri: CONVERSATION, content: 'Eerste vraag', title: 'Eerste vraag' });
const withTitle = state.updates[0];
assert.ok(withTitle.includes('<http://purl.org/dc/terms/title>'), 'the title triple');
resetMu();
await writeMessage({ conversationUri: CONVERSATION, content: 'Tweede vraag' });
const withoutTitle = state.updates[0];
assert.ok(!withoutTitle.includes('<http://purl.org/dc/terms/title>'), 'no title triple');
});
await check('writeMessage: an attachment without url is pending, with url is ready', async () => {
resetMu();
const { documents } = await writeMessage({
conversationUri: CONVERSATION, content: 'Rapport',
attachments: [{ name: 'rapport.csv', mediaType: 'text/csv' }],
});
const pending = state.updates[0];
assert.ok(pending.includes(documents[0].uri), 'the document is named');
assert.ok(pending.includes('<https://www.w3.org/ns/activitystreams#Document>'), 'as:Document');
assert.ok(!pending.includes('<https://www.w3.org/ns/activitystreams#url>'), 'no url yet');
resetMu();
await writeMessage({
conversationUri: CONVERSATION, content: 'Rapport',
attachments: [{ name: 'rapport.csv', mediaType: 'text/csv', url: '/files/1/download' }],
});
const ready = state.updates[0];
assert.ok(ready.includes('/files/1/download'), 'the url is written');
});
await check('setDocumentUrl and dropDocument name the document', async () => {
resetMu();
await setDocumentUrl(
'http://data.lblod.info/id/chat-documents/1', '/files/1/download', CONVERSATION);
assert.ok(state.updates[0].includes('http://data.lblod.info/id/chat-documents/1'));
resetMu();
await dropDocument('http://data.lblod.info/id/chat-documents/1');
assert.ok(state.updates[0].includes('http://data.lblod.info/id/chat-documents/1'));
});
await check('every recorded string parses as a SPARQL update', async () => {
resetMu();
await writeMessage({ conversationUri: CONVERSATION, content: 'Parsen', title: 'Parsen' });
await writeMessage({
conversationUri: CONVERSATION, content: 'Parsen',
attachments: [{ name: 'a.csv', mediaType: 'text/csv', url: '/files/a/download' }],
});
await setDocumentUrl('http://data.lblod.info/id/chat-documents/a', '/files/a/download', CONVERSATION);
await dropDocument('http://data.lblod.info/id/chat-documents/a');
const parser = new Parser();
for (const q of state.updates) {
assert.ok(parser.parse(q), 'parses');
}
});
process.exit(failures ? 1 : 0);

View File

@@ -1,100 +0,0 @@
// Integration check for the turn endpoint (src/chat/index.js). No stack:
// the mu stub records the queries and updates, and a handler steers the
// answers, so the test asserts on the same SPARQL the template's helpers
// would send.
//
// npm test (or: node --import ./dev-stubs/register.js test/chat-turn.mjs)
import assert from 'node:assert/strict';
import express from 'express';
import { resetMu, setQueryHandler, state } from 'mu';
let failures = 0;
async function check(name, fn) {
try {
await fn();
console.log(`ok - ${name}`);
} catch (e) {
failures++;
console.error(`FAIL - ${name}\n ${e.message}`);
}
}
// --- the stub answers -----------------------------------------------------------
const KNOWN_ID = '11111111-1111-1111-1111-111111111111';
const CONVERSATION = 'http://data.lblod.info/id/chat-conversations/9ab4';
const ASSISTANT = 'http://data.lblod.info/id/chat-agents/rapportassistent';
setQueryHandler((sparql) => {
if (sparql.includes('prov#SoftwareAgent')) {
return [{ agent: { value: ASSISTANT } }];
}
if (sparql.includes('sioc/ns#Thread') && sparql.includes(KNOWN_ID)) {
return [{ conversation: { value: CONVERSATION }, creator: { value: 'http://data.lblod.info/id/gebruiker/1' } }];
}
return [];
});
const { mountChat } = await import('../src/chat/index.js');
const app = express();
mountChat(app, {
path: '/assistant',
answer: async (turn) => {
return `Je schreef: ${turn.content}`;
},
});
const server = app.listen(0);
const base = `http://localhost:${server.address().port}`;
const post = (path, body) => fetch(`${base}${path}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'mu-session-id': 'http://mu.semte.ch/sessions/x' },
body: JSON.stringify(body),
});
await check('blank content is 400', async () => {
const res = await post('/assistant/conversations/00000000-0000-0000-0000-000000000000/turns', { content: ' ' });
assert.equal(res.status, 400);
});
await check('an unreadable conversation is 404', async () => {
const res = await post('/assistant/conversations/00000000-0000-0000-0000-000000000000/turns', { content: 'Hallo?' });
assert.equal(res.status, 404);
});
await check('a turn answers 202 with the message id, then writes the echo', async () => {
resetMu();
const res = await post(`/assistant/conversations/${KNOWN_ID}/turns`, { content: 'Hallo?' });
assert.equal(res.status, 202);
const { id } = await res.json();
assert.match(id, /^[0-9a-f-]{36}$/);
// the answer hook runs after the 202; give it a moment
await new Promise((r) => setTimeout(r, 200));
const updates = state.updates.map((u) => u);
assert.ok(updates.length >= 2, 'the question and the echo are written');
assert.ok(updates[0].includes('Hallo?'), 'the question first');
assert.ok(updates.some((u) => u.includes('Je schreef: Hallo?')), 'the echo after');
});
await check('a throwing hook writes the failure line', async () => {
const app2 = express();
mountChat(app2, { path: '/assistant', answer: async () => { throw new Error('boom'); } });
const server2 = app2.listen(0);
const base2 = `http://localhost:${server2.address().port}`;
resetMu();
const res = await fetch(`${base2}/assistant/conversations/${KNOWN_ID}/turns`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'mu-session-id': 'http://mu.semte.ch/sessions/x' },
body: JSON.stringify({ content: 'Breekt dit?' }),
});
assert.equal(res.status, 202);
await new Promise((r) => setTimeout(r, 200));
assert.ok(state.updates.some((u) => u.includes('Er ging iets mis. Probeer het opnieuw.')),
'the failure message is written');
server2.close();
});
server.close();
process.exit(failures ? 1 : 0);

View File

@@ -1,43 +0,0 @@
// node --import ./dev-stubs/register.js <test file>
// (this file itself needs no 'mu', so it runs without the register hook)
//
// The drop-in guarantee (plan step 1.1): src/chat/ imports nothing from the
// rest of this service, and nothing there imports it back.
import { readdirSync, readFileSync, statSync } from 'node:fs';
import { join, dirname, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
const root = join(dirname(fileURLToPath(import.meta.url)), '..');
const CHAT = join(root, 'src', 'chat');
const BACK = join(root, 'src', 'runner'), LLM = join(root, 'src', 'llm'), DB = join(root, 'src', 'db.js');
function walk(dir) {
return readdirSync(dir).flatMap(f => {
const p = join(dir, f);
return statSync(p).isDirectory() ? walk(p) : [p];
});
}
let bad = 0;
for (const f of walk(CHAT)) {
const text = readFileSync(f, 'utf8');
for (const m of text.matchAll(/from\s+['"](\.[^'"]+)['"]/g)) {
const target = new URL(m[1], 'file://' + f).pathname;
if (target === DB || target.startsWith(BACK) || target.startsWith(LLM)) {
console.error(`FAIL - ${relative(root, f)} imports ${relative(root, target)}`);
bad++;
}
}
}
for (const dir of [BACK, LLM]) {
for (const f of walk(dir)) {
const text = readFileSync(f, 'utf8');
if (/from\s+['"]\.\.\/chat/.test(text)) {
console.error(`FAIL - ${relative(root, f)} imports the chat back`);
bad++;
}
}
}
console.log(bad ? `${bad} boundary violations` : 'ok - chat keeps to itself');
process.exit(bad ? 1 : 0);