more semantic.works

This commit is contained in:
felix
2026-09-14 15:34:29 +02:00
parent 120a3d259c
commit 789a676425
18 changed files with 307 additions and 410 deletions

1
.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
node_modules/

View File

@@ -29,21 +29,17 @@ chunk of subjects) → assemble (dedup + `rep:collect`) → CSV → register fil
+ report resource. The job is a `cogs:Job` + `task:Task` we write ourselves,
with `mu-call-scope-id` set so jobs-controller ignores it.
## Two identities
## One identity: mu-authorization's
| Query | Session sent | Reads |
|---|---|---|
| `publicQuery` | none | what the stack makes public |
| `sessionQuery` | the caller's | what that user may read |
All SPARQL goes through the template's own helpers (`mu`'s `query` and
`update`), like ipdc-bookmarks-service. They attach the caller's session from
the request context, and mu-authorization decides graphs and visibility; the
service never touches an auth header. The context lives for the whole run,
also past the `202` the chat turn answers with, so background writes stay
the caller's.
The LLM only ever talks to `publicQuery`. `llm/` never imports `sessionQuery`
(CI check 2), and the `runner/` functions it calls return URIs, counts,
statuses and column names — never a cell value (CI check 3). That split is
the guarantee; it holds by code shape, not by a test that watches it.
After the answer (or the 202 in earlier designs), the request context is
gone. The run carries the caller's `mu-auth-allowed-groups` (captured before
answering) on every later query.
The tools the LLM calls return URIs, counts, statuses and column names —
never a cell value.
## Two endpoints

5
app.js
View File

@@ -1,4 +1,5 @@
import { app } from 'mu';
import express from 'express';
import { loadProfiles } from './src/runner/profile.js';
import { buildSession, stripStoreError } from './src/db.js';
import { ask } from './src/llm/loop.js';
@@ -34,7 +35,9 @@ mountChat(app, { path: '/assistant', answer: reportAssistant(whenProfiles) });
// validate and repair it, run the report) and the answer in Dutch comes back
// with a trace of the tool calls. Follow-ups carry the earlier turns in
// `history`; the conversation lives on the caller's side.
app.post('/ask', async (req, res) => {
// The template parses application/vnd.api+json only; the question comes as
// plain JSON, so the route parses it itself, like the chat route does.
app.post('/ask', express.json(), async (req, res) => {
const { question, history } = req.body || {};
if (!question) return res.status(400).json({ error: 'no question in the request body' });
if (!process.env.LLM_BASE_URL) return res.status(503).json({ error: 'no LLM configured (set LLM_BASE_URL)' });

View File

@@ -1,17 +1,35 @@
// Byte-for-byte copies of the three mu helpers the chat store uses, so the
// unit test asserts on the same strings the container produces.
// The template's helpers, shaped for the tests: query/update record every
// SPARQL string, and a test steers the answers with setQueryHandler.
// Registered for the 'mu' module by dev-stubs/register.js during tests only.
import express from 'express';
// The template's own app, stripped to what the tests mount routes on.
export const app = express();
export const state = { queries: [], updates: [], queryHandler: null };
// Steer the stub's answers: setQueryHandler(sparql => bindings[])
export function setQueryHandler(fn) { state.queryHandler = fn; }
// Clear the recordings only; the handler stays.
export function resetMu() {
state.queries.length = 0;
state.updates.length = 0;
}
export async function query(sparql) {
state.queries.push(sparql);
return { results: { bindings: state.queryHandler ? state.queryHandler(sparql) : [] }, head: { vars: [] } };
}
export async function update(sparql) {
state.updates.push(sparql);
return {};
}
export function sparqlEscapeString(value) {
return '"""' + value.replace(/[\\"]/g, (m) => '\\' + m) + '"""';
}
// The loop test drives ask() end to end; describe/lookup tools reach the
// session helpers, so the stub implements them as no-ops.
export async function query() {
return { results: { bindings: [] }, head: { vars: [] } };
}
export async function update() {}
export function sparqlEscapeUri(value) {
return '<' + value.replace(/[\\"<>]/g, function (match) { return '\\' + match; }) + '>';
}

57
package-lock.json generated
View File

@@ -11,6 +11,29 @@
"dependencies": {
"express": "^5.2.1",
"n3": "^1.17.0"
},
"devDependencies": {
"sparqljs": "^3.7.4"
}
},
"node_modules/@rdfjs/types": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@rdfjs/types/-/types-1.1.2.tgz",
"integrity": "sha512-wqpOJK1QCbmsGNtyzYnojPU8gRDPid2JO0Q0kMtb4j65xhCK880cnKAfEOwC+dX85VJcCByQx5zOwyyfCjDJsg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/node": {
"version": "26.5.1",
"resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.1.tgz",
"integrity": "sha512-CzNm2FezW4VR/LjG6yUdiEgLE/rAQ9Slj5gCu/C2VrdcW7I0ahNZ8DRbHT7zOZ6r3ONgd/bsQIeSaoDGrd1C6g==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~8.9.0"
}
},
"node_modules/abort-controller": {
@@ -781,6 +804,16 @@
"node": ">= 0.10"
}
},
"node_modules/rdf-data-factory": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/rdf-data-factory/-/rdf-data-factory-1.1.3.tgz",
"integrity": "sha512-ny6CI7m2bq4lfQQmDYvcb2l1F9KtGwz9chipX4oWu2aAtVoXjb7k3d8J1EsgAsEbMXnBipB/iuRen5H2fwRWWQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@rdfjs/types": "^1.0.0"
}
},
"node_modules/readable-stream": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz",
@@ -962,6 +995,23 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/sparqljs": {
"version": "3.7.4",
"resolved": "https://registry.npmjs.org/sparqljs/-/sparqljs-3.7.4.tgz",
"integrity": "sha512-hb4C84gf7KM7vz+iG595mPwvqxOvBJCm9L3dCxtV2zZDht6ZMmMG0tHeeqFeqwb9875yU9U4lhYe4LYRvWj0BQ==",
"deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.",
"dev": true,
"license": "MIT",
"dependencies": {
"rdf-data-factory": "^1.1.2"
},
"bin": {
"sparqljs": "bin/sparql-to-json"
},
"engines": {
"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
}
},
"node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
@@ -1020,6 +1070,13 @@
"url": "https://opencollective.com/express"
}
},
"node_modules/undici-types": {
"version": "8.9.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz",
"integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==",
"dev": true,
"license": "MIT"
},
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

View File

@@ -7,10 +7,13 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
"test": "node --import ./dev-stubs/register.js test/chat-store.mjs && node --import ./dev-stubs/register.js test/chat-turn.mjs && node test/import-boundaries.mjs"
"test": "node --import ./dev-stubs/register.js test/chat-store.mjs && node --import ./dev-stubs/register.js test/chat-turn.mjs && node --import ./dev-stubs/register.js test/ask-parse.mjs && node test/import-boundaries.mjs"
},
"dependencies": {
"express": "^5.2.1",
"n3": "^1.17.0"
},
"devDependencies": {
"sparqljs": "^3.7.4"
}
}

View File

@@ -5,10 +5,10 @@
// answer(turn) is the only thing the service writes. See the plan, §3, for
// the turn object. This file knows nothing about what the answer is.
import express from 'express';
import { query as muQuery, update as muUpdate } from 'mu';
import {
readConversation, readHistory, findAssistant,
writeMessage, setDocumentUrl, dropDocument,
groupsQuery, groupsUpdate,
} from './store.js';
const TEXT = {
@@ -23,25 +23,22 @@ export function mountChat(app, { path = '/assistant', answer }) {
const content = String(req.body?.content ?? '').trim();
if (!content) return res.status(400).json({ error: 'content is required' });
// 1. the access check: can this session read the conversation?
// 1. the access check: the query carries the caller's session, and the
// auth layer answers with what they may read
let conversation;
try {
conversation = await readConversation(req.params.id, req.get('mu-session-id'));
conversation = await readConversation(req.params.id);
} catch (e) {
console.error('[chat] conversation read failed:', e);
return res.status(500).json({ error: e.message });
}
if (!conversation) return res.status(404).json({ error: 'no such conversation' });
// 2. everything from here on carries the captured groups
const query = sparql => groupsQuery(sparql, conversation.groups);
const update = sparql => groupsUpdate(sparql, conversation.groups);
let assistant, history, userMessage;
try {
assistant = process.env.CHAT_ASSISTANT_URI || await findAssistant(query);
history = await readHistory(query, conversation.uri, assistant);
userMessage = await writeMessage(update, {
assistant = process.env.CHAT_ASSISTANT_URI || await findAssistant();
history = await readHistory(conversation.uri, assistant);
userMessage = await writeMessage({
conversationUri: conversation.uri,
content,
maker: conversation.creator,
@@ -52,22 +49,24 @@ export function mountChat(app, { path = '/assistant', answer }) {
return res.status(500).json({ error: e.message });
}
// 3. answer now; the assistant's message comes when it comes
// 2. answer now; the assistant's message comes when it comes. The
// template's helpers keep attaching this request's session for the
// whole continuation, so the writes below stay the caller's;
// mu-authorization is in charge of what lands where.
res.status(202).json({ id: userMessage.id });
const turn = {
conversation: { uri: conversation.uri, id: conversation.id, title: conversation.title },
history,
content,
query,
update,
groups: conversation.groups,
query: muQuery,
update: muUpdate,
finished: false,
say: (text, attachments = []) => writeMessage(update, {
say: (text, attachments = []) => writeMessage({
conversationUri: conversation.uri, content: text, maker: assistant, attachments,
}),
setUrl: (documentUri, url) => setDocumentUrl(update, documentUri, url, conversation.uri),
drop: (documentUri) => dropDocument(update, documentUri),
setUrl: (documentUri, url) => setDocumentUrl(documentUri, url, conversation.uri),
drop: (documentUri) => dropDocument(documentUri),
};
try {

View File

@@ -2,23 +2,11 @@
// service it lives in. Full URIs everywhere: the auth layer rejects a prefixed
// name with no PREFIX line with an opaque 500.
import { randomUUID } from 'crypto';
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
import { query as muQuery, update as muUpdate,
sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
const ENDPOINT = process.env.MU_SPARQL_ENDPOINT;
const HISTORY_LIMIT = Number(process.env.CHAT_HISTORY_LIMIT || 20);
// The mu template logs mu.query/mu.update the same way (helpers/mu/sparql.js);
// these fetch paths bypass it, so log here under the same switches.
const LOG_QUERIES = process.env.LOG_SPARQL_QUERIES != undefined
? /^(true|1)$/i.test(process.env.LOG_SPARQL_QUERIES)
: /^(true|1)$/i.test(process.env.LOG_SPARQL_ALL || '');
const LOG_UPDATES = process.env.LOG_SPARQL_UPDATES != undefined
? /^(true|1)$/i.test(process.env.LOG_SPARQL_UPDATES)
: /^(true|1)$/i.test(process.env.LOG_SPARQL_ALL || '');
function logSparql(kind, sparql) {
console.log(`[sparql] ${kind}:\n${sparql}`);
}
const T = {
type: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type',
uuid: 'http://mu.semte.ch/vocabularies/core/uuid',
@@ -48,77 +36,29 @@ const s = sparqlEscapeString;
const dt = (value) => sparqlEscapeDateTime(value)
.replace('xsd:dateTime', '<http://www.w3.org/2001/XMLSchema#dateTime>');
// --- the two ways to talk to the auth layer ---------------------------------
// With the caller's session, inside the request. Returns the JSON result and
// the allowed-groups header, which is the identity we carry past the 202.
// A bare fetch rather than mu.query: the header is on the response, and this
// way we do not depend on what the template's helper chooses to return.
export async function sessionQuery(sparql, sessionId) {
if (LOG_QUERIES) logSparql('query', sparql);
const res = await fetch(ENDPOINT, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
Accept: 'application/sparql-results+json',
'mu-session-id': sessionId,
},
body: new URLSearchParams({ query: sparql }).toString(),
});
if (!res.ok) throw new Error(`query failed (${res.status}): ${firstLine(await res.text())}`);
return { json: await res.json(), groups: res.headers.get('mu-auth-allowed-groups') };
}
// With the captured groups, after the 202.
export async function groupsQuery(sparql, groups) {
if (LOG_QUERIES) logSparql('query', sparql);
const res = await fetch(ENDPOINT, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
Accept: 'application/sparql-results+json',
'mu-auth-allowed-groups': groups,
},
body: new URLSearchParams({ query: sparql }).toString(),
});
if (!res.ok) throw new Error(`query failed (${res.status}): ${firstLine(await res.text())}`);
return res.json();
}
export async function groupsUpdate(sparql, groups) {
if (LOG_UPDATES) logSparql('update', sparql);
const res = await fetch(ENDPOINT, {
method: 'POST',
headers: { 'Content-Type': 'application/sparql-update', 'mu-auth-allowed-groups': groups },
body: sparql,
});
if (!res.ok) throw new Error(`update failed (${res.status}): ${firstLine(await res.text())}`);
}
function firstLine(text) { return String(text).split('\n')[0].slice(0, 300); }
const bindings = (json) => json.results.bindings;
// --- reads --------------------------------------------------------------------
// The conversation by uuid, as the caller sees it. null when not readable.
// Also returns the allowed groups from the same round trip.
export async function readConversation(id, sessionId) {
const { json, groups } = await sessionQuery(`
// The conversation by uuid, as the caller sees it (the query carries the
// caller's session; the auth layer answers with what they may read).
// null when not readable.
export async function readConversation(id) {
const json = await muQuery(`
SELECT ?conversation ?title ?creator WHERE {
?conversation ${u(T.type)} ${u(T.thread)} ;
${u(T.uuid)} ${s(id)} .
OPTIONAL { ?conversation ${u(T.title)} ?title . }
OPTIONAL { ?conversation ${u(T.maker)} ?creator . }
} LIMIT 1`, sessionId);
} LIMIT 1`);
const b = bindings(json)[0];
if (!b) return null;
if (!groups) throw new Error('no mu-auth-allowed-groups header on the conversation read; cannot carry the identity past the 202');
return { uri: b.conversation.value, id, title: b.title?.value, creator: b.creator?.value, groups };
return { uri: b.conversation.value, id, title: b.title?.value, creator: b.creator?.value };
}
// The last HISTORY_LIMIT messages, oldest first, as { role, content }.
export async function readHistory(queryFn, conversationUri, assistantUri) {
const json = await queryFn(`
export async function readHistory(conversationUri, assistantUri) {
const json = await muQuery(`
SELECT ?content ?maker ?created WHERE {
?message ${u(T.hasContainer)} ${u(conversationUri)} ;
${u(T.content)} ?content ;
@@ -132,8 +72,8 @@ export async function readHistory(queryFn, conversationUri, assistantUri) {
}
// The one assistant the caller can read. Throws unless there is exactly one.
export async function findAssistant(queryFn) {
const json = await queryFn(`
export async function findAssistant() {
const json = await muQuery(`
SELECT ?agent WHERE { ?agent ${u(T.type)} ${u(T.softwareAgent)} . } LIMIT 2`);
const rows = bindings(json);
if (rows.length !== 1) {
@@ -147,7 +87,7 @@ export async function findAssistant(queryFn) {
// One message, both types, its attachments, and the conversation touched, in
// one update. `title` is only written when given (the first question names
// an unnamed conversation). Returns { uri, id, documents: [{ uri, id }] }.
export async function writeMessage(updateFn, { conversationUri, content, maker, attachments = [], title }) {
export async function writeMessage({ conversationUri, content, maker, attachments = [], title }) {
const id = randomUUID();
const uri = `${MESSAGE_BASE}${id}`;
const now = dt(new Date());
@@ -171,7 +111,7 @@ export async function writeMessage(updateFn, { conversationUri, content, maker,
const conversationPairs = [`${u(T.lastActivity)} ${now}`];
if (title) conversationPairs.push(`${u(T.title)} ${s(title)}`);
await updateFn(`
await muUpdate(`
DELETE {
${u(conversationUri)} ${u(T.lastActivity)} ?old .
}
@@ -195,9 +135,9 @@ export async function writeMessage(updateFn, { conversationUri, content, maker,
}
// The file is ready.
export async function setDocumentUrl(updateFn, documentUri, url, conversationUri) {
export async function setDocumentUrl(documentUri, url, conversationUri) {
const now = dt(new Date());
await updateFn(`
await muUpdate(`
DELETE {
${u(documentUri)} ${u(T.url)} ?oldUrl .
${u(conversationUri)} ${u(T.lastActivity)} ?old .
@@ -213,8 +153,8 @@ export async function setDocumentUrl(updateFn, documentUri, url, conversationUri
}
// The file failed. The card disappears; the caller says why in a message.
export async function dropDocument(updateFn, documentUri) {
await updateFn(`
export async function dropDocument(documentUri) {
await muUpdate(`
DELETE {
?message ${u(T.attachment)} ${u(documentUri)} .
${u(documentUri)} ?p ?o .

128
src/db.js
View File

@@ -5,129 +5,23 @@ import { query as muQuery, update as muUpdate,
export { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt,
sparqlEscapeDateTime, sparqlEscapeDate, sparqlEscapeBool };
// The caller's session. Everything that touches report data.
// mu attaches mu-session-id / mu-auth-allowed-groups from the incoming
// request itself; we never set headers here.
// All SPARQL goes through the template's own helpers, like
// ipdc-bookmarks-service. They attach the caller's session
// (mu-session-id, mu-call-id) from the request context, and
// mu-authorization decides graphs and visibility. The service never
// touches an auth header itself.
export async function sessionQuery(sparql) { return muQuery(sparql); }
export async function sessionUpdate(sparql) { return muUpdate(sparql); }
// The fetch paths below bypass the template's query helpers, so their
// logging (LOG_SPARQL_ALL / LOG_SPARQL_QUERIES / LOG_SPARQL_UPDATES, the
// same switches helpers/mu/sparql.js reads) is mirrored here.
const LOG_QUERIES = process.env.LOG_SPARQL_QUERIES != undefined
? /^(true|1)$/i.test(process.env.LOG_SPARQL_QUERIES)
: /^(true|1)$/i.test(process.env.LOG_SPARQL_ALL || '');
const LOG_UPDATES = process.env.LOG_SPARQL_UPDATES != undefined
? /^(true|1)$/i.test(process.env.LOG_SPARQL_UPDATES)
: /^(true|1)$/i.test(process.env.LOG_SPARQL_ALL || '');
function logSparql(kind, sparql) {
console.log(`[sparql] ${kind}:\n${sparql}`);
}
// No session. Only what the stack makes public. Everything shown to the LLM.
// Bare HTTP POST: muQuery always attaches the request's session, which would
// widen or narrow the answer depending on the caller.
export async function publicQuery(sparql) {
if (LOG_QUERIES) logSparql('query', sparql);
const res = await fetch(`${process.env.MU_SPARQL_ENDPOINT}`, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/sparql-results+json',
},
body: new URLSearchParams({ query: sparql }).toString(),
});
if (!res.ok) {
const text = await res.text();
throw new Error(`public query failed (${res.status}): ${stripStoreError(text)}`);
}
return res.json();
}
// A bare update, the same trick as publicQuery. muUpdate would attach the
// request's session; job writes after the 202 must not depend on it.
export async function plainUpdate(sparql, extraHeaders = {}) {
if (LOG_UPDATES) logSparql('update', sparql);
const res = await fetch(`${process.env.MU_SPARQL_ENDPOINT}`, {
method: 'POST',
headers: {
'Content-Type': 'application/sparql-update',
...extraHeaders,
},
body: sparql,
});
if (!res.ok) {
const text = await res.text();
throw new Error(`update failed (${res.status}): ${stripStoreError(text)}`);
}
}
// Background half of a run. Call captureGroups() once inside the request,
// then send the captured header on every query after the 202: the template
// stops attaching the session once the request is answered, and the auth
// layer would silently answer from the public graph.
export async function captureGroups(sessionQueryFn) {
const probe = await sessionQueryFn('SELECT ?s WHERE { ?s ?p ?o } LIMIT 1');
const header = probe.headers?.['mu-auth-allowed-groups'];
if (!header) {
throw new Error('no mu-auth-allowed-groups header on the first response; cannot carry the identity past the 202');
}
return header;
}
export async function groupsQuery(sparql, allowedGroups) {
if (LOG_QUERIES) logSparql('query', sparql);
const res = await fetch(`${process.env.MU_SPARQL_ENDPOINT}`, {
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'application/sparql-results+json',
'mu-auth-allowed-groups': allowedGroups,
},
body: new URLSearchParams({ query: sparql }).toString(),
});
if (!res.ok) {
const text = await res.text();
throw new Error(`query failed (${res.status}): ${stripStoreError(text)}`);
}
return res.json();
}
export async function groupsUpdate(sparql, allowedGroups, extraHeaders = {}) {
if (LOG_UPDATES) logSparql('update', sparql);
const res = await fetch(`${process.env.MU_SPARQL_ENDPOINT}`, {
method: 'POST',
headers: {
'Content-Type': 'application/sparql-update',
'mu-auth-allowed-groups': allowedGroups,
...extraHeaders,
},
body: sparql,
});
if (!res.ok) {
const text = await res.text();
throw new Error(`update failed (${res.status}): ${stripStoreError(text)}`);
}
return res.json();
}
// The store's error text may name graphs or internals we do not pass on.
export function stripStoreError(text) {
return String(text).split('\n')[0].slice(0, 300);
}
// The per-request session bundle the MCP tools use. Built here (next to the
// other mu-touching code) so llm/ never imports from 'mu' or sessionQuery
// (CI checks 1 and 2). Read tools use the caller's session through muQuery;
// run_report captures the groups once and runs in the background with them.
// The per-request bundle the tools use: the template's query and update.
// The request context lives for the whole run, also past the 202 the chat
// turn answers with, so mu-authorization keeps rewriting with the caller's
// session. We add nothing to it.
export async function buildSession() {
const query = async (sparql) => muQuery(sparql);
let groupsCache = null;
return {
query,
get groups() {
if (!groupsCache) groupsCache = captureGroups(query);
return groupsCache;
},
};
}
return { query: sessionQuery, update: sessionUpdate };
}

View File

@@ -1,11 +1,11 @@
// Turn a shapes graph into the flat list an LLM reads well (plan §7.3). Not
// Turtle: a plain text menu of entities, their fields and the code lists.
// Code lists under INLINE_VALUES_MAX are inlined at boot from the public
// graph (publicQuery) and refreshed every VALUES_TTL, so the LLM never has
// to search the short ones.
// Code lists under INLINE_VALUES_MAX are inlined at request time
// (sessionQuery; mu-authorization decides what is visible) and refreshed
// every VALUES_TTL, so the LLM never has to search the short ones.
import { fieldsOf, entityForClass } from '../runner/profile.js';
import { publicQuery } from '../db.js';
import { sessionQuery } from '../db.js';
const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50);
const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000;
@@ -34,7 +34,7 @@ async function loadCodeList(profile, f, key) {
try {
values = await queryCodeList(f.class);
} catch (e) {
// public graph read failed; report nothing, the LLM will use lookup_values
// the read failed; report nothing, the LLM will use lookup_values
values = [];
}
cache.set(key, { values, at: Date.now() });
@@ -46,7 +46,7 @@ async function queryCodeList(type) {
?uri a <${type}> .
OPTIONAL { ?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label . }
} ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`;
const r = await publicQuery(q);
const r = await sessionQuery(q);
const bindings = r.results.bindings;
return bindings.slice(0, INLINE_VALUES_MAX).map(b => ({
uri: b.uri.value,

View File

@@ -1,9 +1,10 @@
// lookup_values: search a code list too long to inline. One publicQuery,
// regex on str(?label) so language-tagged labels still match. Returns the
// matches, the total, and an `exact` flag when a label equals the term (case
// aside) so the LLM can settle disambiguation on its own.
// lookup_values: search a code list too long to inline. One query through
// the template helper, regex on str(?label) so language-tagged labels still
// match. Returns the matches, the total, and an `exact` flag when a label
// equals the term (case aside) so the LLM can settle disambiguation on its
// own.
import { publicQuery, sparqlEscapeString } from '../db.js';
import { sessionQuery, sparqlEscapeString } from '../db.js';
import { fieldsOf, entityForClass } from '../runner/profile.js';
const LIMIT = 25;
@@ -18,7 +19,7 @@ export async function lookupValues(profile, term, fieldSpec) {
?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label ; a ?type .
FILTER(REGEX(str(?label), ${sparqlEscapeString(escapeRegex(term))}, "i"))
} ORDER BY ?label LIMIT ${LIMIT * 8}`;
const r = await publicQuery(q);
const r = await sessionQuery(q);
const bindings = r.results.bindings;
// The profile's sh:class is the type the data should carry, but code lists
// sometimes type their entries differently (MAR codes are
@@ -59,7 +60,7 @@ async function codeQuery(profile, type, term) {
FILTER(CONTAINS(LCASE(str(?code)), LCASE(${sparqlEscapeString(term)})))
} ORDER BY ?label LIMIT ${LIMIT}`;
try {
const r = await publicQuery(q);
const r = await sessionQuery(q);
return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value }));
} catch {
return [];

View File

@@ -1,9 +1,7 @@
// The eight MCP tools the LLM calls. Each maps to a runner/ function or a
// pure profile/db read. Tools that touch session data (run_report,
// report_status, export_spec) live here and call into runner/; they never
// import sessionQuery themselves (CI check 2). What they return is always a
// job URI, a status, a count, a link, column names or an error we wrote,
// never a cell value (CI check 3, the §8 guarantee).
// profile/db read. What they return is always a job URI, a status, a count,
// a link, column names or an error we wrote, never a cell value (the §8
// guarantee).
//
// The shape every tool returns is { content: [{ type: 'text', text }] } so the
// loop can pass it straight back to the model.
@@ -19,10 +17,10 @@ const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
const RUN_TIMEOUT = Number(process.env.RUN_TIMEOUT || 60) * 1000;
// buildTools(profiles, session) → { tools: Tool[], handlers: Map<name, fn> }
// `session` is the request's { query, update, groups, capture } bundle the app
// builds once per /mcp request. run_report captures the identity and runs in
// the background exactly like POST /reports does, but waits up to
// RUN_TIMEOUT so the LLM gets the result in one turn when it is quick.
// `session` is the request's { query, update } bundle: the template's own
// helpers, which attach the caller's session from the request context.
// run_report starts the run and waits up to RUN_TIMEOUT so the LLM gets the
// result in one turn when it is quick.
export function buildTools(profiles, session) {
const profileList = () => [...profiles.values()].map(p => ({ id: p.uri, title: p.title }));
@@ -77,11 +75,10 @@ export function buildTools(profiles, session) {
const errors = checkSpec(parsed, p, MAX_PATH_DEPTH, profiles);
if (errors.length) return text(`the spec is not valid: ${errors[0]}`);
try {
const g = await session.groups;
// The spec's own dct:title names the report; there is no other input.
const name = parsed.title || 'report';
await session.onReportStart?.({ title: name, fileName: `${slug(name)}.csv` });
const running = run(g, parsed, p, name);
const running = run(session.query, session.update, parsed, p, name);
// The end callback fires even when withTimeout below has given up: the
// run is not cancelled, and the file arrives when it arrives. The
// rejection handler also keeps a late failure from crashing Node 20.

View File

@@ -19,7 +19,7 @@ export function reportAssistant(whenProfiles) {
const session = {
query: turn.query,
groups: turn.groups,
update: turn.update,
async onReportStart({ fileName }) {
const said = await turn.say(TEXT.making, [{ name: fileName, mediaType: 'text/csv' }]);
pendingDocument = said.documents[0].uri;

View File

@@ -4,29 +4,27 @@ import { assemble } from './assemble.js';
import { rowsToCsv, writeCsv } from './csv.js';
import { registerFile, registerReport } from './report.js';
import { touch } from './job.js';
import { groupsQuery, groupsUpdate } from '../db.js';
// Ties the six steps together. Runs with the captured groups (see
// captureGroups), returns { reportUri, fileUri, rowCount }. The task is
// touched once per column group so the dashboard shows movement.
// Ties the six steps together. `query` and `update` are the template's own
// helpers: they attach the caller's session from the request context, which
// lives for the whole run (also past the 202), so mu-authorization keeps
// deciding graphs and visibility. Returns { reportUri, fileUri, rowCount }.
// The task is touched once per column group so the dashboard shows movement.
export async function run(allowedGroups, parsed, profile, title, taskUri, extra = {}) {
const queryFn = sparql => groupsQuery(sparql, allowedGroups);
const updateFn = (sparql, extra) => groupsUpdate(sparql, allowedGroups, extra);
const subjects = await seed(queryFn, parsed);
export async function run(query, update, parsed, profile, title, taskUri, extra = {}) {
const subjects = await seed(query, parsed);
const values = new Map();
await fetchColumns(queryFn, subjects, parsed, values,
taskUri ? () => touch(updateFn, taskUri) : null);
await fetchColumns(query, subjects, parsed, values,
taskUri ? () => touch(update, taskUri) : null);
const rows = assemble(subjects, values, parsed);
const fileName = `${slug(title)}.csv`;
const csv = rowsToCsv(rows);
const filePath = writeCsv(fileName, csv);
const fileUri = await registerFile((sparql) => updateFn(sparql), fileName, filePath);
const reportUri = await registerReport((sparql) => updateFn(sparql), title, fileUri, extra);
const fileUri = await registerFile(update, fileName, filePath);
const reportUri = await registerReport(update, title, fileUri, extra);
return { reportUri, fileUri, rowCount: rows.length - 1, filePath };
}
@@ -38,4 +36,4 @@ export function slug(title) {
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80) || 'report';
}
}

45
test/ask-parse.mjs Normal file
View File

@@ -0,0 +1,45 @@
// POST /ask reads req.body, but the mu template only parses
// application/vnd.api+json; the route mounts express.json() for plain
// application/json, like the chat route does. Without an LLM configured a
// well-formed question must answer 503, not the "no question" 400.
//
// npm test (or: node --import ./dev-stubs/register.js test/ask-parse.mjs)
import assert from 'node:assert/strict';
delete process.env.LLM_BASE_URL;
let failures = 0;
async function check(name, fn) {
try {
await fn();
console.log(`ok - ${name}`);
} catch (e) {
failures++;
console.error(`FAIL - ${name}\n ${e.message}`);
}
}
const { app } = await import('mu');
await import('../app.js');
const server = app.listen(0);
const base = `http://localhost:${server.address().port}`;
const post = (path, body) => fetch(`${base}${path}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
await check('a plain JSON question parses and answers 503 without an LLM', async () => {
const res = await post('/ask', { question: 'meldingen met MAR-code 7300' });
assert.equal(res.status, 503);
});
await check('an empty body is still the 400', async () => {
const res = await post('/ask', {});
assert.equal(res.status, 400);
});
server.close();
process.exit(failures ? 1 : 0);

View File

@@ -1,12 +1,15 @@
// Unit check for src/chat/store.js. No stack, no network: writeMessage,
// setDocumentUrl and dropDocument are handed a recording updateFn and the
// SPARQL they produce is asserted on and parsed (sparqljs), so a syntax slip
// fails here and not against the auth layer.
// The chat store's SPARQL, asserted on the strings the mu stub records and
// parsed with sparqljs, so a syntax slip fails in the test and not against
// the auth layer.
//
// npm test (or: node --import ./dev-stubs/register.js test/chat-store.mjs)
import assert from 'node:assert/strict';
import { resetMu, state } from 'mu';
import { writeMessage, setDocumentUrl, dropDocument } from '../src/chat/store.js';
import { Parser } from 'sparqljs';
const CONVERSATION = 'http://data.lblod.info/id/chat-conversations/9ab4';
let failures = 0;
async function check(name, fn) {
@@ -19,100 +22,72 @@ async function check(name, fn) {
}
}
const CONVERSATION = 'http://data.lblod.info/id/chat-conversations/9ab4';
const ASSISTANT = 'http://data.lblod.info/id/chat-agents/rapportassistent';
const DOCUMENT = 'http://data.lblod.info/id/chat-documents/7d02';
// A recording update fn. `Parser` (sparqljs) is asserted later: every string
// the store builds must be a valid SPARQL update.
const { Parser } = await import('sparqljs');
const parse = (sparql) => new Parser().parse(sparql);
function recorder() {
const calls = [];
return {
calls,
async updateFn(sparql) { calls.push(sparql); },
};
}
await check('writeMessage: both types, the container, the uuid', async () => {
const { calls, updateFn } = recorder();
const { uri, id, documents } = await writeMessage(updateFn, {
conversationUri: CONVERSATION, content: 'Hallo?', maker: ASSISTANT,
});
const q = calls[0];
assert.ok(q.includes('<http://rdfs.org/sioc/ns#Post>'), 'has sioc:Post');
assert.ok(q.includes('<http://rdfs.org/sioc/types#InstantMessage>'), 'has sioct:InstantMessage');
assert.ok(q.includes('<http://rdfs.org/sioc/ns#has_container>'), 'has sioc:has_container');
assert.ok(q.includes(`<${CONVERSATION}>`), 'names the conversation');
assert.ok(q.includes(id), 'carries the message uuid');
assert.ok(q.trim().startsWith('DELETE'), 'touches the conversation');
assert.ok(!q.includes('http://purl.org/dc/terms/title'), 'no title when none given');
assert.ok(!q.includes('activitystreams'), 'no attachments when none given');
assert.equal(documents.length, 0);
assert.ok(uri.endsWith(id));
resetMu();
const { id } = await writeMessage({ conversationUri: CONVERSATION, content: 'Hallo?' });
const q = state.updates[0];
assert.ok(q.includes('<http://rdfs.org/sioc/ns#Post>'), 'sioc:Post');
assert.ok(q.includes('<http://rdfs.org/sioc/types#InstantMessage>'), 'sioct:InstantMessage');
assert.ok(q.includes(`<${CONVERSATION}>`), 'the container');
assert.ok(q.includes(id), 'the uuid');
assert.ok(q.includes('Hallo?'), 'the content');
});
await check('writeMessage: the title on the first question', async () => {
const { calls, updateFn } = recorder();
await writeMessage(updateFn, {
conversationUri: CONVERSATION, content: 'Hoeveel meldingen?',
maker: ASSISTANT, title: 'Hoeveel meldingen?',
});
assert.ok(calls[0].includes('http://purl.org/dc/terms/title'), 'writes dct:title');
resetMu();
await writeMessage({ conversationUri: CONVERSATION, content: 'Eerste vraag', title: 'Eerste vraag' });
const withTitle = state.updates[0];
assert.ok(withTitle.includes('<http://purl.org/dc/terms/title>'), 'the title triple');
resetMu();
await writeMessage({ conversationUri: CONVERSATION, content: 'Tweede vraag' });
const withoutTitle = state.updates[0];
assert.ok(!withoutTitle.includes('<http://purl.org/dc/terms/title>'), 'no title triple');
});
await check('writeMessage: an attachment without url is pending, with url is ready', async () => {
const { calls, updateFn } = recorder();
const { documents } = await writeMessage(updateFn, {
conversationUri: CONVERSATION, content: 'Ik maak het rapport.',
maker: ASSISTANT, attachments: [{ name: 'meldingen.csv', mediaType: 'text/csv' }],
resetMu();
const { documents } = await writeMessage({
conversationUri: CONVERSATION, content: 'Rapport',
attachments: [{ name: 'rapport.csv', mediaType: 'text/csv' }],
});
const pending = calls[0];
assert.ok(pending.includes('https://www.w3.org/ns/activitystreams#name'), 'names the file');
assert.ok(pending.includes('https://www.w3.org/ns/activitystreams#mediaType'), 'types the file');
assert.ok(!pending.includes('https://www.w3.org/ns/activitystreams#url'), 'a pending file has no url');
assert.ok(documents[0].uri.endsWith(documents[0].id), 'the document uri ends in its uuid');
const pending = state.updates[0];
assert.ok(pending.includes(documents[0].uri), 'the document is named');
assert.ok(pending.includes('<https://www.w3.org/ns/activitystreams#Document>'), 'as:Document');
assert.ok(!pending.includes('<https://www.w3.org/ns/activitystreams#url>'), 'no url yet');
const r2 = recorder();
await writeMessage(r2.updateFn, {
conversationUri: CONVERSATION, content: 'Klaar.',
maker: ASSISTANT,
attachments: [{ name: 'meldingen.csv', mediaType: 'text/csv', url: '/files/8c31/download' }],
resetMu();
await writeMessage({
conversationUri: CONVERSATION, content: 'Rapport',
attachments: [{ name: 'rapport.csv', mediaType: 'text/csv', url: '/files/1/download' }],
});
assert.ok(r2.calls[0].includes('https://www.w3.org/ns/activitystreams#url'), 'a ready file has a url');
const ready = state.updates[0];
assert.ok(ready.includes('/files/1/download'), 'the url is written');
});
await check('setDocumentUrl and dropDocument name the document', async () => {
const r1 = recorder();
await setDocumentUrl(r1.updateFn, DOCUMENT, '/files/8c31/download', CONVERSATION);
assert.ok(r1.calls[0].includes(`<${DOCUMENT}>`), 'setDocumentUrl names the document');
assert.ok(r1.calls[0].includes('sioc/ns#last_activity_date'), 'touches the conversation');
resetMu();
await setDocumentUrl(
'http://data.lblod.info/id/chat-documents/1', '/files/1/download', CONVERSATION);
assert.ok(state.updates[0].includes('http://data.lblod.info/id/chat-documents/1'));
const r2 = recorder();
await dropDocument(r2.updateFn, DOCUMENT);
assert.ok(r2.calls[0].includes(`<${DOCUMENT}>`), 'dropDocument names the document');
assert.ok(r2.calls[0].includes('?message'), 'drops the link from the message');
resetMu();
await dropDocument('http://data.lblod.info/id/chat-documents/1');
assert.ok(state.updates[0].includes('http://data.lblod.info/id/chat-documents/1'));
});
await check('every recorded string parses as a SPARQL update', async () => {
for (const maker of [recorder]) {
const r = maker();
await writeMessage(r.updateFn, {
conversationUri: CONVERSATION, content: 'x', maker: ASSISTANT, title: 'x',
attachments: [{ name: 'a.csv', mediaType: 'text/csv' }],
});
await writeMessage(r.updateFn, { conversationUri: CONVERSATION, content: 'y', maker: ASSISTANT });
await setDocumentUrl(r.updateFn, DOCUMENT, '/files/8c31/download', CONVERSATION);
await dropDocument(r.updateFn, DOCUMENT);
for (const [i, sparql] of r.calls.entries()) {
try {
parse(sparql);
} catch (e) {
assert.fail(`update ${i} does not parse: ${e.message}\n${sparql}`);
}
}
resetMu();
await writeMessage({ conversationUri: CONVERSATION, content: 'Parsen', title: 'Parsen' });
await writeMessage({
conversationUri: CONVERSATION, content: 'Parsen',
attachments: [{ name: 'a.csv', mediaType: 'text/csv', url: '/files/a/download' }],
});
await setDocumentUrl('http://data.lblod.info/id/chat-documents/a', '/files/a/download', CONVERSATION);
await dropDocument('http://data.lblod.info/id/chat-documents/a');
const parser = new Parser();
for (const q of state.updates) {
assert.ok(parser.parse(q), 'parses');
}
});

View File

@@ -1,15 +1,13 @@
// Integration check for the turn endpoint (src/chat/index.js). No stack: the
// SPARQL endpoint is a stub in this process that answers like the auth layer
// (JSON bindings plus the mu-auth-allowed-groups response header), and the
// module is mounted on a real express app.
// Integration check for the turn endpoint (src/chat/index.js). No stack:
// the mu stub records the queries and updates, and a handler steers the
// answers, so the test asserts on the same SPARQL the template's helpers
// would send.
//
// npm test (or: node --import ./dev-stubs/register.js test/chat-turn.mjs)
import assert from 'node:assert/strict';
import http from 'node:http';
import express from 'express';
const PORT = 8899;
import { resetMu, setQueryHandler, state } from 'mu';
let failures = 0;
async function check(name, fn) {
@@ -22,58 +20,36 @@ async function check(name, fn) {
}
}
// --- the stub auth layer ------------------------------------------------------
// --- the stub answers -----------------------------------------------------------
const KNOWN_ID = '11111111-1111-1111-1111-111111111111';
const CONVERSATION = 'http://data.lblod.info/id/chat-conversations/9ab4';
const ASSISTANT = 'http://data.lblod.info/id/chat-agents/rapportassistent';
const GROUPS = '[{"variables":["account"],"name":"chat-owner"}]';
const requests = [];
const stub = http.createServer((req, res) => {
let body = '';
req.on('data', (c) => { body += c; });
req.on('end', () => {
requests.push({ method: req.method, url: req.url, headers: req.headers, body });
res.setHeader('Content-Type', 'application/sparql-results+json');
res.setHeader('mu-auth-allowed-groups', GROUPS);
if (req.url === '/sparql' && /SELECT/.test(body)) {
if (/prov%23SoftwareAgent|prov#SoftwareAgent/.test(body)) {
res.end(JSON.stringify({ results: { bindings: [
{ agent: { value: ASSISTANT } },
] } }));
} else if (body.includes(KNOWN_ID)) {
res.end(JSON.stringify({ results: { bindings: [
{ conversation: { value: CONVERSATION }, creator: { value: 'http://data.lblod.info/id/gebruiker/1' } },
] } }));
} else {
res.end(JSON.stringify({ results: { bindings: [] } }));
}
} else {
res.end('null');
}
});
setQueryHandler((sparql) => {
if (sparql.includes('prov#SoftwareAgent')) {
return [{ agent: { value: ASSISTANT } }];
}
if (sparql.includes('sioc/ns#Thread') && sparql.includes(KNOWN_ID)) {
return [{ conversation: { value: CONVERSATION }, creator: { value: 'http://data.lblod.info/id/gebruiker/1' } }];
}
return [];
});
await new Promise((r) => stub.listen(PORT, r));
process.env.MU_SPARQL_ENDPOINT = `http://localhost:${PORT}/sparql`;
const { mountChat } = await import('../src/chat/index.js');
const app = express();
const writes = [];
mountChat(app, {
path: '/assistant',
answer: async (turn) => {
writes.push(turn.content);
return `Je schreef: ${turn.content}`;
},
});
const server = app.listen(0);
const base = `http://localhost:${server.address().port}`;
const post = (path, body, headers = {}) => fetch(`${base}${path}`, {
const post = (path, body) => fetch(`${base}${path}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'mu-session-id': 'http://mu.semte.ch/sessions/x', ...headers },
headers: { 'Content-Type': 'application/json', 'mu-session-id': 'http://mu.semte.ch/sessions/x' },
body: JSON.stringify(body),
});
@@ -88,7 +64,7 @@ await check('an unreadable conversation is 404', async () => {
});
await check('a turn answers 202 with the message id, then writes the echo', async () => {
requests.length = 0;
resetMu();
const res = await post(`/assistant/conversations/${KNOWN_ID}/turns`, { content: 'Hallo?' });
assert.equal(res.status, 202);
const { id } = await res.json();
@@ -96,15 +72,10 @@ await check('a turn answers 202 with the message id, then writes the echo', asyn
// the answer hook runs after the 202; give it a moment
await new Promise((r) => setTimeout(r, 200));
const updates = requests.filter((r) => r.method === 'POST' && /DELETE|INSERT/.test(r.body)).map((r) => r.body);
const updates = state.updates.map((u) => u);
assert.ok(updates.length >= 2, 'the question and the echo are written');
assert.ok(updates[0].includes('Hallo?'), 'the question first');
assert.ok(updates.some((u) => u.includes('Je schreef: Hallo?')), 'the echo after');
// the identity carries past the 202
assert.ok(requests.slice(1).every((r) => r.headers['mu-auth-allowed-groups'] === GROUPS),
'every write carries the captured groups');
const sessionRead = requests[0];
assert.equal(sessionRead.headers['mu-session-id'], 'http://mu.semte.ch/sessions/x');
});
await check('a throwing hook writes the failure line', async () => {
@@ -112,9 +83,7 @@ await check('a throwing hook writes the failure line', async () => {
mountChat(app2, { path: '/assistant', answer: async () => { throw new Error('boom'); } });
const server2 = app2.listen(0);
const base2 = `http://localhost:${server2.address().port}`;
const requests2 = [];
// route the stub's answers through a fresh recorder by watching requests on the shared stub
const before = requests.length;
resetMu();
const res = await fetch(`${base2}/assistant/conversations/${KNOWN_ID}/turns`, {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'mu-session-id': 'http://mu.semte.ch/sessions/x' },
@@ -122,12 +91,10 @@ await check('a throwing hook writes the failure line', async () => {
});
assert.equal(res.status, 202);
await new Promise((r) => setTimeout(r, 200));
const after = requests.slice(before).filter((r) => /DELETE|INSERT/.test(r.body)).map((r) => r.body);
assert.ok(after.some((u) => u.includes('Er ging iets mis. Probeer het opnieuw.')),
assert.ok(state.updates.some((u) => u.includes('Er ging iets mis. Probeer het opnieuw.')),
'the failure message is written');
server2.close();
});
server.close();
stub.close();
process.exit(failures ? 1 : 0);

View File

@@ -1,3 +1,6 @@
// node --import ./dev-stubs/register.js <test file>
// (this file itself needs no 'mu', so it runs without the register hook)
//
// The drop-in guarantee (plan step 1.1): src/chat/ imports nothing from the
// rest of this service, and nothing there imports it back.
import { readdirSync, readFileSync, statSync } from 'node:fs';