delete

README.md

@@ -29,8 +29,8 @@ a fixed Dutch text with the CSV and the spec as bijlagen. No spec yet, a spec
 that no longer checks out, or a failed run each give a fixed Dutch failure
 message. Nothing is retried and no spec is rewritten.
 
-Only refinement has tools: `list_profiles`, `describe_profile`,
+Only refinement has tools: `describe_profile`, `validate_spec`, `read_spec`
-`validate_spec`, `read_spec` and `lookup_values`.
+and `lookup_values`.
 
 ## How it works
 
@@ -411,6 +411,7 @@ The scope URI is `SERVICE_SCOPE` (default
 
 ```
 POST /assistant/conversations/:id/turns      { content }           → 202 { id }
+DELETE /assistant/conversations/:id                                 → 204
 ```
 
 This is the chat. The route is in `app.js`: it records the question, answers
@@ -424,6 +425,12 @@ that way. Failure is a message, never a silent loader.
 The chat needs `LLM_BASE_URL`; without it every turn answers with a failure
 message. A full-org report run takes minutes.
 
+The delete is the trash bin in the chat's sidebar. It removes the
+conversation, its messages, their bijlagen (the triples and the files on
+the share) and the reports of those files. It runs as the caller: the auth
+layer keeps a user to their own conversations, and a conversation the caller
+cannot read is a 404.
+
 ### The internal tools
 
 The refinement loop drives one tool set (`src/llm/tools.js`). The tools are
@@ -431,6 +438,27 @@ machinery, not an external surface. All LLM-facing text (system prompts,
 profile menu, tool descriptions, validator errors) is English; the answer the
 user sees is Dutch.
 
+### What the model sees
+
+The system prompt ends with the profile list and, when there is a current
+spec, the menu of its profile (`src/llm/loop.js`). A turn that stays on that
+profile needs no `describe_profile` call; the model calls it only for
+another profile.
+
+Each request to the model holds one spec in full: the newest.
+
+- The history is text only. A spec bijlage shows as its file name, and a
+  spec the model printed in its answer is cut out.
+- The current spec, the last spec bijlage in the conversation, comes in full
+  under the user's new message. When the user refines, the model changes it;
+  when the user asks for a report that lists another kind of thing per row,
+  it starts a new spec.
+- Within a turn, each newer spec (a `validate_spec` call, or an older spec
+  opened with `read_spec`) replaces the older one with "Left out: a newer
+  spec follows."
+- The model opens an older spec with `read_spec` only when the user asks for
+  it.
+
 ## Environment
 
 | Var | Default | Meaning |

app.js

@@ -1,7 +1,7 @@
 import { app, errorHandler } from 'mu';
 import bodyParser from 'body-parser';
 import { loadProfiles } from './src/runner/profile.js';
-import { readConversation, readHistory, writeMessage } from './src/chat.js';
+import { readConversation, readHistory, writeMessage, deleteConversation } from './src/chat.js';
 import { wantsExecution } from './src/llm/mode.js';
 import { executeSpec, refineSpec } from './src/report-assistant.js';
 
@@ -64,4 +64,21 @@ app.post('/assistant/conversations/:id/turns', bodyParser.json(), async function
   }
 });
 
+// Delete a conversation with everything that hangs off it: the messages,
+// their bijlagen (triples and share files) and the reports. The read is the
+// access check: the caller's session decides what exists, so another user's
+// conversation is a 404.
+app.delete('/assistant/conversations/:id', async function(req, res) {
+  try {
+    const conversation = await readConversation(req.params.id);
+    if (!conversation) return res.status(404).json({ error: 'no such conversation' });
+    const removed = await deleteConversation(conversation.uri);
+    console.log(`[chat] deleted conversation ${req.params.id}, ${removed} share file(s) removed`);
+    res.status(204).send();
+  } catch (e) {
+    console.error('[chat] could not delete the conversation:', e);
+    res.status(500).json({ error: e.message });
+  }
+});
+
 app.use(errorHandler);

src/chat.js

@@ -6,7 +6,7 @@
 // A bijlage is a real file: a logical nfo:FileDataObject the file service
 // serves at /files/<uuid>/download, over a physical share:// object with
 // nie:dataSource back to the logical one.
-import { readFileSync, writeFileSync, statSync } from 'fs';
+import { readFileSync, writeFileSync, statSync, unlinkSync } from 'fs';
 import { query, update, uuid,
          sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt, sparqlEscapeDateTime } from 'mu';
 
@@ -32,6 +32,7 @@ const T = {
   format: 'http://purl.org/dc/terms/format',
   fileType: 'http://purl.org/dc/terms/type',
   dataSource: 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource',
+  generated: 'http://www.w3.org/ns/prov#generated',
 };
 const MESSAGE_BASE = 'http://data.lblod.info/id/chat-messages/';
 const FILE_BASE = 'http://data.lblod.info/files/';
@@ -210,3 +211,68 @@ export async function registerFile(shareName, format, type, name = shareName) {
 
   return logical;
 }
+
+// --- deletes -------------------------------------------------------------------
+
+// deleteConversation(uri) → the share files it removed (a number). Deletes
+// the conversation, its messages, their bijlagen (the logical file and the
+// physical one under it) and the reports of those files, then removes the
+// share files. Everything runs as the caller, so the auth layer keeps a
+// user to their own conversation.
+export async function deleteConversation(conversationUri) {
+  // The bijlagen first: once the triples are gone their names cannot be
+  // found back.
+  const json = await query(`
+    SELECT DISTINCT ?doc ?name WHERE {
+      ?message ${u(T.hasContainer)} ${u(conversationUri)} ;
+        ${u(T.attachment)} ?doc .
+      OPTIONAL {
+        ?physical ${u(T.dataSource)} ?doc ;
+          ${u(T.fileName)} ?name .
+      }
+    }`);
+  const rows = bindings(json);
+  const docs = [...new Set(rows.map(b => b.doc.value))];
+  const names = [...new Set(rows.filter(b => b.name).map(b => b.name.value))];
+
+  await update(`
+    DELETE {
+      ?conversation ?cp ?co .
+    } WHERE {
+      VALUES ?conversation { ${u(conversationUri)} }
+      ?conversation ?cp ?co .
+    }`);
+
+  await update(`
+    DELETE {
+      ?message ?mp ?mo .
+    } WHERE {
+      ?message ${u(T.hasContainer)} ${u(conversationUri)} ;
+        ?mp ?mo .
+    }`);
+
+  if (docs.length) {
+    await update(`
+      DELETE {
+        ?doc ?dp ?do .
+        ?physical ?pp ?po .
+        ?report ?rp ?ro .
+      } WHERE {
+        VALUES ?doc { ${docs.map(u).join(' ')} }
+        ?doc ?dp ?do .
+        OPTIONAL { ?physical ${u(T.dataSource)} ?doc ; ?pp ?po . }
+        OPTIONAL { ?report ${u(T.generated)} ?doc ; ?rp ?ro . }
+      }`);
+  }
+
+  let removed = 0;
+  for (const name of names) {
+    try {
+      unlinkSync(`${SHARE_DIR}/${name}`);
+      removed++;
+    } catch {
+      // the file was already gone
+    }
+  }
+  return removed;
+}

src/llm/loop.js

@@ -8,10 +8,14 @@
 // confirm before anything executes; execution runs the agreed spec without
 // the LLM (../report-assistant.js).
 
-import { TOOLS, runTool } from './tools.js';
+import { TOOLS, runTool, listProfiles } from './tools.js';
-import { SPEC_MEDIA_TYPE } from '../chat.js';
+import { describeProfile } from './describe.js';
+import { parseSpec } from '../runner/spec.js';
+import { SPEC_MEDIA_TYPE, readSpecFile } from '../chat.js';
 
 const MAX_ROUNDS = Number(process.env.MAX_ROUNDS || 12);
+// What an older spec becomes: only the newest spec goes to the model in full.
+const LEFT_OUT = 'Left out: a newer spec follows.';
 const LLM_BASE_URL = process.env.LLM_BASE_URL;
 const LLM_MODEL = process.env.LLM_MODEL;
 const LLM_API_KEY = process.env.LLM_API_KEY;
@@ -25,7 +29,10 @@ const LOG_LLM = /^(true|1)$/i.test(process.env.LOG_LLM || '');
 // Dutch and asks the user to confirm.
 const REFINE_PROMPT = `You refine report specs with the user. You never write SPARQL and you never run a report.
 
-The profile lists what can be asked. Compose paths by chaining fields.
+The profile lists what can be asked. Compose paths by chaining fields. The
+profiles are listed at the end of this prompt. When there is a current spec,
+the menu of its profile follows them: use it, and call describe_profile only
+for another profile.
 
 Rules:
 - A column must end on a value, not on a link. Chain one more hop. Only when
@@ -61,9 +68,14 @@ for a filter, so your proposal names real values instead of guesses. That is
 the only database read you have, and it only suggests values: it never tells
 you whether a spec is good or how much it matches. The database you read may
 differ from the one the report runs on, so never treat a lookup as a check.
-Iterate with the user on the spec itself: paths, filters, columns. When the
+Iterate with the user on the spec itself: paths, filters, columns. The
-user wants an earlier spec changed, open it with read_spec and change what
+current spec, the last one in the conversation, comes with the user's
-they asked for. Never call
+message. When the user refines (other columns, filters or title), change the
+current spec. When the user asks for a different report, one that lists
+another kind of thing per row, start a new spec. Older specs show only as a
+bijlage name, and a spec you wrote before in this turn shows as "left out"
+once a newer one follows. Only when the user asks for an older spec, open it
+with read_spec and change that one. Never call
 run_report; there is no such tool here. When the spec is ready, ask the user
 to confirm execution. Then stop.
 
@@ -85,14 +97,17 @@ Reply to the user in Dutch. Earlier turns are context only. Answer the last one.
 // [{ role, content, attachments }], newest last. spec is the last spec
 // validate_spec accepted in this turn, or null.
 export async function ask(turns, profiles) {
-  const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns.map(withSpecHint)];
+  const current = currentSpec(turns);
+  const messages = [{ role: 'system', content: await systemPrompt(current, profiles) },
+    ...withCurrentSpec(turns.map(withSpecHint), current)];
   let spec = null;
 
   for (let round = 0; round < MAX_ROUNDS; round++) {
-    const { message } = await chat(messages, TOOLS);
+    const { message } = await chat(newestSpecOnly(messages), TOOLS);
     messages.push(message);
     const calls = message.tool_calls || [];
     if (!calls.length) return { text: message.content || '', spec };
+    if (calls.some(c => c.function.name === 'validate_spec')) message.leftOut = { tool_calls: calls.map(leaveOutSpec) };
 
     for (const call of calls) {
       const args = parseArgs(call.function.arguments);
@@ -111,20 +126,85 @@ export async function ask(turns, profiles) {
         }
         spec = args.spec;
       }
-      messages.push({ role: 'tool', tool_call_id: call.id, content: result });
+      const reply = { role: 'tool', tool_call_id: call.id, content: result };
+      if (call.function.name === 'read_spec' && result.includes('ReportSpec')) reply.leftOut = { content: LEFT_OUT };
+      messages.push(reply);
     }
   }
   return { text: 'Sorry, dit rapport is te moeilijk. Contacteer de developers.', spec };
 }
 
-// The history carries the spec as a bijlage. Paste one hint line per spec
+// The history carries the spec as a bijlage, never as text. Paste one hint
-// under the message text, so the model knows a spec is there and opens it
+// line per spec under the message text, so the model knows a spec is there
-// with read_spec; the chat itself still keeps the Turtle out of the text.
+// and can open it with read_spec when the user asks for it. A spec the model
+// printed in its answer anyway is cut from the text.
 function withSpecHint(turn) {
+  const content = turn.role === 'assistant' ? withoutSpecText(turn.content) : turn.content;
   const specs = (turn.attachments || []).filter(a => a.mediaType === SPEC_MEDIA_TYPE);
-  if (!specs.length) return turn;
+  if (!specs.length) return { ...turn, content };
-  const hints = specs.map(a => `Bijlage "${a.name}", a spec file (read it with read_spec).`);
+  const hints = specs.map(a => `Bijlage "${a.name}", a spec file.`);
-  return { ...turn, content: `${turn.content}\n\n${hints.join('\n\n')}` };
+  return { ...turn, content: `${content}\n\n${hints.join('\n\n')}` };
+}
+
+function withoutSpecText(text) {
+  return text.replace(/```[\s\S]*?```/g, block => (block.includes('ReportSpec') ? '(spec left out)' : block));
+}
+
+// The current spec: the last spec bijlage in the conversation, or null.
+function currentSpec(turns) {
+  const name = turns.flatMap(t => t.attachments || [])
+    .filter(a => a.mediaType === SPEC_MEDIA_TYPE).at(-1)?.name;
+  const turtle = name && readSpecFile(name);
+  return turtle ? { name, turtle } : null;
+}
+
+// The system prompt also carries what the model would otherwise fetch at the
+// start of every turn: the profile list, and the menu of the current spec's
+// profile. Without a menu the model calls describe_profile itself.
+async function systemPrompt(current, profiles) {
+  const parts = [REFINE_PROMPT, `The profiles:\n${listProfiles(profiles)}`];
+  const uri = current && profileOf(current.turtle);
+  if (profiles.has(uri)) {
+    try {
+      parts.push(`The menu of the current spec's profile <${uri}>:\n\n${await describeProfile(profiles.get(uri))}`);
+    } catch (e) {
+      console.error('[llm] could not describe the current profile:', e.message);
+    }
+  }
+  return parts.join('\n\n');
+}
+
+function profileOf(turtle) {
+  try {
+    return parseSpec(turtle).profileUri;
+  } catch {
+    return null;
+  }
+}
+
+// The current spec goes in full under the user's new message: the one spec
+// the history holds as text.
+function withCurrentSpec(messages, current) {
+  if (!current) return messages;
+  const { name, turtle } = current;
+  const last = messages.at(-1);
+  return [...messages.slice(0, -1), {
+    ...last,
+    content: `${last.content}\n\nThe current spec, bijlage "${name}":\n\n${turtle}`,
+    leftOut: { content: `${last.content}\n\nThe current spec, bijlage "${name}": ${LEFT_OUT}` },
+  }];
+}
+
+// Only the newest spec goes to the model in full. A message that holds an
+// older one goes in its leftOut form; read_spec opens that spec again.
+function newestSpecOnly(messages) {
+  const newest = messages.findLastIndex(m => m.leftOut);
+  return messages.map((m, i) => (m.leftOut && i !== newest ? { ...m, ...m.leftOut } : m));
+}
+
+function leaveOutSpec(call) {
+  if (call.function.name !== 'validate_spec') return call;
+  return { ...call, function: { ...call.function, arguments: JSON.stringify({ spec: LEFT_OUT }) } };
 }
 
 // chat(messages, tools) → { message }. The one place that talks to the

src/llm/tools.js

@@ -1,5 +1,5 @@
-// The tools the LLM calls, refinement only: list_profiles,
+// The tools the LLM calls, refinement only: describe_profile,
-// describe_profile, validate_spec, read_spec and lookup_values. Each returns
+// validate_spec, read_spec and lookup_values. Each returns
 // plain text for the model. No report is created, no spec is evaluated; the
 // database reads run under the service scope (public graph).
 import { parseSpec } from '../runner/spec.js';
@@ -11,22 +11,19 @@ import { readSpecFile } from '../chat.js';
 // The tool definitions sent to the model. The descriptions are what the LLM
 // reads to decide which tool to call.
 export const TOOLS = [
-  tool('list_profiles',
-    'List the report profiles available, with their id and title.',
-    {}),
   tool('describe_profile',
     'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.',
-    { profile_id: 'The profile URI returned by list_profiles' }),
+    { profile_id: 'The profile URI from the profile list' }),
   tool('validate_spec',
     'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.',
     { spec: 'The spec as Turtle text' }),
   tool('read_spec',
-    'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.',
+    'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. The current spec already comes with the user\'s message; use this only when the user asks for an older spec. Read-only.',
     { file_name: 'The file name of the spec bijlage, e.g. specificatie-<uuid>.ttl' }),
   tool('lookup_values',
     'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.',
     {
-      profile_id: 'The profile URI returned by list_profiles',
+      profile_id: 'The profile URI from the profile list',
       field: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"',
       term: 'The Dutch word or name to search for',
     }),
@@ -49,7 +46,6 @@ function tool(name, description, params) {
 
 export async function runTool(name, args, profiles) {
   switch (name) {
-    case 'list_profiles': return listProfiles(profiles);
     case 'describe_profile': return describe(args, profiles);
     case 'validate_spec': return validateSpec(args, profiles);
     case 'read_spec': return readSpec(args);
@@ -58,7 +54,8 @@ export async function runTool(name, args, profiles) {
   }
 }
 
-function listProfiles(profiles) {
+// The profile list, in the system prompt: one line per profile.
+export function listProfiles(profiles) {
   return [...profiles.values()].map(p => `${p.title}  <${p.uri}>`).join('\n');
 }