diff --git a/README.md b/README.md index edcc757..f1f36b2 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,11 @@ not. The answer picks one of two deterministic code paths. **Refinement (Modus A).** The LLM proposes a spec and explains in Dutch what it will list and filter. It may call `lookup_values` to search code lists for candidate values, so its filter suggestions name real values instead of -guesses. It never runs a report and never learns what a spec would match: the -database it reads may differ from the one the report runs on, so a lookup is -a suggestion, not a check. The user must confirm before anything executes. +guesses, `explore_data` to see the predicates, values and links of a target +class, and `sample_spec` to see a few rows a draft spec finds. All three read +the public graph only, under the service scope. The LLM never runs a report +and never learns what the report would hold: the report runs as the user, on +data the LLM may not see, so a sample is a hint, not a check. The user must confirm before anything executes. As long as the user has not given a clear command, the turn stays in this mode. @@ -30,7 +32,8 @@ that no longer checks out, or a failed run each give a fixed Dutch failure message. Nothing is retried and no spec is rewritten. Only refinement has tools: `list_profiles`, `describe_profile`, -`validate_spec`, `read_spec` and `lookup_values`. +`validate_spec`, `read_spec`, `lookup_values`, `explore_data` and +`sample_spec`. ## How it works @@ -78,21 +81,23 @@ never a cell value. No report row, row count or run error ever reaches the LLM. A run starts after the turn's only LLM call (the mode check) and answers with fixed text. What the LLM does see: the conversation, the profiles, validator messages, -spec files, and public reference data read under the service scope (inlined -code lists and `lookup_values`, see below). +spec files, and public data read under the service scope (inlined code +lists, `lookup_values`, `explore_data` and `sample_spec`, see below). -Keep it that way. Do not add a tool that previews, samples or counts what a -spec would match, not even to catch a `rep:where` that hangs on the wrong -node. That check stays on the profile: `validate_spec` says where each +Keep it that way. Every database read the LLM drives goes through +`publicQuery` (the service scope), never the caller's session, and none of +them counts. `sample_spec` runs a draft spec's queries on the public graph +only; it is not the report. `validate_spec` still says where each `rep:where` applies, and the proposal repeats it to the user. ### The service scope -The LLM's own reads during refinement (`describe_profile`'s code lists and -`lookup_values`) run under a service scope, not the caller's session. mu's +The LLM's own reads during refinement (`describe_profile`'s code lists, +`lookup_values`, `explore_data` and `sample_spec`) run under a service +scope, not the caller's session: `publicQuery` in `src/llm/lookup.js`. mu's `query(q, { scope })` sends `mu-auth-scope`; the sparql-parser config grants that scope read access to `http://mu.semte.ch/graphs/public` only. So -refinement reads only public code lists, regardless of who calls. Execution +refinement reads only the public graph, regardless of who calls. Execution still runs as the caller, so the report contains what they may see, and the LLM sees none of it. @@ -146,7 +151,7 @@ user sees is Dutch. | `SHARE_DIR` | `/share` | where CSVs are written | | `CHAT_ASSISTANT_URI` | — | the `prov:SoftwareAgent` that signs assistant messages; found by lookup when unset | | `CHAT_HISTORY_LIMIT` | `20` | messages of history a turn reads | -| `SERVICE_SCOPE` | `http://services.semantic.works/natural-language-report` | the mu-auth-scope for the LLM's own reads (code lists, `lookup_values`). Must match the `with-scope` grant in the app's sparql-parser config. | +| `SERVICE_SCOPE` | `http://services.semantic.works/natural-language-report` | the mu-auth-scope for the LLM's own reads (code lists, `lookup_values`, `explore_data`, `sample_spec`). Must match the `with-scope` grant in the app's sparql-parser config. | Do not set `ALLOW_MU_AUTH_SUDO`; the template then refuses sudo queries. diff --git a/src/llm/explore.js b/src/llm/explore.js new file mode 100644 index 0000000..8a128c6 --- /dev/null +++ b/src/llm/explore.js @@ -0,0 +1,136 @@ +// The LLM's exploration of the public data, beyond the code lists. Both +// read under the service scope only (publicQuery, see lookup.js) and count +// nothing: +// - explore_data: the predicates, values and links of one target class; +// - sample_spec: the queries of a draft spec, for a few subjects. +// A suggestion, not a check: the report runs as the user and may find other +// data. +import { sparqlEscapeUri } from 'mu'; +import { publicQuery } from './lookup.js'; +import { startShape } from '../runner/profile.js'; +import { seedPageQuery } from '../runner/seed.js'; +import { groupColumns, groupQuery, collect } from '../runner/columns.js'; +import { assemble } from '../runner/assemble.js'; + +const SAMPLE_SIZE = 10; +const MAX_ROWS = 30; +const EXAMPLES = 3; +const MAX_VALUE_LENGTH = 60; +const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type'; +const XSD = 'http://www.w3.org/2001/XMLSchema#'; + +// exploreClass(profile, cls) → text for the LLM. cls must be a +// sh:targetClass of the profile. +export async function exploreClass(profile, cls) { + const shapes = profile.shapes.filter(s => s.targetClass === cls); + if (!shapes.length) { + const classes = [...new Set(profile.shapes.map(s => s.targetClass).filter(Boolean))]; + return `<${cls}> is no sh:targetClass in "${profile.title}". You can explore: ${classes.map(c => `<${c}>`).join(', ')}.`; + } + + const sample = await publicQuery(`SELECT DISTINCT ?s WHERE { + ?s a ${sparqlEscapeUri(cls)} . + } LIMIT ${SAMPLE_SIZE}`); + const subjects = sample.results.bindings.map(b => sparqlEscapeUri(b.s.value)); + if (!subjects.length) { + return `no <${cls}> in the public data. That says nothing about the report: it runs as the user.`; + } + + const outgoing = await publicQuery(`SELECT DISTINCT ?p ?o ?type WHERE { + VALUES ?s { ${subjects.join(' ')} } + ?s ?p ?o . + OPTIONAL { ?o a ?type . } + }`); + const incoming = await publicQuery(`SELECT DISTINCT ?p ?type WHERE { + VALUES ?s { ${subjects.join(' ')} } + ?x ?p ?s . + OPTIONAL { ?x a ?type . } + }`); + + const lines = [ + `A sample of ${subjects.length} <${cls}> from the public data. It shows what the data looks like;`, + 'it is no check of a spec, and the report may run on other data.', + 'Out = the instance points to it, In = something points to the instance.', + '→ / ← give the class of the linked node: explore it to follow the path.', + 'A predicate "not in the profile" cannot be used in a spec.', + '', + ]; + + const out = new Map(); + for (const b of outgoing.results.bindings) { + if (b.p.value === RDF_TYPE) continue; + const entry = out.get(b.p.value) || { examples: new Set(), types: new Set() }; + if (entry.examples.size < EXAMPLES) entry.examples.add(showValue(b.o)); + if (b.type) entry.types.add(`<${b.type.value}>`); + out.set(b.p.value, entry); + } + for (const [p, { examples, types }] of out) { + const linked = types.size ? ` → ${[...types].join(', ')}` : ''; + lines.push(`Out <${p}> ${fieldName(shapes, p, false)}${linked} e.g. ${[...examples].join(' | ')}`); + } + + const inc = new Map(); + for (const b of incoming.results.bindings) { + const types = inc.get(b.p.value) || new Set(); + if (b.type) types.add(`<${b.type.value}>`); + inc.set(b.p.value, types); + } + for (const [p, types] of inc) { + const linked = types.size ? ` ← ${[...types].join(', ')}` : ''; + lines.push(`In <${p}> ${fieldName(shapes, p, true)}${linked}`); + } + + return lines.join('\n'); +} + +// sampleSpec(spec, profile) → text for the LLM: the rows of up to +// SAMPLE_SIZE subjects the spec finds in the public data. The spec has +// passed checkSpec. Same query builders as the run, other identity. +export async function sampleSpec(spec, profile) { + const seed = await publicQuery(seedPageQuery(spec, SAMPLE_SIZE, 0, startShape(profile, spec))); + const subjects = seed.results.bindings.map(b => b.s.value); + if (!subjects.length) { + return 'the spec finds nothing in the public data. That says nothing about the report: it runs as the user, on data you cannot see.'; + } + + const values = new Map(); + for (const group of groupColumns(spec.columns)) { + collect(await publicQuery(groupQuery(group, subjects)), group, values); + } + const [header, ...rows] = assemble(subjects, values, spec); + + return [ + `A sample from the public data: rows for up to ${SAMPLE_SIZE} subjects the spec finds there.`, + 'The report runs as the user and may find other rows. Use it to see whether paths and filters', + 'land where you expect, not to tell the user what the report will hold.', + '', + header.join(' | '), + ...rows.slice(0, MAX_ROWS).map(row => row.map(showCell).join(' | ')), + ].join('\n'); +} + +function showCell(cell) { + const text = String(cell ?? ''); + return text.length > MAX_VALUE_LENGTH ? `${text.slice(0, MAX_VALUE_LENGTH)}…` : text; +} + +// The profile field a predicate is, on any entity with this class. +function fieldName(shapes, predicate, inverse) { + const names = []; + for (const shape of shapes) { + const f = shape.fields.find(x => x.path === predicate && !!x.inverse === inverse); + if (f) names.push(shapes.length > 1 ? `"${f.name}" of ${shape.label}` : `"${f.name}"`); + } + return names.length ? `field ${names.join(', ')}${inverse ? ' (inverse)' : ''}` : 'not in the profile'; +} + +// A SPARQL JSON term as the LLM would write it in Turtle. +function showValue(term) { + if (term.type === 'uri') return `<${term.value}>`; + if (term.type === 'bnode') return '[blank node]'; + const text = term.value.length > MAX_VALUE_LENGTH ? `${term.value.slice(0, MAX_VALUE_LENGTH)}…` : term.value; + if (term['xml:lang']) return `${JSON.stringify(text)}@${term['xml:lang']}`; + if (!term.datatype) return JSON.stringify(text); + const datatype = term.datatype.startsWith(XSD) ? term.datatype.replace(XSD, 'xsd:') : `<${term.datatype}>`; + return `${JSON.stringify(text)}^^${datatype}`; +} diff --git a/src/llm/lookup.js b/src/llm/lookup.js index 9793ee5..afe4100 100644 --- a/src/llm/lookup.js +++ b/src/llm/lookup.js @@ -1,7 +1,8 @@ -// The LLM's own reads: code lists. They run under the service scope, not -// the caller's session; the sparql-parser config grants that scope read -// access to the public graph only, so they read the same for every caller. -// Execution still runs as the caller. +// The LLM's own reads: code lists here, the data samples in explore.js. +// They run under the service scope, not the caller's session; the +// sparql-parser config grants that scope read access to the public graph +// only, so they read the same for every caller. Execution still runs as the +// caller. // // lookup_values is a suggest helper, not a spec check. It reads candidate // values the LLM can put in a filter; it never runs the spec and never sees @@ -17,7 +18,7 @@ const LIMIT = 25; // class → { values, at }. Refreshed lazily when stale. const cache = new Map(); -function publicQuery(q) { +export function publicQuery(q) { return query(q, { scope: SERVICE_SCOPE }); } diff --git a/src/llm/loop.js b/src/llm/loop.js index 2ce38a7..14ad58f 100644 --- a/src/llm/loop.js +++ b/src/llm/loop.js @@ -3,8 +3,9 @@ // SDKs. Stops when the model answers without tool calls. // // Refinement only: the LLM proposes a spec, may look up candidate values in -// code lists (under the service scope) and iterates with the user. It never -// runs a report and never learns what a spec would match. The user must +// code lists, explore the data and sample a draft spec (all on the public +// graph, under the service scope) and iterates with the user. It never +// runs a report and never learns what the report would hold. The user must // confirm before anything executes; execution runs the agreed spec without // the LLM (../report-assistant.js). @@ -19,9 +20,10 @@ const LLM_API_KEY = process.env.LLM_API_KEY; const LOG_LLM = /^(true|1)$/i.test(process.env.LOG_LLM || ''); // Refinement (Modus A). The LLM and the user iterate on a spec. The LLM may -// look up candidate values in code lists (lookup_values, under the service -// scope) to make concrete suggestions, but it may not run a report and never -// learns how many subjects a spec would match. It ends with a proposal in +// look up candidate values in code lists (lookup_values), explore the data +// of a target class (explore_data) and sample a draft spec (sample_spec), +// all on the public graph under the service scope, but it may not run a +// report and never learns how many subjects the report would match. It ends with a proposal in // Dutch and asks the user to confirm. const REFINE_PROMPT = `You refine report specs with the user. You never write SPARQL and you never run a report. @@ -54,10 +56,15 @@ Rules: You are in refinement mode. You propose a spec and explain in Dutch what it will list and filter. Use lookup_values to find concrete values (names, codes) -for a filter, so your proposal names real values instead of guesses. That is -the only database read you have, and it only suggests values: it never tells -you whether a spec is good or how much it matches. The database you read may -differ from the one the report runs on, so never treat a lookup as a check. +for a filter, so your proposal names real values instead of guesses. Use +explore_data to see what the data of a target class holds: its predicates, +example values and links; call it on a linked class to follow a path. A +predicate that is not in the profile cannot go in a spec. Use sample_spec on +a spec that validates to see a few of its rows, and fix paths or filters that +do not land where you expect. All three read only public data. The report +runs as the user, on data you may not see, so a lookup or a sample is a hint, +never a count and never a promise: do not tell the user how many rows the +report will have, or that it will be empty. Iterate with the user on the spec itself: paths, filters, columns. When the user wants an earlier spec changed, open it with read_spec and change what they asked for. Never call diff --git a/src/llm/tools.js b/src/llm/tools.js index 55abf1c..0ce4ac5 100644 --- a/src/llm/tools.js +++ b/src/llm/tools.js @@ -1,11 +1,12 @@ // The tools the LLM calls, refinement only: list_profiles, -// describe_profile, validate_spec, read_spec and lookup_values. Each returns -// plain text for the model. No report is created, no spec is evaluated; -// the code-list reads run under the service scope (public graph). +// describe_profile, validate_spec, read_spec, lookup_values, explore_data +// and sample_spec. Each returns plain text for the model. No report is +// created; the database reads run under the service scope (public graph). import { parseSpec } from '../runner/spec.js'; import { checkSpec, whereNotes, profileError } from '../runner/check.js'; import { describeProfile } from './describe.js'; import { lookupValues } from './lookup.js'; +import { exploreClass, sampleSpec } from './explore.js'; import { readSpecFile } from '../chat.js'; // The tool definitions sent to the model. The descriptions are what the LLM @@ -30,6 +31,15 @@ export const TOOLS = [ field: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"', term: 'The Dutch word or name to search for', }), + tool('explore_data', + 'Show a sample of the public data of one sh:targetClass of the profile: the predicates its instances use (outgoing and incoming), a few example values, the class of linked nodes, and which profile field each predicate is. Call it again on a linked class to follow a path. A predicate not in the profile cannot be used in a spec. A sample, not a check: it never runs a spec and counts nothing.', + { + profile_id: 'The profile URI returned by list_profiles', + class: 'The sh:targetClass URI, as describe_profile shows it', + }), + tool('sample_spec', + 'Run a spec that passes validate_spec on the public data and return the rows of up to 10 subjects it finds there. Use it to see whether its paths and filters land where you expect. Public data only: the report runs as the user and may find other rows, so an empty or short sample says nothing about the report. It does not attach the spec; validate_spec does.', + { spec: 'The spec as Turtle text' }), ]; function tool(name, description, params) { @@ -54,6 +64,8 @@ export async function runTool(name, args, profiles) { case 'validate_spec': return validateSpec(args, profiles); case 'read_spec': return readSpec(args); case 'lookup_values': return lookup(args, profiles); + case 'explore_data': return explore(args, profiles); + case 'sample_spec': return sample(args, profiles); default: return `unknown tool "${name}"`; } } @@ -89,3 +101,21 @@ async function lookup({ profile_id, field, term }, profiles) { if (!term) return 'no term to search for.'; return lookupValues(profiles.get(profile_id), field, term); } + +async function explore({ profile_id, class: cls }, profiles) { + const error = profileError(profiles, profile_id); + if (error) return error; + return exploreClass(profiles.get(profile_id), cls); +} + +async function sample({ spec }, profiles) { + let parsed; + try { + parsed = parseSpec(spec); + } catch (e) { + return `the spec did not parse: ${e.message}`; + } + const error = checkSpec(parsed, profiles); + if (error) return error; + return sampleSpec(parsed, profiles.get(parsed.profileUri)); +} diff --git a/src/runner/columns.js b/src/runner/columns.js index d916199..b550350 100644 --- a/src/runner/columns.js +++ b/src/runner/columns.js @@ -52,7 +52,7 @@ export function groupQuery(group, chunk) { return `SELECT ?s ?v WHERE {\n VALUES ?s { ${subjects} }\n ${lines.join('\n ')}\n}`; } -function collect(result, group, values) { +export function collect(result, group, values) { for (const b of result.results.bindings) { let perColumn = values.get(b.s.value); if (!perColumn) { perColumn = new Map(); values.set(b.s.value, perColumn); }