felix.ruizdearcaute/natural-language-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
felix
2026-09-15 19:12:03 +02:00
parent f128255226
commit 8334512d1e
11 changed files with 116 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@@ -84,17 +84,17 @@ sees of them.
| `MU_SPARQL_ENDPOINT` | — | the auth layer, e.g. `http://database:8890/sparql`. Never virtuoso. |
| `REPORT_CLASS` | `…/reporting/Report` | type of the report resource |
| `CSV_SEPARATOR` | `;` | CSV column separator |
| `SUBJECT_CHUNK_SIZE` | `250` | subjects per column query |
| `SUBJECT_CHUNK_SIZE` | `100` | subjects per column query (VALUES batch size); lower it when the auth layer rejects long queries |
| `ROW_LIMIT` | `200000` | seed limit; failing the job beats truncating |
| `MAX_PATH_DEPTH` | `8` | longest allowed spec path |
| `RUN_TIMEOUT` | `60` | seconds `run_report` waits before giving up. Full-org runs need 300. |
| `RUN_TIMEOUT` | `60` | seconds of silence before `run_report` gives up. Every answered query (seed page, column batch) restarts the clock, so a big report only needs the window per batch, not for the whole run. The run itself is never cancelled: the file still lands in the chat. |
| `INLINE_VALUES_MAX` | `50` | code lists up to this size are inlined in `describe_profile` |
| `VALUES_TTL` | `3600` | seconds the inlined lists are cached |
| `LLM_BASE_URL` | — | enables `/ask`; OpenAI-compatible (`https://ollama.com/v1`) |
| `LLM_MODEL` | — | e.g. `gpt-oss:120b` |
| `LLM_API_KEY` | — | bearer token, optional for local Ollama |
| `SEED_PAGE_SIZE` | `5000` | seed page size |
| `SUBJECT_CHUNK_SIZE` | `250` | see above |
| `SUBJECT_CHUNK_SIZE` | `100` | see above |
| `SHARE_DIR` | `/share` | where CSVs are written |
| `CHAT_ASSISTANT_URI` | — | the `prov:SoftwareAgent` that signs assistant messages; found by lookup when unset |
| `CHAT_HISTORY_LIMIT` | `20` | messages of history a turn reads |

1
src/llm/describe.js
View File

@@ -70,6 +70,7 @@ export function describeProfile(profile, codeLists = {}) {
lines.push('');
lines.push('sh:targetClass is de volledige URI tussen haakjes achter de entiteit hieronder.');
lines.push('Gebruik in sh:path de predicate-URI die achter elke veldnaam staat, niet de veldnaam zelf.');
lines.push('Een omgekeerde hop (het veld staat "(omgekeerd)" achter) schrijf je in de lijst als [ sh:inversePath <predicate> ] .');
lines.push('Een datumfilter is sh:minInclusive of sh:maxInclusive met "jjjj-mm-dd"^^xsd:dateTime');
lines.push('(kopieer ook het xsd-prefix hierboven niet: voeg @prefix xsd: <http://www.w3.org/2001/XMLSchema#> toe).');
lines.push('');

37
src/llm/lookup.js
View File

@@ -13,29 +13,20 @@ export async function lookupValues(profile, term, fieldSpec) {
const type = resolveClass(profile, fieldSpec);
if (!type) return { total: 0, exact: null, matches: [], error: `field "${fieldSpec}" not found in profile "${profile.title}"` };
const q = `SELECT ?uri ?label ?type WHERE {
?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label ; a ?type .
FILTER(REGEX(str(?label), ${sparqlEscapeString(escapeRegex(term))}, "i"))
} ORDER BY ?label LIMIT ${LIMIT * 8}`;
const r = await sessionQuery(q);
const bindings = r.results.bindings;
// The profile's sh:class is the type the data should carry, but code lists
// sometimes type their entries differently (MAR codes are
// ext:supervision/Nomenclature, the profile says skos:Concept). Prefer rows
// matching the asked class, fall back to all rows. The query reads more
// rows than LIMIT so an exact hit deeper in the alphabet still surfaces.
const wanted = bindings.filter(b => type && b.type?.value === type);
const rows = wanted.length ? wanted : bindings;
// Search within the class first. A common word ("gemeente") also sits in
// hundreds of unrelated labels; an unscoped scan drowns the code.
let rows = await labelQuery(term, type);
// Code lists sometimes type their entries differently than the profile
// says (MAR codes are ext:supervision/Nomenclature, the profile says
// skos:Concept). Fall back to a scan over everything.
if (!rows.length) rows = await labelQuery(term, null);
// Order: an exact label (case aside) first, then the rest as the store
// ordered them. "Gent" must surface before "AGB Erfgoed Gent".
const lower = term.toLowerCase();
const exactRow = rows.find(b => b.label.value.toLowerCase() === lower);
const exactRow = rows.find(b => b.label.toLowerCase() === lower);
const rest = rows.filter(b => b !== exactRow);
const ordered = exactRow ? [exactRow, ...rest] : rest;
let matches = ordered.slice(0, LIMIT).map(b => ({
label: b.label.value,
uri: b.uri.value,
}));
let matches = ordered.slice(0, LIMIT);
if (!matches.length) {
const byCode = await codeQuery(profile, type, term);
matches = byCode;
@@ -48,6 +39,16 @@ export async function lookupValues(profile, term, fieldSpec) {
};
}
async function labelQuery(term, type) {
const q = `SELECT DISTINCT ?uri ?label WHERE {
?uri a ${type ? `<${type}>` : '?anyType'} ;
<http://www.w3.org/2004/02/skos/core#prefLabel> ?label .
FILTER(REGEX(str(?label), ${sparqlEscapeString(escapeRegex(term))}, "i"))
} ORDER BY ?label LIMIT ${LIMIT * 8}`;
const r = await sessionQuery(q);
return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value }));
}
// MAR codes: the label says "MAR7300 - …" but the searchable code sits on
// ext:supervision/nomenclatureCode. One extra query, only when the label
// search came up empty.

2
src/llm/schemas.js
View File

@@ -50,7 +50,7 @@ export function toolSchemas() {
},
{
name: 'run_report',
description: 'Run a valid spec and return the report URI, the row count and the file URI. Runs as the caller, so it sees only what they may see. Waits up to RUN_TIMEOUT. The report is named by the dct:title in the spec.',
description: 'Run a valid spec and return the report URI, the row count and the file URI. Runs as the caller, so it sees only what they may see. Waits as long as the run keeps answering; gives up after RUN_TIMEOUT seconds of silence. The report is named by the dct:title in the spec.',
inputSchema: {
type: 'object',
properties: { spec: { type: 'string' } },

52
src/llm/tools.js
View File

@@ -9,12 +9,17 @@ import { seedPageQuery } from '../runner/seed.js';
import { describeProfile, codeListValues } from './describe.js';
import { run, slug } from '../runner/run.js';
import { lookupValues } from './lookup.js';
import { readSpec } from '../runner/csv.js';
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
const RUN_TIMEOUT = Number(process.env.RUN_TIMEOUT || 60) * 1000;
// buildTools(profiles, session) → { handlers }
// run_report waits up to RUN_TIMEOUT so a quick run returns in one turn.
// run_report waits while the run keeps answering: every query that comes
// back (a seed page, a column batch) restarts the RUN_TIMEOUT clock, and
// only that much silence gives up. The batch count is known upfront — the
// subject list is complete after the seed — so a slow run with many small
// batches finishes, however long it takes in total.
export function buildTools(profiles, session) {
const profileList = () => [...profiles.values()].map(p => ({ id: p.uri, title: p.title }));
@@ -71,14 +76,15 @@ export function buildTools(profiles, session) {
// The spec's dct:title names the report.
const name = parsed.title || 'report';
await session.onReportStart?.({ title: name, fileName: `${slug(name)}.csv` });
const running = run(session.query, session.update, parsed, p, name);
// The end callback also fires after the timeout gave up: the run is not
const waiter = newRunWaiter(RUN_TIMEOUT);
const running = run(session.query, session.update, parsed, p, name, null, { spec }, waiter.tick);
// The end callback also fires after the wait gave up: the run is not
// cancelled, the file arrives when it arrives. The .catch keeps a late
// failure from crashing the process.
running
.then((r) => session.onReportEnd?.(null, r), (e) => session.onReportEnd?.(e))
.catch((e) => console.error('[reports] end callback failed:', e));
const result = await withTimeout(running, RUN_TIMEOUT);
const result = await waiter.finish(running);
return text(`done. report <${result.reportUri}>, ${result.rowCount} rows, file <${result.fileUri}>.`);
} catch (e) {
return text(`the report failed: ${String(e.message || e).split('\n')[0]}`);
@@ -103,9 +109,14 @@ export function buildTools(profiles, session) {
async function export_spec({ report_uri }) {
const r = await session.query(
`SELECT ?spec WHERE { <${report_uri}> <http://mu.semte.ch/vocabularies/ext/spec> ?spec } LIMIT 1`);
`SELECT ?specFile WHERE { <${report_uri}> <http://mu.semte.ch/vocabularies/ext/specFile> ?specFile } LIMIT 1`);
const b = r.results.bindings[0];
return text(b ? b.spec.value : `no spec stored on <${report_uri}>.`);
if (!b) return text(`no spec stored on <${report_uri}>.`);
try {
return text(readSpec(b.specFile.value));
} catch {
return text(`the spec file ${b.specFile.value} is not on disk anymore.`);
}
}
const handlers = new Map([
@@ -129,6 +140,31 @@ function buildQueriesText(spec, profile) {
return ['-- seed query', seed, ''].join('\n');
}
function withTimeout(p, ms) {
return Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error(`report took longer than ${ms / 1000}s`)), ms))]);
// Waits for a report run while it makes progress. tick restarts the silence
// clock; idleMs without a single tick rejects. The run is never cancelled:
// finish's handlers only pass its own settlement on, so a run that answers
// after the wait gave up still delivers through the end callback.
function newRunWaiter(idleMs) {
let deferred;
const promise = new Promise((resolve, reject) => { deferred = { resolve, reject }; });
let settled = false;
let timer = setTimeout(fail, idleMs);
function fail() {
if (settled) return;
settled = true;
deferred.reject(new Error(`no query answered for ${idleMs / 1000}s`));
}
return {
tick() {
if (settled) return;
clearTimeout(timer);
timer = setTimeout(fail, idleMs);
},
finish(running) {
running.then(
(r) => { settled = true; clearTimeout(timer); deferred.resolve(r); },
(e) => { settled = true; clearTimeout(timer); deferred.reject(e); });
return promise;
},
};
}

12
src/runner/columns.js
View File

@@ -5,16 +5,22 @@ import { sparqlEscapeUri } from '../db.js';
// before the leaf is identical, and the leaf is read the same way.
// rep:self columns (empty path) are filled by assemble.js.
const SUBJECT_CHUNK_SIZE = Number(process.env.SUBJECT_CHUNK_SIZE || 250);
const SUBJECT_CHUNK_SIZE = Math.max(1, Number(process.env.SUBJECT_CHUNK_SIZE) || 100);
export async function fetchColumns(sessionQuery, subjects, spec, values = new Map(), onChunk = null) {
export async function fetchColumns(sessionQuery, subjects, spec, values = new Map(), onChunk = null, onProgress = null) {
const groups = groupColumns(spec.columns);
// The subject list is complete and the chunk size is fixed, so the number
// of batch queries is known before the first one flies. onProgress fires
// after every answered batch with { done, total }.
const total = groups.length * Math.ceil(subjects.length / SUBJECT_CHUNK_SIZE);
let done = 0;
for (const group of groups) {
for (let i = 0; i < subjects.length; i += SUBJECT_CHUNK_SIZE) {
const chunk = subjects.slice(i, i + SUBJECT_CHUNK_SIZE);
const result = await sessionQuery(groupQuery(group, chunk));
collect(result, group, values);
done += 1;
onProgress?.({ done, total });
}
if (onChunk) await onChunk(group);
}

16
src/runner/csv.js
View File

@@ -1,4 +1,4 @@
import { mkdirSync, writeFileSync } from 'fs';
import { mkdirSync, writeFileSync, readFileSync } from 'fs';
// Rows to a CSV file. CSV_SEPARATOR is ';' by default. A cell is quoted
// only when it holds the separator, a quote or a newline; a quote inside a
@@ -23,4 +23,18 @@ export function writeCsv(name, csv) {
const file = `${SHARE_DIR}/${name}`;
writeFileSync(file, csv, 'utf8');
return file;
}
// The spec of the run, next to its CSV (same base name, .ttl), so a report
// can be debugged afterwards. The store keeps only the pointer.
export function writeSpec(csvName, spec) {
mkdirSync(SHARE_DIR, { recursive: true });
const name = csvName.replace(/\.csv$/, '') + '.ttl';
writeFileSync(`${SHARE_DIR}/${name}`, spec, 'utf8');
return `share://${name}`;
}
export function readSpec(pointer) {
if (!pointer.startsWith('share://')) throw new Error(`not a share:// pointer: ${pointer}`);
return readFileSync(`${SHARE_DIR}/${pointer.slice('share://'.length)}`, 'utf8');
}

2
src/runner/report.js
View File

@@ -44,7 +44,7 @@ export async function registerReport(sessionUpdateFn, title, fileUri, extra = {}
const extraTriples = [];
if (extra.question) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/question')} ${sparqlEscapeString(extra.question)} .`);
if (extra.spec) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/spec')} ${sparqlEscapeString(extra.spec)} .`);
if (extra.specFile) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/specFile')} ${sparqlEscapeString(extra.specFile)} .`);
if (extra.creator) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://purl.org/dc/terms/creator')} ${sparqlEscapeUri(extra.creator)} .`);
await (sessionUpdateFn || sessionUpdate)(`

18
src/runner/run.js
View File

@@ -1,28 +1,34 @@
import { seed } from './seed.js';
import { fetchColumns } from './columns.js';
import { assemble } from './assemble.js';
import { rowsToCsv, writeCsv } from './csv.js';
import { rowsToCsv, writeCsv, writeSpec } from './csv.js';
import { registerFile, registerReport } from './report.js';
import { touch } from './job.js';
// Ties the steps together: seed → columns → assemble → CSV → register.
// Returns { reportUri, fileUri, rowCount }. The task is touched once per
// column group so the dashboard shows movement.
// column group so the dashboard shows movement. onProgress fires on every
// answered query with what the phase knows — { subjects } per seed page,
// { done, total } per column batch — so a waiter can tell a live run from
// a stalled one.
export async function run(query, update, parsed, profile, title, taskUri, extra = {}) {
const subjects = await seed(query, parsed);
export async function run(query, update, parsed, profile, title, taskUri, extra = {}, onProgress = null) {
const subjects = await seed(query, parsed,
onProgress ? (n) => onProgress({ subjects: n }) : null);
const values = new Map();
await fetchColumns(query, subjects, parsed, values,
taskUri ? () => touch(update, taskUri) : null);
taskUri ? () => touch(update, taskUri) : null, onProgress);
const rows = assemble(subjects, values, parsed);
const fileName = `${slug(title)}.csv`;
const csv = rowsToCsv(rows);
const filePath = writeCsv(fileName, csv);
const specFile = extra.spec && writeSpec(fileName, extra.spec);
const fileUri = await registerFile(update, fileName, filePath);
const reportUri = await registerReport(update, title, fileUri, extra);
const reportUri = await registerReport(update, title, fileUri,
specFile ? { ...extra, specFile } : extra);
return { reportUri, fileUri, rowCount: rows.length - 1, filePath };
}

5
src/runner/seed.js
View File

@@ -1,10 +1,12 @@
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from '../db.js';
// Subject selection. One paged query per page; stops at ROW_LIMIT.
// onPage(n) fires after every answered page with the subject count so far,
// so a caller waiting on this run can tell a live one from a stalled one.
const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000);
const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000);
export async function seed(sessionQuery, spec) {
export async function seed(sessionQuery, spec, onPage = null) {
const subjects = [];
let offset = 0;
for (;;) {
@@ -12,6 +14,7 @@ export async function seed(sessionQuery, spec) {
const result = await sessionQuery(query);
const rows = result.results.bindings.map(b => b.s.value);
subjects.push(...rows);
onPage?.(subjects.length);
if (subjects.length > ROW_LIMIT) {
throw new Error(`the question matches more than ${ROW_LIMIT} subjects. Add a filter, for example a date range or a bestuurseenheid.`);
}

9
src/runner/spec.js
View File

@@ -90,8 +90,13 @@ function parsePath(store, pathQuad) {
while (cur && cur.value !== RDF + 'nil') {
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object;
if (!item) throw new Error('sh:path list is broken');
if (item.termType !== 'NamedNode') throw new Error('sh:path list items must be URIs');
hops.push({ predicate: item.value, inverse: false });
if (item.termType === 'BlankNode') {
const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
hops.push({ predicate: inv.value, inverse: true });
} else {
hops.push({ predicate: item.value, inverse: false });
}
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
}
return hops;