Cleanup
This commit is contained in:
felix
@@ -1,15 +1,14 @@
|
||||
// The chat's reads and writes. Nothing here knows the service it lives in.
|
||||
// The chat's reads and writes: conversations, messages and their bijlagen.
|
||||
// Full URIs everywhere: the auth layer rejects a prefixed name with no
|
||||
// PREFIX line with an opaque 500.
|
||||
// PREFIX line with an opaque 500 (the xsd: types of mu's escape helpers are
|
||||
// fine).
|
||||
//
|
||||
// A bijlage is a real file: a logical nfo:FileDataObject the file service
|
||||
// serves at /files/<uuid>/download, over a physical share:// object with
|
||||
// nie:dataSource back to the logical one. Every attachment carries the
|
||||
// file's logical URI (see chat/attachment-file.js how those files are
|
||||
// made); nothing here writes files.
|
||||
import { randomUUID } from 'crypto';
|
||||
import { query as muQuery, update as muUpdate,
|
||||
sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
|
||||
// nie:dataSource back to the logical one.
|
||||
import { readFileSync, writeFileSync, statSync } from 'fs';
|
||||
import { query, update, uuid,
|
||||
sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt, sparqlEscapeDateTime } from 'mu';
|
||||
|
||||
const HISTORY_LIMIT = Number(process.env.CHAT_HISTORY_LIMIT || 20);
|
||||
|
||||
@@ -27,14 +26,28 @@ const T = {
|
||||
maker: 'http://xmlns.com/foaf/0.1/maker',
|
||||
attachment: 'https://www.w3.org/ns/activitystreams#attachment',
|
||||
softwareAgent: 'http://www.w3.org/ns/prov#SoftwareAgent',
|
||||
file: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject',
|
||||
fileName: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName',
|
||||
fileSize: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize',
|
||||
fileCreated: 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated',
|
||||
format: 'http://purl.org/dc/terms/format',
|
||||
fileType: 'http://purl.org/dc/terms/type',
|
||||
dataSource: 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource',
|
||||
};
|
||||
const MESSAGE_BASE = 'http://data.lblod.info/id/chat-messages/';
|
||||
const FILE_BASE = 'http://data.lblod.info/files/';
|
||||
const SHARE_DIR = process.env.SHARE_DIR || '/share';
|
||||
|
||||
export const SPEC_MEDIA_TYPE = 'text/turtle';
|
||||
// The bijlage types. Match the add-chat-attachment-types-codelist migration
|
||||
// in the portal.
|
||||
export const SPEC_TYPE = 'http://lblod.data.gift/concepts/6693014e-7b7d-448c-bb24-80c9758521f3';
|
||||
export const RESULT_TYPE = 'http://lblod.data.gift/concepts/28d97698-e219-4a92-bdbd-7ab367a7524e';
|
||||
// Every spec file is called like this; readSpecFile opens nothing else.
|
||||
const SPEC_FILE_NAME = /^specificatie-[0-9a-f-]{36}\.ttl$/;
|
||||
|
||||
const u = sparqlEscapeUri;
|
||||
const s = sparqlEscapeString;
|
||||
// Full URIs, so spell the dateTime type out.
|
||||
const dt = (value) => sparqlEscapeDateTime(value)
|
||||
.replace('xsd:dateTime', '<http://www.w3.org/2001/XMLSchema#dateTime>');
|
||||
|
||||
const bindings = (json) => json.results.bindings;
|
||||
|
||||
@@ -42,7 +55,7 @@ const bindings = (json) => json.results.bindings;
|
||||
|
||||
// The conversation by uuid, null when the caller may not read it.
|
||||
export async function readConversation(id) {
|
||||
const json = await muQuery(`
|
||||
const json = await query(`
|
||||
SELECT ?conversation ?title ?creator WHERE {
|
||||
?conversation ${u(T.type)} ${u(T.thread)} ;
|
||||
${u(T.uuid)} ${s(id)} .
|
||||
@@ -57,7 +70,7 @@ export async function readConversation(id) {
|
||||
// The last HISTORY_LIMIT messages, oldest first, as
|
||||
// { role, content, attachments: [{ uri, name, mediaType, type }] }.
|
||||
export async function readHistory(conversationUri, assistantUri) {
|
||||
const json = await muQuery(`
|
||||
const json = await query(`
|
||||
SELECT ?message ?content ?maker ?created ?doc ?docName ?docFormat ?docType WHERE {
|
||||
?message ${u(T.hasContainer)} ${u(conversationUri)} ;
|
||||
${u(T.content)} ?content ;
|
||||
@@ -65,9 +78,9 @@ export async function readHistory(conversationUri, assistantUri) {
|
||||
${u(T.created)} ?created .
|
||||
OPTIONAL {
|
||||
?message ${u(T.attachment)} ?doc .
|
||||
OPTIONAL { ?doc ${u('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ?docName . }
|
||||
OPTIONAL { ?doc ${u('http://purl.org/dc/terms/format')} ?docFormat . }
|
||||
OPTIONAL { ?doc ${u('http://purl.org/dc/terms/type')} ?docType . }
|
||||
OPTIONAL { ?doc ${u(T.fileName)} ?docName . }
|
||||
OPTIONAL { ?doc ${u(T.format)} ?docFormat . }
|
||||
OPTIONAL { ?doc ${u(T.fileType)} ?docType . }
|
||||
}
|
||||
} ORDER BY DESC(?created) LIMIT ${HISTORY_LIMIT * 4}`);
|
||||
const messages = [];
|
||||
@@ -98,7 +111,7 @@ export async function readHistory(conversationUri, assistantUri) {
|
||||
|
||||
// The one assistant the caller can read. Throws unless there is exactly one.
|
||||
export async function findAssistant() {
|
||||
const json = await muQuery(`
|
||||
const json = await query(`
|
||||
SELECT ?agent WHERE { ?agent ${u(T.type)} ${u(T.softwareAgent)} . } LIMIT 2`);
|
||||
const rows = bindings(json);
|
||||
if (rows.length !== 1) {
|
||||
@@ -111,12 +124,12 @@ export async function findAssistant() {
|
||||
|
||||
// One message, both types, its bijlagen, and the conversation touched, in
|
||||
// one update. `title` is only written when given (the first question names
|
||||
// an unnamed conversation). Every attachment is the logical uri of a file
|
||||
// that already exists.
|
||||
// an unnamed conversation). attachments are the logical uris of files that
|
||||
// already exist.
|
||||
export async function writeMessage({ conversationUri, content, maker, attachments = [], title }) {
|
||||
const id = randomUUID();
|
||||
const id = uuid();
|
||||
const uri = `${MESSAGE_BASE}${id}`;
|
||||
const now = dt(new Date());
|
||||
const now = sparqlEscapeDateTime(new Date());
|
||||
|
||||
// Predicate pairs joined with ';', never a trailing one: it breaks the Turtle.
|
||||
const pairs = [
|
||||
@@ -128,15 +141,14 @@ export async function writeMessage({ conversationUri, content, maker, attachment
|
||||
`${u(T.created)} ${now}`,
|
||||
];
|
||||
if (maker) pairs.push(`${u(T.maker)} ${u(maker)}`);
|
||||
for (const attachment of attachments) {
|
||||
if (!attachment.uri) throw new Error('an attachment without a file uri');
|
||||
pairs.push(`${u(T.attachment)} ${u(attachment.uri)}`);
|
||||
for (const fileUri of attachments) {
|
||||
pairs.push(`${u(T.attachment)} ${u(fileUri)}`);
|
||||
}
|
||||
|
||||
const conversationPairs = [`${u(T.lastActivity)} ${now}`];
|
||||
if (title) conversationPairs.push(`${u(T.title)} ${s(title)}`);
|
||||
|
||||
await muUpdate(`
|
||||
await update(`
|
||||
DELETE {
|
||||
${u(conversationUri)} ${u(T.lastActivity)} ?old .
|
||||
}
|
||||
@@ -150,3 +162,63 @@ export async function writeMessage({ conversationUri, content, maker, attachment
|
||||
|
||||
return { uri, id };
|
||||
}
|
||||
|
||||
// --- files --------------------------------------------------------------------
|
||||
|
||||
export function writeShareFile(fileName, content) {
|
||||
writeFileSync(`${SHARE_DIR}/${fileName}`, content, 'utf8');
|
||||
}
|
||||
|
||||
// storeSpecFile(spec) → the file uri of a new spec bijlage. A spec file
|
||||
// keeps one name for the logical and the physical file: readSpecFile finds
|
||||
// it by the name in the history.
|
||||
export async function storeSpecFile(spec) {
|
||||
const fileName = `specificatie-${uuid()}.ttl`;
|
||||
writeShareFile(fileName, spec);
|
||||
return registerFile(fileName, SPEC_MEDIA_TYPE, SPEC_TYPE);
|
||||
}
|
||||
|
||||
// readSpecFile(fileName) → the Turtle of a spec bijlage, null when there is
|
||||
// no such spec file.
|
||||
export function readSpecFile(fileName) {
|
||||
if (!SPEC_FILE_NAME.test(fileName || '')) return null;
|
||||
try {
|
||||
return readFileSync(`${SHARE_DIR}/${fileName}`, 'utf8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// registerFile(shareName, format, type, name) → the logical file uri. One
|
||||
// logical file called `name` over the physical share file `shareName`, both
|
||||
// nfo:FileDataObject, both typed with dct:type. A share name must be
|
||||
// unique: two files on one share name overwrite each other.
|
||||
export async function registerFile(shareName, format, type, name = shareName) {
|
||||
const id = uuid();
|
||||
const physicalId = uuid();
|
||||
const logical = `${FILE_BASE}${id}`;
|
||||
const physical = `share://${shareName}`;
|
||||
const size = statSync(`${SHARE_DIR}/${shareName}`).size;
|
||||
const now = sparqlEscapeDateTime(new Date());
|
||||
|
||||
await update(`
|
||||
INSERT DATA {
|
||||
${u(logical)} ${u(T.type)} ${u(T.file)} ;
|
||||
${u(T.uuid)} ${s(id)} ;
|
||||
${u(T.fileName)} ${s(name)} ;
|
||||
${u(T.format)} ${s(format)} ;
|
||||
${u(T.fileSize)} ${sparqlEscapeInt(size)} ;
|
||||
${u(T.created)} ${now} ;
|
||||
${u(T.fileType)} ${u(type)} .
|
||||
${u(physical)} ${u(T.type)} ${u(T.file)} ;
|
||||
${u(T.uuid)} ${s(physicalId)} ;
|
||||
${u(T.fileName)} ${s(shareName)} ;
|
||||
${u(T.format)} ${s(format)} ;
|
||||
${u(T.fileSize)} ${sparqlEscapeInt(size)} ;
|
||||
${u(T.fileCreated)} ${now} ;
|
||||
${u(T.dataSource)} ${u(logical)} ;
|
||||
${u(T.fileType)} ${u(type)} .
|
||||
}`);
|
||||
|
||||
return logical;
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
// A file made to sit on a message: written to the share, registered as a
|
||||
// logical nfo:FileDataObject over a physical share:// object, typed with
|
||||
// dct:type. The file service serves /files/<uuid>/download for it.
|
||||
import { writeFileSync } from 'fs';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString,
|
||||
sparqlEscapeInt, sparqlEscapeDateTime } from '../db.js';
|
||||
import { ATTACHMENT_TYPES, SPEC_MEDIA_TYPE } from './vocab.js';
|
||||
|
||||
const SHARE_DIR = process.env.SHARE_DIR || '/share';
|
||||
const FILE_BASE = 'http://data.lblod.info/files/';
|
||||
const SHARE_BASE = 'share://';
|
||||
const UUID = 'http://mu.semte.ch/vocabularies/core/uuid';
|
||||
|
||||
const NFO = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#';
|
||||
const FILE = `${NFO}FileDataObject`;
|
||||
const FILE_NAME = `${NFO}fileName`;
|
||||
const FILE_SIZE = `${NFO}fileSize`;
|
||||
const FILE_CREATED = `${NFO}fileCreated`;
|
||||
const FORMAT = 'http://purl.org/dc/terms/format';
|
||||
const RDF_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type';
|
||||
const FILE_TYPE = 'http://purl.org/dc/terms/type';
|
||||
const DATA_SOURCE = 'http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource';
|
||||
|
||||
const file = sparqlEscapeUri;
|
||||
const str = sparqlEscapeString;
|
||||
|
||||
// storeSpecFile(update, content) → { uri, name }. One spec, one unique file:
|
||||
// spec bijlagen keep their name apart from the report's ttl-file. Every
|
||||
// spec file starts with "specificatie-", the read_spec tool opens only
|
||||
// those.
|
||||
export async function storeSpecFile(update, content) {
|
||||
const fileName = `specificatie-${randomUUID()}.ttl`;
|
||||
writeFileSync(`${SHARE_DIR}/${fileName}`, content, 'utf8');
|
||||
return registerFileObject(update, {
|
||||
fileName,
|
||||
format: SPEC_MEDIA_TYPE,
|
||||
size: Buffer.byteLength(content),
|
||||
type: ATTACHMENT_TYPES.spec,
|
||||
});
|
||||
}
|
||||
|
||||
// registerFileObject(update, { fileName, format, size, type }) → { uri, name }.
|
||||
// One logical file over one physical share:// file, both nfo:FileDataObject,
|
||||
// both carrying the dct:type when given.
|
||||
export async function registerFileObject(update, { fileName, format, size, type }) {
|
||||
const id = randomUUID();
|
||||
const physicalId = randomUUID();
|
||||
const logical = `${FILE_BASE}${id}`;
|
||||
const physical = `${SHARE_BASE}${fileName}`;
|
||||
const now = sparqlEscapeDateTime(new Date()) // full URI, xsd spelled out
|
||||
.replace('xsd:dateTime', '<http://www.w3.org/2001/XMLSchema#dateTime>');
|
||||
|
||||
const logicalPairs = [
|
||||
`${file(RDF_TYPE)} ${file(FILE)}`,
|
||||
`${file(UUID)} ${str(id)}`,
|
||||
`${file(FILE_NAME)} ${str(fileName)}`,
|
||||
`${file(FORMAT)} ${str(format)}`,
|
||||
`${file(FILE_SIZE)} ${sparqlEscapeInt(size)}`,
|
||||
`${file('http://purl.org/dc/terms/created')} ${now}`,
|
||||
...(type ? [`${file(FILE_TYPE)} ${file(type)}`] : []),
|
||||
];
|
||||
const physicalPairs = [
|
||||
`${file(RDF_TYPE)} ${file(FILE)}`,
|
||||
`${file(UUID)} ${str(physicalId)}`,
|
||||
`${file(FILE_NAME)} ${str(fileName)}`,
|
||||
`${file(FORMAT)} ${str(format)}`,
|
||||
`${file(FILE_SIZE)} ${sparqlEscapeInt(size)}`,
|
||||
`${file(FILE_CREATED)} ${now}`,
|
||||
`${file(DATA_SOURCE)} ${file(logical)}`,
|
||||
...(type ? [`${file(FILE_TYPE)} ${file(type)}`] : []),
|
||||
];
|
||||
|
||||
await (update || sessionUpdate)(`
|
||||
INSERT DATA {
|
||||
${file(logical)} ${logicalPairs.join(' ;\n ')} .
|
||||
${file(physical)} ${physicalPairs.join(' ;\n ')} .
|
||||
}`);
|
||||
|
||||
return { uri: logical, name: fileName };
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
// The turn endpoint. Mount it with mountChat(app, { path, answer });
|
||||
// answer(turn) is the only thing the mounting service writes.
|
||||
import express from 'express';
|
||||
import { query as muQuery, update as muUpdate } from 'mu';
|
||||
import {
|
||||
readConversation, readHistory, findAssistant,
|
||||
writeMessage,
|
||||
} from './store.js';
|
||||
|
||||
const TEXT = {
|
||||
failed: 'Er ging iets mis. Probeer het opnieuw.',
|
||||
};
|
||||
const TITLE_LENGTH = 80;
|
||||
|
||||
export function mountChat(app, { path = '/assistant', answer }) {
|
||||
if (typeof answer !== 'function') throw new Error('mountChat needs an answer(turn) function');
|
||||
|
||||
app.post(`${path}/conversations/:id/turns`, express.json(), async (req, res) => {
|
||||
const content = String(req.body?.content ?? '').trim();
|
||||
if (!content) return res.status(400).json({ error: 'content is required' });
|
||||
|
||||
// 1. the access check: the caller's session decides what is readable
|
||||
let conversation;
|
||||
try {
|
||||
conversation = await readConversation(req.params.id);
|
||||
} catch (e) {
|
||||
console.error('[chat] conversation read failed:', e);
|
||||
return res.status(500).json({ error: e.message });
|
||||
}
|
||||
if (!conversation) return res.status(404).json({ error: 'no such conversation' });
|
||||
|
||||
let assistant, history, userMessage;
|
||||
try {
|
||||
assistant = process.env.CHAT_ASSISTANT_URI || await findAssistant();
|
||||
history = await readHistory(conversation.uri, assistant);
|
||||
userMessage = await writeMessage({
|
||||
conversationUri: conversation.uri,
|
||||
content,
|
||||
maker: conversation.creator,
|
||||
title: conversation.title ? undefined : content.slice(0, TITLE_LENGTH),
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('[chat] could not record the question:', e);
|
||||
return res.status(500).json({ error: e.message });
|
||||
}
|
||||
|
||||
// 2. answer 202 now; the assistant's message comes when it comes. The
|
||||
// continuation still runs as the caller.
|
||||
res.status(202).json({ id: userMessage.id });
|
||||
|
||||
const turn = {
|
||||
conversation: { uri: conversation.uri, id: conversation.id, title: conversation.title },
|
||||
history,
|
||||
content,
|
||||
query: muQuery,
|
||||
update: muUpdate,
|
||||
finished: false,
|
||||
say: (text, attachments = []) => writeMessage({
|
||||
conversationUri: conversation.uri, content: text, maker: assistant, attachments,
|
||||
}),
|
||||
};
|
||||
|
||||
try {
|
||||
// answer may return a string, or { text, attachments } for answers the
|
||||
// assistant writes with an attachment, such as the report spec.
|
||||
const result = await answer(turn);
|
||||
const text = typeof result === 'string' ? result : result?.text;
|
||||
const attachments
|
||||
= typeof result === 'string' ? [] : (result?.attachments ?? []);
|
||||
if (typeof text === 'string' && text.trim()) {
|
||||
await turn.say(text.trim(), attachments);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('[chat] turn failed:', e);
|
||||
try {
|
||||
await turn.say(TEXT.failed);
|
||||
} catch (e2) {
|
||||
console.error('[chat] could not write the failure message:', e2);
|
||||
}
|
||||
} finally {
|
||||
turn.finished = true;
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[chat] turns at POST ${path}/conversations/:id/turns`);
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
// The bijlage types of the report assistant. Match the
|
||||
// add-chat-attachment-types-codelist migration in the portal.
|
||||
export const ATTACHMENT_TYPES = {
|
||||
spec: 'http://lblod.data.gift/concepts/6693014e-7b7d-448c-bb24-80c9758521f3',
|
||||
result: 'http://lblod.data.gift/concepts/28d97698-e219-4a92-bdbd-7ab367a7524e',
|
||||
};
|
||||
export const SPEC_BASE_NAME = 'specificatie';
|
||||
export const SPEC_MEDIA_TYPE = 'text/turtle';
|
||||
37
src/db.js
37
src/db.js
@@ -1,37 +0,0 @@
|
||||
// Thin re-export of mu's SPARQL helpers so the runner files share one import
|
||||
// site. mu's query/update attach the caller's session from the request
|
||||
// context, and mu-authorization decides graphs and visibility; the service
|
||||
// never touches an auth header.
|
||||
import {
|
||||
query as muQuery,
|
||||
update as muUpdate,
|
||||
sparqlEscapeUri,
|
||||
sparqlEscapeString,
|
||||
sparqlEscapeInt,
|
||||
sparqlEscapeDateTime,
|
||||
sparqlEscapeDate,
|
||||
sparqlEscapeBool,
|
||||
} from 'mu';
|
||||
|
||||
export const sessionUpdate = muUpdate;
|
||||
export {
|
||||
sparqlEscapeUri,
|
||||
sparqlEscapeString,
|
||||
sparqlEscapeInt,
|
||||
sparqlEscapeDateTime,
|
||||
sparqlEscapeDate,
|
||||
sparqlEscapeBool,
|
||||
};
|
||||
|
||||
// The service's own scope. Reads the LLM drives during refinement (code
|
||||
// lists, lookup_values) run under this scope, not the caller's session. The
|
||||
// sparql-parser config grants this scope read access to the public graph
|
||||
// only, so refinement reads only public data. Execution still runs as the
|
||||
// caller. Set to match the with-scope grant in config.lisp.
|
||||
export const SERVICE_SCOPE =
|
||||
process.env.SERVICE_SCOPE || 'http://services.semantic.works/natural-language-report';
|
||||
|
||||
// Wrap a mu query/update so it carries the service scope. mu's query/update
|
||||
// accept { scope } on mu-javascript-template >= 1.9.0.
|
||||
export function scopedQuery(q) { return muQuery(q, { scope: SERVICE_SCOPE }); }
|
||||
export function scopedUpdate(q) { return muUpdate(q, { scope: SERVICE_SCOPE }); }
|
||||
@@ -1,59 +1,9 @@
|
||||
// A profile as a plain text menu: entities, fields, code lists. Code lists up
|
||||
// to INLINE_VALUES_MAX are inlined and cached for VALUES_TTL, so the LLM
|
||||
// never has to search the short ones.
|
||||
// to INLINE_VALUES_MAX are inlined (and cached), so the LLM never has to
|
||||
// search the short ones.
|
||||
import { codeList, INLINE_VALUES_MAX } from './lookup.js';
|
||||
|
||||
import { fieldsOf, entityForClass } from '../runner/profile.js';
|
||||
|
||||
const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50);
|
||||
const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000;
|
||||
|
||||
// In-memory cache: profile+field → { values, at }. Refreshed lazily when stale.
|
||||
const cache = new Map();
|
||||
|
||||
// codeListValues(profile, queryFn) → map "shapeUri|predicate|inverse"
|
||||
// → [{ uri, label }]. queryFn runs under the service scope (public graph), so
|
||||
// code lists read the same regardless of the caller's rights.
|
||||
export async function codeListValues(profile, queryFn) {
|
||||
const out = {};
|
||||
for (const shape of profile.shapes) {
|
||||
for (const f of shape.fields) {
|
||||
if (!f.class) continue;
|
||||
const key = `${shape.uri}|${f.path}|${f.inverse ? 1 : 0}`;
|
||||
out[key] = await loadCodeList(f, key, queryFn);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function loadCodeList(f, key, queryFn) {
|
||||
if (!f.class) return [];
|
||||
const hit = cache.get(key);
|
||||
if (hit && Date.now() - hit.at < VALUES_TTL) return hit.values;
|
||||
let values = [];
|
||||
try {
|
||||
values = await queryCodeList(f.class, queryFn);
|
||||
} catch (e) {
|
||||
// the read failed; report nothing, the LLM will use rep:anyOf with words
|
||||
values = [];
|
||||
}
|
||||
cache.set(key, { values, at: Date.now() });
|
||||
return values;
|
||||
}
|
||||
|
||||
async function queryCodeList(type, queryFn) {
|
||||
const q = `SELECT ?uri ?label WHERE {
|
||||
?uri a <${type}> .
|
||||
OPTIONAL { ?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label . }
|
||||
} ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`;
|
||||
const r = await queryFn(q);
|
||||
const bindings = r.results.bindings;
|
||||
return bindings.slice(0, INLINE_VALUES_MAX).map(b => ({
|
||||
uri: b.uri.value,
|
||||
label: b.label?.value || b.uri.value,
|
||||
}));
|
||||
}
|
||||
|
||||
export function describeProfile(profile, codeLists = {}) {
|
||||
export async function describeProfile(profile) {
|
||||
const lines = [];
|
||||
lines.push('A spec is Turtle, in exactly this shape (copy these prefixes):');
|
||||
lines.push('@prefix rep: <http://mu.semte.ch/vocabularies/reporting/> .');
|
||||
@@ -80,13 +30,12 @@ export function describeProfile(profile, codeLists = {}) {
|
||||
for (const shape of profile.shapes) {
|
||||
lines.push(`${shape.label || shape.targetClass} — sh:targetClass <${shape.targetClass}> ; rep:entity <${shape.uri}>`);
|
||||
for (const f of shape.fields) {
|
||||
const key = `${shape.uri}|${f.path}|${f.inverse ? 1 : 0}`;
|
||||
const kind = f.datatype ? datatypeName(f.datatype)
|
||||
: f.class ? `→ ${conceptLabel(f.class, codeLists[key])}`
|
||||
: f.class ? `→ ${conceptLabel(f.class, await codeList(f.class))}`
|
||||
: f.node ? `→ ${linkedLabel(profile, f.node)}`
|
||||
: '?';
|
||||
const dir = f.inverse ? ' (inverse)' : '';
|
||||
const pathShown = typeof f.path === 'string' ? f.path : '?';
|
||||
const pathShown = f.path || '?';
|
||||
lines.push(` ${f.name || pathShown} sh:path <${pathShown}>${dir} ${kind}`);
|
||||
}
|
||||
lines.push('');
|
||||
@@ -95,11 +44,10 @@ export function describeProfile(profile, codeLists = {}) {
|
||||
}
|
||||
|
||||
function conceptLabel(type, values) {
|
||||
const head = values.slice(0, INLINE_VALUES_MAX);
|
||||
if (!head.length) return short(type);
|
||||
const names = head.map(v => v.label).join(', ');
|
||||
if (!values.length) return short(type);
|
||||
const names = values.map(v => v.label).join(', ');
|
||||
const more = values.length >= INLINE_VALUES_MAX ? ', …' : '';
|
||||
return `${short(type)} — ${head.length} waarden: ${names}${more}`;
|
||||
return `${short(type)} — ${values.length} waarden: ${names}${more}`;
|
||||
}
|
||||
|
||||
function linkedLabel(profile, nodeUri) {
|
||||
|
||||
@@ -1,98 +1,118 @@
|
||||
// Search a code list too long to inline: regex on str(?label), so
|
||||
// language-tagged labels match. Returns the matches, the total, and an
|
||||
// `exact` flag when a label equals the term (case aside).
|
||||
// The LLM's own reads: code lists. They run under the service scope, not
|
||||
// the caller's session; the sparql-parser config grants that scope read
|
||||
// access to the public graph only, so they read the same for every caller.
|
||||
// Execution still runs as the caller.
|
||||
//
|
||||
// This is a suggest helper, not a spec check. It reads candidate values the
|
||||
// LLM can put in a filter; it never runs the spec and never sees report
|
||||
// rows. queryFn runs under the service scope (public graph only), so the
|
||||
// suggestions come from the same data set the profile describes, regardless
|
||||
// of who calls.
|
||||
|
||||
import { sparqlEscapeString } from '../db.js';
|
||||
// lookup_values is a suggest helper, not a spec check. It reads candidate
|
||||
// values the LLM can put in a filter; it never runs the spec and never sees
|
||||
// report rows.
|
||||
import { query, sparqlEscapeString } from 'mu';
|
||||
|
||||
// Must match the with-scope grant in the app's config.lisp.
|
||||
const SERVICE_SCOPE = process.env.SERVICE_SCOPE || 'http://services.semantic.works/natural-language-report';
|
||||
export const INLINE_VALUES_MAX = Number(process.env.INLINE_VALUES_MAX || 50);
|
||||
const VALUES_TTL = Number(process.env.VALUES_TTL || 3600) * 1000;
|
||||
const LIMIT = 25;
|
||||
|
||||
// lookupValues(profile, term, fieldSpec, queryFn) →
|
||||
// { total, exact, matches } or { error }.
|
||||
// field is "shapeLabel.fieldLabel" as the LLM writes it, or a predicate URI.
|
||||
// We resolve it to the field's sh:class to know what type to search.
|
||||
export async function lookupValues(profile, term, fieldSpec, queryFn) {
|
||||
const type = resolveClass(profile, fieldSpec);
|
||||
if (!type) return { total: 0, exact: null, matches: [], error: `field "${fieldSpec}" not found in profile "${profile.title}"` };
|
||||
// class → { values, at }. Refreshed lazily when stale.
|
||||
const cache = new Map();
|
||||
|
||||
function publicQuery(q) {
|
||||
return query(q, { scope: SERVICE_SCOPE });
|
||||
}
|
||||
|
||||
// codeList(type) → up to INLINE_VALUES_MAX [{ uri, label }], cached for
|
||||
// VALUES_TTL. Empty when the read fails; the LLM then uses rep:anyOf with
|
||||
// words.
|
||||
export async function codeList(type) {
|
||||
const hit = cache.get(type);
|
||||
if (hit && Date.now() - hit.at < VALUES_TTL) return hit.values;
|
||||
let values = [];
|
||||
try {
|
||||
const r = await publicQuery(`SELECT ?uri ?label WHERE {
|
||||
?uri a <${type}> .
|
||||
OPTIONAL { ?uri <http://www.w3.org/2004/02/skos/core#prefLabel> ?label . }
|
||||
} ORDER BY ?label LIMIT ${INLINE_VALUES_MAX + 1}`);
|
||||
values = r.results.bindings.slice(0, INLINE_VALUES_MAX).map(b => ({
|
||||
uri: b.uri.value,
|
||||
label: b.label?.value || b.uri.value,
|
||||
}));
|
||||
} catch (e) {
|
||||
values = [];
|
||||
}
|
||||
cache.set(type, { values, at: Date.now() });
|
||||
return values;
|
||||
}
|
||||
|
||||
// lookupValues(profile, field, term) → the matches as text for the LLM: up
|
||||
// to LIMIT labels and URIs, an exact label (case aside) first. field is
|
||||
// "shapeLabel.fieldLabel" as the LLM writes it; it resolves to the class to
|
||||
// search.
|
||||
export async function lookupValues(profile, field, term) {
|
||||
const type = resolveClass(profile, field);
|
||||
if (!type) return `field "${field}" not found in profile "${profile.title}"`;
|
||||
|
||||
// Search within the class first. A common word ("gemeente") also sits in
|
||||
// hundreds of unrelated labels; an unscoped scan drowns the code.
|
||||
let rows = await labelQuery(term, type, queryFn);
|
||||
let rows = await labelQuery(term, type);
|
||||
// Code lists sometimes type their entries differently than the profile
|
||||
// says (MAR codes are ext:supervision/Nomenclature, the profile says
|
||||
// skos:Concept). Fall back to a scan over everything.
|
||||
if (!rows.length) rows = await labelQuery(term, null, queryFn);
|
||||
// Order: an exact label (case aside) first, then the rest as the store
|
||||
// ordered them. "Gent" must surface before "AGB Erfgoed Gent".
|
||||
if (!rows.length) rows = await labelQuery(term, null);
|
||||
// "Gent" must surface before "AGB Erfgoed Gent".
|
||||
const lower = term.toLowerCase();
|
||||
const exactRow = rows.find(b => b.label.toLowerCase() === lower);
|
||||
const rest = rows.filter(b => b !== exactRow);
|
||||
const ordered = exactRow ? [exactRow, ...rest] : rest;
|
||||
let matches = ordered.slice(0, LIMIT);
|
||||
if (!matches.length) {
|
||||
const byCode = await codeQuery(profile, type, term, queryFn);
|
||||
matches = byCode;
|
||||
}
|
||||
const exact = matches.find(m => m.label.toLowerCase() === term.toLowerCase());
|
||||
return {
|
||||
total: matches.length >= LIMIT ? `${LIMIT}+` : matches.length,
|
||||
exact: exact ? exact.uri : null,
|
||||
matches,
|
||||
};
|
||||
const exactRow = rows.find(r => r.label.toLowerCase() === lower);
|
||||
let matches = (exactRow ? [exactRow, ...rows.filter(r => r !== exactRow)] : rows).slice(0, LIMIT);
|
||||
if (!matches.length) matches = await codeQuery(term);
|
||||
if (!matches.length) return `no value matches "${term}" in ${field}.`;
|
||||
|
||||
const lines = matches.map(m => `${m.label} <${m.uri}>`);
|
||||
if (matches.length >= LIMIT) lines.push(`(${LIMIT}+ matches, narrow the term)`);
|
||||
const exact = matches.find(m => m.label.toLowerCase() === lower);
|
||||
if (exact) lines.push(`exact: <${exact.uri}>`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function labelQuery(term, type, queryFn) {
|
||||
// Regex on str(?label), so language-tagged labels match.
|
||||
async function labelQuery(term, type) {
|
||||
const q = `SELECT DISTINCT ?uri ?label WHERE {
|
||||
?uri a ${type ? `<${type}>` : '?anyType'} ;
|
||||
<http://www.w3.org/2004/02/skos/core#prefLabel> ?label .
|
||||
FILTER(REGEX(str(?label), ${sparqlEscapeString(escapeRegex(term))}, "i"))
|
||||
} ORDER BY ?label LIMIT ${LIMIT * 8}`;
|
||||
const r = await queryFn(q);
|
||||
const r = await publicQuery(q);
|
||||
return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value }));
|
||||
}
|
||||
|
||||
// MAR codes: the label says "MAR7300 - …" but the searchable code sits on
|
||||
// ext:supervision/nomenclatureCode. One extra query, only when the label
|
||||
// search came up empty.
|
||||
async function codeQuery(profile, type, term, queryFn) {
|
||||
async function codeQuery(term) {
|
||||
const q = `SELECT ?uri ?label WHERE {
|
||||
?uri <http://mu.semte.ch/vocabularies/ext/supervision/nomenclatureCode> ?code ;
|
||||
<http://www.w3.org/2004/02/skos/core#prefLabel> ?label .
|
||||
FILTER(CONTAINS(LCASE(str(?code)), LCASE(${sparqlEscapeString(term)})))
|
||||
} ORDER BY ?label LIMIT ${LIMIT}`;
|
||||
try {
|
||||
const r = await queryFn(q);
|
||||
const r = await publicQuery(q);
|
||||
return r.results.bindings.map(b => ({ label: b.label.value, uri: b.uri.value }));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function resolveClass(profile, fieldSpec) {
|
||||
if (fieldSpec.includes('/')) return null;
|
||||
// "shapeLabel.fieldLabel" → find the field carrying sh:class
|
||||
const dot = fieldSpec.lastIndexOf('.');
|
||||
if (dot < 0) {
|
||||
// bare predicate URI
|
||||
for (const shape of profile.shapes) {
|
||||
for (const f of shape.fields) {
|
||||
if (f.path === fieldSpec && f.class) return f.class;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
const shapeLabel = fieldSpec.slice(0, dot);
|
||||
const fieldLabel = fieldSpec.slice(dot + 1);
|
||||
// "shapeLabel.fieldLabel" → the class to search: the field's sh:class, or
|
||||
// for a link (sh:node) the linked shape's sh:targetClass
|
||||
// ("melding.bestuurseenheid" → besluit:Bestuurseenheid).
|
||||
function resolveClass(profile, field) {
|
||||
if (field.includes('/')) return null;
|
||||
const dot = field.lastIndexOf('.');
|
||||
if (dot < 0) return null;
|
||||
const shapeLabel = field.slice(0, dot);
|
||||
const fieldLabel = field.slice(dot + 1);
|
||||
const shape = profile.shapes.find(s => s.label === shapeLabel);
|
||||
if (!shape) return null;
|
||||
const f = shape.fields.find(x => x.name === fieldLabel);
|
||||
// sh:node marks a link, not a code list; the searchable type is the linked
|
||||
// shape's sh:targetClass ("melding.bestuurseenheid" → besluit:Bestuurseenheid).
|
||||
if (f?.class) return f.class;
|
||||
if (f?.node) {
|
||||
const linked = profile.shapes.find(s => s.uri === f.node);
|
||||
@@ -114,4 +134,4 @@ function shortType(cls) {
|
||||
|
||||
function escapeRegex(s) {
|
||||
return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
}
|
||||
|
||||
114
src/llm/loop.js
114
src/llm/loop.js
@@ -8,7 +8,8 @@
|
||||
// confirm before anything executes; execution runs the agreed spec without
|
||||
// the LLM (../report-assistant.js).
|
||||
|
||||
import { buildTools } from './tools.js';
|
||||
import { TOOLS, runTool } from './tools.js';
|
||||
import { SPEC_MEDIA_TYPE } from '../chat.js';
|
||||
|
||||
const MAX_ROUNDS = Number(process.env.MAX_ROUNDS || 12);
|
||||
const LLM_BASE_URL = process.env.LLM_BASE_URL;
|
||||
@@ -77,70 +78,49 @@ what it says, up to three rounds. Then propose and stop.
|
||||
|
||||
Reply to the user in Dutch. Earlier turns are context only. Answer the last one.`;
|
||||
|
||||
// ask(questionOrMessages, profiles, session) → { text }
|
||||
// A string is one question; an array is the conversation so far,
|
||||
// [{ role, content }], newest last. The system prompt is always this file's
|
||||
// own; a system message in the input is dropped.
|
||||
export async function ask(questionOrMessages, profiles, session) {
|
||||
const turns = typeof questionOrMessages === 'string'
|
||||
? [{ role: 'user', content: questionOrMessages }]
|
||||
: questionOrMessages.filter(m => m.role !== 'system');
|
||||
// ask(turns, profiles) → { text, spec }. turns is the conversation so far,
|
||||
// [{ role, content, attachments }], newest last. spec is the last spec
|
||||
// validate_spec accepted in this turn, or null.
|
||||
export async function ask(turns, profiles) {
|
||||
const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns.map(withSpecHint)];
|
||||
let spec = null;
|
||||
|
||||
return runLoop(withAttachmentSpecs(turns), profiles, session);
|
||||
for (let round = 0; round < MAX_ROUNDS; round++) {
|
||||
const { message } = await chat(messages, TOOLS);
|
||||
messages.push(message);
|
||||
const calls = message.tool_calls || [];
|
||||
if (!calls.length) return { text: message.content || '', spec };
|
||||
|
||||
for (const call of calls) {
|
||||
const args = parseArgs(call.function.arguments);
|
||||
let result;
|
||||
try {
|
||||
result = await runTool(call.function.name, args, profiles);
|
||||
} catch (e) {
|
||||
result = `tool failed: ${String(e.message || e).split('\n')[0]}`;
|
||||
}
|
||||
// validate_spec answers "ok" (plus notes) when the spec checks out;
|
||||
// that spec becomes the bijlage of the answer.
|
||||
if (call.function.name === 'validate_spec' && /^ok(\n|$)/.test(result)) spec = args.spec;
|
||||
messages.push({ role: 'tool', tool_call_id: call.id, content: result });
|
||||
}
|
||||
}
|
||||
return { text: 'Sorry, dit rapport is te moeilijk. Contacteer de developers.', spec };
|
||||
}
|
||||
|
||||
// The history carries the spec as a bijlage. Paste one hint line per spec
|
||||
// under the message text, so the model knows a spec is there and opens it
|
||||
// with read_spec; the chat itself still keeps the Turtle out of the text.
|
||||
function withAttachmentSpecs(turns) {
|
||||
return turns.map(t => {
|
||||
const specs = (t.attachments || []).filter(a => a.mediaType === 'text/turtle' && a.uri);
|
||||
if (!specs.length) return t;
|
||||
return {
|
||||
...t,
|
||||
content: `${t.content}\n\n${specs.map(a =>
|
||||
`Bijlage "${a.name}", a spec file (read it with read_spec): ${a.uri}`).join('\n\n')}`,
|
||||
};
|
||||
});
|
||||
function withSpecHint(turn) {
|
||||
const specs = (turn.attachments || []).filter(a => a.mediaType === SPEC_MEDIA_TYPE);
|
||||
if (!specs.length) return turn;
|
||||
const hints = specs.map(a => `Bijlage "${a.name}", a spec file (read it with read_spec).`);
|
||||
return { ...turn, content: `${turn.content}\n\n${hints.join('\n\n')}` };
|
||||
}
|
||||
|
||||
// runLoop drives the agent loop with the refinement prompt and tool set.
|
||||
async function runLoop(turns, profiles, session) {
|
||||
const { handlers } = buildTools(profiles, session);
|
||||
const messages = [{ role: 'system', content: REFINE_PROMPT }, ...turns];
|
||||
|
||||
for (let round = 0; round < MAX_ROUNDS; round++) {
|
||||
const reply = await chat(messages, { tools: toolDefs() });
|
||||
messages.push(reply.message);
|
||||
const calls = reply.message.tool_calls || [];
|
||||
if (!calls.length) {
|
||||
return { text: reply.message.content || '' };
|
||||
}
|
||||
for (const call of calls) {
|
||||
const args = parseArgs(call.function.arguments);
|
||||
const fn = handlers.get(call.function.name);
|
||||
let resultText;
|
||||
if (!fn) {
|
||||
resultText = `unknown tool "${call.function.name}"`;
|
||||
} else {
|
||||
try {
|
||||
const out = await fn(args);
|
||||
resultText = out.content?.[0]?.text ?? JSON.stringify(out);
|
||||
} catch (e) {
|
||||
resultText = `tool failed: ${String(e.message || e).split('\n')[0]}`;
|
||||
}
|
||||
}
|
||||
messages.push({ role: 'tool', tool_call_id: call.id, content: resultText });
|
||||
}
|
||||
}
|
||||
return { text: '(loop did not finish)' };
|
||||
}
|
||||
|
||||
// chat(messages, { tools, toolChoice }) → { message }
|
||||
// The one place that talks to the provider. tools null (the default) means a
|
||||
// plain completion (the mode classifier); runLoop passes the tool
|
||||
// definitions; toolChoice defaults to 'auto'.
|
||||
export async function chat(messages, { tools = null, toolChoice = 'auto' } = {}) {
|
||||
// chat(messages, tools) → { message }. The one place that talks to the
|
||||
// provider. Without tools it is a plain completion (the mode check).
|
||||
export async function chat(messages, tools = null) {
|
||||
// Send only the fields every provider accepts back.
|
||||
const wire = messages.map(m => {
|
||||
const out = { role: m.role, content: m.content ?? null };
|
||||
@@ -154,13 +134,10 @@ export async function chat(messages, { tools = null, toolChoice = 'auto' } = {})
|
||||
if (m.role === 'tool' && m.tool_call_id) out.tool_call_id = m.tool_call_id;
|
||||
return out;
|
||||
});
|
||||
const body = {
|
||||
model: LLM_MODEL,
|
||||
messages: wire,
|
||||
};
|
||||
const body = { model: LLM_MODEL, messages: wire };
|
||||
if (tools) {
|
||||
body.tools = tools;
|
||||
body.tool_choice = toolChoice === 'auto' ? 'auto' : toolChoice;
|
||||
body.tool_choice = 'auto';
|
||||
}
|
||||
if (LOG_LLM) {
|
||||
console.log(`[llm] POST ${LLM_BASE_URL}/chat/completions request:\n${
|
||||
@@ -191,16 +168,3 @@ function parseArgs(s) {
|
||||
if (!s) return {};
|
||||
try { return JSON.parse(s); } catch { return {}; }
|
||||
}
|
||||
|
||||
// The tool definitions sent to the model.
|
||||
import { toolSchemas } from './schemas.js';
|
||||
function toolDefs() {
|
||||
return toolSchemas().map(t => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
name: t.name,
|
||||
description: t.description,
|
||||
parameters: t.inputSchema,
|
||||
},
|
||||
}));
|
||||
}
|
||||
@@ -25,43 +25,18 @@ Rules:
|
||||
Reply with exactly one token: true or false. No other words, no punctuation,
|
||||
no markdown.`;
|
||||
|
||||
// classifyMode(turns) → boolean (true = execute now)
|
||||
// turns: [{ role, content }], newest last. A leading system message is
|
||||
// dropped; the assistant's own system prompt is never sent to the model.
|
||||
export async function classifyMode(turns) {
|
||||
const convo = (turns || [])
|
||||
.filter(m => m.role !== 'system')
|
||||
.map(m => ({ role: m.role === 'assistant' ? 'assistant' : 'user', content: m.content ?? '' }));
|
||||
if (!convo.length) return false;
|
||||
|
||||
// Keep the last few turns; the decision is about the latest message.
|
||||
const recent = convo.slice(-6);
|
||||
const messages = [{ role: 'system', content: MODE_PROMPT }, ...recent];
|
||||
let raw;
|
||||
// wantsExecution(turns) → true when the user's last message says "run it
|
||||
// now". turns: [{ role, content }], newest last; the last few are enough.
|
||||
export async function wantsExecution(turns) {
|
||||
let message;
|
||||
try {
|
||||
const reply = await chat(messages, { tools: null });
|
||||
raw = firstText(reply.message);
|
||||
({ message } = await chat([{ role: 'system', content: MODE_PROMPT }, ...turns.slice(-6)]));
|
||||
} catch (e) {
|
||||
// The classifier call failed (no LLM, provider down). Default to refine:
|
||||
// safer than running a report on a guess, and the refine loop will
|
||||
// surface the real error to the user.
|
||||
// The call failed (no LLM, provider down). Refine: safer than running a
|
||||
// report on a guess, and the refine loop will surface the real error.
|
||||
return false;
|
||||
}
|
||||
const text = stripMarkdown(raw).trim().toLowerCase();
|
||||
if (!text) return false;
|
||||
if (text.startsWith('true')) return true;
|
||||
if (text.startsWith('false')) return false;
|
||||
// A JSON {"execute": ...} shape some models emit.
|
||||
try {
|
||||
const m = text.match(/\{[\s\S]*\}/);
|
||||
if (m) {
|
||||
const v = JSON.parse(m[0]);
|
||||
if (typeof v === 'boolean') return v;
|
||||
if (v && typeof v.execute === 'boolean') return v.execute;
|
||||
if (v && typeof v.refine === 'boolean') return !v.refine;
|
||||
}
|
||||
} catch { /* ignore */ }
|
||||
return false;
|
||||
return stripMarkdown(firstText(message)).trim().toLowerCase().startsWith('true');
|
||||
}
|
||||
|
||||
// Reasoning models may leave `content` empty and put the visible text in a
|
||||
@@ -78,4 +53,4 @@ function firstText(message) {
|
||||
|
||||
function stripMarkdown(s) {
|
||||
return String(s).replace(/[*_`>#]/g, ' ');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
// The JSON schemas of the tools. The descriptions are what the LLM reads to
|
||||
// decide which tool to call. Refinement only: list_profiles,
|
||||
// describe_profile, validate_spec, read_spec and lookup_values.
|
||||
|
||||
const COMMON = [
|
||||
{
|
||||
name: 'list_profiles',
|
||||
description: 'List the report profiles available, with their id and title.',
|
||||
inputSchema: { type: 'object', properties: {}, required: [] },
|
||||
},
|
||||
{
|
||||
name: 'describe_profile',
|
||||
description: 'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: { profile_id: { type: 'string', description: 'The profile URI returned by list_profiles' } },
|
||||
required: ['profile_id'],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'validate_spec',
|
||||
description: 'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: { spec: { type: 'string', description: 'The spec as Turtle text' } },
|
||||
required: ['spec'],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const LOOKUP_VALUES = {
|
||||
name: 'lookup_values',
|
||||
description: 'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
profile_id: { type: 'string', description: 'The profile URI returned by list_profiles' },
|
||||
field: { type: 'string', description: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"' },
|
||||
term: { type: 'string', description: 'The Dutch word or name to search for' },
|
||||
},
|
||||
required: ['profile_id', 'field', 'term'],
|
||||
},
|
||||
};
|
||||
|
||||
const READ_SPEC = {
|
||||
name: 'read_spec',
|
||||
description: 'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: { file_uri: { type: 'string', description: 'The file URI of the spec bijlage, e.g. http://data.lblod.info/files/<uuid>' } },
|
||||
required: ['file_uri'],
|
||||
},
|
||||
};
|
||||
|
||||
export function toolSchemas() {
|
||||
return [...COMMON, READ_SPEC, LOOKUP_VALUES];
|
||||
}
|
||||
171
src/llm/tools.js
171
src/llm/tools.js
@@ -1,94 +1,91 @@
|
||||
// The tools the LLM calls, refinement only: list_profiles, describe_profile,
|
||||
// validate_spec, lookup_values, read_spec. lookup_values searches a code list
|
||||
// under the service scope (public graph) and returns candidate values, so the
|
||||
// LLM can suggest concrete filter values. No report is created, no spec is
|
||||
// evaluated.
|
||||
//
|
||||
// The shape every tool returns is { content: [{ type: 'text', text }] },
|
||||
// which the loop passes straight back to the model.
|
||||
|
||||
import { checkSpec, whereNotes } from '../runner/check.js';
|
||||
// The tools the LLM calls, refinement only: list_profiles,
|
||||
// describe_profile, validate_spec, read_spec and lookup_values. Each returns
|
||||
// plain text for the model. No report is created, no spec is evaluated;
|
||||
// the code-list reads run under the service scope (public graph).
|
||||
import { parseSpec } from '../runner/spec.js';
|
||||
import { describeProfile, codeListValues } from './describe.js';
|
||||
import { checkSpec, whereNotes, profileError } from '../runner/check.js';
|
||||
import { describeProfile } from './describe.js';
|
||||
import { lookupValues } from './lookup.js';
|
||||
import { scopedQuery } from '../db.js';
|
||||
import { readFileSync } from 'fs';
|
||||
import { readSpecFile } from '../chat.js';
|
||||
|
||||
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
|
||||
// The tool definitions sent to the model. The descriptions are what the LLM
|
||||
// reads to decide which tool to call.
|
||||
export const TOOLS = [
|
||||
tool('list_profiles',
|
||||
'List the report profiles available, with their id and title.',
|
||||
{}),
|
||||
tool('describe_profile',
|
||||
'Describe one profile: its entities, fields, relations and labels. Inline the values of short code lists.',
|
||||
{ profile_id: 'The profile URI returned by list_profiles' }),
|
||||
tool('validate_spec',
|
||||
'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.',
|
||||
{ spec: 'The spec as Turtle text' }),
|
||||
tool('read_spec',
|
||||
'Return the Turtle of a spec bijlage, a ttl file attached to an earlier message in this conversation. Use it to reuse the agreed spec, or to read back an earlier proposal before you change it. Read-only.',
|
||||
{ file_name: 'The file name of the spec bijlage, e.g. specificatie-<uuid>.ttl' }),
|
||||
tool('lookup_values',
|
||||
'Search a code list for a term so you can suggest concrete values for a filter (for sh:hasValue or sh:in). Returns up to 25 matches with their URIs and labels, a total, and an exact flag when a label equals the term. Reads candidate values only; it never runs the spec and never tells you how many subjects a spec would match. Read-only, no report is created.',
|
||||
{
|
||||
profile_id: 'The profile URI returned by list_profiles',
|
||||
field: 'The field as "entityLabel.fieldLabel", e.g. "bestuurseenheid.naam"',
|
||||
term: 'The Dutch word or name to search for',
|
||||
}),
|
||||
];
|
||||
|
||||
// buildTools(profiles, session) → { handlers }
|
||||
export function buildTools(profiles, session) {
|
||||
const profileList = () => [...profiles.values()].map(p => ({ id: p.uri, title: p.title }));
|
||||
|
||||
const findProfile = (id) => profiles.get(id);
|
||||
|
||||
async function describe_profile({ profile_id }) {
|
||||
const p = findProfile(profile_id);
|
||||
if (!p) return text(`no profile <${profile_id}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
|
||||
const inlined = await codeListValues(p, scopedQuery);
|
||||
return text(describeProfile(p, inlined));
|
||||
function tool(name, description, params) {
|
||||
const properties = {};
|
||||
for (const [key, text] of Object.entries(params)) {
|
||||
properties[key] = { type: 'string', description: text };
|
||||
}
|
||||
|
||||
async function validate_spec({ spec }) {
|
||||
let parsed;
|
||||
try { parsed = parseSpec(spec); }
|
||||
catch (e) { return text(`the spec did not parse: ${e.message}`); }
|
||||
const p = findProfile(parsed.profileUri);
|
||||
if (!p) return text(`no profile <${parsed.profileUri}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
|
||||
const errors = checkSpec(parsed, p, MAX_PATH_DEPTH, profiles);
|
||||
if (errors.length) return text(errors[0]);
|
||||
// The spec that checks out becomes the attached proposal.
|
||||
session.onSpecValidated?.(spec);
|
||||
return text(['ok', ...whereNotes(parsed, p)].join('\n'));
|
||||
}
|
||||
|
||||
async function list_profiles() {
|
||||
return text(profileList().map(x => `${x.title} <${x.id}>`).join('\n'));
|
||||
}
|
||||
|
||||
async function lookup_values({ profile_id, field, term }) {
|
||||
const p = findProfile(profile_id);
|
||||
if (!p) return text(`no profile <${profile_id}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
|
||||
if (!term) return text('no term to search for.');
|
||||
try {
|
||||
const out = await lookupValues(p, term, field, scopedQuery);
|
||||
if (out.error) return text(out.error);
|
||||
if (!out.matches.length) return text(`no value matches "${term}" in ${field}.`);
|
||||
const lines = out.matches.map(m => `${m.label} <${m.uri}>`);
|
||||
if (out.total === '25+') lines.push('(25+ matches, narrow the term)');
|
||||
if (out.exact) lines.push(`exact: <${out.exact}>`);
|
||||
return text(lines.join('\n'));
|
||||
} catch (e) {
|
||||
return text(`the lookup failed: ${String(e.message || e).split('\n')[0]}`);
|
||||
}
|
||||
}
|
||||
|
||||
// A spec bijlage next to a message. The model opens it with read_spec:
|
||||
// only files this service made are readable, hte spec files under share.
|
||||
async function read_spec({ file_uri }) {
|
||||
const fileName = String(file_uri || '').split('/').pop();
|
||||
if (!fileName || !/^specificatie-[0-9a-f]{8}-[0-9a-f-]{27}\.ttl$/.test(fileName)) {
|
||||
return text('no such spec file.');
|
||||
}
|
||||
const shareDir = process.env.SHARE_DIR || '/share';
|
||||
try {
|
||||
return text(readFileSync(`${shareDir}/${fileName}`, 'utf8'));
|
||||
} catch (e) {
|
||||
return text(`the spec failed to load: ${String(e.message || e).split('\n')[0]}`);
|
||||
}
|
||||
}
|
||||
|
||||
const handlers = new Map(
|
||||
[
|
||||
['list_profiles', list_profiles],
|
||||
['describe_profile', describe_profile],
|
||||
['validate_spec', validate_spec],
|
||||
['read_spec', read_spec],
|
||||
['lookup_values', lookup_values],
|
||||
],
|
||||
);
|
||||
|
||||
return { handlers };
|
||||
return {
|
||||
type: 'function',
|
||||
function: {
|
||||
name,
|
||||
description,
|
||||
parameters: { type: 'object', properties, required: Object.keys(params) },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function text(s) { return { content: [{ type: 'text', text: s }] }; }
|
||||
export async function runTool(name, args, profiles) {
|
||||
switch (name) {
|
||||
case 'list_profiles': return listProfiles(profiles);
|
||||
case 'describe_profile': return describe(args, profiles);
|
||||
case 'validate_spec': return validateSpec(args, profiles);
|
||||
case 'read_spec': return readSpec(args);
|
||||
case 'lookup_values': return lookup(args, profiles);
|
||||
default: return `unknown tool "${name}"`;
|
||||
}
|
||||
}
|
||||
|
||||
function listProfiles(profiles) {
|
||||
return [...profiles.values()].map(p => `${p.title} <${p.uri}>`).join('\n');
|
||||
}
|
||||
|
||||
async function describe({ profile_id }, profiles) {
|
||||
return profileError(profiles, profile_id) || describeProfile(profiles.get(profile_id));
|
||||
}
|
||||
|
||||
function validateSpec({ spec }, profiles) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = parseSpec(spec);
|
||||
} catch (e) {
|
||||
return `the spec did not parse: ${e.message}`;
|
||||
}
|
||||
const error = checkSpec(parsed, profiles);
|
||||
if (error) return error;
|
||||
return ['ok', ...whereNotes(parsed, profiles.get(parsed.profileUri))].join('\n');
|
||||
}
|
||||
|
||||
// Only spec files this service made are readable.
|
||||
function readSpec({ file_name }) {
|
||||
return readSpecFile(file_name) || 'no such spec file.';
|
||||
}
|
||||
|
||||
async function lookup({ profile_id, field, term }, profiles) {
|
||||
const error = profileError(profiles, profile_id);
|
||||
if (error) return error;
|
||||
if (!term) return 'no term to search for.';
|
||||
return lookupValues(profiles.get(profile_id), field, term);
|
||||
}
|
||||
|
||||
@@ -1,129 +1,71 @@
|
||||
// What this service answers a chat turn with. The service is synchronous
|
||||
// per turn: when the answer is written, every bijlage already exists.
|
||||
// What the assistant answers a chat turn with: { text, attachments }. When
|
||||
// the answer is written, every bijlage already exists.
|
||||
//
|
||||
// Execute turns skip the LLM loop. The mode classifier says the user wants
|
||||
// to run the report now; the agreed spec already sits in the conversation
|
||||
// as a bijlage, so it is read from disk and fed straight to the runner.
|
||||
// The answer is hard-coded: no extra prompt runs for it. When the run cannot
|
||||
// happen or fails, the answer is a hard-coded failure message: nothing is
|
||||
// retried and no spec is rewritten.
|
||||
import { classifyMode } from './llm/mode.js';
|
||||
// Refine: the LLM proposes a spec; the last spec that checked out goes on
|
||||
// the answer as a bijlage. The prompt never carries the Turtle.
|
||||
//
|
||||
// Execute: no LLM runs. The last spec bijlage in the conversation is the
|
||||
// agreed spec; it is read from the share and fed straight to the runner.
|
||||
// The answer is fixed text. When the run cannot happen or fails, the answer
|
||||
// is a fixed failure message: nothing is retried and no spec is rewritten.
|
||||
import { uuid } from 'mu';
|
||||
import { ask } from './llm/loop.js';
|
||||
import { parseSpec } from './runner/spec.js';
|
||||
import { checkSpec } from './runner/check.js';
|
||||
import { run, slug } from './runner/run.js';
|
||||
import { readFileSync } from 'fs';
|
||||
import { storeSpecFile, registerFileObject } from './chat/attachment-file.js';
|
||||
import { SPEC_MEDIA_TYPE, ATTACHMENT_TYPES } from './chat/vocab.js';
|
||||
import { run, registerReport, slug } from './runner/run.js';
|
||||
import { storeSpecFile, readSpecFile, writeShareFile, registerFile,
|
||||
SPEC_MEDIA_TYPE, SPEC_TYPE, RESULT_TYPE } from './chat.js';
|
||||
|
||||
const TEXT = {
|
||||
executed: 'Het rapport is uitgevoerd. De bijlagen staan erbij.',
|
||||
noSpec: 'Er is nog geen voorstel om uit te voeren. Beschrijf eerst welk rapport je wilt.',
|
||||
invalid: 'Het voorstel is niet meer geldig. Vraag een nieuw voorstel.',
|
||||
failed: 'Het rapport kon niet uitgevoerd worden. Probeer het later opnieuw.',
|
||||
noAnswer: 'Er kwam geen antwoord. Probeer het opnieuw.',
|
||||
};
|
||||
|
||||
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
|
||||
const SHARE_DIR = process.env.SHARE_DIR || '/share';
|
||||
|
||||
// The chat mounts before the profiles are in; whenProfiles waits for the load.
|
||||
export function reportAssistant(whenProfiles) {
|
||||
return async function answer(turn) {
|
||||
const turns = [...turn.history, { role: 'user', content: turn.content }];
|
||||
const profiles = await whenProfiles();
|
||||
|
||||
if (await classifyMode(turns)) {
|
||||
try {
|
||||
return await runStoredSpec(turn, profiles);
|
||||
} catch (e) {
|
||||
console.error('[reports] execution failed:', e);
|
||||
return TEXT.failed;
|
||||
}
|
||||
}
|
||||
return llmAnswer(turn, turns, profiles);
|
||||
};
|
||||
// An empty answer (a reasoning model can end with no text) gets a fixed
|
||||
// text and no spec: only a spec the user saw explained may run.
|
||||
export async function refineSpec(turns, profiles) {
|
||||
const { text, spec } = await ask(turns, profiles);
|
||||
if (!text.trim()) return { text: TEXT.noAnswer };
|
||||
const attachments = spec ? [await storeSpecFile(spec)] : [];
|
||||
return { text: text.trim(), attachments };
|
||||
}
|
||||
|
||||
// Execution: the last spec bijlage in the conversation is the agreed spec.
|
||||
// No prompt runs; the runner gets it straight away.
|
||||
async function runStoredSpec(turn, profiles) {
|
||||
const attachment = lastSpecAttachment(turn.history);
|
||||
export async function executeSpec(history, conversationTitle, profiles) {
|
||||
const attachment = lastSpecAttachment(history);
|
||||
const content = attachment && readSpecFile(attachment.name);
|
||||
if (!content) return TEXT.noSpec;
|
||||
if (!content) return { text: TEXT.noSpec };
|
||||
|
||||
const parsed = parseSpec(content);
|
||||
const profile = profiles.get(parsed.profileUri);
|
||||
if (!profile) {
|
||||
console.error(`[reports] no profile <${parsed.profileUri}>`);
|
||||
return TEXT.invalid;
|
||||
try {
|
||||
const spec = parseSpec(content);
|
||||
const error = checkSpec(spec, profiles);
|
||||
if (error) {
|
||||
console.error(`[reports] the agreed spec is not valid: ${error}`);
|
||||
return { text: TEXT.invalid };
|
||||
}
|
||||
|
||||
// The conversation's title (named by the first question) names the
|
||||
// file; the share name is unique, so a second run never overwrites the
|
||||
// first. The report itself keeps the dct:title of the spec.
|
||||
const shareName = `${uuid()}.csv`;
|
||||
writeShareFile(shareName, await run(spec, profiles.get(spec.profileUri)));
|
||||
const csvFile = await registerFile(shareName, 'text/csv', RESULT_TYPE, `${slug(conversationTitle)}.csv`);
|
||||
await registerReport(spec.title, csvFile);
|
||||
|
||||
// Same physical spec file, registered as a bijlage for this message.
|
||||
const specFile = await registerFile(attachment.name, SPEC_MEDIA_TYPE, SPEC_TYPE);
|
||||
|
||||
return { text: TEXT.executed, attachments: [csvFile, specFile] };
|
||||
} catch (e) {
|
||||
console.error('[reports] execution failed:', e);
|
||||
return { text: TEXT.failed };
|
||||
}
|
||||
const errors = checkSpec(parsed, profile, MAX_PATH_DEPTH, profiles);
|
||||
if (errors.length) {
|
||||
console.error(`[reports] the agreed spec is not valid: ${errors[0]}`);
|
||||
return TEXT.invalid;
|
||||
}
|
||||
|
||||
// The conversation's title (named by the first question) names the file.
|
||||
// The report itself keeps the dct:title of the spec.
|
||||
const { fileUri } = await run(turn.query, turn.update, parsed, profile,
|
||||
parsed.title, {
|
||||
spec: content,
|
||||
fileType: ATTACHMENT_TYPES.result,
|
||||
fileName: `${slug(turn.conversation.title)}.csv`,
|
||||
});
|
||||
|
||||
// Same physical spec file, registered as a bijlage for this message.
|
||||
const specFile = await registerFileObject(turn.update, {
|
||||
fileName: attachment.name,
|
||||
format: SPEC_MEDIA_TYPE,
|
||||
size: Buffer.byteLength(content),
|
||||
type: ATTACHMENT_TYPES.spec,
|
||||
});
|
||||
|
||||
return {
|
||||
text: TEXT.executed,
|
||||
attachments: [
|
||||
{ uri: fileUri },
|
||||
{ ...specFile, mediaType: SPEC_MEDIA_TYPE },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function lastSpecAttachment(history) {
|
||||
for (const message of [...(history || [])].reverse()) {
|
||||
const specs = (message.attachments || []).filter(a =>
|
||||
a.mediaType === SPEC_MEDIA_TYPE
|
||||
&& /^specificatie-[0-9a-f-]+\.ttl$/.test(a.name || ''));
|
||||
if (specs.length) return specs[specs.length - 1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function readSpecFile(fileName) {
|
||||
try {
|
||||
return readFileSync(`${SHARE_DIR}/${fileName}`, 'utf8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// The LLM loop, refinement only. The spec that checked out goes on the
|
||||
// answer as a bijlage. The prompt never carries the Turtle.
|
||||
async function llmAnswer(turn, turns, profiles) {
|
||||
// The last spec that checked out; stored as a file for this turn's
|
||||
// final message.
|
||||
let spec = null;
|
||||
|
||||
const session = {
|
||||
onSpecValidated(validated) { spec = validated; },
|
||||
};
|
||||
|
||||
const { text } = await ask(turns, profiles, session);
|
||||
|
||||
const attachments = [];
|
||||
if (spec) {
|
||||
const stored = await storeSpecFile(turn.update, spec);
|
||||
attachments.push({ ...stored, mediaType: SPEC_MEDIA_TYPE });
|
||||
}
|
||||
return { text, attachments };
|
||||
const specs = history.flatMap(message => message.attachments)
|
||||
.filter(a => a.mediaType === SPEC_MEDIA_TYPE && a.name?.startsWith('specificatie-'));
|
||||
return specs.at(-1) || null;
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ export const NUMERIC_DATATYPES = new Set([
|
||||
'nonNegativeInteger', 'positiveInteger', 'nonPositiveInteger', 'negativeInteger',
|
||||
'unsignedLong', 'unsignedInt', 'unsignedShort', 'unsignedByte',
|
||||
].map(d => XSD + d));
|
||||
export const TEMPORAL_DATATYPES = new Set([XSD + 'date', XSD + 'dateTime']);
|
||||
|
||||
export function assemble(subjects, values, spec) {
|
||||
const sorted = [...subjects].sort();
|
||||
@@ -57,19 +56,18 @@ function collect(terms, col) {
|
||||
}
|
||||
}
|
||||
|
||||
// min/max over values that share one datatype. Numbers compare as numbers,
|
||||
// dates and times as instants. Mixed datatypes throw: check.js should have
|
||||
// caught it, and a silently wrong cell is worse than a failed job.
|
||||
// min/max. Numbers compare as numbers. Dates and times compare as text:
|
||||
// xsd:date and xsd:dateTime are ISO strings, so text order is date order.
|
||||
// Numbers mixed with other values throw: check.js should have caught it,
|
||||
// and a silently wrong cell is worse than a failed job. The terms are SPARQL
|
||||
// JSON bindings, so the datatype is a plain string.
|
||||
function extreme(terms, col, direction) {
|
||||
if (!terms.length) return '';
|
||||
const datatypes = [...new Set(terms.map(t => t.datatype?.value || ''))];
|
||||
if (datatypes.length > 1) {
|
||||
throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but its values carry ${datatypes.length} datatypes (${datatypes.join(', ')}).`);
|
||||
const numbers = terms.filter(t => NUMERIC_DATATYPES.has(t.datatype)).length;
|
||||
if (numbers && numbers < terms.length) {
|
||||
throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but mixes numbers with other values.`);
|
||||
}
|
||||
const dt = datatypes[0];
|
||||
const key = NUMERIC_DATATYPES.has(dt) ? t => Number(t.value)
|
||||
: TEMPORAL_DATATYPES.has(dt) ? t => new Date(t.value).getTime()
|
||||
: t => t.value;
|
||||
const key = numbers ? t => Number(t.value) : t => t.value;
|
||||
const cmp = (a, b) => a < b ? -1 : a > b ? 1 : 0;
|
||||
const best = terms.reduce((a, b) => (cmp(key(b), key(a)) * direction > 0 ? b : a));
|
||||
return best.value;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { fieldsOf } from './profile.js';
|
||||
import { startShape, fieldsOf, field } from './profile.js';
|
||||
import { NUMERIC_DATATYPES } from './assemble.js';
|
||||
import { sharedSteps } from './spec.js';
|
||||
|
||||
@@ -9,23 +9,18 @@ const XSD_DATE = XSD + 'date';
|
||||
const XSD_DATETIME = XSD + 'dateTime';
|
||||
|
||||
// The validator. checkSpec runs the checks in order and returns the first
|
||||
// error as a one-element array (empty if ok). Every message names the thing
|
||||
// that is wrong in the profile's own words and says what to write instead.
|
||||
// No database call anywhere in this file.
|
||||
// error, or null when the spec is fine. Every message names the thing that
|
||||
// is wrong in the profile's own words and says what to write instead. No
|
||||
// database call anywhere in this file.
|
||||
|
||||
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
|
||||
const ANYOF_MAX_TERMS = 24;
|
||||
const ANYOF_MAX_TERM_LENGTH = 100;
|
||||
|
||||
// The spec names a profile we have. knownProfiles is the profile Map loaded
|
||||
// at boot; the check is skipped when it is omitted.
|
||||
function profileNamed(spec, knownProfiles) {
|
||||
if (!spec.profileUri) {
|
||||
return `no rep:profile named. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'} .`;
|
||||
}
|
||||
if (knownProfiles && !knownProfiles.has(spec.profileUri)) {
|
||||
return `no profile <${spec.profileUri}>. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'}.`;
|
||||
}
|
||||
return null;
|
||||
// null when the profile exists, else what to write instead.
|
||||
export function profileError(profiles, uri) {
|
||||
if (profiles.has(uri)) return null;
|
||||
return `no profile <${uri}>. Available: ${[...profiles.values()].map(p => p.title).join(', ')}.`;
|
||||
}
|
||||
|
||||
// The spec names an entity we have: by rep:entity, or by a targetClass only
|
||||
@@ -74,16 +69,6 @@ function conditions(spec) {
|
||||
return out;
|
||||
}
|
||||
|
||||
// The shape a spec starts from, after startEntity has approved it: rep:entity
|
||||
// wins, otherwise the one shape carrying the targetClass.
|
||||
function startShapeOf(spec, profile) {
|
||||
if (spec.entity) {
|
||||
return profile.shapes.find(s => s.uri === spec.entity)?.uri || null;
|
||||
}
|
||||
const matches = profile.shapes.filter(s => s.targetClass === spec.targetClass);
|
||||
return matches.length === 1 ? matches[0].uri : null;
|
||||
}
|
||||
|
||||
// Columns exist
|
||||
function columnsPresent(spec) {
|
||||
if (!spec.columns.length) return 'no columns. Add at least one rep:columns entry.';
|
||||
@@ -100,26 +85,26 @@ function columnLabel(spec) {
|
||||
|
||||
// Labels are unique
|
||||
function columnLabelsUnique(spec) {
|
||||
const seen = new Map();
|
||||
const seen = new Set();
|
||||
for (const col of spec.columns) {
|
||||
if (seen.has(col.label)) {
|
||||
return `two columns are called "${col.label}". Labels must differ.`;
|
||||
}
|
||||
seen.set(col.label, true);
|
||||
seen.add(col.label);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Every path resolves, columns end on a value, depth capped
|
||||
function pathsResolve(spec, profile, maxPathDepth) {
|
||||
const startShapeUri = startShapeOf(spec, profile);
|
||||
function pathsResolve(spec, profile) {
|
||||
const startShapeUri = startShape(profile, spec).uri;
|
||||
for (const col of spec.columns) {
|
||||
const err = walk(profile, startShapeUri, col.path, maxPathDepth,
|
||||
const err = walk(profile, startShapeUri, col.path,
|
||||
`column "${col.label}"`, true, col.nodeKind === SH + 'IRI');
|
||||
if (err) return err;
|
||||
}
|
||||
for (const [i, filter] of spec.filters.entries()) {
|
||||
const err = walk(profile, startShapeUri, filter.path, maxPathDepth,
|
||||
const err = walk(profile, startShapeUri, filter.path,
|
||||
`filter ${i + 1}`, false);
|
||||
if (err) return err;
|
||||
}
|
||||
@@ -128,7 +113,7 @@ function pathsResolve(spec, profile, maxPathDepth) {
|
||||
|
||||
// sh:min/sh:max only on numbers and dates
|
||||
function collectMinMaxTyped(spec, profile) {
|
||||
const startShapeUri = startShapeOf(spec, profile);
|
||||
const startShapeUri = startShape(profile, spec).uri;
|
||||
for (const col of spec.columns) {
|
||||
if (col.collect !== SH + 'min' && col.collect !== SH + 'max') continue;
|
||||
const name = col.collect === SH + 'min' ? 'sh:min' : 'sh:max';
|
||||
@@ -136,8 +121,7 @@ function collectMinMaxTyped(spec, profile) {
|
||||
const last = col.path[col.path.length - 1];
|
||||
const shapeUri = shapeAfter(profile, startShapeUri, col.path);
|
||||
if (!shapeUri) continue; // pathsResolve already reported the real error
|
||||
const f = fieldsOf(profile, shapeUri)
|
||||
.find(x => x.path === last.predicate && !!x.inverse === !!last.inverse);
|
||||
const f = field(profile, shapeUri, last);
|
||||
if (!f) continue;
|
||||
const kind = f.datatype === XSD_DATETIME || f.datatype === XSD_DATE ? 'date'
|
||||
: NUMERIC_DATATYPES.has(f.datatype) ? 'number'
|
||||
@@ -205,10 +189,10 @@ function filterConstraint(spec) {
|
||||
function maxCountZero(spec) {
|
||||
for (const { cond } of conditions(spec)) {
|
||||
const c = cond.constraints;
|
||||
if (c.maxCount !== null && c.maxCount !== undefined && c.maxCount > 0) {
|
||||
if (c.maxCount > 0) {
|
||||
return `sh:maxCount ${c.maxCount} needs counting, which this service cannot do. Use sh:maxCount 0 for "has none", or drop it.`;
|
||||
}
|
||||
if (c.minCount !== null && c.minCount !== undefined && c.minCount > 2) {
|
||||
if (c.minCount > 2) {
|
||||
return `sh:minCount ${c.minCount} needs counting. Use sh:minCount 1 (at least one) or sh:minCount 2 (two distinct values); anything higher is beyond this service.`;
|
||||
}
|
||||
}
|
||||
@@ -272,14 +256,14 @@ function literalsParse(spec) {
|
||||
|
||||
// rep:where: walks from the row through the profile, shares at least one
|
||||
// step with its filter or column, and holds no rep:where of its own.
|
||||
function whereConditions(spec, profile, maxPathDepth) {
|
||||
const startShapeUri = startShapeOf(spec, profile);
|
||||
function whereConditions(spec, profile) {
|
||||
const startShapeUri = startShape(profile, spec).uri;
|
||||
for (const { what, cond, host } of conditions(spec)) {
|
||||
if (!host) continue;
|
||||
if (cond.where.length) {
|
||||
return `${what} holds a rep:where of its own. Put every rep:where directly on the filter or column.`;
|
||||
}
|
||||
const err = walk(profile, startShapeUri, cond.path, maxPathDepth, what, false);
|
||||
const err = walk(profile, startShapeUri, cond.path, what, false);
|
||||
if (err) return err;
|
||||
const k = sharedSteps(host.path, cond.path);
|
||||
if (!k) {
|
||||
@@ -302,14 +286,13 @@ function whereConditions(spec, profile, maxPathDepth) {
|
||||
// code) it compares a URI and never matches, so its path must end on a field
|
||||
// with a datatype.
|
||||
function rangeOnValue(spec, profile) {
|
||||
const startShapeUri = startShapeOf(spec, profile);
|
||||
const startShapeUri = startShape(profile, spec).uri;
|
||||
for (const { what, cond } of conditions(spec)) {
|
||||
const key = ['minInclusive', 'maxInclusive', 'minExclusive', 'maxExclusive']
|
||||
.find(k => cond.constraints[k]);
|
||||
if (!key || !cond.path.length) continue;
|
||||
const last = cond.path[cond.path.length - 1];
|
||||
const f = fieldsOf(profile, shapeAfter(profile, startShapeUri, cond.path))
|
||||
.find(x => x.path === last.predicate && !!x.inverse === !!last.inverse);
|
||||
const f = field(profile, shapeAfter(profile, startShapeUri, cond.path), last);
|
||||
if (!f || f.datatype) continue; // pathsResolve reports a missing step
|
||||
const target = f.node
|
||||
? `"${profile.shapes.find(s => s.uri === f.node)?.label || f.node}"`
|
||||
@@ -326,7 +309,7 @@ function rangeOnValue(spec, profile) {
|
||||
// Where each rep:where applies, in the profile's words, so the proposal can
|
||||
// say it. Profile only, no database.
|
||||
export function whereNotes(spec, profile) {
|
||||
const startShapeUri = startShapeOf(spec, profile);
|
||||
const startShapeUri = startShape(profile, spec).uri;
|
||||
return conditions(spec).filter(x => x.host).map(({ what, cond, host }) => {
|
||||
const k = sharedSteps(host.path, cond.path);
|
||||
return `${what} applies at ${nodeName(profile, startShapeUri, host.path.slice(0, k))}, after step ${k} of ${host.what}.`;
|
||||
@@ -337,7 +320,7 @@ export function whereNotes(spec, profile) {
|
||||
function nodeName(profile, startShapeUri, hops) {
|
||||
let shapeUri = startShapeUri;
|
||||
for (const [i, hop] of hops.entries()) {
|
||||
const f = fieldsOf(profile, shapeUri).find(x => x.path === hop.predicate && !!x.inverse === !!hop.inverse);
|
||||
const f = field(profile, shapeUri, hop);
|
||||
if (!f) return `step ${hops.length}`;
|
||||
if (!f.node) return i === hops.length - 1 ? `the value "${f.name}"` : `step ${hops.length}`;
|
||||
shapeUri = f.node;
|
||||
@@ -351,33 +334,37 @@ function parsesAsDate(value, withTime) {
|
||||
return withTime ? value.includes('T') : /^\d{4}-\d{2}-\d{2}$/.test(value);
|
||||
}
|
||||
|
||||
// In this order: startEntity comes first, the checks after it rely on a
|
||||
// start shape.
|
||||
const CHECKS = [
|
||||
['no entity or unknown class', startEntity],
|
||||
['no columns', columnsPresent],
|
||||
['column without rdfs:label', columnLabel],
|
||||
['two columns with one label', columnLabelsUnique],
|
||||
['column carries constraints', columnPure],
|
||||
['filter carries rdfs:label', filterLabel],
|
||||
['filter without a condition', filterConstraint],
|
||||
['sh:maxCount above 0', maxCountZero],
|
||||
['rep:anyOf too big', anyOfLimits],
|
||||
['sh:in mixes URIs and literals', inHomogeneous],
|
||||
['typed literal does not parse', literalsParse],
|
||||
['two rep:row columns', oneRowColumn],
|
||||
['path does not resolve', pathsResolve],
|
||||
['rep:where does not fit', whereConditions],
|
||||
['date or number condition on a link', rangeOnValue],
|
||||
['sh:min/sh:max on a non-number', collectMinMaxTyped],
|
||||
startEntity,
|
||||
columnsPresent,
|
||||
columnLabel,
|
||||
columnLabelsUnique,
|
||||
columnPure,
|
||||
filterLabel,
|
||||
filterConstraint,
|
||||
maxCountZero,
|
||||
anyOfLimits,
|
||||
inHomogeneous,
|
||||
literalsParse,
|
||||
oneRowColumn,
|
||||
pathsResolve,
|
||||
whereConditions,
|
||||
rangeOnValue,
|
||||
collectMinMaxTyped,
|
||||
];
|
||||
|
||||
export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) {
|
||||
const profileErr = profileNamed(spec, knownProfiles);
|
||||
if (profileErr) return [profileErr];
|
||||
for (const [name, check] of CHECKS) {
|
||||
const err = check(spec, profile, maxPathDepth);
|
||||
if (err) return [err];
|
||||
// checkSpec(spec, profiles) → the first error, or null.
|
||||
export function checkSpec(spec, profiles) {
|
||||
const error = profileError(profiles, spec.profileUri);
|
||||
if (error) return error;
|
||||
const profile = profiles.get(spec.profileUri);
|
||||
for (const check of CHECKS) {
|
||||
const err = check(spec, profile);
|
||||
if (err) return err;
|
||||
}
|
||||
return [];
|
||||
return null;
|
||||
}
|
||||
|
||||
// The shape a path lands in: follow sh:node through every hop but the
|
||||
@@ -386,17 +373,16 @@ export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) {
|
||||
function shapeAfter(profile, startShapeUri, hops) {
|
||||
let shapeUri = startShapeUri;
|
||||
for (let i = 0; i < hops.length - 1; i++) {
|
||||
const fields = fieldsOf(profile, shapeUri);
|
||||
const f = fields.find(x => x.path === hops[i].predicate && !!x.inverse === !!hops[i].inverse);
|
||||
const f = field(profile, shapeUri, hops[i]);
|
||||
if (!f || !f.node) return null;
|
||||
shapeUri = f.node;
|
||||
}
|
||||
return shapeUri;
|
||||
}
|
||||
|
||||
function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsIri = false) {
|
||||
if (hops.length > maxPathDepth) {
|
||||
return `${what} walks ${hops.length} hops. The limit is ${maxPathDepth}.`;
|
||||
function walk(profile, startShapeUri, hops, what, isColumn, wantsIri = false) {
|
||||
if (hops.length > MAX_PATH_DEPTH) {
|
||||
return `${what} walks ${hops.length} hops. The limit is ${MAX_PATH_DEPTH}.`;
|
||||
}
|
||||
if (!hops.length) return null;
|
||||
let shapeUri = startShapeUri;
|
||||
@@ -416,20 +402,15 @@ function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsI
|
||||
shapeUri = f.node;
|
||||
} else if (isColumn) {
|
||||
if (wantsIri && f.datatype) {
|
||||
return `column "${colLabel(what)}" has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`;
|
||||
return `${what} has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`;
|
||||
}
|
||||
if (!wantsIri && !f.datatype && !f.class) {
|
||||
const valueFields = fieldsOf(profile, f.node)
|
||||
.filter(x => x.datatype || x.class)
|
||||
.map(x => `${x.name} (${x.path})`);
|
||||
return `column "${colLabel(what)}" ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`;
|
||||
return `${what} ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function colLabel(what) {
|
||||
const m = what.match(/"([^"]+)"/);
|
||||
return m ? m[1] : what;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { sparqlEscapeUri } from '../db.js';
|
||||
import { wherePatterns } from './seed.js';
|
||||
import { query, sparqlEscapeUri } from 'mu';
|
||||
import { hopsPattern, wherePatterns } from './seed.js';
|
||||
|
||||
// Fetch column values for known subjects. Columns are grouped by their
|
||||
// shared non-leaf hops; two columns share a query only when every hop
|
||||
@@ -8,12 +8,14 @@ import { wherePatterns } from './seed.js';
|
||||
|
||||
const SUBJECT_CHUNK_SIZE = Math.max(1, Number(process.env.SUBJECT_CHUNK_SIZE) || 100);
|
||||
|
||||
export async function fetchColumns(sessionQuery, subjects, spec, values = new Map()) {
|
||||
// fetchColumns(subjects, spec) → subject → column index → [term]
|
||||
export async function fetchColumns(subjects, spec) {
|
||||
const values = new Map();
|
||||
const groups = groupColumns(spec.columns);
|
||||
for (const group of groups) {
|
||||
for (let i = 0; i < subjects.length; i += SUBJECT_CHUNK_SIZE) {
|
||||
const chunk = subjects.slice(i, i + SUBJECT_CHUNK_SIZE);
|
||||
const result = await sessionQuery(groupQuery(group, chunk));
|
||||
const result = await query(groupQuery(group, chunk));
|
||||
collect(result, group, values);
|
||||
}
|
||||
}
|
||||
@@ -41,21 +43,11 @@ export function groupColumns(columns) {
|
||||
|
||||
export function groupQuery(group, chunk) {
|
||||
const subjects = chunk.map(sparqlEscapeUri).join(' ');
|
||||
const lines = [];
|
||||
let prev = '?s';
|
||||
// every hop except the last gets its own variable; the leaf is ?v
|
||||
group.hops.slice(0, -1).forEach((hop, i) => {
|
||||
const v = `?x${i}`;
|
||||
if (hop.inverse) lines.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
|
||||
else lines.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`);
|
||||
prev = v;
|
||||
});
|
||||
const leaf = group.hops[group.hops.length - 1];
|
||||
if (leaf.inverse) lines.push(`?v ${sparqlEscapeUri(leaf.predicate)} ${prev} .`);
|
||||
else lines.push(`${prev} ${sparqlEscapeUri(leaf.predicate)} ?v .`);
|
||||
// rep:where hangs on the column's own nodes: ?x<k-1> after k hops, ?v at
|
||||
const lines = [hopsPattern(group.hops, 'x', '?s', '?v')];
|
||||
// rep:where hangs on the column's own nodes: ?x_<k> after k hops, ?v at
|
||||
// the end.
|
||||
const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x${k - 1}`);
|
||||
const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x_${k}`);
|
||||
lines.push(...wherePatterns(group.where, group.hops, nodeVar, 'c'));
|
||||
return `SELECT ?s ?v WHERE {\n VALUES ?s { ${subjects} }\n ${lines.join('\n ')}\n}`;
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
import { mkdirSync, writeFileSync } from 'fs';
|
||||
|
||||
// Rows to a CSV file. CSV_SEPARATOR is ';' by default. A cell is quoted
|
||||
// only when it holds the separator, a quote or a newline; a quote inside a
|
||||
// cell is doubled, per RFC 4180.
|
||||
|
||||
const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
|
||||
const SHARE_DIR = process.env.SHARE_DIR || '/share';
|
||||
|
||||
export function rowsToCsv(rows) {
|
||||
return rows.map(row => row.map(escapeCell).join(CSV_SEPARATOR)).join('\n');
|
||||
}
|
||||
|
||||
function escapeCell(cell) {
|
||||
const s = String(cell ?? '');
|
||||
return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s)
|
||||
? `"${s.replace(/"/g, '""')}"`
|
||||
: s;
|
||||
}
|
||||
|
||||
export function writeCsv(name, csv) {
|
||||
mkdirSync(SHARE_DIR, { recursive: true });
|
||||
const file = `${SHARE_DIR}/${name}`;
|
||||
writeFileSync(file, csv, 'utf8');
|
||||
return file;
|
||||
}
|
||||
|
||||
// The spec of the run, next to its CSV (same base name, .ttl), so a report
|
||||
// can be debugged afterwards. The store keeps only the pointer.
|
||||
export function writeSpec(csvName, spec) {
|
||||
mkdirSync(SHARE_DIR, { recursive: true });
|
||||
const name = csvName.replace(/\.csv$/, '') + '.ttl';
|
||||
writeFileSync(`${SHARE_DIR}/${name}`, spec, 'utf8');
|
||||
return `share://${name}`;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import N3 from 'n3';
|
||||
import { readFileSync } from 'fs';
|
||||
import { readFileSync, readdirSync } from 'fs';
|
||||
|
||||
// Parse a profile (a SHACL shapes graph) and answer questions about it.
|
||||
|
||||
@@ -8,13 +8,11 @@ const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
|
||||
const OWL_ONTOLOGY = 'http://www.w3.org/2002/07/owl#Ontology';
|
||||
const REP = 'http://mu.semte.ch/vocabularies/reporting/';
|
||||
|
||||
export async function loadProfiles(dir) {
|
||||
const fs = await import('fs');
|
||||
const path = await import('path');
|
||||
const files = fs.readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
|
||||
export function loadProfiles(dir) {
|
||||
const files = readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
|
||||
const profiles = new Map();
|
||||
for (const file of files) {
|
||||
const profile = await loadProfile(path.join(dir, file));
|
||||
const profile = loadProfile(`${dir}/${file}`);
|
||||
profiles.set(profile.uri, profile);
|
||||
console.log(`[profile] "${profile.title}" (${file}): ${profile.shapes.length} entities, ` +
|
||||
`${profile.shapes.reduce((n, s) => n + s.fields.length, 0)} fields`);
|
||||
@@ -23,29 +21,19 @@ export async function loadProfiles(dir) {
|
||||
return profiles;
|
||||
}
|
||||
|
||||
export async function loadProfile(file) {
|
||||
const parser = new N3.Parser();
|
||||
const store = new N3.Store();
|
||||
await new Promise((resolve, reject) => {
|
||||
parser.parse(readFileSync(file, 'utf8'), (err, quad) => {
|
||||
if (err) return reject(new Error(`${file}: ${err.message}`));
|
||||
if (quad) store.addQuad(quad); else resolve();
|
||||
});
|
||||
});
|
||||
function loadProfile(file) {
|
||||
let store;
|
||||
try {
|
||||
store = new N3.Store(new N3.Parser().parse(readFileSync(file, 'utf8')));
|
||||
} catch (e) {
|
||||
throw new Error(`${file}: ${e.message}`);
|
||||
}
|
||||
|
||||
const ontology = store.getQuads(null, RDF + 'type', OWL_ONTOLOGY)[0];
|
||||
if (!ontology) throw new Error(`${file}: no owl:Ontology. A profile file must name itself.`);
|
||||
const uri = ontology.subject.value;
|
||||
const title = store.getQuads(ontology.subject, 'http://purl.org/dc/terms/title', null)[0]?.object.value || uri;
|
||||
|
||||
const prefixQuads = store.getQuads(null, SH + 'declare', null);
|
||||
const prefixes = {};
|
||||
for (const q of prefixQuads) {
|
||||
const p = store.getQuads(q.object, SH + 'prefix', null)[0]?.object.value;
|
||||
const ns = store.getQuads(q.object, SH + 'namespace', null)[0]?.object.value;
|
||||
if (p && ns) prefixes[p] = ns;
|
||||
}
|
||||
|
||||
const shapes = store.getQuads(null, RDF + 'type', SH + 'NodeShape').map(q => q.subject);
|
||||
if (!shapes.length) throw new Error(`${file}: no sh:NodeShape. A profile without entities is useless.`);
|
||||
|
||||
@@ -57,30 +45,26 @@ export async function loadProfile(file) {
|
||||
return { uri: shape.value, targetClass, label, fields, discriminator };
|
||||
});
|
||||
|
||||
return { uri, title, prefixes, shapes: shapeList, store };
|
||||
return { uri, title, shapes: shapeList };
|
||||
}
|
||||
|
||||
// A field's sh:path: one predicate, or [ sh:inversePath <predicate> ].
|
||||
function readPath(store, node) {
|
||||
const pathNode = store.getQuads(node, SH + 'path', null)[0]?.object;
|
||||
if (!pathNode) return { path: null, inverse: false };
|
||||
if (pathNode.termType !== 'BlankNode') return { path: pathNode.value, inverse: false };
|
||||
const inv = store.getQuads(pathNode, SH + 'inversePath', null)[0]?.object;
|
||||
return inv ? { path: inv.value, inverse: true } : { path: null, inverse: false };
|
||||
}
|
||||
|
||||
function parseField(store, p) {
|
||||
let path = null, inverse = false;
|
||||
const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object;
|
||||
if (pathQuad) {
|
||||
if (pathQuad.termType === 'BlankNode') {
|
||||
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
|
||||
if (inv) { path = inv.value; inverse = true; }
|
||||
} else {
|
||||
path = pathQuad.value;
|
||||
}
|
||||
}
|
||||
const one = (pred) => store.getQuads(p, pred, null)[0]?.object || null;
|
||||
const one = (pred) => store.getQuads(p, pred, null)[0]?.object?.value || null;
|
||||
return {
|
||||
path, inverse,
|
||||
name: one(SH + 'name')?.value || null,
|
||||
description: one(SH + 'description')?.value || null,
|
||||
datatype: one(SH + 'datatype')?.value || null,
|
||||
class: one(SH + 'class')?.value || null,
|
||||
node: one(SH + 'node')?.value || null,
|
||||
nodeKind: one(SH + 'nodeKind')?.value || null,
|
||||
maxCount: one(SH + 'maxCount')?.value || null,
|
||||
...readPath(store, p),
|
||||
name: one(SH + 'name'),
|
||||
datatype: one(SH + 'datatype'),
|
||||
class: one(SH + 'class'),
|
||||
node: one(SH + 'node'),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -89,16 +73,7 @@ function parseField(store, p) {
|
||||
function parseDiscriminator(store, shape) {
|
||||
const node = store.getQuads(shape, REP + 'discriminator', null)[0]?.object;
|
||||
if (!node) return null;
|
||||
let path = null, inverse = false;
|
||||
const pathQuad = store.getQuads(node, SH + 'path', null)[0]?.object;
|
||||
if (pathQuad) {
|
||||
if (pathQuad.termType === 'BlankNode') {
|
||||
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
|
||||
if (inv) { path = inv.value; inverse = true; }
|
||||
} else {
|
||||
path = pathQuad.value;
|
||||
}
|
||||
}
|
||||
const { path, inverse } = readPath(store, node);
|
||||
const one = (pred) => store.getQuads(node, pred, null)[0]?.object?.value || null;
|
||||
const minCount = one(SH + 'minCount');
|
||||
const maxCount = one(SH + 'maxCount');
|
||||
@@ -110,11 +85,6 @@ function parseDiscriminator(store, shape) {
|
||||
};
|
||||
}
|
||||
|
||||
export function entityForClass(profile, cls) {
|
||||
const shape = profile.shapes.find(s => s.targetClass === cls);
|
||||
return shape ? shape.uri : null;
|
||||
}
|
||||
|
||||
// The shape a spec starts from. rep:entity wins; a targetClass only resolves
|
||||
// when exactly one shape carries it, so a class two entities share is a
|
||||
// validator error, not a silent first match.
|
||||
@@ -132,10 +102,7 @@ export function fieldsOf(profile, shapeUri) {
|
||||
return shape(profile, shapeUri)?.fields || [];
|
||||
}
|
||||
|
||||
export function field(profile, shapeUri, predicate, inverse) {
|
||||
return fieldsOf(profile, shapeUri).find(f => f.path === predicate && !!f.inverse === !!inverse) || null;
|
||||
// The field a hop takes from a shape, or null.
|
||||
export function field(profile, shapeUri, hop) {
|
||||
return fieldsOf(profile, shapeUri).find(f => f.path === hop.predicate && !!f.inverse === !!hop.inverse) || null;
|
||||
}
|
||||
|
||||
export function prefixes(profile) {
|
||||
return profile.prefixes;
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { statSync } from 'fs';
|
||||
import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt,
|
||||
sparqlEscapeDateTime } from '../db.js';
|
||||
|
||||
const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
|
||||
|
||||
// Register the CSV: a logical file (http://data.lblod.info/files/<uuid>) plus
|
||||
// a physical share:// file pointing back with nie:dataSource. The logical
|
||||
// URI is what you link to. When given, the file gets dct:type.
|
||||
export async function registerFile(sessionUpdateFn, fileName, filePath, { type } = {}) {
|
||||
const size = statSync(filePath).size;
|
||||
const uuid = randomUUID();
|
||||
const uuid2 = randomUUID();
|
||||
const now = new Date();
|
||||
const logicalUri = `http://data.lblod.info/files/${uuid}`;
|
||||
const physicalUri = `share://${fileName}`;
|
||||
|
||||
await (sessionUpdateFn || sessionUpdate)(`
|
||||
INSERT DATA {
|
||||
${sparqlEscapeUri(logicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ;
|
||||
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ;
|
||||
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ;
|
||||
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)}${type ? ` ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} .
|
||||
${sparqlEscapeUri(physicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ;
|
||||
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid2)} ;
|
||||
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ;
|
||||
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ;
|
||||
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated')} ${sparqlEscapeDateTime(now)} ;
|
||||
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource')} ${sparqlEscapeUri(logicalUri)}${type ? ` ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} .
|
||||
}`);
|
||||
|
||||
return logicalUri;
|
||||
}
|
||||
|
||||
// One report pointing at the logical file URI.
|
||||
export async function registerReport(sessionUpdateFn, title, fileUri, extra = {}) {
|
||||
const uuid = randomUUID();
|
||||
const now = new Date();
|
||||
const reportUri = `http://data.lblod.info/id/reports/${uuid}`;
|
||||
|
||||
const extraTriples = [];
|
||||
if (extra.question) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/question')} ${sparqlEscapeString(extra.question)} .`);
|
||||
if (extra.specFile) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/specFile')} ${sparqlEscapeString(extra.specFile)} .`);
|
||||
if (extra.creator) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://purl.org/dc/terms/creator')} ${sparqlEscapeUri(extra.creator)} .`);
|
||||
|
||||
await (sessionUpdateFn || sessionUpdate)(`
|
||||
INSERT DATA {
|
||||
${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ;
|
||||
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)} ;
|
||||
${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} .
|
||||
${extraTriples.join('\n ')}
|
||||
}`);
|
||||
|
||||
return reportUri;
|
||||
}
|
||||
@@ -1,38 +1,49 @@
|
||||
import { update, uuid, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
|
||||
import { seed } from './seed.js';
|
||||
import { fetchColumns } from './columns.js';
|
||||
import { assemble } from './assemble.js';
|
||||
import { rowsToCsv, writeCsv, writeSpec } from './csv.js';
|
||||
import { registerFile, registerReport } from './report.js';
|
||||
import { startShape } from './profile.js';
|
||||
|
||||
// Ties the steps together: seed → columns → assemble → CSV → register.
|
||||
// Returns { reportUri, fileUri, rowCount }.
|
||||
const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
|
||||
const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
|
||||
|
||||
export async function run(query, update, parsed, profile, title, extra = {}) {
|
||||
const shape = startShape(profile, parsed);
|
||||
const subjects = await seed(query, parsed, shape);
|
||||
|
||||
const values = new Map();
|
||||
await fetchColumns(query, subjects, parsed, values);
|
||||
|
||||
const rows = assemble(subjects, values, parsed);
|
||||
const fileName = extra.fileName || `${slug(title)}.csv`;
|
||||
const csv = rowsToCsv(rows);
|
||||
const filePath = writeCsv(fileName, csv);
|
||||
const specFile = extra.spec && writeSpec(fileName, extra.spec);
|
||||
|
||||
const fileUri = await registerFile(update, fileName, filePath, { type: extra.fileType });
|
||||
const reportUri = await registerReport(update, title, fileUri,
|
||||
specFile ? { ...extra, specFile } : extra);
|
||||
|
||||
return { reportUri, fileUri, rowCount: rows.length - 1, filePath };
|
||||
// run(spec, profile) → the CSV text. seed → columns → assemble → CSV.
|
||||
export async function run(spec, profile) {
|
||||
const subjects = await seed(spec, startShape(profile, spec));
|
||||
const values = await fetchColumns(subjects, spec);
|
||||
const rows = assemble(subjects, values, spec);
|
||||
return rows.map(row => row.map(csvCell).join(CSV_SEPARATOR)).join('\n');
|
||||
}
|
||||
|
||||
// A cell is quoted only when it holds the separator, a quote or a newline;
|
||||
// a quote inside a cell is doubled, per RFC 4180.
|
||||
function csvCell(cell) {
|
||||
const s = String(cell ?? '');
|
||||
return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s)
|
||||
? `"${s.replace(/"/g, '""')}"`
|
||||
: s;
|
||||
}
|
||||
|
||||
// One report pointing at the logical file URI of its CSV.
|
||||
export async function registerReport(title, fileUri) {
|
||||
const id = uuid();
|
||||
const reportUri = `http://data.lblod.info/id/reports/${id}`;
|
||||
|
||||
await update(`
|
||||
INSERT DATA {
|
||||
${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ;
|
||||
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(id)} ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ;
|
||||
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(new Date())} ;
|
||||
${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} .
|
||||
}`);
|
||||
|
||||
return reportUri;
|
||||
}
|
||||
|
||||
// The file name must be known before the run starts, so the chat can show a
|
||||
// pending card named after the report.
|
||||
export function slug(title) {
|
||||
return (title || 'report').toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, 80) || 'report';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from '../db.js';
|
||||
import { query, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from 'mu';
|
||||
import { sharedSteps } from './spec.js';
|
||||
|
||||
// Subject selection. One paged query per page; stops at ROW_LIMIT.
|
||||
const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000);
|
||||
const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000);
|
||||
|
||||
export async function seed(sessionQuery, spec, shape = null) {
|
||||
export async function seed(spec, shape) {
|
||||
const subjects = [];
|
||||
let offset = 0;
|
||||
for (;;) {
|
||||
const query = seedPageQuery(spec, PAGE_SIZE, offset, shape);
|
||||
const result = await sessionQuery(query);
|
||||
const result = await query(seedPageQuery(spec, PAGE_SIZE, offset, shape));
|
||||
const rows = result.results.bindings.map(b => b.s.value);
|
||||
subjects.push(...rows);
|
||||
if (subjects.length > ROW_LIMIT) {
|
||||
@@ -49,13 +48,13 @@ function discriminatorPattern(d) {
|
||||
// Writes out every hop as its own triple pattern.
|
||||
// Forward: ?s <p1> ?f0_1 . ?f0_1 <p2> ?f0_v
|
||||
// Inverse: ?s ^<p1> → ?f0_1 <p1> ?s
|
||||
export function hopsPattern(hops, varPrefix, startVar) {
|
||||
export function hopsPattern(hops, varPrefix, startVar, lastVar = `?${varPrefix}_v`) {
|
||||
let prev = startVar;
|
||||
const patterns = [];
|
||||
for (let i = 0; i < hops.length; i++) {
|
||||
const hop = hops[i];
|
||||
const isLast = i === hops.length - 1;
|
||||
const v = isLast ? `?${varPrefix}_v` : `?${varPrefix}_${i + 1}`;
|
||||
const v = isLast ? lastVar : `?${varPrefix}_${i + 1}`;
|
||||
if (hop.inverse) {
|
||||
patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
|
||||
} else {
|
||||
@@ -114,29 +113,14 @@ export function conditionPattern(c, varPrefix, hops, startVar) {
|
||||
parts.push(`FILTER(REGEX(str(${v}), ${sparqlEscapeString(alternation)}${flags}))`);
|
||||
}
|
||||
if (c.minCount === 2) {
|
||||
// two distinct leaf values on the same path
|
||||
// two distinct leaf values: the same path again, to a second leaf
|
||||
const w = `?${varPrefix}_w`;
|
||||
parts.push(hopsPattern2(hops, varPrefix, startVar, w));
|
||||
parts.push(hopsPattern(hops, `${varPrefix}_2`, startVar, w));
|
||||
parts.push(`FILTER(${v} != ${w})`);
|
||||
}
|
||||
return parts.join('\n ');
|
||||
}
|
||||
|
||||
function hopsPattern2(hops, varPrefix, startVar, wVar) {
|
||||
// repeat the path to a second leaf variable
|
||||
let prev = startVar;
|
||||
const patterns = [];
|
||||
for (let i = 0; i < hops.length; i++) {
|
||||
const hop = hops[i];
|
||||
const isLast = i === hops.length - 1;
|
||||
const v = isLast ? wVar : `?${varPrefix}_2_${i + 1}`;
|
||||
if (hop.inverse) patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
|
||||
else patterns.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`);
|
||||
prev = v;
|
||||
}
|
||||
return patterns.join('\n ');
|
||||
}
|
||||
|
||||
function escapeTerm(t) {
|
||||
if (t.type === 'uri') return sparqlEscapeUri(t.value);
|
||||
if (t.datatype === 'http://www.w3.org/2001/XMLSchema#dateTime')
|
||||
|
||||
@@ -23,22 +23,21 @@ export function parseSpec(turtle) {
|
||||
const profileUri = one(specNode, REP + 'profile')?.value || null;
|
||||
const title = one(specNode, 'http://purl.org/dc/terms/title')?.value || null;
|
||||
|
||||
const filters = store.getQuads(specNode, SH + 'property', null).map(q =>
|
||||
parseFilter(store, q.object, { withLabel: true }));
|
||||
const filters = store.getQuads(specNode, SH + 'property', null).map(q => parseFilter(store, q.object));
|
||||
|
||||
const columnsQuad = one(specNode, REP + 'columns');
|
||||
const columns = columnsQuad ? parseColumnList(store, columnsQuad) : [];
|
||||
const columns = columnsQuad ? readRdfList(store, columnsQuad).map(col => parseColumn(store, col)) : [];
|
||||
|
||||
return { uri: specNode.value, title, profileUri, targetClass, entity, filters, columns };
|
||||
}
|
||||
|
||||
function parseFilter(store, p, { withLabel = false } = {}) {
|
||||
function parseFilter(store, p) {
|
||||
let path = [];
|
||||
const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object;
|
||||
if (pathQuad) path = parsePath(store, pathQuad);
|
||||
return {
|
||||
path,
|
||||
constraints: readConstraints(store, p, { withLabel }),
|
||||
constraints: readConstraints(store, p, { withLabel: true }),
|
||||
where: readWhere(store, p),
|
||||
};
|
||||
}
|
||||
@@ -47,8 +46,7 @@ function parseFilter(store, p, { withLabel = false } = {}) {
|
||||
// is written like a filter, from the row; check.js refuses one nested in
|
||||
// another.
|
||||
function readWhere(store, node) {
|
||||
return store.getQuads(node, REP + 'where', null).map(q =>
|
||||
parseFilter(store, q.object, { withLabel: true }));
|
||||
return store.getQuads(node, REP + 'where', null).map(q => parseFilter(store, q.object));
|
||||
}
|
||||
|
||||
// The number of steps two paths have in common from the row. A rep:where
|
||||
@@ -60,18 +58,6 @@ export function sharedSteps(a, b) {
|
||||
return k;
|
||||
}
|
||||
|
||||
function parseColumnList(store, listNode) {
|
||||
const columns = [];
|
||||
let cur = listNode;
|
||||
while (cur && cur.value !== RDF + 'nil') {
|
||||
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object;
|
||||
if (item) columns.push(parseColumn(store, item));
|
||||
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
|
||||
if (!cur) throw new Error('rep:columns list is broken (no rdf:rest terminator)');
|
||||
}
|
||||
return columns;
|
||||
}
|
||||
|
||||
function parseColumn(store, col) {
|
||||
let path = [];
|
||||
const pathQuad = store.getQuads(col, SH + 'path', null)[0]?.object;
|
||||
@@ -107,21 +93,12 @@ function parsePath(store, pathQuad) {
|
||||
return [{ predicate: pathQuad.value, inverse: false }];
|
||||
}
|
||||
// RDF list of predicates
|
||||
const hops = [];
|
||||
let cur = pathQuad;
|
||||
while (cur && cur.value !== RDF + 'nil') {
|
||||
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object;
|
||||
if (!item) throw new Error('sh:path list is broken');
|
||||
if (item.termType === 'BlankNode') {
|
||||
const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
|
||||
if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
|
||||
hops.push({ predicate: inv.value, inverse: true });
|
||||
} else {
|
||||
hops.push({ predicate: item.value, inverse: false });
|
||||
}
|
||||
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
|
||||
}
|
||||
return hops;
|
||||
return readRdfList(store, pathQuad).map(item => {
|
||||
if (item.termType !== 'BlankNode') return { predicate: item.value, inverse: false };
|
||||
const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
|
||||
if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
|
||||
return { predicate: inv.value, inverse: true };
|
||||
});
|
||||
}
|
||||
|
||||
function readConstraints(store, p, { withLabel = false } = {}) {
|
||||
|
||||
Reference in New Issue
Block a user