felix.ruizdearcaute/natural-language-report-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup

Browse Source
This commit is contained in:
felix
2026-09-27 00:15:18 +02:00
parent ad278dd680
commit 9b0b8a85c9
25 changed files with 588 additions and 1959 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/runner/assemble.js
View File

@@ -14,7 +14,6 @@ export const NUMERIC_DATATYPES = new Set([
'nonNegativeInteger', 'positiveInteger', 'nonPositiveInteger', 'negativeInteger',
'unsignedLong', 'unsignedInt', 'unsignedShort', 'unsignedByte',
].map(d => XSD + d));
export const TEMPORAL_DATATYPES = new Set([XSD + 'date', XSD + 'dateTime']);
export function assemble(subjects, values, spec) {
const sorted = [...subjects].sort();
@@ -57,19 +56,18 @@ function collect(terms, col) {
}
}
// min/max over values that share one datatype. Numbers compare as numbers,
// dates and times as instants. Mixed datatypes throw: check.js should have
// caught it, and a silently wrong cell is worse than a failed job.
// min/max. Numbers compare as numbers. Dates and times compare as text:
// xsd:date and xsd:dateTime are ISO strings, so text order is date order.
// Numbers mixed with other values throw: check.js should have caught it,
// and a silently wrong cell is worse than a failed job. The terms are SPARQL
// JSON bindings, so the datatype is a plain string.
function extreme(terms, col, direction) {
if (!terms.length) return '';
const datatypes = [...new Set(terms.map(t => t.datatype?.value || ''))];
if (datatypes.length > 1) {
throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but its values carry ${datatypes.length} datatypes (${datatypes.join(', ')}).`);
const numbers = terms.filter(t => NUMERIC_DATATYPES.has(t.datatype)).length;
if (numbers && numbers < terms.length) {
throw new Error(`column "${col.label}" asks for sh:${direction < 0 ? 'min' : 'max'} but mixes numbers with other values.`);
}
const dt = datatypes[0];
const key = NUMERIC_DATATYPES.has(dt) ? t => Number(t.value)
: TEMPORAL_DATATYPES.has(dt) ? t => new Date(t.value).getTime()
: t => t.value;
const key = numbers ? t => Number(t.value) : t => t.value;
const cmp = (a, b) => a < b ? -1 : a > b ? 1 : 0;
const best = terms.reduce((a, b) => (cmp(key(b), key(a)) * direction > 0 ? b : a));
return best.value;

137
src/runner/check.js
View File

@@ -1,4 +1,4 @@
import { fieldsOf } from './profile.js';
import { startShape, fieldsOf, field } from './profile.js';
import { NUMERIC_DATATYPES } from './assemble.js';
import { sharedSteps } from './spec.js';
@@ -9,23 +9,18 @@ const XSD_DATE = XSD + 'date';
const XSD_DATETIME = XSD + 'dateTime';
// The validator. checkSpec runs the checks in order and returns the first
// error as a one-element array (empty if ok). Every message names the thing
// that is wrong in the profile's own words and says what to write instead.
// No database call anywhere in this file.
// error, or null when the spec is fine. Every message names the thing that
// is wrong in the profile's own words and says what to write instead. No
// database call anywhere in this file.
const MAX_PATH_DEPTH = Number(process.env.MAX_PATH_DEPTH || 8);
const ANYOF_MAX_TERMS = 24;
const ANYOF_MAX_TERM_LENGTH = 100;
// The spec names a profile we have. knownProfiles is the profile Map loaded
// at boot; the check is skipped when it is omitted.
function profileNamed(spec, knownProfiles) {
if (!spec.profileUri) {
return `no rep:profile named. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'} .`;
}
if (knownProfiles && !knownProfiles.has(spec.profileUri)) {
return `no profile <${spec.profileUri}>. Available: ${knownProfiles ? [...knownProfiles.values()].map(p => p.title).join(', ') : '(none)'}.`;
}
return null;
// null when the profile exists, else what to write instead.
export function profileError(profiles, uri) {
if (profiles.has(uri)) return null;
return `no profile <${uri}>. Available: ${[...profiles.values()].map(p => p.title).join(', ')}.`;
}
// The spec names an entity we have: by rep:entity, or by a targetClass only
@@ -74,16 +69,6 @@ function conditions(spec) {
return out;
}
// The shape a spec starts from, after startEntity has approved it: rep:entity
// wins, otherwise the one shape carrying the targetClass.
function startShapeOf(spec, profile) {
if (spec.entity) {
return profile.shapes.find(s => s.uri === spec.entity)?.uri || null;
}
const matches = profile.shapes.filter(s => s.targetClass === spec.targetClass);
return matches.length === 1 ? matches[0].uri : null;
}
// Columns exist
function columnsPresent(spec) {
if (!spec.columns.length) return 'no columns. Add at least one rep:columns entry.';
@@ -100,26 +85,26 @@ function columnLabel(spec) {
// Labels are unique
function columnLabelsUnique(spec) {
const seen = new Map();
const seen = new Set();
for (const col of spec.columns) {
if (seen.has(col.label)) {
return `two columns are called "${col.label}". Labels must differ.`;
}
seen.set(col.label, true);
seen.add(col.label);
}
return null;
}
// Every path resolves, columns end on a value, depth capped
function pathsResolve(spec, profile, maxPathDepth) {
const startShapeUri = startShapeOf(spec, profile);
function pathsResolve(spec, profile) {
const startShapeUri = startShape(profile, spec).uri;
for (const col of spec.columns) {
const err = walk(profile, startShapeUri, col.path, maxPathDepth,
const err = walk(profile, startShapeUri, col.path,
`column "${col.label}"`, true, col.nodeKind === SH + 'IRI');
if (err) return err;
}
for (const [i, filter] of spec.filters.entries()) {
const err = walk(profile, startShapeUri, filter.path, maxPathDepth,
const err = walk(profile, startShapeUri, filter.path,
`filter ${i + 1}`, false);
if (err) return err;
}
@@ -128,7 +113,7 @@ function pathsResolve(spec, profile, maxPathDepth) {
// sh:min/sh:max only on numbers and dates
function collectMinMaxTyped(spec, profile) {
const startShapeUri = startShapeOf(spec, profile);
const startShapeUri = startShape(profile, spec).uri;
for (const col of spec.columns) {
if (col.collect !== SH + 'min' && col.collect !== SH + 'max') continue;
const name = col.collect === SH + 'min' ? 'sh:min' : 'sh:max';
@@ -136,8 +121,7 @@ function collectMinMaxTyped(spec, profile) {
const last = col.path[col.path.length - 1];
const shapeUri = shapeAfter(profile, startShapeUri, col.path);
if (!shapeUri) continue; // pathsResolve already reported the real error
const f = fieldsOf(profile, shapeUri)
.find(x => x.path === last.predicate && !!x.inverse === !!last.inverse);
const f = field(profile, shapeUri, last);
if (!f) continue;
const kind = f.datatype === XSD_DATETIME || f.datatype === XSD_DATE ? 'date'
: NUMERIC_DATATYPES.has(f.datatype) ? 'number'
@@ -205,10 +189,10 @@ function filterConstraint(spec) {
function maxCountZero(spec) {
for (const { cond } of conditions(spec)) {
const c = cond.constraints;
if (c.maxCount !== null && c.maxCount !== undefined && c.maxCount > 0) {
if (c.maxCount > 0) {
return `sh:maxCount ${c.maxCount} needs counting, which this service cannot do. Use sh:maxCount 0 for "has none", or drop it.`;
}
if (c.minCount !== null && c.minCount !== undefined && c.minCount > 2) {
if (c.minCount > 2) {
return `sh:minCount ${c.minCount} needs counting. Use sh:minCount 1 (at least one) or sh:minCount 2 (two distinct values); anything higher is beyond this service.`;
}
}
@@ -272,14 +256,14 @@ function literalsParse(spec) {
// rep:where: walks from the row through the profile, shares at least one
// step with its filter or column, and holds no rep:where of its own.
function whereConditions(spec, profile, maxPathDepth) {
const startShapeUri = startShapeOf(spec, profile);
function whereConditions(spec, profile) {
const startShapeUri = startShape(profile, spec).uri;
for (const { what, cond, host } of conditions(spec)) {
if (!host) continue;
if (cond.where.length) {
return `${what} holds a rep:where of its own. Put every rep:where directly on the filter or column.`;
}
const err = walk(profile, startShapeUri, cond.path, maxPathDepth, what, false);
const err = walk(profile, startShapeUri, cond.path, what, false);
if (err) return err;
const k = sharedSteps(host.path, cond.path);
if (!k) {
@@ -302,14 +286,13 @@ function whereConditions(spec, profile, maxPathDepth) {
// code) it compares a URI and never matches, so its path must end on a field
// with a datatype.
function rangeOnValue(spec, profile) {
const startShapeUri = startShapeOf(spec, profile);
const startShapeUri = startShape(profile, spec).uri;
for (const { what, cond } of conditions(spec)) {
const key = ['minInclusive', 'maxInclusive', 'minExclusive', 'maxExclusive']
.find(k => cond.constraints[k]);
if (!key || !cond.path.length) continue;
const last = cond.path[cond.path.length - 1];
const f = fieldsOf(profile, shapeAfter(profile, startShapeUri, cond.path))
.find(x => x.path === last.predicate && !!x.inverse === !!last.inverse);
const f = field(profile, shapeAfter(profile, startShapeUri, cond.path), last);
if (!f || f.datatype) continue; // pathsResolve reports a missing step
const target = f.node
? `"${profile.shapes.find(s => s.uri === f.node)?.label || f.node}"`
@@ -326,7 +309,7 @@ function rangeOnValue(spec, profile) {
// Where each rep:where applies, in the profile's words, so the proposal can
// say it. Profile only, no database.
export function whereNotes(spec, profile) {
const startShapeUri = startShapeOf(spec, profile);
const startShapeUri = startShape(profile, spec).uri;
return conditions(spec).filter(x => x.host).map(({ what, cond, host }) => {
const k = sharedSteps(host.path, cond.path);
return `${what} applies at ${nodeName(profile, startShapeUri, host.path.slice(0, k))}, after step ${k} of ${host.what}.`;
@@ -337,7 +320,7 @@ export function whereNotes(spec, profile) {
function nodeName(profile, startShapeUri, hops) {
let shapeUri = startShapeUri;
for (const [i, hop] of hops.entries()) {
const f = fieldsOf(profile, shapeUri).find(x => x.path === hop.predicate && !!x.inverse === !!hop.inverse);
const f = field(profile, shapeUri, hop);
if (!f) return `step ${hops.length}`;
if (!f.node) return i === hops.length - 1 ? `the value "${f.name}"` : `step ${hops.length}`;
shapeUri = f.node;
@@ -351,33 +334,37 @@ function parsesAsDate(value, withTime) {
return withTime ? value.includes('T') : /^\d{4}-\d{2}-\d{2}$/.test(value);
}
// In this order: startEntity comes first, the checks after it rely on a
// start shape.
const CHECKS = [
['no entity or unknown class', startEntity],
['no columns', columnsPresent],
['column without rdfs:label', columnLabel],
['two columns with one label', columnLabelsUnique],
['column carries constraints', columnPure],
['filter carries rdfs:label', filterLabel],
['filter without a condition', filterConstraint],
['sh:maxCount above 0', maxCountZero],
['rep:anyOf too big', anyOfLimits],
['sh:in mixes URIs and literals', inHomogeneous],
['typed literal does not parse', literalsParse],
['two rep:row columns', oneRowColumn],
['path does not resolve', pathsResolve],
['rep:where does not fit', whereConditions],
['date or number condition on a link', rangeOnValue],
['sh:min/sh:max on a non-number', collectMinMaxTyped],
startEntity,
columnsPresent,
columnLabel,
columnLabelsUnique,
columnPure,
filterLabel,
filterConstraint,
maxCountZero,
anyOfLimits,
inHomogeneous,
literalsParse,
oneRowColumn,
pathsResolve,
whereConditions,
rangeOnValue,
collectMinMaxTyped,
];
export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) {
const profileErr = profileNamed(spec, knownProfiles);
if (profileErr) return [profileErr];
for (const [name, check] of CHECKS) {
const err = check(spec, profile, maxPathDepth);
if (err) return [err];
// checkSpec(spec, profiles) → the first error, or null.
export function checkSpec(spec, profiles) {
const error = profileError(profiles, spec.profileUri);
if (error) return error;
const profile = profiles.get(spec.profileUri);
for (const check of CHECKS) {
const err = check(spec, profile);
if (err) return err;
}
return [];
return null;
}
// The shape a path lands in: follow sh:node through every hop but the
@@ -386,17 +373,16 @@ export function checkSpec(spec, profile, maxPathDepth, knownProfiles = null) {
function shapeAfter(profile, startShapeUri, hops) {
let shapeUri = startShapeUri;
for (let i = 0; i < hops.length - 1; i++) {
const fields = fieldsOf(profile, shapeUri);
const f = fields.find(x => x.path === hops[i].predicate && !!x.inverse === !!hops[i].inverse);
const f = field(profile, shapeUri, hops[i]);
if (!f || !f.node) return null;
shapeUri = f.node;
}
return shapeUri;
}
function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsIri = false) {
if (hops.length > maxPathDepth) {
return `${what} walks ${hops.length} hops. The limit is ${maxPathDepth}.`;
function walk(profile, startShapeUri, hops, what, isColumn, wantsIri = false) {
if (hops.length > MAX_PATH_DEPTH) {
return `${what} walks ${hops.length} hops. The limit is ${MAX_PATH_DEPTH}.`;
}
if (!hops.length) return null;
let shapeUri = startShapeUri;
@@ -416,20 +402,15 @@ function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn, wantsI
shapeUri = f.node;
} else if (isColumn) {
if (wantsIri && f.datatype) {
return `column "${colLabel(what)}" has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`;
return `${what} has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`;
}
if (!wantsIri && !f.datatype && !f.class) {
const valueFields = fieldsOf(profile, f.node)
.filter(x => x.datatype || x.class)
.map(x => `${x.name} (${x.path})`);
return `column "${colLabel(what)}" ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`;
return `${what} ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`;
}
}
}
return null;
}
function colLabel(what) {
const m = what.match(/"([^"]+)"/);
return m ? m[1] : what;
}

26
src/runner/columns.js
View File

@@ -1,5 +1,5 @@
import { sparqlEscapeUri } from '../db.js';
import { wherePatterns } from './seed.js';
import { query, sparqlEscapeUri } from 'mu';
import { hopsPattern, wherePatterns } from './seed.js';
// Fetch column values for known subjects. Columns are grouped by their
// shared non-leaf hops; two columns share a query only when every hop
@@ -8,12 +8,14 @@ import { wherePatterns } from './seed.js';
const SUBJECT_CHUNK_SIZE = Math.max(1, Number(process.env.SUBJECT_CHUNK_SIZE) || 100);
export async function fetchColumns(sessionQuery, subjects, spec, values = new Map()) {
// fetchColumns(subjects, spec) → subject → column index → [term]
export async function fetchColumns(subjects, spec) {
const values = new Map();
const groups = groupColumns(spec.columns);
for (const group of groups) {
for (let i = 0; i < subjects.length; i += SUBJECT_CHUNK_SIZE) {
const chunk = subjects.slice(i, i + SUBJECT_CHUNK_SIZE);
const result = await sessionQuery(groupQuery(group, chunk));
const result = await query(groupQuery(group, chunk));
collect(result, group, values);
}
}
@@ -41,21 +43,11 @@ export function groupColumns(columns) {
export function groupQuery(group, chunk) {
const subjects = chunk.map(sparqlEscapeUri).join(' ');
const lines = [];
let prev = '?s';
// every hop except the last gets its own variable; the leaf is ?v
group.hops.slice(0, -1).forEach((hop, i) => {
const v = `?x${i}`;
if (hop.inverse) lines.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
else lines.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`);
prev = v;
});
const leaf = group.hops[group.hops.length - 1];
if (leaf.inverse) lines.push(`?v ${sparqlEscapeUri(leaf.predicate)} ${prev} .`);
else lines.push(`${prev} ${sparqlEscapeUri(leaf.predicate)} ?v .`);
// rep:where hangs on the column's own nodes: ?x<k-1> after k hops, ?v at
const lines = [hopsPattern(group.hops, 'x', '?s', '?v')];
// rep:where hangs on the column's own nodes: ?x_<k> after k hops, ?v at
// the end.
const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x${k - 1}`);
const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x_${k}`);
lines.push(...wherePatterns(group.where, group.hops, nodeVar, 'c'));
return `SELECT ?s ?v WHERE {\n VALUES ?s { ${subjects} }\n ${lines.join('\n ')}\n}`;
}

35
src/runner/csv.js
View File

@@ -1,35 +0,0 @@
import { mkdirSync, writeFileSync } from 'fs';
// Rows to a CSV file. CSV_SEPARATOR is ';' by default. A cell is quoted
// only when it holds the separator, a quote or a newline; a quote inside a
// cell is doubled, per RFC 4180.
const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
const SHARE_DIR = process.env.SHARE_DIR || '/share';
export function rowsToCsv(rows) {
return rows.map(row => row.map(escapeCell).join(CSV_SEPARATOR)).join('\n');
}
function escapeCell(cell) {
const s = String(cell ?? '');
return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s)
? `"${s.replace(/"/g, '""')}"`
: s;
}
export function writeCsv(name, csv) {
mkdirSync(SHARE_DIR, { recursive: true });
const file = `${SHARE_DIR}/${name}`;
writeFileSync(file, csv, 'utf8');
return file;
}
// The spec of the run, next to its CSV (same base name, .ttl), so a report
// can be debugged afterwards. The store keeps only the pointer.
export function writeSpec(csvName, spec) {
mkdirSync(SHARE_DIR, { recursive: true });
const name = csvName.replace(/\.csv$/, '') + '.ttl';
writeFileSync(`${SHARE_DIR}/${name}`, spec, 'utf8');
return `share://${name}`;
}

95
src/runner/profile.js
View File

@@ -1,5 +1,5 @@
import N3 from 'n3';
import { readFileSync } from 'fs';
import { readFileSync, readdirSync } from 'fs';
// Parse a profile (a SHACL shapes graph) and answer questions about it.
@@ -8,13 +8,11 @@ const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
const OWL_ONTOLOGY = 'http://www.w3.org/2002/07/owl#Ontology';
const REP = 'http://mu.semte.ch/vocabularies/reporting/';
export async function loadProfiles(dir) {
const fs = await import('fs');
const path = await import('path');
const files = fs.readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
export function loadProfiles(dir) {
const files = readdirSync(dir).filter(f => f.endsWith('.ttl')).sort();
const profiles = new Map();
for (const file of files) {
const profile = await loadProfile(path.join(dir, file));
const profile = loadProfile(`${dir}/${file}`);
profiles.set(profile.uri, profile);
console.log(`[profile] "${profile.title}" (${file}): ${profile.shapes.length} entities, ` +
`${profile.shapes.reduce((n, s) => n + s.fields.length, 0)} fields`);
@@ -23,29 +21,19 @@ export async function loadProfiles(dir) {
return profiles;
}
export async function loadProfile(file) {
const parser = new N3.Parser();
const store = new N3.Store();
await new Promise((resolve, reject) => {
parser.parse(readFileSync(file, 'utf8'), (err, quad) => {
if (err) return reject(new Error(`${file}: ${err.message}`));
if (quad) store.addQuad(quad); else resolve();
});
});
function loadProfile(file) {
let store;
try {
store = new N3.Store(new N3.Parser().parse(readFileSync(file, 'utf8')));
} catch (e) {
throw new Error(`${file}: ${e.message}`);
}
const ontology = store.getQuads(null, RDF + 'type', OWL_ONTOLOGY)[0];
if (!ontology) throw new Error(`${file}: no owl:Ontology. A profile file must name itself.`);
const uri = ontology.subject.value;
const title = store.getQuads(ontology.subject, 'http://purl.org/dc/terms/title', null)[0]?.object.value || uri;
const prefixQuads = store.getQuads(null, SH + 'declare', null);
const prefixes = {};
for (const q of prefixQuads) {
const p = store.getQuads(q.object, SH + 'prefix', null)[0]?.object.value;
const ns = store.getQuads(q.object, SH + 'namespace', null)[0]?.object.value;
if (p && ns) prefixes[p] = ns;
}
const shapes = store.getQuads(null, RDF + 'type', SH + 'NodeShape').map(q => q.subject);
if (!shapes.length) throw new Error(`${file}: no sh:NodeShape. A profile without entities is useless.`);
@@ -57,30 +45,26 @@ export async function loadProfile(file) {
return { uri: shape.value, targetClass, label, fields, discriminator };
});
return { uri, title, prefixes, shapes: shapeList, store };
return { uri, title, shapes: shapeList };
}
// A field's sh:path: one predicate, or [ sh:inversePath <predicate> ].
function readPath(store, node) {
const pathNode = store.getQuads(node, SH + 'path', null)[0]?.object;
if (!pathNode) return { path: null, inverse: false };
if (pathNode.termType !== 'BlankNode') return { path: pathNode.value, inverse: false };
const inv = store.getQuads(pathNode, SH + 'inversePath', null)[0]?.object;
return inv ? { path: inv.value, inverse: true } : { path: null, inverse: false };
}
function parseField(store, p) {
let path = null, inverse = false;
const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object;
if (pathQuad) {
if (pathQuad.termType === 'BlankNode') {
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
if (inv) { path = inv.value; inverse = true; }
} else {
path = pathQuad.value;
}
}
const one = (pred) => store.getQuads(p, pred, null)[0]?.object || null;
const one = (pred) => store.getQuads(p, pred, null)[0]?.object?.value || null;
return {
path, inverse,
name: one(SH + 'name')?.value || null,
description: one(SH + 'description')?.value || null,
datatype: one(SH + 'datatype')?.value || null,
class: one(SH + 'class')?.value || null,
node: one(SH + 'node')?.value || null,
nodeKind: one(SH + 'nodeKind')?.value || null,
maxCount: one(SH + 'maxCount')?.value || null,
...readPath(store, p),
name: one(SH + 'name'),
datatype: one(SH + 'datatype'),
class: one(SH + 'class'),
node: one(SH + 'node'),
};
}
@@ -89,16 +73,7 @@ function parseField(store, p) {
function parseDiscriminator(store, shape) {
const node = store.getQuads(shape, REP + 'discriminator', null)[0]?.object;
if (!node) return null;
let path = null, inverse = false;
const pathQuad = store.getQuads(node, SH + 'path', null)[0]?.object;
if (pathQuad) {
if (pathQuad.termType === 'BlankNode') {
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
if (inv) { path = inv.value; inverse = true; }
} else {
path = pathQuad.value;
}
}
const { path, inverse } = readPath(store, node);
const one = (pred) => store.getQuads(node, pred, null)[0]?.object?.value || null;
const minCount = one(SH + 'minCount');
const maxCount = one(SH + 'maxCount');
@@ -110,11 +85,6 @@ function parseDiscriminator(store, shape) {
};
}
export function entityForClass(profile, cls) {
const shape = profile.shapes.find(s => s.targetClass === cls);
return shape ? shape.uri : null;
}
// The shape a spec starts from. rep:entity wins; a targetClass only resolves
// when exactly one shape carries it, so a class two entities share is a
// validator error, not a silent first match.
@@ -132,10 +102,7 @@ export function fieldsOf(profile, shapeUri) {
return shape(profile, shapeUri)?.fields || [];
}
export function field(profile, shapeUri, predicate, inverse) {
return fieldsOf(profile, shapeUri).find(f => f.path === predicate && !!f.inverse === !!inverse) || null;
// The field a hop takes from a shape, or null.
export function field(profile, shapeUri, hop) {
return fieldsOf(profile, shapeUri).find(f => f.path === hop.predicate && !!f.inverse === !!hop.inverse) || null;
}
export function prefixes(profile) {
return profile.prefixes;
}

63
src/runner/report.js
View File

@@ -1,63 +0,0 @@
import { randomUUID } from 'crypto';
import { statSync } from 'fs';
import { sessionUpdate, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeInt,
sparqlEscapeDateTime } from '../db.js';
const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
// Register the CSV: a logical file (http://data.lblod.info/files/<uuid>) plus
// a physical share:// file pointing back with nie:dataSource. The logical
// URI is what you link to. When given, the file gets dct:type.
export async function registerFile(sessionUpdateFn, fileName, filePath, { type } = {}) {
const size = statSync(filePath).size;
const uuid = randomUUID();
const uuid2 = randomUUID();
const now = new Date();
const logicalUri = `http://data.lblod.info/files/${uuid}`;
const physicalUri = `share://${fileName}`;
await (sessionUpdateFn || sessionUpdate)(`
INSERT DATA {
${sparqlEscapeUri(logicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)}${type ? ` ;
${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} .
${sparqlEscapeUri(physicalUri)} a ${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#FileDataObject')} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid2)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileName')} ${sparqlEscapeString(fileName)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/format')} ${sparqlEscapeString('text/csv')} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileSize')} ${sparqlEscapeInt(size)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#fileCreated')} ${sparqlEscapeDateTime(now)} ;
${sparqlEscapeUri('http://www.semanticdesktop.org/ontologies/2007/01/19/nie#dataSource')} ${sparqlEscapeUri(logicalUri)}${type ? ` ;
${sparqlEscapeUri('http://purl.org/dc/terms/type')} ${sparqlEscapeUri(type)}` : ''} .
}`);
return logicalUri;
}
// One report pointing at the logical file URI.
export async function registerReport(sessionUpdateFn, title, fileUri, extra = {}) {
const uuid = randomUUID();
const now = new Date();
const reportUri = `http://data.lblod.info/id/reports/${uuid}`;
const extraTriples = [];
if (extra.question) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/question')} ${sparqlEscapeString(extra.question)} .`);
if (extra.specFile) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/specFile')} ${sparqlEscapeString(extra.specFile)} .`);
if (extra.creator) extraTriples.push(`${sparqlEscapeUri(reportUri)} ${sparqlEscapeUri('http://purl.org/dc/terms/creator')} ${sparqlEscapeUri(extra.creator)} .`);
await (sessionUpdateFn || sessionUpdate)(`
INSERT DATA {
${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(uuid)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(now)} ;
${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} .
${extraTriples.join('\n ')}
}`);
return reportUri;
}

61
src/runner/run.js
View File

@@ -1,38 +1,49 @@
import { update, uuid, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from 'mu';
import { seed } from './seed.js';
import { fetchColumns } from './columns.js';
import { assemble } from './assemble.js';
import { rowsToCsv, writeCsv, writeSpec } from './csv.js';
import { registerFile, registerReport } from './report.js';
import { startShape } from './profile.js';
// Ties the steps together: seed → columns → assemble → CSV → register.
// Returns { reportUri, fileUri, rowCount }.
const REPORT_CLASS = process.env.REPORT_CLASS || 'http://lblod.data.gift/vocabularies/reporting/Report';
const CSV_SEPARATOR = process.env.CSV_SEPARATOR || ';';
export async function run(query, update, parsed, profile, title, extra = {}) {
const shape = startShape(profile, parsed);
const subjects = await seed(query, parsed, shape);
const values = new Map();
await fetchColumns(query, subjects, parsed, values);
const rows = assemble(subjects, values, parsed);
const fileName = extra.fileName || `${slug(title)}.csv`;
const csv = rowsToCsv(rows);
const filePath = writeCsv(fileName, csv);
const specFile = extra.spec && writeSpec(fileName, extra.spec);
const fileUri = await registerFile(update, fileName, filePath, { type: extra.fileType });
const reportUri = await registerReport(update, title, fileUri,
specFile ? { ...extra, specFile } : extra);
return { reportUri, fileUri, rowCount: rows.length - 1, filePath };
// run(spec, profile) → the CSV text. seed → columns → assemble → CSV.
export async function run(spec, profile) {
const subjects = await seed(spec, startShape(profile, spec));
const values = await fetchColumns(subjects, spec);
const rows = assemble(subjects, values, spec);
return rows.map(row => row.map(csvCell).join(CSV_SEPARATOR)).join('\n');
}
// A cell is quoted only when it holds the separator, a quote or a newline;
// a quote inside a cell is doubled, per RFC 4180.
function csvCell(cell) {
const s = String(cell ?? '');
return s.includes(CSV_SEPARATOR) || s.includes('"') || /[\n\r]/.test(s)
? `"${s.replace(/"/g, '""')}"`
: s;
}
// One report pointing at the logical file URI of its CSV.
export async function registerReport(title, fileUri) {
const id = uuid();
const reportUri = `http://data.lblod.info/id/reports/${id}`;
await update(`
INSERT DATA {
${sparqlEscapeUri(reportUri)} a ${sparqlEscapeUri(REPORT_CLASS)} ;
${sparqlEscapeUri('http://mu.semte.ch/vocabularies/core/uuid')} ${sparqlEscapeString(id)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/title')} ${sparqlEscapeString(title)} ;
${sparqlEscapeUri('http://purl.org/dc/terms/created')} ${sparqlEscapeDateTime(new Date())} ;
${sparqlEscapeUri('http://www.w3.org/ns/prov#generated')} ${sparqlEscapeUri(fileUri)} .
}`);
return reportUri;
}
// The file name must be known before the run starts, so the chat can show a
// pending card named after the report.
export function slug(title) {
return (title || 'report').toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80) || 'report';
}
}

30
src/runner/seed.js
View File

@@ -1,16 +1,15 @@
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from '../db.js';
import { query, sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from 'mu';
import { sharedSteps } from './spec.js';
// Subject selection. One paged query per page; stops at ROW_LIMIT.
const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000);
const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000);
export async function seed(sessionQuery, spec, shape = null) {
export async function seed(spec, shape) {
const subjects = [];
let offset = 0;
for (;;) {
const query = seedPageQuery(spec, PAGE_SIZE, offset, shape);
const result = await sessionQuery(query);
const result = await query(seedPageQuery(spec, PAGE_SIZE, offset, shape));
const rows = result.results.bindings.map(b => b.s.value);
subjects.push(...rows);
if (subjects.length > ROW_LIMIT) {
@@ -49,13 +48,13 @@ function discriminatorPattern(d) {
// Writes out every hop as its own triple pattern.
// Forward: ?s <p1> ?f0_1 . ?f0_1 <p2> ?f0_v
// Inverse: ?s ^<p1> → ?f0_1 <p1> ?s
export function hopsPattern(hops, varPrefix, startVar) {
export function hopsPattern(hops, varPrefix, startVar, lastVar = `?${varPrefix}_v`) {
let prev = startVar;
const patterns = [];
for (let i = 0; i < hops.length; i++) {
const hop = hops[i];
const isLast = i === hops.length - 1;
const v = isLast ? `?${varPrefix}_v` : `?${varPrefix}_${i + 1}`;
const v = isLast ? lastVar : `?${varPrefix}_${i + 1}`;
if (hop.inverse) {
patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
} else {
@@ -114,29 +113,14 @@ export function conditionPattern(c, varPrefix, hops, startVar) {
parts.push(`FILTER(REGEX(str(${v}), ${sparqlEscapeString(alternation)}${flags}))`);
}
if (c.minCount === 2) {
// two distinct leaf values on the same path
// two distinct leaf values: the same path again, to a second leaf
const w = `?${varPrefix}_w`;
parts.push(hopsPattern2(hops, varPrefix, startVar, w));
parts.push(hopsPattern(hops, `${varPrefix}_2`, startVar, w));
parts.push(`FILTER(${v} != ${w})`);
}
return parts.join('\n ');
}
function hopsPattern2(hops, varPrefix, startVar, wVar) {
// repeat the path to a second leaf variable
let prev = startVar;
const patterns = [];
for (let i = 0; i < hops.length; i++) {
const hop = hops[i];
const isLast = i === hops.length - 1;
const v = isLast ? wVar : `?${varPrefix}_2_${i + 1}`;
if (hop.inverse) patterns.push(`${v} ${sparqlEscapeUri(hop.predicate)} ${prev} .`);
else patterns.push(`${prev} ${sparqlEscapeUri(hop.predicate)} ${v} .`);
prev = v;
}
return patterns.join('\n ');
}
function escapeTerm(t) {
if (t.type === 'uri') return sparqlEscapeUri(t.value);
if (t.datatype === 'http://www.w3.org/2001/XMLSchema#dateTime')

45
src/runner/spec.js
View File

@@ -23,22 +23,21 @@ export function parseSpec(turtle) {
const profileUri = one(specNode, REP + 'profile')?.value || null;
const title = one(specNode, 'http://purl.org/dc/terms/title')?.value || null;
const filters = store.getQuads(specNode, SH + 'property', null).map(q =>
parseFilter(store, q.object, { withLabel: true }));
const filters = store.getQuads(specNode, SH + 'property', null).map(q => parseFilter(store, q.object));
const columnsQuad = one(specNode, REP + 'columns');
const columns = columnsQuad ? parseColumnList(store, columnsQuad) : [];
const columns = columnsQuad ? readRdfList(store, columnsQuad).map(col => parseColumn(store, col)) : [];
return { uri: specNode.value, title, profileUri, targetClass, entity, filters, columns };
}
function parseFilter(store, p, { withLabel = false } = {}) {
function parseFilter(store, p) {
let path = [];
const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object;
if (pathQuad) path = parsePath(store, pathQuad);
return {
path,
constraints: readConstraints(store, p, { withLabel }),
constraints: readConstraints(store, p, { withLabel: true }),
where: readWhere(store, p),
};
}
@@ -47,8 +46,7 @@ function parseFilter(store, p, { withLabel = false } = {}) {
// is written like a filter, from the row; check.js refuses one nested in
// another.
function readWhere(store, node) {
return store.getQuads(node, REP + 'where', null).map(q =>
parseFilter(store, q.object, { withLabel: true }));
return store.getQuads(node, REP + 'where', null).map(q => parseFilter(store, q.object));
}
// The number of steps two paths have in common from the row. A rep:where
@@ -60,18 +58,6 @@ export function sharedSteps(a, b) {
return k;
}
function parseColumnList(store, listNode) {
const columns = [];
let cur = listNode;
while (cur && cur.value !== RDF + 'nil') {
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object;
if (item) columns.push(parseColumn(store, item));
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
if (!cur) throw new Error('rep:columns list is broken (no rdf:rest terminator)');
}
return columns;
}
function parseColumn(store, col) {
let path = [];
const pathQuad = store.getQuads(col, SH + 'path', null)[0]?.object;
@@ -107,21 +93,12 @@ function parsePath(store, pathQuad) {
return [{ predicate: pathQuad.value, inverse: false }];
}
// RDF list of predicates
const hops = [];
let cur = pathQuad;
while (cur && cur.value !== RDF + 'nil') {
const item = store.getQuads(cur, RDF + 'first', null)[0]?.object;
if (!item) throw new Error('sh:path list is broken');
if (item.termType === 'BlankNode') {
const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
hops.push({ predicate: inv.value, inverse: true });
} else {
hops.push({ predicate: item.value, inverse: false });
}
cur = store.getQuads(cur, RDF + 'rest', null)[0]?.object;
}
return hops;
return readRdfList(store, pathQuad).map(item => {
if (item.termType !== 'BlankNode') return { predicate: item.value, inverse: false };
const inv = store.getQuads(item, SH + 'inversePath', null)[0]?.object;
if (!inv) throw new Error('sh:path list items must be URIs or [ sh:inversePath <predicate> ]');
return { predicate: inv.value, inverse: true };
});
}
function readConstraints(store, p, { withLabel = false } = {}) {