Implement rep:where

README.md

@@ -49,6 +49,13 @@ A **spec** is `rep:ReportSpec` + `sh:NodeShape`: `sh:property` selects the
 subjects, `rep:columns` says what the CSV holds. The two never interact. The
 namespace is `http://mu.semte.ch/vocabularies/reporting/` (`rep:`).
 
+A filter tests only the end of its own path. To test a node in the middle of
+a path ("the mandate whose role is Burgemeester"), a filter or column carries
+`rep:where`: a condition written like a filter, from the row, that shares the
+nodes of every step it has in common with its host. On a column it drops
+values from the cell, never rows. The app's `report-spec-readme.md` has the
+details and an example.
+
 The run: check → seed (`SELECT DISTINCT ?s`, paged) → columns (per group, per
 chunk of subjects) → assemble (dedup + `rep:collect`) → CSV → register file
 + report resource. One run per confirmed spec.
@@ -65,6 +72,19 @@ the caller's.
 The tools the LLM calls return URIs, counts, statuses and column names —
 never a cell value.
 
+### The rule: no report data reaches the LLM
+
+No report row, row count or run error ever reaches the LLM. A run starts
+after the turn's only LLM call (the mode check) and answers with fixed text.
+What the LLM does see: the conversation, the profiles, validator messages,
+spec files, and public reference data read under the service scope (inlined
+code lists and `lookup_values`, see below).
+
+Keep it that way. Do not add a tool that previews, samples or counts what a
+spec would match, not even to catch a `rep:where` that hangs on the wrong
+node. That check stays on the profile: `validate_spec` says where each
+`rep:where` applies, and the proposal repeats it to the user.
+
 ### The service scope
 
 The LLM's own reads during refinement (`describe_profile`'s code lists and

src/llm/loop.js

@@ -33,6 +33,18 @@ Rules:
 - Filters go in sh:property. Columns go in rep:columns, in the order you want.
 - To match a topic rather than a value, list the Dutch words and compounds you
   would expect and put them all in rep:anyOf.
+- A filter tests only the end of its own path, and two filters never share a
+  node. To test a node in the middle of a path (for example "the mandate
+  whose role is Burgemeester"), give that filter or column a rep:where:
+  rep:where [ sh:path ( <the same first steps> <more steps> ) ; <one condition> ].
+  Write its path from the row, like a filter. The steps it has in common with
+  its filter or column are the same nodes, and its condition tests where its
+  own path ends. On a column, rep:where drops values from the cell, never
+  rows. Several rep:where on one filter or column all apply.
+- validate_spec answers "ok" plus one line per rep:where saying where it
+  applies. Say that in Dutch in your proposal, so the user can check it.
+- Only offer what validate_spec accepted. Before you suggest an alternative,
+  write it and validate it; if it does not validate, say it cannot be done.
 - You cannot count, sort, take the first N, or compare two subjects. Say so.
 - The report's title is the dct:title you write in the spec. There is no other
   title input.

src/llm/schemas.js

@@ -19,7 +19,7 @@ const COMMON = [
   },
   {
     name: 'validate_spec',
-    description: 'Validate a report spec (Turtle). Returns "ok" or one clear sentence saying what is wrong and what to write instead. No database is touched.',
+    description: 'Validate a report spec (Turtle). Returns "ok", followed by one line per rep:where saying where it applies, or one clear sentence saying what is wrong and what to write instead. No database is touched.',
     inputSchema: {
       type: 'object',
       properties: { spec: { type: 'string', description: 'The spec as Turtle text' } },

src/llm/tools.js

@@ -7,7 +7,7 @@
 // The shape every tool returns is { content: [{ type: 'text', text }] },
 // which the loop passes straight back to the model.
 
-import { checkSpec } from '../runner/check.js';
+import { checkSpec, whereNotes } from '../runner/check.js';
 import { parseSpec } from '../runner/spec.js';
 import { describeProfile, codeListValues } from './describe.js';
 import { lookupValues } from './lookup.js';
@@ -36,9 +36,10 @@ export function buildTools(profiles, session) {
     const p = findProfile(parsed.profileUri);
     if (!p) return text(`no profile <${parsed.profileUri}>. Available: ${profileList().map(x => x.title).join(', ')}.`);
     const errors = checkSpec(parsed, p, MAX_PATH_DEPTH, profiles);
+    if (errors.length) return text(errors[0]);
     // The spec that checks out becomes the attached proposal.
-    if (!errors.length) session.onSpecValidated?.(spec);
+    session.onSpecValidated?.(spec);
-    return text(errors.length ? errors[0] : 'ok');
+    return text(['ok', ...whereNotes(parsed, p)].join('\n'));
   }
 
   async function list_profiles() {

src/runner/check.js

@@ -1,5 +1,6 @@
 import { fieldsOf } from './profile.js';
 import { NUMERIC_DATATYPES } from './assemble.js';
+import { sharedSteps } from './spec.js';
 
 const SH = 'http://www.w3.org/ns/shacl#';
 const REP = 'http://mu.semte.ch/vocabularies/reporting/';
@@ -57,6 +58,22 @@ function shapeList(profile) {
   return profile.shapes.map(s => s.label || s.targetClass).join(', ');
 }
 
+// Every condition in the spec: the filters, and the rep:where conditions on
+// filters and columns, with the words that name each one in a message. A
+// rep:where also carries its host, the filter or column it hangs on.
+function conditions(spec) {
+  const out = spec.filters.map((f, i) => ({ what: `filter ${i + 1}`, cond: f }));
+  spec.filters.forEach((f, i) => f.where.forEach((w, j) => out.push({
+    what: `rep:where ${j + 1} on filter ${i + 1}`, cond: w,
+    host: { what: `filter ${i + 1}`, path: f.path },
+  })));
+  spec.columns.forEach(c => c.where.forEach((w, j) => out.push({
+    what: `rep:where ${j + 1} on column "${c.label}"`, cond: w,
+    host: { what: `column "${c.label}"`, path: c.path },
+  })));
+  return out;
+}
+
 // The shape a spec starts from, after startEntity has approved it: rep:entity
 // wins, otherwise the one shape carrying the targetClass.
 function startShapeOf(spec, profile) {
@@ -147,7 +164,7 @@ function columnPure(spec) {
   for (const col of spec.columns) {
     if (Object.keys(col.constraints || {}).length) {
       const which = Object.keys(col.constraints)[0];
-      return `a column carries ${which}. Constraints belong in sh:property.`;
+      return `a column carries ${which}. Constraints belong in sh:property, or in rep:where for a condition on a step of this column.`;
     }
     if (col.collect && ![SH + 'groupConcat', SH + 'min', SH + 'max', REP + 'row'].includes(col.collect)) {
       return `column "${col.label}" has an unknown rep:collect <${col.collect}>. Use sh:groupConcat (joins the values), rep:row or nothing (each value gets its own row), sh:min or sh:max.`;
@@ -156,11 +173,11 @@ function columnPure(spec) {
   return null;
 }
 
-// No label inside a filter
+// No label inside a filter or rep:where
 function filterLabel(spec) {
-  for (const filter of spec.filters) {
+  for (const { what, cond } of conditions(spec)) {
-    if ((filter.constraints || {}).label) {
+    if ((cond.constraints || {}).label) {
-      return 'a filter carries rdfs:label. Move it to rep:columns to show it.';
+      return `${what} carries rdfs:label. Move it to rep:columns to show it.`;
     }
   }
   return null;
@@ -171,11 +188,11 @@ function filterLabel(spec) {
 function filterConstraint(spec) {
   const MEANINGFUL = ['minCount', 'maxCount', 'hasValue', 'in', 'minInclusive',
     'maxInclusive', 'minExclusive', 'maxExclusive', 'pattern', 'flags', 'anyOf'];
-  for (const [i, filter] of spec.filters.entries()) {
+  for (const { what, cond } of conditions(spec)) {
-    const c = filter.constraints || {};
+    const c = cond.constraints || {};
     if (!MEANINGFUL.some(k => c[k] !== undefined && c[k] !== null)) {
-      const p = filter.path.map(h => h.predicate).join(' → ');
+      const p = cond.path.map(h => h.predicate).join(' → ');
-      return `filter ${i + 1} (path ${p}) has no condition. Add one: sh:hasValue for an exact value, sh:in for a list, rep:anyOf for words, or remove the filter.`;
+      return `${what} (path ${p}) has no condition. Add one: sh:hasValue for an exact value, sh:in for a list, rep:anyOf for words, or remove the filter.`;
     }
   }
   return null;
@@ -183,8 +200,8 @@ function filterConstraint(spec) {
 
 // sh:maxCount above 0 needs counting
 function maxCountZero(spec) {
-  for (const [i, filter] of spec.filters.entries()) {
+  for (const { cond } of conditions(spec)) {
-    const c = filter.constraints;
+    const c = cond.constraints;
     if (c.maxCount !== null && c.maxCount !== undefined && c.maxCount > 0) {
       return `sh:maxCount ${c.maxCount} needs counting, which this service cannot do. Use sh:maxCount 0 for "has none", or drop it.`;
     }
@@ -197,8 +214,8 @@ function maxCountZero(spec) {
 
 // rep:anyOf size limits
 function anyOfLimits(spec) {
-  for (const [i, filter] of spec.filters.entries()) {
+  for (const { cond } of conditions(spec)) {
-    const c = filter.constraints;
+    const c = cond.constraints;
     if (!c.anyOf) continue;
     if (c.anyOf.length > ANYOF_MAX_TERMS) {
       return `rep:anyOf has ${c.anyOf.length} terms. The limit is ${ANYOF_MAX_TERMS}. Narrow the words, or split the filter into two sh:property blocks.`;
@@ -217,8 +234,8 @@ function anyOfLimits(spec) {
 
 // sh:in must not mix URIs and literals
 function inHomogeneous(spec) {
-  for (const filter of spec.filters) {
+  for (const { cond } of conditions(spec)) {
-    const c = filter.constraints;
+    const c = cond.constraints;
     if (!c.in) continue;
     const uris = c.in.filter(t => t.type === 'uri').length;
     if (uris > 0 && uris < c.in.length) {
@@ -231,8 +248,8 @@ function inHomogeneous(spec) {
 // Typed literals must parse
 function literalsParse(spec) {
   const constraintTerms = [];
-  for (const filter of spec.filters) {
+  for (const { cond } of conditions(spec)) {
-    const c = filter.constraints;
+    const c = cond.constraints;
     for (const key of ['hasValue', 'minInclusive', 'maxInclusive', 'minExclusive', 'maxExclusive']) {
       if (c[key]) constraintTerms.push(c[key]);
     }
@@ -250,6 +267,56 @@ function literalsParse(spec) {
   return null;
 }
 
+// rep:where: walks from the row through the profile, shares at least one
+// step with its filter or column, and holds no rep:where of its own.
+function whereConditions(spec, profile, maxPathDepth) {
+  const startShapeUri = startShapeOf(spec, profile);
+  for (const { what, cond, host } of conditions(spec)) {
+    if (!host) continue;
+    if (cond.where.length) {
+      return `${what} holds a rep:where of its own. Put every rep:where directly on the filter or column.`;
+    }
+    const err = walk(profile, startShapeUri, cond.path, maxPathDepth, what, false);
+    if (err) return err;
+    const k = sharedSteps(host.path, cond.path);
+    if (!k) {
+      return `${what} shares no step with ${host.what}. Start its path with the same steps as ${host.what}; a condition on the row itself goes in sh:property.`;
+    }
+    const c = cond.constraints;
+    if (k === cond.path.length && (c.maxCount === 0 || c.minCount === 2)) {
+      return `${what} ends on a step of ${host.what}, so ${c.maxCount === 0 ? 'sh:maxCount 0' : 'sh:minCount 2'} has nothing to count. Add the step that must be missing or repeated.`;
+    }
+  }
+  for (const [i, filter] of spec.filters.entries()) {
+    if (filter.where.length && filter.constraints.minCount === 2) {
+      return `filter ${i + 1} combines sh:minCount 2 with rep:where. Use one of the two.`;
+    }
+  }
+  return null;
+}
+
+// Where each rep:where applies, in the profile's words, so the proposal can
+// say it. Profile only, no database.
+export function whereNotes(spec, profile) {
+  const startShapeUri = startShapeOf(spec, profile);
+  return conditions(spec).filter(x => x.host).map(({ what, cond, host }) => {
+    const k = sharedSteps(host.path, cond.path);
+    return `${what} applies at ${nodeName(profile, startShapeUri, host.path.slice(0, k))}, after step ${k} of ${host.what}.`;
+  });
+}
+
+// The entity a path lands on, or the value field when it ends on one.
+function nodeName(profile, startShapeUri, hops) {
+  let shapeUri = startShapeUri;
+  for (const [i, hop] of hops.entries()) {
+    const f = fieldsOf(profile, shapeUri).find(x => x.path === hop.predicate && !!x.inverse === !!hop.inverse);
+    if (!f) return `step ${hops.length}`;
+    if (!f.node) return i === hops.length - 1 ? `the value "${f.name}"` : `step ${hops.length}`;
+    shapeUri = f.node;
+  }
+  return `"${profile.shapes.find(s => s.uri === shapeUri)?.label || shapeUri}"`;
+}
+
 function parsesAsDate(value, withTime) {
   const d = new Date(value);
   if (Number.isNaN(d.getTime())) return false;
@@ -270,6 +337,7 @@ const CHECKS = [
   ['typed literal does not parse', literalsParse],
   ['two rep:row columns', oneRowColumn],
   ['path does not resolve', pathsResolve],
+  ['rep:where does not fit', whereConditions],
   ['sh:min/sh:max on a non-number', collectMinMaxTyped],
 ];
 
@@ -308,7 +376,7 @@ function walk(profile, startShapeUri, hops, maxPathDepth, what, isColumn) {
     const fields = fieldsOf(profile, shapeUri);
     const f = fields.find(x => x.path === hop.predicate && !!x.inverse === !!hop.inverse);
     if (!f) {
-      const have = fields.map(x => (x.inverse ? `geen ${x.name} (${x.path}, omgekeerd)` : `${x.name} (${x.path})`)).join(', ');
+      const have = fields.map(x => (x.inverse ? `${x.name} (${x.path}, inverse)` : `${x.name} (${x.path})`)).join(', ');
       const shapeLabel = profile.shapes.find(s => s.uri === shapeUri)?.label || shapeUri;
       return `at "${shapeLabel}" there is no "${hop.predicate}". It has: ${have || '(nothing)'}.`;
     }

src/runner/columns.js

@@ -1,4 +1,5 @@
 import { sparqlEscapeUri } from '../db.js';
+import { wherePatterns } from './seed.js';
 
 // Fetch column values for known subjects. Columns are grouped by their
 // shared non-leaf hops; two columns share a query only when every hop
@@ -21,14 +22,17 @@ export async function fetchColumns(sessionQuery, subjects, spec, values = new Ma
 
 // Group columns by their shared prefix (every hop before the leaf) and the
 // direction of the leaf. A column with an empty path (rep:self) joins no group.
+// A column with rep:where only shares a group with the same conditions.
 export function groupColumns(columns) {
   const groups = new Map();
   columns.forEach((col, index) => {
     if (!col.path.length) return;   // rep:self
     const leaf = col.path[col.path.length - 1];
-    const key = col.path.map(h => `${h.inverse ? '^' : ''}${h.predicate}`).join(' > ');
+    const where = col.where || [];
+    const key = col.path.map(h => `${h.inverse ? '^' : ''}${h.predicate}`).join(' > ')
+      + (where.length ? ` | ${JSON.stringify(where)}` : '');
     if (!groups.has(key)) {
-      groups.set(key, { hops: col.path, columns: [] });
+      groups.set(key, { hops: col.path, where, columns: [] });
     }
     groups.get(key).columns.push({ index, leaf: leaf.predicate });
   });
@@ -49,6 +53,10 @@ export function groupQuery(group, chunk) {
   const leaf = group.hops[group.hops.length - 1];
   if (leaf.inverse) lines.push(`?v ${sparqlEscapeUri(leaf.predicate)} ${prev} .`);
   else lines.push(`${prev} ${sparqlEscapeUri(leaf.predicate)} ?v .`);
+  // rep:where hangs on the column's own nodes: ?x<k-1> after k hops, ?v at
+  // the end.
+  const nodeVar = (k) => (k === group.hops.length ? '?v' : `?x${k - 1}`);
+  lines.push(...wherePatterns(group.where, group.hops, nodeVar, 'c'));
   return `SELECT ?s ?v WHERE {\n  VALUES ?s { ${subjects} }\n  ${lines.join('\n  ')}\n}`;
 }
 

src/runner/seed.js

@@ -1,4 +1,5 @@
 import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscapeDate } from '../db.js';
+import { sharedSteps } from './spec.js';
 
 // Subject selection. One paged query per page; stops at ROW_LIMIT.
 const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000);
@@ -27,8 +28,7 @@ export function seedPageQuery(spec, limit, offset, shape = null) {
   parts.push(`?s a ${sparqlEscapeUri(targetClass)} .`);
   if (shape?.discriminator) parts.push(discriminatorPattern(shape.discriminator));
   spec.filters.forEach((filter, fi) => {
-    parts.push(hopsPattern(filter.path, `f${fi}`, '?s'));
+    parts.push(filterPattern(filter, `f${fi}`));
-    parts.push(constraintsPattern(filter.constraints, `f${fi}`));
   });
   return `SELECT DISTINCT ?s WHERE {
   ${parts.join('\n  ')}
@@ -66,9 +66,37 @@ export function hopsPattern(hops, varPrefix, startVar) {
   return patterns.join('\n  ');
 }
 
-export function constraintsPattern(c, varPrefix) {
+// One filter: its condition, plus its rep:where conditions on the nodes it
-  const parts = [];
+// walks through. With sh:maxCount 0 the whole of it becomes "none such".
-  const v = `?${varPrefix}_v`;
+function filterPattern(filter, varPrefix) {
+  const nodeVar = (k) => (k === filter.path.length ? `?${varPrefix}_v` : `?${varPrefix}_${k}`);
+  const wheres = wherePatterns(filter.where, filter.path, nodeVar, varPrefix);
+  if (filter.constraints.maxCount === 0) {
+    return `FILTER NOT EXISTS {\n    ${[hopsPattern(filter.path, varPrefix, '?s'), ...wheres].join('\n    ')}\n  }`;
+  }
+  return [conditionPattern(filter.constraints, varPrefix, filter.path, '?s'), ...wheres].join('\n  ');
+}
+
+// The rep:where conditions of a filter or column. Each starts at the node
+// after the steps it shares with its host; nodeVar(k) names that node in the
+// host's own query.
+export function wherePatterns(where, hostHops, nodeVar, varPrefix) {
+  return (where || []).map((w, wi) => {
+    const k = sharedSteps(hostHops, w.path);
+    return conditionPattern(w.constraints, `${varPrefix}_w${wi}`, w.path.slice(k), nodeVar(k));
+  });
+}
+
+// A condition: the hops from startVar and the test on where they end. A
+// filter starts at ?s; a rep:where starts at a node of its host and, with no
+// hops left, tests that node itself. sh:maxCount 0 means "none such", so its
+// hops only live inside the NOT EXISTS.
+export function conditionPattern(c, varPrefix, hops, startVar) {
+  if (c.maxCount === 0) {
+    return `FILTER NOT EXISTS { ${hopsPattern(hops, varPrefix + '_nex', startVar)} }`;
+  }
+  const v = hops.length ? `?${varPrefix}_v` : startVar;
+  const parts = hops.length ? [hopsPattern(hops, varPrefix, startVar)] : [];
 
   if (c.hasValue) parts.push(`FILTER(${v} = ${escapeTerm(c.hasValue)})`);
   if (c.in) parts.push(`FILTER(${v} IN (${c.in.map(escapeTerm).join(', ')}))`);
@@ -88,19 +116,15 @@ export function constraintsPattern(c, varPrefix) {
   if (c.minCount === 2) {
     // two distinct leaf values on the same path
     const w = `?${varPrefix}_w`;
-    parts.push(hopsPattern2(c, varPrefix, w));
+    parts.push(hopsPattern2(hops, varPrefix, startVar, w));
     parts.push(`FILTER(${v} != ${w})`);
   }
-  if (c.maxCount === 0) {
-    parts.push(`FILTER NOT EXISTS { ${hopsPattern(c.path, varPrefix + '_nex', '?s')} }`);
-  }
   return parts.join('\n  ');
 }
 
-function hopsPattern2(filter, varPrefix, wVar) {
+function hopsPattern2(hops, varPrefix, startVar, wVar) {
   // repeat the path to a second leaf variable
-  const hops = filter.path.map(h => ({ ...h }));
+  let prev = startVar;
-  let prev = '?s';
   const patterns = [];
   for (let i = 0; i < hops.length; i++) {
     const hop = hops[i];

src/runner/spec.js

@@ -39,9 +39,27 @@ function parseFilter(store, p, { withLabel = false } = {}) {
   return {
     path,
     constraints: readConstraints(store, p, { withLabel }),
+    where: readWhere(store, p),
   };
 }
 
+// rep:where: conditions on the nodes a filter or column walks through. Each
+// is written like a filter, from the row; check.js refuses one nested in
+// another.
+function readWhere(store, node) {
+  return store.getQuads(node, REP + 'where', null).map(q =>
+    parseFilter(store, q.object, { withLabel: true }));
+}
+
+// The number of steps two paths have in common from the row. A rep:where
+// shares those nodes with its filter or column.
+export function sharedSteps(a, b) {
+  let k = 0;
+  while (k < a.length && k < b.length
+    && a[k].predicate === b[k].predicate && !!a[k].inverse === !!b[k].inverse) k++;
+  return k;
+}
+
 function parseColumnList(store, listNode) {
   const columns = [];
   let cur = listNode;
@@ -70,6 +88,7 @@ function parseColumn(store, col) {
     collect: store.getQuads(col, REP + 'collect', null)[0]?.object.value || null,
     separator: store.getQuads(col, SH + 'separator', null)[0]?.object.value || null,
     constraints: readConstraints(store, col),   // kept so check.js can refuse it
+    where: readWhere(store, col),
   };
 }