diff --git a/src/llm/describe.js b/src/llm/describe.js index 37a49f8..f573c78 100644 --- a/src/llm/describe.js +++ b/src/llm/describe.js @@ -23,6 +23,7 @@ sh:targetClass is the full URI in brackets after the entity below. In sh:path, use the predicate URI shown after each field name, not the field name itself. An inverse hop (the field shows "(inverse)") goes in the list as [ sh:inversePath ] . When two entities share one sh:targetClass, pick one with rep:entity . +A field that links to several entities ("→ a or b") reaches all of them; the steps after it decide. To keep one kind, add a rep:where on a step only that kind has. A date filter is sh:minInclusive or sh:maxInclusive with "yyyy-mm-dd"^^xsd:dateTime (the xsd prefix above is not copied: add @prefix xsd: ). `]; @@ -31,7 +32,7 @@ A date filter is sh:minInclusive or sh:maxInclusive with "yyyy-mm-dd"^^xsd:dateT for (const f of shape.fields) { const kind = f.datatype ? datatypeName(f.datatype) : f.class ? `→ ${conceptLabel(f.class, await codeList(f.class))}` - : f.node ? `→ ${linkedLabel(profile, f.node)}` + : f.nodes.length ? `→ ${f.nodes.map(n => linkedLabel(profile, n)).join(' or ')}` : '?'; const dir = f.inverse ? ' (inverse)' : ''; const pathShown = f.path || '?'; diff --git a/src/llm/explore.js b/src/llm/explore.js index 427a988..338ea2c 100644 --- a/src/llm/explore.js +++ b/src/llm/explore.js @@ -116,9 +116,10 @@ function resolveClass(profile, field) { if (!shape) return null; const f = shape.fields.find(x => x.name === fieldLabel); if (f?.class) return f.class; - if (f?.node) { - const linked = profile.shapes.find(s => s.uri === f.node); - return linked?.targetClass || null; + if (f?.nodes.length) { + // sh:or of several entities: only when they share one class + const classes = new Set(f.nodes.map(n => profile.shapes.find(s => s.uri === n)?.targetClass || null)); + return classes.size === 1 ? [...classes][0] : null; } // The model sometimes writes shape.field on a linked shape // ("bestuurseenheid.naam"); resolve it to that shape's target class. diff --git a/src/runner/check.js b/src/runner/check.js index 1a44598..8190411 100644 --- a/src/runner/check.js +++ b/src/runner/check.js @@ -1,4 +1,4 @@ -import { startShape, fieldsOf, field } from './profile.js'; +import { startShape, fieldsOf } from './profile.js'; import { NUMERIC_DATATYPES } from './assemble.js'; import { sharedSteps } from './spec.js'; @@ -119,14 +119,10 @@ function collectMinMaxTyped(spec, profile) { const name = col.collect === SH + 'min' ? 'sh:min' : 'sh:max'; if (!col.path.length) return `${name} on a rep:self column does not apply. A subject URI is not a number or date.`; const last = col.path[col.path.length - 1]; - const shapeUri = shapeAfter(profile, startShapeUri, col.path); - if (!shapeUri) continue; // pathsResolve already reported the real error - const f = field(profile, shapeUri, last); - if (!f) continue; - const kind = f.datatype === XSD_DATETIME || f.datatype === XSD_DATE ? 'date' - : NUMERIC_DATATYPES.has(f.datatype) ? 'number' - : null; - if (!kind) { + // no end field: pathsResolve already reported the real error + const f = endFields(profile, startShapeUri, col.path) + .find(x => x.datatype !== XSD_DATETIME && x.datatype !== XSD_DATE && !NUMERIC_DATATYPES.has(x.datatype)); + if (f) { return `"${f.name || last.predicate}" is text, so ${name} does not apply. Use rep:row (same as leaving rep:collect out: every value gets its own row).`; } } @@ -291,15 +287,15 @@ function rangeOnValue(spec, profile) { const key = ['minInclusive', 'maxInclusive', 'minExclusive', 'maxExclusive'] .find(k => cond.constraints[k]); if (!key || !cond.path.length) continue; - const last = cond.path[cond.path.length - 1]; - const f = field(profile, shapeAfter(profile, startShapeUri, cond.path), last); - if (!f || f.datatype) continue; // pathsResolve reports a missing step - const target = f.node - ? `"${profile.shapes.find(s => s.uri === f.node)?.label || f.node}"` + // pathsResolve reports a missing step + const f = endFields(profile, startShapeUri, cond.path).find(x => !x.datatype); + if (!f) continue; + const target = f.nodes.length + ? f.nodes.map(uri => shapeName(profile, uri)).join(' or ') : `the code "${f.name}"`; - const steps = fieldsOf(profile, f.node) + const steps = [...new Set(f.nodes.flatMap(uri => fieldsOf(profile, uri)) .filter(x => x.datatype === XSD_DATE || x.datatype === XSD_DATETIME || NUMERIC_DATATYPES.has(x.datatype)) - .map(x => `${x.name} (${x.path})`); + .map(x => `${x.name} (${x.path})`))]; const add = steps.length ? `Add a step: ${steps.join(' or ')}.` : 'End the path on a date or number field.'; return `${what} ends on ${target}, not on a value, so sh:${key} never matches. ${add}`; } @@ -316,15 +312,18 @@ export function whereNotes(spec, profile) { }); } -// The entity a path lands on, or the value field when it ends on one. +// The entity a path lands on, or the value field when it ends on one. A +// step that leads to several entities names them all. function nodeName(profile, startShapeUri, hops) { - let shapeUri = startShapeUri; - for (const [i, hop] of hops.entries()) { - const f = field(profile, shapeUri, hop); - if (!f) return `step ${hops.length}`; - if (!f.node) return i === hops.length - 1 ? `the value "${f.name}"` : `step ${hops.length}`; - shapeUri = f.node; - } + if (!hops.length) return shapeName(profile, startShapeUri); + const ends = endFields(profile, startShapeUri, hops); + if (!ends.length) return `step ${hops.length}`; + const value = ends.find(f => !f.nodes.length); + if (value) return `the value "${value.name}"`; + return [...new Set(ends.flatMap(f => f.nodes))].map(uri => shapeName(profile, uri)).join(' or '); +} + +function shapeName(profile, shapeUri) { return `"${profile.shapes.find(s => s.uri === shapeUri)?.label || shapeUri}"`; } @@ -367,17 +366,24 @@ export function checkSpec(spec, profiles) { return null; } -// The shape a path lands in: follow sh:node through every hop but the -// last. walk() has already validated the path, so a gap here means the walk -// reported the real error and this can bail. -function shapeAfter(profile, startShapeUri, hops) { - let shapeUri = startShapeUri; - for (let i = 0; i < hops.length - 1; i++) { - const f = field(profile, shapeUri, hops[i]); - if (!f || !f.node) return null; - shapeUri = f.node; +// The fields a hop takes from any of the shapes a path has reached. A field +// may link to several entities (sh:or of sh:node: an eenheid's political and +// leidinggevende organen): the walk follows them all and the next steps +// decide. The query follows predicates only, so this is what the data does. +function hopFields(profile, shapeUris, hop) { + return shapeUris.flatMap(uri => fieldsOf(profile, uri) + .filter(f => f.path === hop.predicate && !!f.inverse === !!hop.inverse)); +} + +// The fields a path can end on: follow sh:node through every hop but the +// last. walk() has already validated the path, so none here means the walk +// reported the real error and the caller can bail. +function endFields(profile, startShapeUri, hops) { + let shapeUris = [startShapeUri]; + for (const hop of hops.slice(0, -1)) { + shapeUris = [...new Set(hopFields(profile, shapeUris, hop).flatMap(f => f.nodes))]; } - return shapeUri; + return hopFields(profile, shapeUris, hops[hops.length - 1]); } function walk(profile, startShapeUri, hops, what, isColumn, wantsIri = false) { @@ -385,30 +391,34 @@ function walk(profile, startShapeUri, hops, what, isColumn, wantsIri = false) { return `${what} walks ${hops.length} hops. The limit is ${MAX_PATH_DEPTH}.`; } if (!hops.length) return null; - let shapeUri = startShapeUri; + let shapeUris = [startShapeUri]; for (let i = 0; i < hops.length; i++) { const hop = hops[i]; - const fields = fieldsOf(profile, shapeUri); - const f = fields.find(x => x.path === hop.predicate && !!x.inverse === !!hop.inverse); - if (!f) { - const have = fields.map(x => (x.inverse ? `${x.name} (${x.path}, inverse)` : `${x.name} (${x.path})`)).join(', '); - const shapeLabel = profile.shapes.find(s => s.uri === shapeUri)?.label || shapeUri; - return `at "${shapeLabel}" there is no "${hop.predicate}". It has: ${have || '(nothing)'}.`; + const found = hopFields(profile, shapeUris, hop); + if (!found.length) { + const have = [...new Set(shapeUris.flatMap(uri => fieldsOf(profile, uri)) + .map(x => (x.inverse ? `${x.name} (${x.path}, inverse)` : `${x.name} (${x.path})`)))].join(', '); + const shapeLabel = shapeUris.map(uri => shapeName(profile, uri)).join(' or '); + return `at ${shapeLabel} there is no "${hop.predicate}". It has: ${have || '(nothing)'}.`; } if (i < hops.length - 1) { - if (!f.node) { + const links = found.filter(f => f.nodes.length); + if (!links.length) { return `${what} hop ${i + 1} <${hop.predicate}> holds a value, not a link. You cannot walk past it.`; } - shapeUri = f.node; + shapeUris = [...new Set(links.flatMap(f => f.nodes))]; } else if (isColumn) { - if (wantsIri && f.datatype) { - return `${what} has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`; - } - if (!wantsIri && !f.datatype && !f.class) { - const valueFields = fieldsOf(profile, f.node) - .filter(x => x.datatype || x.class) - .map(x => `${x.name} (${x.path})`); - return `${what} ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`; + // every field the path can end on must suit the column + for (const f of found) { + if (wantsIri && f.datatype) { + return `${what} has sh:nodeKind sh:IRI but ends on the value "${f.name}", which is no URI. Drop sh:nodeKind.`; + } + if (!wantsIri && !f.datatype && !f.class) { + const valueFields = [...new Set(f.nodes.flatMap(uri => fieldsOf(profile, uri)) + .filter(x => x.datatype || x.class) + .map(x => `${x.name} (${x.path})`))]; + return `${what} ends on a link. Add a hop: ${valueFields.join(' or ')}. Only if the user asked for the URI of that node itself, add sh:nodeKind sh:IRI to the column instead.`; + } } } } diff --git a/src/runner/profile.js b/src/runner/profile.js index 04ae572..20814f8 100644 --- a/src/runner/profile.js +++ b/src/runner/profile.js @@ -1,5 +1,6 @@ import N3 from 'n3'; import { readFileSync, readdirSync } from 'fs'; +import { parseFilter, readRdfList } from './spec.js'; // Parse a profile (a SHACL shapes graph) and answer questions about it. @@ -41,8 +42,8 @@ function loadProfile(file) { const targetClass = store.getQuads(shape, SH + 'targetClass', null)[0]?.object.value || null; const label = store.getQuads(shape, 'http://www.w3.org/2000/01/rdf-schema#label', null)[0]?.object.value || null; const fields = store.getQuads(shape, SH + 'property', null).map(q => parseField(store, q.object)); - const discriminator = parseDiscriminator(store, shape); - return { uri: shape.value, targetClass, label, fields, discriminator }; + const discriminators = parseDiscriminators(store, shape); + return { uri: shape.value, targetClass, label, fields, discriminators }; }); return { uri, title, shapes: shapeList }; @@ -64,25 +65,49 @@ function parseField(store, p) { name: one(SH + 'name'), datatype: one(SH + 'datatype'), class: one(SH + 'class'), - node: one(SH + 'node'), + nodes: linkedShapes(store, p, one(SH + 'name')), }; } -// rep:discriminator tells shapes that share a targetClass apart. sh:minCount 1 -// becomes a plain triple in the seed, sh:maxCount 0 a FILTER NOT EXISTS. -function parseDiscriminator(store, shape) { - const node = store.getQuads(shape, REP + 'discriminator', null)[0]?.object; - if (!node) return null; - const { path, inverse } = readPath(store, node); - const one = (pred) => store.getQuads(node, pred, null)[0]?.object?.value || null; - const minCount = one(SH + 'minCount'); - const maxCount = one(SH + 'maxCount'); - if (!path || (minCount === null && maxCount === null)) return null; - return { - path, inverse, - minCount: minCount !== null ? Number(minCount) : null, - maxCount: maxCount !== null ? Number(maxCount) : null, +// The entities a field links to: its sh:node, or the sh:node of each item of +// its sh:or (SHACL for "each value is one of these"). +function linkedShapes(store, p, name) { + const node = store.getQuads(p, SH + 'node', null)[0]?.object; + if (node) return [node.value]; + const or = store.getQuads(p, SH + 'or', null)[0]?.object; + if (!or) return []; + return readRdfList(store, or).map(item => { + const n = store.getQuads(item, SH + 'node', null)[0]?.object; + if (!n) throw new Error(`field "${name}": every sh:or item needs a sh:node.`); + return n.value; + }); +} + +// rep:discriminator tells shapes that share a targetClass apart. A shape may +// carry several; a subject meets them all. +function parseDiscriminators(store, shape) { + return store.getQuads(shape, REP + 'discriminator', null) + .map(q => parseDiscriminator(store, q.object, shape.value)); +} + +// One discriminator: a condition written like a spec filter (sh:path plus +// sh:minCount, sh:maxCount, sh:in, sh:hasValue, ...), or sh:and / sh:or (a +// list) or sh:not (one) over discriminators, as in SHACL. +function parseDiscriminator(store, node, shapeUri) { + const one = (pred) => store.getQuads(node, SH + pred, null)[0]?.object; + const list = (pred) => { + const items = readRdfList(store, one(pred)).map(x => parseDiscriminator(store, x, shapeUri)); + if (!items.length) throw new Error(`rep:discriminator on <${shapeUri}> has an empty sh:${pred}.`); + return items; }; + if (one('and')) return { and: list('and') }; + if (one('or')) return { or: list('or') }; + if (one('not')) return { not: parseDiscriminator(store, one('not'), shapeUri) }; + const d = parseFilter(store, node); + if (!d.path.length || !Object.keys(d.constraints).length) { + throw new Error(`rep:discriminator on <${shapeUri}> needs a sh:path with a condition, or sh:and, sh:or or sh:not.`); + } + return d; } // The shape a spec starts from. rep:entity wins; a targetClass only resolves @@ -101,8 +126,3 @@ export function shape(profile, shapeUri) { export function fieldsOf(profile, shapeUri) { return shape(profile, shapeUri)?.fields || []; } - -// The field a hop takes from a shape, or null. -export function field(profile, shapeUri, hop) { - return fieldsOf(profile, shapeUri).find(f => f.path === hop.predicate && !!f.inverse === !!hop.inverse) || null; -} diff --git a/src/runner/seed.js b/src/runner/seed.js index 4739394..facb8d7 100644 --- a/src/runner/seed.js +++ b/src/runner/seed.js @@ -25,7 +25,9 @@ export function seedPageQuery(spec, limit, offset, shape = null) { const parts = []; const targetClass = spec.targetClass || shape?.targetClass; parts.push(`?s a ${sparqlEscapeUri(targetClass)} .`); - if (shape?.discriminator) parts.push(discriminatorPattern(shape.discriminator)); + (shape?.discriminators || []).forEach((d, di) => { + parts.push(discriminatorPattern(d, `disc${di}`)); + }); spec.filters.forEach((filter, fi) => { parts.push(filterPattern(filter, `f${fi}`)); }); @@ -36,13 +38,20 @@ LIMIT ${limit} OFFSET ${offset}`; } // A shape that shares its class with another says here what sets its -// subjects apart: sh:minCount 1 is a plain triple, sh:maxCount 0 a NOT EXISTS. -function discriminatorPattern(d) { - const p = sparqlEscapeUri(d.path); - const triple = d.inverse ? `?disc ${p} ?s .` : `?s ${p} ?disc .`; - if (d.minCount !== null && d.minCount >= 1) return triple; - if (d.maxCount === 0) return `FILTER NOT EXISTS { ${triple} }`; - return null; +// subjects apart. A plain discriminator is a filter; sh:and, sh:or and sh:not +// become one FILTER over EXISTS tests, which see ?s from outside. +function discriminatorPattern(d, varPrefix) { + if (!d.and && !d.or && !d.not) return filterPattern(d, varPrefix); + return `FILTER(${discriminatorTest(d, varPrefix)})`; +} + +function discriminatorTest(d, varPrefix) { + if (d.and) return `(${d.and.map((x, i) => discriminatorTest(x, `${varPrefix}_${i}`)).join(' && ')})`; + if (d.or) return `(${d.or.map((x, i) => discriminatorTest(x, `${varPrefix}_${i}`)).join(' || ')})`; + if (d.not) return `!(${discriminatorTest(d.not, `${varPrefix}_n`)})`; + const { maxCount, ...test } = d.constraints; + if (maxCount === 0) return `NOT EXISTS { ${filterPattern({ ...d, constraints: test }, varPrefix)} }`; + return `EXISTS { ${filterPattern(d, varPrefix)} }`; } // Writes out every hop as its own triple pattern. @@ -66,13 +75,15 @@ export function hopsPattern(hops, varPrefix, startVar, lastVar = `?${varPrefix}_ } // One filter: its condition, plus its rep:where conditions on the nodes it -// walks through. With sh:maxCount 0 the whole of it becomes "none such". +// walks through. With sh:maxCount 0 the whole of it becomes "none such"; +// with a value test too (sh:in, sh:hasValue, ...), "none with such a value". function filterPattern(filter, varPrefix) { + const { maxCount, ...test } = filter.constraints; + if (maxCount === 0) { + return `FILTER NOT EXISTS {\n ${filterPattern({ ...filter, constraints: test }, varPrefix)}\n }`; + } const nodeVar = (k) => (k === filter.path.length ? `?${varPrefix}_v` : `?${varPrefix}_${k}`); const wheres = wherePatterns(filter.where, filter.path, nodeVar, varPrefix); - if (filter.constraints.maxCount === 0) { - return `FILTER NOT EXISTS {\n ${[hopsPattern(filter.path, varPrefix, '?s'), ...wheres].join('\n ')}\n }`; - } return [conditionPattern(filter.constraints, varPrefix, filter.path, '?s'), ...wheres].join('\n '); } @@ -89,10 +100,11 @@ export function wherePatterns(where, hostHops, nodeVar, varPrefix) { // A condition: the hops from startVar and the test on where they end. A // filter starts at ?s; a rep:where starts at a node of its host and, with no // hops left, tests that node itself. sh:maxCount 0 means "none such", so its -// hops only live inside the NOT EXISTS. +// hops and any value test only live inside the NOT EXISTS. export function conditionPattern(c, varPrefix, hops, startVar) { if (c.maxCount === 0) { - return `FILTER NOT EXISTS { ${hopsPattern(hops, varPrefix + '_nex', startVar)} }`; + const { maxCount, ...test } = c; + return `FILTER NOT EXISTS { ${conditionPattern(test, varPrefix + '_nex', hops, startVar)} }`; } const v = hops.length ? `?${varPrefix}_v` : startVar; const parts = hops.length ? [hopsPattern(hops, varPrefix, startVar)] : []; diff --git a/src/runner/spec.js b/src/runner/spec.js index 801fe57..a2a0bfc 100644 --- a/src/runner/spec.js +++ b/src/runner/spec.js @@ -31,7 +31,7 @@ export function parseSpec(turtle) { return { uri: specNode.value, title, profileUri, targetClass, entity, filters, columns }; } -function parseFilter(store, p) { +export function parseFilter(store, p) { let path = []; const pathQuad = store.getQuads(p, SH + 'path', null)[0]?.object; if (pathQuad) path = parsePath(store, pathQuad); @@ -134,7 +134,7 @@ function readConstraints(store, p, { withLabel = false } = {}) { return constraints; } -function readRdfList(store, node) { +export function readRdfList(store, node) { const items = []; let cur = node; while (cur && cur.value !== RDF + 'nil') {