simplificaiton

This commit is contained in:
felix
2026-09-15 21:48:26 +02:00
parent 8334512d1e
commit 776b50c7d4
16 changed files with 168 additions and 455 deletions

View File

@@ -1,4 +1,4 @@
import { entityForClass, fieldsOf } from './profile.js';
import { fieldsOf } from './profile.js';
import { NUMERIC_DATATYPES } from './assemble.js';
const SH = 'http://www.w3.org/ns/shacl#';
@@ -27,21 +27,46 @@ function profileNamed(spec, knownProfiles) {
return null;
}
// The target class is in the profile
function targetClass(spec, profile) {
// The spec names an entity we have: by rep:entity, or by a targetClass only
// one entity carries. A class two entities share is refused with both names,
// so the model can add rep:entity instead of guessing.
function startEntity(spec, profile) {
if (!spec.targetClass) {
return `no sh:targetClass. In "${profile.title}" you can list: ${shapeList(profile)}.`;
}
if (!entityForClass(profile, spec.targetClass)) {
if (spec.entity) {
const s = profile.shapes.find(x => x.uri === spec.entity);
if (!s) {
return `no entity <${spec.entity}> in "${profile.title}". It has: ${shapeList(profile)}.`;
}
if (s.targetClass !== spec.targetClass) {
return `rep:entity <${spec.entity}> is a <${s.targetClass}>, but sh:targetClass says <${spec.targetClass}>. Make them agree.`;
}
return null;
}
const matches = profile.shapes.filter(s => s.targetClass === spec.targetClass);
if (matches.length === 1) return null;
if (!matches.length) {
return `"${profile.title}" cannot list <${spec.targetClass}>. It has: ${shapeList(profile)}.`;
}
return null;
const names = matches.map(m => `${m.label || m.uri} <${m.uri}>`).join(', ');
return `<${spec.targetClass}> is the class of ${matches.length} entities: ${names}. Add rep:entity <...> to say which one.`;
}
function shapeList(profile) {
return profile.shapes.map(s => s.label || s.targetClass).join(', ');
}
// The shape a spec starts from, after startEntity has approved it: rep:entity
// wins, otherwise the one shape carrying the targetClass.
function startShapeOf(spec, profile) {
if (spec.entity) {
return profile.shapes.find(s => s.uri === spec.entity)?.uri || null;
}
const matches = profile.shapes.filter(s => s.targetClass === spec.targetClass);
return matches.length === 1 ? matches[0].uri : null;
}
// Columns exist
function columnsPresent(spec) {
if (!spec.columns.length) return 'no columns. Add at least one rep:columns entry.';
@@ -70,7 +95,7 @@ function columnLabelsUnique(spec) {
// Every path resolves, columns end on a value, depth capped
function pathsResolve(spec, profile, maxPathDepth) {
const startShapeUri = entityForClass(profile, spec.targetClass);
const startShapeUri = startShapeOf(spec, profile);
for (const col of spec.columns) {
const err = walk(profile, startShapeUri, col.path, maxPathDepth,
`column "${col.label}"`, true);
@@ -86,7 +111,7 @@ function pathsResolve(spec, profile, maxPathDepth) {
// sh:min/sh:max only on numbers and dates
function collectMinMaxTyped(spec, profile) {
const startShapeUri = entityForClass(profile, spec.targetClass);
const startShapeUri = startShapeOf(spec, profile);
for (const col of spec.columns) {
if (col.collect !== SH + 'min' && col.collect !== SH + 'max') continue;
const name = col.collect === SH + 'min' ? 'sh:min' : 'sh:max';
@@ -176,11 +201,11 @@ function anyOfLimits(spec) {
const c = filter.constraints;
if (!c.anyOf) continue;
if (c.anyOf.length > ANYOF_MAX_TERMS) {
return `rep:anyOf has ${c.anyOf.length} terms. The limit is ${ANYOF_MAX_TERMS}. Use sh:in with exact URIs instead (find them with lookup_values).`;
return `rep:anyOf has ${c.anyOf.length} terms. The limit is ${ANYOF_MAX_TERMS}. Narrow the words, or split the filter into two sh:property blocks.`;
}
for (const term of c.anyOf) {
if (term.type === 'uri') {
return 'a rep:anyOf term is a URI. rep:anyOf takes Dutch words, not URIs. Use sh:in with the exact URI instead.';
return 'a rep:anyOf term is a URI. rep:anyOf takes Dutch words, not URIs.';
}
if (term.value && term.value.length > ANYOF_MAX_TERM_LENGTH) {
return `a rep:anyOf term is ${term.value.length} characters. The limit is ${ANYOF_MAX_TERM_LENGTH}.`;
@@ -232,7 +257,7 @@ function parsesAsDate(value, withTime) {
}
const CHECKS = [
['no sh:targetClass or unknown', targetClass],
['no entity or unknown class', startEntity],
['no columns', columnsPresent],
['column without rdfs:label', columnLabel],
['two columns with one label', columnLabelsUnique],

View File

@@ -7,7 +7,7 @@ import { sparqlEscapeUri } from '../db.js';
const SUBJECT_CHUNK_SIZE = Math.max(1, Number(process.env.SUBJECT_CHUNK_SIZE) || 100);
export async function fetchColumns(sessionQuery, subjects, spec, values = new Map(), onChunk = null, onProgress = null) {
export async function fetchColumns(sessionQuery, subjects, spec, values = new Map(), onProgress = null) {
const groups = groupColumns(spec.columns);
// The subject list is complete and the chunk size is fixed, so the number
// of batch queries is known before the first one flies. onProgress fires
@@ -22,7 +22,6 @@ export async function fetchColumns(sessionQuery, subjects, spec, values = new Ma
done += 1;
onProgress?.({ done, total });
}
if (onChunk) await onChunk(group);
}
return values;
}

View File

@@ -1,4 +1,4 @@
import { mkdirSync, writeFileSync, readFileSync } from 'fs';
import { mkdirSync, writeFileSync } from 'fs';
// Rows to a CSV file. CSV_SEPARATOR is ';' by default. A cell is quoted
// only when it holds the separator, a quote or a newline; a quote inside a
@@ -32,9 +32,4 @@ export function writeSpec(csvName, spec) {
const name = csvName.replace(/\.csv$/, '') + '.ttl';
writeFileSync(`${SHARE_DIR}/${name}`, spec, 'utf8');
return `share://${name}`;
}
export function readSpec(pointer) {
if (!pointer.startsWith('share://')) throw new Error(`not a share:// pointer: ${pointer}`);
return readFileSync(`${SHARE_DIR}/${pointer.slice('share://'.length)}`, 'utf8');
}

View File

@@ -1,152 +0,0 @@
import { randomUUID } from 'crypto';
import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime } from '../db.js';
// cogs:Job + task:Task, written by us; no other service is involved. Every
// write carries mu-call-scope-id (REPORT_SCOPE) so the delta notifier lets
// jobs-controller ignore our statuses — it throws on unknown operations.
const REPORT_SCOPE = 'http://redpencil.data.gift/id/concept/muScope/report-generation';
const JOB_BASE = process.env.JOB_URI_BASE || 'http://redpencil.data.gift/id/job/';
const TASK_BASE = process.env.TASK_URI_BASE || 'http://redpencil.data.gift/id/task/';
const STATUS = {
scheduled: 'http://redpencil.data.gift/id/concept/JobStatus/scheduled',
busy: 'http://redpencil.data.gift/id/concept/JobStatus/busy',
success: 'http://redpencil.data.gift/id/concept/JobStatus/success',
failed: 'http://redpencil.data.gift/id/concept/JobStatus/failed',
};
const JOB_OPERATION = 'http://lblod.data.gift/id/jobs/concept/JobOperation/reportGeneration';
const TASK_OPERATION = 'http://lblod.data.gift/id/jobs/concept/TaskOperation/reportGeneration';
const COGS_JOB = 'http://vocab.deri.ie/cogs#Job';
const TASK_TASK = 'http://redpencil.data.gift/vocabularies/tasks/Task';
const DATA_CONTAINER = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#DataContainer';
const MU_UUID = 'http://mu.semte.ch/vocabularies/core/uuid';
const ADMS_STATUS = 'http://www.w3.org/ns/adms#status';
const DCT = 'http://purl.org/dc/terms/';
const TASK = 'http://redpencil.data.gift/vocabularies/tasks/';
const OSLC_MESSAGE = 'http://open-services.net/ns/core#message';
const OSLC_ERROR_URI = 'http://open-services.net/ns/core#Error';
// Adds the scope header to every job write.
function withScope(updateFn) {
return async sparql => updateFn(sparql, { 'mu-call-scope-id': REPORT_SCOPE });
}
// Create job + task + result container, all at status scheduled.
// Returns { jobUri, taskUri, containerUri }.
export async function createJob(updateFn, title, question, creator) {
const upd = withScope(updateFn);
const jobId = randomUUID();
const taskId = randomUUID();
const containerId = randomUUID();
const jobUri = `${JOB_BASE}${jobId}`;
const taskUri = `${TASK_BASE}${taskId}`;
const containerUri = `http://redpencil.data.gift/id/dataContainers/${containerId}`;
const now = sparqlEscapeDateTime(new Date());
const creatorTriple = creator
? `${sparqlEscapeUri(jobUri)} ${sparqlEscapeUri(DCT + 'creator')} ${sparqlEscapeUri(creator)} .`
: '';
const questionTriple = question
? `${sparqlEscapeUri(jobUri)} ${sparqlEscapeUri('http://mu.semte.ch/vocabularies/ext/question')} ${sparqlEscapeString(question)} .`
: '';
await upd(`
INSERT DATA {
${sparqlEscapeUri(jobUri)} a ${sparqlEscapeUri(COGS_JOB)} ;
${sparqlEscapeUri(MU_UUID)} ${sparqlEscapeString(jobId)} ;
${sparqlEscapeUri(DCT + 'title')} ${sparqlEscapeString(title)} ;
${sparqlEscapeUri(ADMS_STATUS)} ${sparqlEscapeUri(STATUS.scheduled)} ;
${sparqlEscapeUri(TASK + 'operation')} ${sparqlEscapeUri(JOB_OPERATION)} ;
${sparqlEscapeUri(DCT + 'created')} ${now} ;
${sparqlEscapeUri(DCT + 'modified')} ${now} .
${creatorTriple}
${questionTriple}
${sparqlEscapeUri(taskUri)} a ${sparqlEscapeUri(TASK_TASK)} ;
${sparqlEscapeUri(MU_UUID)} ${sparqlEscapeString(taskId)} ;
${sparqlEscapeUri(DCT + 'isPartOf')} ${sparqlEscapeUri(jobUri)} ;
${sparqlEscapeUri(ADMS_STATUS)} ${sparqlEscapeUri(STATUS.scheduled)} ;
${sparqlEscapeUri(TASK + 'operation')} ${sparqlEscapeUri(TASK_OPERATION)} ;
${sparqlEscapeUri(TASK + 'index')} ${sparqlEscapeString('0')} ;
${sparqlEscapeUri(DCT + 'created')} ${now} ;
${sparqlEscapeUri(DCT + 'modified')} ${now} ;
${sparqlEscapeUri(TASK + 'resultsContainer')} ${sparqlEscapeUri(containerUri)} .
${sparqlEscapeUri(containerUri)} a ${sparqlEscapeUri(DATA_CONTAINER)} ;
${sparqlEscapeUri(MU_UUID)} ${sparqlEscapeString(containerId)} .
}`);
return { jobUri, taskUri, containerUri };
}
export async function setStatus(updateFn, uri, status) {
const s = STATUS[status];
if (!s) throw new Error(`unknown job status "${status}"`);
const now = sparqlEscapeDateTime(new Date());
await withScope(updateFn)(`
DELETE {
${sparqlEscapeUri(uri)} ${sparqlEscapeUri(ADMS_STATUS)} ?old ;
${sparqlEscapeUri(DCT + 'modified')} ?modified .
}
INSERT {
${sparqlEscapeUri(uri)} ${sparqlEscapeUri(ADMS_STATUS)} ${sparqlEscapeUri(s)} ;
${sparqlEscapeUri(DCT + 'modified')} ${now} .
}
WHERE {
${sparqlEscapeUri(uri)} ${sparqlEscapeUri(ADMS_STATUS)} ?old .
OPTIONAL { ${sparqlEscapeUri(uri)} ${sparqlEscapeUri(DCT + 'modified')} ?modified . }
}`);
}
export async function touch(updateFn, taskUri) {
const now = sparqlEscapeDateTime(new Date());
await withScope(updateFn)(`
DELETE {
${sparqlEscapeUri(taskUri)} ${sparqlEscapeUri(DCT + 'modified')} ?modified .
}
INSERT {
${sparqlEscapeUri(taskUri)} ${sparqlEscapeUri(DCT + 'modified')} ${now} .
}
WHERE {
OPTIONAL { ${sparqlEscapeUri(taskUri)} ${sparqlEscapeUri(DCT + 'modified')} ?modified . }
}`);
}
// Link the finished CSV to the task's result container. The container is
// created with the job; the file URI is the logical file from registerFile.
export async function attachResult(updateFn, taskUri, fileUri) {
await withScope(updateFn)(`
INSERT {
?container ${sparqlEscapeUri(TASK + 'hasFile')} ${sparqlEscapeUri(fileUri)} .
}
WHERE {
${sparqlEscapeUri(taskUri)} ${sparqlEscapeUri(TASK + 'resultsContainer')} ?container .
}`);
}
// An oslc:Error with the message, job and task both failed.
export async function fail(updateFn, jobUri, taskUri, message) {
const errorId = randomUUID();
const errorUri = `http://redpencil.data.gift/id/jobs/error/${errorId}`;
const now = sparqlEscapeDateTime(new Date());
await withScope(updateFn)(`
DELETE {
${sparqlEscapeUri(jobUri)} ${sparqlEscapeUri(ADMS_STATUS)} ?js .
${sparqlEscapeUri(taskUri)} ${sparqlEscapeUri(ADMS_STATUS)} ?ts .
}
INSERT DATA {
${sparqlEscapeUri(errorUri)} a ${sparqlEscapeUri(OSLC_ERROR_URI)} ;
${sparqlEscapeUri(MU_UUID)} ${sparqlEscapeString(errorId)} ;
${sparqlEscapeUri(OSLC_MESSAGE)} ${sparqlEscapeString(message)} ;
${sparqlEscapeUri(DCT + 'created')} ${now} .
${sparqlEscapeUri(taskUri)} ${sparqlEscapeUri(TASK + 'error')} ${sparqlEscapeUri(errorUri)} ;
${sparqlEscapeUri(ADMS_STATUS)} ${sparqlEscapeUri(STATUS.failed)} ;
${sparqlEscapeUri(DCT + 'modified')} ${now} .
${sparqlEscapeUri(jobUri)} ${sparqlEscapeUri(ADMS_STATUS)} ${sparqlEscapeUri(STATUS.failed)} ;
${sparqlEscapeUri(DCT + 'modified')} ${now} .
}
WHERE {
${sparqlEscapeUri(jobUri)} ${sparqlEscapeUri(ADMS_STATUS)} ?js .
${sparqlEscapeUri(taskUri)} ${sparqlEscapeUri(ADMS_STATUS)} ?ts .
}`);
}

View File

@@ -6,6 +6,7 @@ import { readFileSync } from 'fs';
const SH = 'http://www.w3.org/ns/shacl#';
const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
const OWL_ONTOLOGY = 'http://www.w3.org/2002/07/owl#Ontology';
const REP = 'http://mu.semte.ch/vocabularies/reporting/';
export async function loadProfiles(dir) {
const fs = await import('fs');
@@ -52,7 +53,8 @@ export async function loadProfile(file) {
const targetClass = store.getQuads(shape, SH + 'targetClass', null)[0]?.object.value || null;
const label = store.getQuads(shape, 'http://www.w3.org/2000/01/rdf-schema#label', null)[0]?.object.value || null;
const fields = store.getQuads(shape, SH + 'property', null).map(q => parseField(store, q.object));
return { uri: shape.value, targetClass, label, fields };
const discriminator = parseDiscriminator(store, shape);
return { uri: shape.value, targetClass, label, fields, discriminator };
});
return { uri, title, prefixes, shapes: shapeList, store };
@@ -82,11 +84,46 @@ function parseField(store, p) {
};
}
// rep:discriminator tells shapes that share a targetClass apart. sh:minCount 1
// becomes a plain triple in the seed, sh:maxCount 0 a FILTER NOT EXISTS.
function parseDiscriminator(store, shape) {
const node = store.getQuads(shape, REP + 'discriminator', null)[0]?.object;
if (!node) return null;
let path = null, inverse = false;
const pathQuad = store.getQuads(node, SH + 'path', null)[0]?.object;
if (pathQuad) {
if (pathQuad.termType === 'BlankNode') {
const inv = store.getQuads(pathQuad, SH + 'inversePath', null)[0]?.object;
if (inv) { path = inv.value; inverse = true; }
} else {
path = pathQuad.value;
}
}
const one = (pred) => store.getQuads(node, pred, null)[0]?.object?.value || null;
const minCount = one(SH + 'minCount');
const maxCount = one(SH + 'maxCount');
if (!path || (minCount === null && maxCount === null)) return null;
return {
path, inverse,
minCount: minCount !== null ? Number(minCount) : null,
maxCount: maxCount !== null ? Number(maxCount) : null,
};
}
export function entityForClass(profile, cls) {
const shape = profile.shapes.find(s => s.targetClass === cls);
return shape ? shape.uri : null;
}
// The shape a spec starts from. rep:entity wins; a targetClass only resolves
// when exactly one shape carries it, so a class two entities share is a
// validator error, not a silent first match.
export function startShape(profile, spec) {
if (spec.entity) return shape(profile, spec.entity);
const matches = profile.shapes.filter(s => s.targetClass === spec.targetClass);
return matches.length === 1 ? matches[0] : null;
}
export function shape(profile, shapeUri) {
return profile.shapes.find(s => s.uri === shapeUri) || null;
}

View File

@@ -3,22 +3,21 @@ import { fetchColumns } from './columns.js';
import { assemble } from './assemble.js';
import { rowsToCsv, writeCsv, writeSpec } from './csv.js';
import { registerFile, registerReport } from './report.js';
import { touch } from './job.js';
import { startShape } from './profile.js';
// Ties the steps together: seed → columns → assemble → CSV → register.
// Returns { reportUri, fileUri, rowCount }. The task is touched once per
// column group so the dashboard shows movement. onProgress fires on every
// Returns { reportUri, fileUri, rowCount }. onProgress fires on every
// answered query with what the phase knows — { subjects } per seed page,
// { done, total } per column batch — so a waiter can tell a live run from
// a stalled one.
export async function run(query, update, parsed, profile, title, taskUri, extra = {}, onProgress = null) {
const subjects = await seed(query, parsed,
export async function run(query, update, parsed, profile, title, extra = {}, onProgress = null) {
const shape = startShape(profile, parsed);
const subjects = await seed(query, parsed, shape,
onProgress ? (n) => onProgress({ subjects: n }) : null);
const values = new Map();
await fetchColumns(query, subjects, parsed, values,
taskUri ? () => touch(update, taskUri) : null, onProgress);
await fetchColumns(query, subjects, parsed, values, null, onProgress);
const rows = assemble(subjects, values, parsed);
const fileName = `${slug(title)}.csv`;
@@ -40,4 +39,4 @@ export function slug(title) {
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80) || 'report';
}
}

View File

@@ -6,11 +6,11 @@ import { sparqlEscapeUri, sparqlEscapeString, sparqlEscapeDateTime, sparqlEscape
const ROW_LIMIT = Number(process.env.ROW_LIMIT || 200000);
const PAGE_SIZE = Number(process.env.SEED_PAGE_SIZE || 5000);
export async function seed(sessionQuery, spec, onPage = null) {
export async function seed(sessionQuery, spec, shape = null, onPage = null) {
const subjects = [];
let offset = 0;
for (;;) {
const query = seedPageQuery(spec, PAGE_SIZE, offset);
const query = seedPageQuery(spec, PAGE_SIZE, offset, shape);
const result = await sessionQuery(query);
const rows = result.results.bindings.map(b => b.s.value);
subjects.push(...rows);
@@ -24,9 +24,11 @@ export async function seed(sessionQuery, spec, onPage = null) {
return subjects;
}
export function seedPageQuery(spec, limit, offset) {
export function seedPageQuery(spec, limit, offset, shape = null) {
const parts = [];
parts.push(`?s a ${sparqlEscapeUri(spec.targetClass)} .`);
const targetClass = spec.targetClass || shape?.targetClass;
parts.push(`?s a ${sparqlEscapeUri(targetClass)} .`);
if (shape?.discriminator) parts.push(discriminatorPattern(shape.discriminator));
spec.filters.forEach((filter, fi) => {
parts.push(hopsPattern(filter.path, `f${fi}`, '?s'));
parts.push(constraintsPattern(filter.constraints, `f${fi}`));
@@ -37,6 +39,16 @@ export function seedPageQuery(spec, limit, offset) {
LIMIT ${limit} OFFSET ${offset}`;
}
// A shape that shares its class with another says here what sets its
// subjects apart: sh:minCount 1 is a plain triple, sh:maxCount 0 a NOT EXISTS.
function discriminatorPattern(d) {
const p = sparqlEscapeUri(d.path);
const triple = d.inverse ? `?disc ${p} ?s .` : `?s ${p} ?disc .`;
if (d.minCount !== null && d.minCount >= 1) return triple;
if (d.maxCount === 0) return `FILTER NOT EXISTS { ${triple} }`;
return null;
}
// Writes out every hop as its own triple pattern.
// Forward: ?s <p1> ?f0_1 . ?f0_1 <p2> ?f0_v
// Inverse: ?s ^<p1> → ?f0_1 <p1> ?s

View File

@@ -19,6 +19,7 @@ export function parseSpec(turtle) {
const one = (s, pred) => store.getQuads(s, pred, null)[0]?.object || null;
const targetClass = one(specNode, SH + 'targetClass')?.value || null;
const entity = one(specNode, REP + 'entity')?.value || null;
const profileUri = one(specNode, REP + 'profile')?.value || null;
const title = one(specNode, 'http://purl.org/dc/terms/title')?.value || null;
@@ -28,7 +29,7 @@ export function parseSpec(turtle) {
const columnsQuad = one(specNode, REP + 'columns');
const columns = columnsQuad ? parseColumnList(store, columnsQuad) : [];
return { uri: specNode.value, title, profileUri, targetClass, filters, columns };
return { uri: specNode.value, title, profileUri, targetClass, entity, filters, columns };
}
function parseFilter(store, p, { withLabel = false } = {}) {