feat: phase 5 of AI pipeline hardening — R42 retrieval & telemetry #7
@@ -6,6 +6,8 @@
|
||||
|
||||
This plan upgrades the platform's interaction with the Anthropic API: how prompts are built, how responses are parsed, how the model is retried, and how outputs are validated. It is broken into six phases that can be implemented and shipped independently. Each phase ends with verifiable acceptance criteria.
|
||||
|
||||
> **Status (2026-05-20):** Phases 1–5 implemented and shipped. Phase 6 (eval harness) is intentionally **out of scope** for this initiative — the production pipeline is hardened to the level the platform needs, and a golden-set runner can be reopened later as a stand-alone task if regression risk grows. The repo no longer carries any TODO from this plan.
|
||||
|
||||
---
|
||||
|
||||
## 0. Operating principles
|
||||
@@ -461,7 +463,9 @@ Wire `callLLM` to write to it (best-effort, never throws). Add a minimal `Admin
|
||||
|
||||
---
|
||||
|
||||
## Phase 6 — Eval harness (optional, high-leverage)
|
||||
## Phase 6 — Eval harness (NOT IMPLEMENTED — out of scope)
|
||||
|
||||
> Phase 6 was deliberately skipped. The acceptance criteria for Phases 1–5 give enough confidence in extraction, content, quiz, R42, and telemetry that a golden-set harness is not currently load-bearing. Leave this section intact as a starting point for a future, separately-scoped initiative.
|
||||
|
||||
**Goal:** prompt or model changes can be measured before they ship.
|
||||
|
||||
|
||||
209
pb_migrations/1780500002_created_llm_calls.js
Normal file
209
pb_migrations/1780500002_created_llm_calls.js
Normal file
@@ -0,0 +1,209 @@
|
||||
/// <reference path="../pb_data/types.d.ts" />
|
||||
migrate((app) => {
|
||||
const collection = new Collection({
|
||||
"createRule": "",
|
||||
"deleteRule": "",
|
||||
"fields": [
|
||||
{
|
||||
"autogeneratePattern": "[a-z0-9]{15}",
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "text3208210256",
|
||||
"max": 15,
|
||||
"min": 15,
|
||||
"name": "id",
|
||||
"pattern": "^[a-z0-9]+$",
|
||||
"presentable": false,
|
||||
"primaryKey": true,
|
||||
"required": true,
|
||||
"system": true,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"autogeneratePattern": "",
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "text_llm_task",
|
||||
"max": 0,
|
||||
"min": 0,
|
||||
"name": "task",
|
||||
"pattern": "",
|
||||
"presentable": false,
|
||||
"primaryKey": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"autogeneratePattern": "",
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "text_llm_model",
|
||||
"max": 0,
|
||||
"min": 0,
|
||||
"name": "model",
|
||||
"pattern": "",
|
||||
"presentable": false,
|
||||
"primaryKey": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"autogeneratePattern": "",
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "text_llm_tier",
|
||||
"max": 0,
|
||||
"min": 0,
|
||||
"name": "tier",
|
||||
"pattern": "",
|
||||
"presentable": false,
|
||||
"primaryKey": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "number_llm_duration",
|
||||
"max": null,
|
||||
"min": null,
|
||||
"name": "duration_ms",
|
||||
"onlyInt": false,
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "number_llm_input",
|
||||
"max": null,
|
||||
"min": null,
|
||||
"name": "input_tokens",
|
||||
"onlyInt": false,
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "number_llm_output",
|
||||
"max": null,
|
||||
"min": null,
|
||||
"name": "output_tokens",
|
||||
"onlyInt": false,
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "number_llm_cache_r",
|
||||
"max": null,
|
||||
"min": null,
|
||||
"name": "cache_read_tokens",
|
||||
"onlyInt": false,
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "number_llm_cache_c",
|
||||
"max": null,
|
||||
"min": null,
|
||||
"name": "cache_create_tokens",
|
||||
"onlyInt": false,
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"autogeneratePattern": "",
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "text_llm_stop",
|
||||
"max": 0,
|
||||
"min": 0,
|
||||
"name": "stop_reason",
|
||||
"pattern": "",
|
||||
"presentable": false,
|
||||
"primaryKey": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "bool_llm_ok",
|
||||
"name": "ok",
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "bool"
|
||||
},
|
||||
{
|
||||
"autogeneratePattern": "",
|
||||
"help": "",
|
||||
"hidden": false,
|
||||
"id": "text_llm_err",
|
||||
"max": 0,
|
||||
"min": 0,
|
||||
"name": "error_msg",
|
||||
"pattern": "",
|
||||
"presentable": false,
|
||||
"primaryKey": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "autodate_llm_created",
|
||||
"name": "created",
|
||||
"onCreate": true,
|
||||
"onUpdate": false,
|
||||
"presentable": false,
|
||||
"system": false,
|
||||
"type": "autodate"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "autodate_llm_updated",
|
||||
"name": "updated",
|
||||
"onCreate": true,
|
||||
"onUpdate": true,
|
||||
"presentable": false,
|
||||
"system": false,
|
||||
"type": "autodate"
|
||||
}
|
||||
],
|
||||
"id": "pbc_llm_calls_001",
|
||||
"indexes": [
|
||||
"CREATE INDEX `idx_llm_calls_created` ON `llm_calls` (`created`)",
|
||||
"CREATE INDEX `idx_llm_calls_task` ON `llm_calls` (`task`)"
|
||||
],
|
||||
"listRule": "",
|
||||
"name": "llm_calls",
|
||||
"system": false,
|
||||
"type": "base",
|
||||
"updateRule": "",
|
||||
"viewRule": ""
|
||||
});
|
||||
|
||||
return app.save(collection);
|
||||
}, (app) => {
|
||||
const collection = app.findCollectionByNameOrId("pbc_llm_calls_001");
|
||||
return app.delete(collection);
|
||||
})
|
||||
162
src/components/admin/Diagnostics.jsx
Normal file
162
src/components/admin/Diagnostics.jsx
Normal file
@@ -0,0 +1,162 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { RefreshCw, AlertCircle, CheckCircle2 } from 'lucide-react';
|
||||
import Card from '../ui/Card';
|
||||
import Button from '../ui/Button';
|
||||
import Tag from '../ui/Tag';
|
||||
import * as db from '../../lib/db';
|
||||
|
||||
// Public Anthropic pricing per 1M tokens. Update manually when prices change.
|
||||
const PRICES = {
|
||||
'claude-haiku-4-5-20251001': { input: 1.0, output: 5.0, cache_read: 0.10, cache_write: 1.25 },
|
||||
'claude-haiku-4-5': { input: 1.0, output: 5.0, cache_read: 0.10, cache_write: 1.25 },
|
||||
'claude-sonnet-4-6': { input: 3.0, output: 15.0, cache_read: 0.30, cache_write: 3.75 },
|
||||
'claude-opus-4-7': { input: 15.0, output: 75.0, cache_read: 1.50, cache_write: 18.75 },
|
||||
};
|
||||
|
||||
function pricesFor(model) {
|
||||
if (!model) return null;
|
||||
if (PRICES[model]) return PRICES[model];
|
||||
if (model.includes('haiku')) return PRICES['claude-haiku-4-5'];
|
||||
if (model.includes('sonnet')) return PRICES['claude-sonnet-4-6'];
|
||||
if (model.includes('opus')) return PRICES['claude-opus-4-7'];
|
||||
return null;
|
||||
}
|
||||
|
||||
function costUsd(row) {
|
||||
const p = pricesFor(row.model);
|
||||
if (!p) return null;
|
||||
const inTok = (row.input_tokens || 0) - (row.cache_read_tokens || 0) - (row.cache_create_tokens || 0);
|
||||
const out = row.output_tokens || 0;
|
||||
const cr = row.cache_read_tokens || 0;
|
||||
const cc = row.cache_create_tokens || 0;
|
||||
const usd = (Math.max(inTok, 0) * p.input + out * p.output + cr * p.cache_read + cc * p.cache_write) / 1_000_000;
|
||||
return usd;
|
||||
}
|
||||
|
||||
function fmtUsd(n) {
|
||||
if (n == null) return '—';
|
||||
if (n < 0.0001) return '<$0.0001';
|
||||
return `$${n.toFixed(4)}`;
|
||||
}
|
||||
|
||||
function fmtMs(n) {
|
||||
if (n == null) return '—';
|
||||
if (n < 1000) return `${Math.round(n)}ms`;
|
||||
return `${(n / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
const Diagnostics = () => {
|
||||
const [rows, setRows] = useState([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const r = await db.getRecentLlmCalls(100);
|
||||
setRows(r);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => { load(); }, []);
|
||||
|
||||
const totals = rows.reduce((acc, r) => {
|
||||
acc.input += r.input_tokens || 0;
|
||||
acc.output += r.output_tokens || 0;
|
||||
acc.cacheRead += r.cache_read_tokens || 0;
|
||||
acc.cacheCreate += r.cache_create_tokens || 0;
|
||||
const c = costUsd(r);
|
||||
if (c != null) acc.cost += c;
|
||||
acc.duration += r.duration_ms || 0;
|
||||
if (r.ok) acc.ok++;
|
||||
else acc.fail++;
|
||||
return acc;
|
||||
}, { input: 0, output: 0, cacheRead: 0, cacheCreate: 0, cost: 0, duration: 0, ok: 0, fail: 0 });
|
||||
|
||||
const cacheHitRate = totals.input > 0
|
||||
? Math.round((totals.cacheRead / totals.input) * 100)
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<p className="text-fg-muted text-sm">
|
||||
Laatste 100 LLM-aanroepen. Kosten worden lokaal berekend met publieke Anthropic-prijzen (handmatig verversen).
|
||||
</p>
|
||||
<Button onClick={load} disabled={loading}>
|
||||
<RefreshCw size={16} className={`mr-2 ${loading ? 'animate-spin' : ''}`} /> Vernieuwen
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-3 mb-6">
|
||||
<Card className="p-4 border border-bg-warm">
|
||||
<p className="text-xs text-fg-muted uppercase">Calls</p>
|
||||
<p className="text-2xl font-bold mt-1">{rows.length}</p>
|
||||
<p className="text-xs text-fg-muted mt-1">{totals.ok} ok · {totals.fail} fail</p>
|
||||
</Card>
|
||||
<Card className="p-4 border border-bg-warm">
|
||||
<p className="text-xs text-fg-muted uppercase">Tokens (in/out)</p>
|
||||
<p className="text-2xl font-bold mt-1">{totals.input.toLocaleString()} / {totals.output.toLocaleString()}</p>
|
||||
<p className="text-xs text-fg-muted mt-1">cache read: {totals.cacheRead.toLocaleString()} ({cacheHitRate}%)</p>
|
||||
</Card>
|
||||
<Card className="p-4 border border-bg-warm">
|
||||
<p className="text-xs text-fg-muted uppercase">Geschatte kosten</p>
|
||||
<p className="text-2xl font-bold mt-1">{fmtUsd(totals.cost)}</p>
|
||||
<p className="text-xs text-fg-muted mt-1">over deze 100 aanroepen</p>
|
||||
</Card>
|
||||
<Card className="p-4 border border-bg-warm">
|
||||
<p className="text-xs text-fg-muted uppercase">Gem. duur</p>
|
||||
<p className="text-2xl font-bold mt-1">{rows.length ? fmtMs(totals.duration / rows.length) : '—'}</p>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<Card className="p-0 border border-bg-warm overflow-hidden">
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-bg-warm/40 text-fg-muted text-xs uppercase">
|
||||
<tr>
|
||||
<th className="text-left p-3">Tijd</th>
|
||||
<th className="text-left p-3">Task</th>
|
||||
<th className="text-left p-3">Tier</th>
|
||||
<th className="text-left p-3">Model</th>
|
||||
<th className="text-right p-3">In</th>
|
||||
<th className="text-right p-3">Out</th>
|
||||
<th className="text-right p-3">Cache</th>
|
||||
<th className="text-right p-3">$</th>
|
||||
<th className="text-right p-3">Duur</th>
|
||||
<th className="text-left p-3">Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-bg-warm">
|
||||
{rows.length === 0 ? (
|
||||
<tr><td colSpan={10} className="p-8 text-center text-fg-muted">Nog geen aanroepen geregistreerd.</td></tr>
|
||||
) : rows.map(r => (
|
||||
<tr key={r.id} className="hover:bg-bg-warm/20">
|
||||
<td className="p-3 text-xs text-fg-muted whitespace-nowrap">
|
||||
{r.created ? new Date(r.created).toLocaleString() : '—'}
|
||||
</td>
|
||||
<td className="p-3 font-mono text-xs">{r.task || '—'}</td>
|
||||
<td className="p-3 text-xs">{r.tier || '—'}</td>
|
||||
<td className="p-3 font-mono text-xs">{r.model || '—'}</td>
|
||||
<td className="p-3 text-right">{(r.input_tokens || 0).toLocaleString()}</td>
|
||||
<td className="p-3 text-right">{(r.output_tokens || 0).toLocaleString()}</td>
|
||||
<td className="p-3 text-right text-fg-muted">{(r.cache_read_tokens || 0).toLocaleString()}</td>
|
||||
<td className="p-3 text-right">{fmtUsd(costUsd(r))}</td>
|
||||
<td className="p-3 text-right">{fmtMs(r.duration_ms)}</td>
|
||||
<td className="p-3">
|
||||
{r.ok
|
||||
? <Tag variant="success" className="flex items-center gap-1 w-fit"><CheckCircle2 size={12}/> ok</Tag>
|
||||
: <Tag variant="dark" className="bg-red-100 text-red-800 flex items-center gap-1 w-fit"><AlertCircle size={12}/> {(r.error_msg || r.stop_reason || 'fail').slice(0, 24)}</Tag>}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default Diagnostics;
|
||||
@@ -68,6 +68,22 @@ export function buildSystemPrompt({ userName, isAdmin, kbContext }) {
|
||||
];
|
||||
}
|
||||
|
||||
export const LOOKUP_TOPIC_TOOL = {
|
||||
name: 'lookup_topic',
|
||||
description:
|
||||
'Haal de volledige beschrijving en eventuele leerinhoud van één topic op uit de kennisgraaf. Gebruik dit wanneer de samenvattende KENNISGRAAF in de systeemprompt niet genoeg informatie bevat om de vraag te beantwoorden.',
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
id: {
|
||||
type: 'string',
|
||||
description: 'Het exacte topic id (kebab-case) zoals het in de kennisgraaf staat.',
|
||||
},
|
||||
},
|
||||
required: ['id'],
|
||||
},
|
||||
};
|
||||
|
||||
export const PROPOSE_GRAPH_DELTA_TOOL = {
|
||||
name: 'propose_graph_delta',
|
||||
description:
|
||||
|
||||
@@ -1,68 +1,119 @@
|
||||
import * as db from '../../lib/db';
|
||||
import { buildIndex, retrieveTopK } from '../../lib/retrieval';
|
||||
|
||||
const TOP_K = 10;
|
||||
|
||||
async function sha256Hex(input) {
|
||||
const enc = new TextEncoder().encode(input);
|
||||
if (globalThis.crypto?.subtle?.digest) {
|
||||
const buf = await globalThis.crypto.subtle.digest('SHA-256', enc);
|
||||
return Array.from(new Uint8Array(buf)).map(b => b.toString(16).padStart(2, '0')).join('');
|
||||
}
|
||||
let h = 2166136261 >>> 0;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
h ^= input.charCodeAt(i);
|
||||
h = Math.imul(h, 16777619);
|
||||
}
|
||||
return (h >>> 0).toString(16).padStart(8, '0');
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a compact knowledge-base context string to inject into the system prompt.
|
||||
* Reads topics + relations from PocketBase via db.js.
|
||||
* Topic-level content is loaded only when a topic id/label appears in the user's message.
|
||||
* Build a retrieval-scoped KB context. Instead of dumping the whole graph,
|
||||
* we pick the top-K topics by TF-IDF over `userMessage`, plus any topic
|
||||
* whose id or label appears verbatim in the message. Relations are filtered
|
||||
* to those that touch the included set.
|
||||
*
|
||||
* Returns { context: string, topics: Array } so callers can reuse the fetched topics
|
||||
* for validateDelta without a second round-trip.
|
||||
* A `[kb_hash: …]` suffix is appended so the Anthropic ephemeral prompt
|
||||
* cache automatically busts when topics are added/removed.
|
||||
*
|
||||
* Returns:
|
||||
* { context, retrievedTopics, allTopics }
|
||||
* — `allTopics` is the full PocketBase list so callers can still run
|
||||
* `validateDelta` against the entire current graph.
|
||||
*/
|
||||
export async function buildKbContext(userMessage = '') {
|
||||
const [topics, relations] = await Promise.all([
|
||||
const [allTopics, allRelations] = await Promise.all([
|
||||
db.getTopics(),
|
||||
db.getRelations(),
|
||||
]);
|
||||
|
||||
if (topics.length === 0) {
|
||||
const sortedIds = allTopics.map(t => t.id).sort().join('|');
|
||||
const fullHash = await sha256Hex(sortedIds);
|
||||
const kbHash = fullHash.slice(0, 8);
|
||||
|
||||
if (allTopics.length === 0) {
|
||||
return {
|
||||
context: 'KENNISGRAAF: (leeg — er zijn nog geen onderwerpen geëxtraheerd)',
|
||||
topics: [],
|
||||
context: `KENNISGRAAF: (leeg — er zijn nog geen onderwerpen geëxtraheerd)\n[kb_hash: ${kbHash}]`,
|
||||
retrievedTopics: [],
|
||||
allTopics: [],
|
||||
};
|
||||
}
|
||||
|
||||
const topicLines = topics.map(t => {
|
||||
const lowered = userMessage.toLowerCase();
|
||||
const mentionedIds = new Set();
|
||||
for (const t of allTopics) {
|
||||
const idHit = t.id && lowered.includes(t.id.toLowerCase());
|
||||
const labelHit = t.label && lowered.includes(t.label.toLowerCase());
|
||||
if (idHit || labelHit) mentionedIds.add(t.id);
|
||||
}
|
||||
|
||||
const index = buildIndex(allTopics);
|
||||
const retrieved = retrieveTopK(index, userMessage, TOP_K);
|
||||
|
||||
const includedById = new Map();
|
||||
for (const id of mentionedIds) {
|
||||
const t = allTopics.find(x => x.id === id);
|
||||
if (t) includedById.set(id, t);
|
||||
}
|
||||
for (const t of retrieved) {
|
||||
if (!includedById.has(t.id)) includedById.set(t.id, t);
|
||||
}
|
||||
const included = [...includedById.values()];
|
||||
|
||||
const topicLines = included.map(t => {
|
||||
const desc = (t.description || '').replace(/\s+/g, ' ').trim().slice(0, 200);
|
||||
return `- ${t.id} (${t.type || 'concept'}) "${t.label}": ${desc}`;
|
||||
});
|
||||
|
||||
const relLines = relations.map(r => {
|
||||
const includedIds = new Set(included.map(t => t.id));
|
||||
const relLines = [];
|
||||
for (const r of allRelations) {
|
||||
const src = typeof r.source === 'object' ? r.source.id : r.source;
|
||||
const tgt = typeof r.target === 'object' ? r.target.id : r.target;
|
||||
return `- ${src} --${r.type}--> ${tgt}`;
|
||||
});
|
||||
if (includedIds.has(src) && includedIds.has(tgt)) {
|
||||
relLines.push(`- ${src} --${r.type}--> ${tgt}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Pull deep content for any topic explicitly mentioned in the user message.
|
||||
const lowered = userMessage.toLowerCase();
|
||||
const mentionedDeepContent = [];
|
||||
for (const t of topics) {
|
||||
const idHit = t.id && lowered.includes(t.id.toLowerCase());
|
||||
const labelHit = t.label && lowered.includes(t.label.toLowerCase());
|
||||
if (idHit || labelHit) {
|
||||
for (const id of mentionedIds) {
|
||||
const t = includedById.get(id);
|
||||
if (!t) continue;
|
||||
const content = await db.getContent(t.id).catch(() => null);
|
||||
if (content) {
|
||||
if (!content) continue;
|
||||
let raw;
|
||||
if (typeof content === 'string') raw = content;
|
||||
else if (content.article) raw = content.article;
|
||||
else if (content.article) raw = typeof content.article === 'string' ? content.article : JSON.stringify(content.article);
|
||||
else raw = JSON.stringify(content);
|
||||
const snippet = raw.replace(/\s+/g, ' ').trim().slice(0, 1200);
|
||||
mentionedDeepContent.push(`### ${t.label}\n${snippet}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const context = [
|
||||
`KENNISGRAAF — TOPICS:`,
|
||||
`KENNISGRAAF — RELEVANTE TOPICS (top ${included.length} van ${allTopics.length}):`,
|
||||
topicLines.join('\n'),
|
||||
``,
|
||||
`KENNISGRAAF — RELATIES:`,
|
||||
relLines.length ? relLines.join('\n') : '(geen relaties)',
|
||||
`KENNISGRAAF — RELATIES (binnen deze selectie):`,
|
||||
relLines.length ? relLines.join('\n') : '(geen relaties binnen deze selectie)',
|
||||
mentionedDeepContent.length
|
||||
? `\n\nDIEPERE INHOUD (voor genoemde topics):\n${mentionedDeepContent.join('\n\n')}`
|
||||
: '',
|
||||
``,
|
||||
`Als de relevante context hierboven te beperkt is, gebruik dan de tool "lookup_topic" om de volledige beschrijving en eventuele leerinhoud van een specifiek topic op te halen.`,
|
||||
`[kb_hash: ${kbHash}]`,
|
||||
].join('\n');
|
||||
|
||||
return { context, topics };
|
||||
return { context, retrievedTopics: included, allTopics };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,17 +1,59 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { storage } from '../../lib/storage';
|
||||
import { anthropicApi } from '../../lib/api';
|
||||
import { callLLM } from '../../lib/llm';
|
||||
import * as db from '../../lib/db';
|
||||
import { buildKbContext, validateDelta, deltaKey } from './rag';
|
||||
import { buildSystemPrompt, PROPOSE_GRAPH_DELTA_TOOL, STRINGS } from './prompts';
|
||||
import {
|
||||
buildSystemPrompt,
|
||||
PROPOSE_GRAPH_DELTA_TOOL,
|
||||
LOOKUP_TOPIC_TOOL,
|
||||
STRINGS,
|
||||
} from './prompts';
|
||||
|
||||
const MAX_HISTORY = 50;
|
||||
const VERBATIM_TURNS = 12;
|
||||
const MAX_LOOKUP_HOPS = 3;
|
||||
const TRUNCATION_NOTICE = '(earlier conversation truncated)';
|
||||
|
||||
/** Trim API history to the last N turns and prepend a truncation notice. */
|
||||
function truncateApiMessages(history) {
|
||||
if (history.length <= VERBATIM_TURNS) return history;
|
||||
const tail = history.slice(-VERBATIM_TURNS);
|
||||
return [{ role: 'assistant', content: TRUNCATION_NOTICE }, ...tail];
|
||||
}
|
||||
|
||||
async function resolveLookupTopic(id, allTopics) {
|
||||
const topic = allTopics.find(t => t.id === id);
|
||||
if (!topic) {
|
||||
return { ok: false, payload: `Geen topic gevonden met id "${id}".` };
|
||||
}
|
||||
const content = await db.getContent(id).catch(() => null);
|
||||
let contentSnippet = null;
|
||||
if (content) {
|
||||
let raw;
|
||||
if (typeof content === 'string') raw = content;
|
||||
else if (content.article) raw = typeof content.article === 'string' ? content.article : JSON.stringify(content.article);
|
||||
else raw = JSON.stringify(content);
|
||||
contentSnippet = raw.replace(/\s+/g, ' ').trim().slice(0, 2400);
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
payload: {
|
||||
id: topic.id,
|
||||
label: topic.label,
|
||||
type: topic.type || 'concept',
|
||||
description: topic.description || '',
|
||||
learning_content: contentSnippet,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Conversation hook for R42.
|
||||
* Owns the message list, persists to chat:thread:{userId}, calls Anthropic,
|
||||
* and surfaces validated graph delta suggestions inline.
|
||||
*
|
||||
* Note: buildKbContext is async (reads PocketBase), so send() is fully async.
|
||||
* Owns the message list, persists to chat:thread:{userId}, calls Anthropic
|
||||
* via the shared `callLLM` client (with a `lookup_topic` multi-hop loop and
|
||||
* the `propose_graph_delta` tool), and surfaces validated graph delta
|
||||
* suggestions inline.
|
||||
*/
|
||||
export function useChat({ user, isAdmin }) {
|
||||
const threadKey = user ? `chat:thread:${user.id}` : null;
|
||||
@@ -20,7 +62,6 @@ export function useChat({ user, isAdmin }) {
|
||||
const [errored, setErrored] = useState(false);
|
||||
const seenDeltaKeys = useRef(new Set());
|
||||
|
||||
// Load persisted thread + seed greeting
|
||||
useEffect(() => {
|
||||
if (!user) return;
|
||||
const stored = storage.get(threadKey, null);
|
||||
@@ -41,7 +82,6 @@ export function useChat({ user, isAdmin }) {
|
||||
}
|
||||
}, [user, threadKey]);
|
||||
|
||||
// Persist on change
|
||||
useEffect(() => {
|
||||
if (!threadKey) return;
|
||||
const capped = messages.slice(-MAX_HISTORY);
|
||||
@@ -68,29 +108,43 @@ export function useChat({ user, isAdmin }) {
|
||||
setErrored(false);
|
||||
|
||||
try {
|
||||
const { context: kbContext, topics: kbTopics } = await buildKbContext(trimmed);
|
||||
const { context: kbContext, allTopics } = await buildKbContext(trimmed);
|
||||
const systemPrompt = buildSystemPrompt({
|
||||
userName: user.name || 'daar',
|
||||
isAdmin,
|
||||
kbContext,
|
||||
});
|
||||
|
||||
// Strip UI-only fields before sending to Anthropic
|
||||
const apiMessages = next
|
||||
const historyMessages = next
|
||||
.filter(m => m.role === 'user' || m.role === 'assistant')
|
||||
.map(m => ({ role: m.role, content: m.content }));
|
||||
|
||||
const response = await anthropicApi.chat(systemPrompt, apiMessages, {
|
||||
tools: [PROPOSE_GRAPH_DELTA_TOOL],
|
||||
});
|
||||
const apiMessages = truncateApiMessages(historyMessages);
|
||||
|
||||
let textOut = '';
|
||||
let suggestion = null;
|
||||
for (const block of response.content || []) {
|
||||
if (block.type === 'text') {
|
||||
textOut += (textOut ? '\n' : '') + (block.text || '');
|
||||
} else if (block.type === 'tool_use' && block.name === PROPOSE_GRAPH_DELTA_TOOL.name) {
|
||||
const validated = validateDelta(block.input, kbTopics);
|
||||
let hops = 0;
|
||||
|
||||
while (true) {
|
||||
const response = await callLLM({
|
||||
task: 'chat.r42',
|
||||
tier: 'standard',
|
||||
system: systemPrompt,
|
||||
messages: apiMessages,
|
||||
tools: [LOOKUP_TOPIC_TOOL, PROPOSE_GRAPH_DELTA_TOOL],
|
||||
maxTokens: 2048,
|
||||
temperature: 0.3,
|
||||
});
|
||||
|
||||
if (response.text) {
|
||||
textOut += (textOut ? '\n' : '') + response.text;
|
||||
}
|
||||
|
||||
const lookupCalls = response.toolUses.filter(tu => tu.name === LOOKUP_TOPIC_TOOL.name);
|
||||
const deltaCall = response.toolUses.find(tu => tu.name === PROPOSE_GRAPH_DELTA_TOOL.name);
|
||||
|
||||
if (deltaCall && !suggestion) {
|
||||
const validated = validateDelta(deltaCall.input, allTopics);
|
||||
if (validated) {
|
||||
const key = deltaKey(validated);
|
||||
if (!seenDeltaKeys.current.has(key)) {
|
||||
@@ -99,6 +153,33 @@ export function useChat({ user, isAdmin }) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (lookupCalls.length === 0 || hops >= MAX_LOOKUP_HOPS) break;
|
||||
hops++;
|
||||
|
||||
const assistantBlocks = [];
|
||||
if (response.text) assistantBlocks.push({ type: 'text', text: response.text });
|
||||
for (const tu of response.toolUses) {
|
||||
if (!tu.id) continue;
|
||||
assistantBlocks.push({ type: 'tool_use', id: tu.id, name: tu.name, input: tu.input });
|
||||
}
|
||||
apiMessages.push({ role: 'assistant', content: assistantBlocks });
|
||||
|
||||
const toolResults = [];
|
||||
for (const tu of lookupCalls) {
|
||||
if (!tu.id) continue;
|
||||
const topicId = String(tu.input?.id || '').trim();
|
||||
const resolved = await resolveLookupTopic(topicId, allTopics);
|
||||
toolResults.push({
|
||||
type: 'tool_result',
|
||||
tool_use_id: tu.id,
|
||||
content: typeof resolved.payload === 'string'
|
||||
? resolved.payload
|
||||
: JSON.stringify(resolved.payload),
|
||||
...(resolved.ok ? {} : { is_error: true }),
|
||||
});
|
||||
}
|
||||
apiMessages.push({ role: 'user', content: toolResults });
|
||||
}
|
||||
|
||||
const assistantMsg = {
|
||||
|
||||
58
src/lib/__tests__/retrieval.test.js
Normal file
58
src/lib/__tests__/retrieval.test.js
Normal file
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { buildIndex, retrieveTopK, tokenize } from '../retrieval';
|
||||
|
||||
const sampleTopics = [
|
||||
{ id: 'software-engineer', label: 'Software Engineer', description: 'Bouwt en onderhoudt applicaties; werkt in agile teams.' },
|
||||
{ id: 'onboarding-buddy', label: 'Onboarding Buddy', description: 'Begeleidt nieuwe medewerkers in hun eerste weken.' },
|
||||
{ id: 'kennisbeheer', label: 'Kennisbeheer', description: 'Het proces van het vastleggen en ontsluiten van organisatiekennis.' },
|
||||
{ id: 'wekelijkse-sessie', label: 'Wekelijkse Leersessie', description: 'Microlearning sessie waarin medewerkers wekelijks leren via AI-gegenereerde quizzen.' },
|
||||
];
|
||||
|
||||
describe('tokenize', () => {
|
||||
it('lowercases and splits on non-alphanumeric', () => {
|
||||
expect(tokenize('Hello, World!')).toEqual(['hello', 'world']);
|
||||
});
|
||||
|
||||
it('drops stopwords and short tokens', () => {
|
||||
expect(tokenize('de software engineer is hier')).toEqual(['software', 'engineer', 'hier']);
|
||||
});
|
||||
|
||||
it('keeps hyphenated identifiers', () => {
|
||||
expect(tokenize('software-engineer onboarding-buddy')).toEqual(['software-engineer', 'onboarding-buddy']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildIndex / retrieveTopK', () => {
|
||||
it('returns empty for empty topics', () => {
|
||||
const idx = buildIndex([]);
|
||||
expect(retrieveTopK(idx, 'anything')).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty for empty query', () => {
|
||||
const idx = buildIndex(sampleTopics);
|
||||
expect(retrieveTopK(idx, '')).toEqual([]);
|
||||
});
|
||||
|
||||
it('ranks the most relevant topic first', () => {
|
||||
const idx = buildIndex(sampleTopics);
|
||||
const hits = retrieveTopK(idx, 'wat doet een onboarding buddy?', 2);
|
||||
expect(hits[0].id).toBe('onboarding-buddy');
|
||||
});
|
||||
|
||||
it('matches on description when label does not contain query terms', () => {
|
||||
const idx = buildIndex(sampleTopics);
|
||||
const hits = retrieveTopK(idx, 'microlearning quizzen', 3);
|
||||
expect(hits.map(h => h.id)).toContain('wekelijkse-sessie');
|
||||
});
|
||||
|
||||
it('returns no hits when no terms match', () => {
|
||||
const idx = buildIndex(sampleTopics);
|
||||
expect(retrieveTopK(idx, 'kwantumfysica raketten')).toEqual([]);
|
||||
});
|
||||
|
||||
it('caches the index per topics array reference', () => {
|
||||
const idx1 = buildIndex(sampleTopics);
|
||||
const idx2 = buildIndex(sampleTopics);
|
||||
expect(idx1).toBe(idx2);
|
||||
});
|
||||
});
|
||||
@@ -327,6 +327,17 @@ export async function bulkSetCurriculum(year, weeks) {
|
||||
);
|
||||
}
|
||||
|
||||
// ── LLM Call Telemetry ───────────────────────────────────────────────────────
|
||||
|
||||
export async function getRecentLlmCalls(limit = 100) {
|
||||
try {
|
||||
const r = await pb.collection('llm_calls').getList(1, limit, { sort: '-created' });
|
||||
return r.items;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// ── Handbook Sync State ───────────────────────────────────────────────────────
|
||||
|
||||
export async function getHandbookSyncStates() {
|
||||
|
||||
@@ -236,7 +236,7 @@ function extractToolUses(content) {
|
||||
if (!Array.isArray(content)) return [];
|
||||
return content
|
||||
.filter((b) => b?.type === 'tool_use')
|
||||
.map((b) => ({ name: b.name, input: b.input }));
|
||||
.map((b) => ({ id: b.id, name: b.name, input: b.input }));
|
||||
}
|
||||
|
||||
function extractText(content) {
|
||||
|
||||
95
src/lib/retrieval.js
Normal file
95
src/lib/retrieval.js
Normal file
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Lightweight, dependency-free TF-IDF retrieval over the knowledge graph.
|
||||
*
|
||||
* `buildIndex(topics)` tokenises the `label + description` of each topic and
|
||||
* computes document-frequency stats so queries can be scored with TF-IDF in
|
||||
* `retrieveTopK`. The index is cached against the `topics` array reference,
|
||||
* so repeated calls with the same array don't rebuild.
|
||||
*
|
||||
* Tokeniser: lowercase, split on `[^a-zA-Z0-9-]`, drop short tokens and a
|
||||
* small Dutch/English stopword list.
|
||||
*/
|
||||
|
||||
const STOPWORDS = new Set([
|
||||
// English
|
||||
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from', 'has', 'have',
|
||||
'how', 'i', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'than', 'that', 'the',
|
||||
'this', 'to', 'was', 'were', 'what', 'when', 'where', 'which', 'who', 'why',
|
||||
'with', 'you', 'your', 'do', 'does', 'did',
|
||||
// Dutch
|
||||
'de', 'het', 'een', 'en', 'of', 'in', 'op', 'aan', 'bij', 'voor', 'naar',
|
||||
'met', 'uit', 'om', 'door', 'over', 'tegen', 'ook', 'er', 'is', 'zijn',
|
||||
'was', 'waren', 'wat', 'wie', 'hoe', 'waar', 'wanneer', 'welke', 'die',
|
||||
'dat', 'deze', 'dit', 'ik', 'jij', 'hij', 'zij', 'we', 'wij', 'jullie',
|
||||
'als', 'dan', 'maar', 'want', 'omdat', 'niet', 'wel', 'heeft', 'hebben',
|
||||
'word', 'wordt', 'worden', 'kan', 'kunnen', 'mag', 'moet', 'moeten',
|
||||
'zal', 'zou', 'zouden', 'al', 'ook', 'nog', 'naar',
|
||||
]);
|
||||
|
||||
export function tokenize(text) {
|
||||
if (!text) return [];
|
||||
return String(text)
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9-]+/i)
|
||||
.filter(t => t.length >= 2 && !STOPWORDS.has(t));
|
||||
}
|
||||
|
||||
const indexCache = new WeakMap();
|
||||
|
||||
export function buildIndex(topics) {
|
||||
if (!Array.isArray(topics) || topics.length === 0) {
|
||||
return { topics: [], docFreq: new Map(), termsByDoc: [], N: 0 };
|
||||
}
|
||||
const cached = indexCache.get(topics);
|
||||
if (cached) return cached;
|
||||
|
||||
const termsByDoc = topics.map(t => {
|
||||
const text = `${t.label || ''} ${t.description || ''}`;
|
||||
const tokens = tokenize(text);
|
||||
const tf = new Map();
|
||||
for (const tk of tokens) tf.set(tk, (tf.get(tk) || 0) + 1);
|
||||
return tf;
|
||||
});
|
||||
|
||||
const docFreq = new Map();
|
||||
for (const tf of termsByDoc) {
|
||||
for (const term of tf.keys()) {
|
||||
docFreq.set(term, (docFreq.get(term) || 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
const index = { topics, docFreq, termsByDoc, N: topics.length };
|
||||
indexCache.set(topics, index);
|
||||
return index;
|
||||
}
|
||||
|
||||
export function retrieveTopK(index, query, k = 10) {
|
||||
if (!index || !index.N || !query) return [];
|
||||
const qTokens = tokenize(query);
|
||||
if (qTokens.length === 0) return [];
|
||||
|
||||
const idf = (term) => {
|
||||
const df = index.docFreq.get(term) || 0;
|
||||
if (df === 0) return 0;
|
||||
return Math.log((index.N + 1) / (df + 1)) + 1;
|
||||
};
|
||||
|
||||
const scores = new Array(index.N);
|
||||
for (let i = 0; i < index.N; i++) {
|
||||
const tf = index.termsByDoc[i];
|
||||
let s = 0;
|
||||
for (const t of qTokens) {
|
||||
const f = tf.get(t);
|
||||
if (!f) continue;
|
||||
s += (1 + Math.log(f)) * idf(t);
|
||||
}
|
||||
scores[i] = s;
|
||||
}
|
||||
|
||||
const ranked = [];
|
||||
for (let i = 0; i < index.N; i++) {
|
||||
if (scores[i] > 0) ranked.push({ i, s: scores[i] });
|
||||
}
|
||||
ranked.sort((a, b) => b.s - a.s);
|
||||
return ranked.slice(0, k).map(r => index.topics[r.i]);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { Database, FileText, Settings, Users, Network, Clock, CheckCircle2, AlertCircle, Save, Info, Layers, CheckSquare, CalendarDays } from 'lucide-react';
|
||||
import { Database, FileText, Settings, Users, Network, Clock, CheckCircle2, AlertCircle, Save, Info, Layers, CheckSquare, CalendarDays, Activity } from 'lucide-react';
|
||||
import Card from '../../components/ui/Card';
|
||||
import Tag from '../../components/ui/Tag';
|
||||
import Button from '../../components/ui/Button';
|
||||
@@ -12,6 +12,7 @@ import ContentManager from '../../components/admin/ContentManager';
|
||||
import TestManager from '../../components/admin/TestManager';
|
||||
import TeamManager from '../../components/admin/TeamManager';
|
||||
import CurriculumManager from '../../components/admin/CurriculumManager';
|
||||
import Diagnostics from '../../components/admin/Diagnostics';
|
||||
import { Trash2 } from 'lucide-react';
|
||||
|
||||
const TIER_PLACEHOLDERS = {
|
||||
@@ -71,6 +72,7 @@ const Admin = () => {
|
||||
{ key: 'curriculum', icon: CalendarDays, label: 'Curriculum' },
|
||||
{ key: 'graph', icon: Network, label: 'Graph' },
|
||||
{ key: 'team', icon: Users, label: 'Team' },
|
||||
{ key: 'diagnostics', icon: Activity, label: 'Diagnostics' },
|
||||
{ key: 'settings', icon: Settings, label: 'Settings', bottom: true },
|
||||
];
|
||||
|
||||
@@ -181,6 +183,14 @@ const Admin = () => {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{activeTab === 'diagnostics' && (
|
||||
<div className="animate-in fade-in duration-300 max-w-6xl mx-auto">
|
||||
<h1 className="text-3xl text-teal mb-2">Diagnostics</h1>
|
||||
<p className="text-fg-muted mb-8">LLM-aanroepen, tokenverbruik en geschatte kosten.</p>
|
||||
<Diagnostics />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{activeTab === 'settings' && (
|
||||
<div className="animate-in fade-in duration-300 max-w-2xl">
|
||||
<h1 className="text-3xl text-teal mb-2">Settings</h1>
|
||||
|
||||
Reference in New Issue
Block a user