learning-platform/src/components/chat/rag.js

import * as db from '../../lib/db';
import { buildIndex, retrieveTopK } from '../../lib/retrieval';

const TOP_K = 10;
// How many topics get their full article body injected (not just the short
// description). Verbatim-mentioned topics come first, then the highest-ranked
// retrieved ones, so a query that never names a topic exactly still gets rich
// content for what it matched.
const DEEP_CONTENT_LIMIT = 5;
const DEEP_SNIPPET_CHARS = 1000;
const DESC_SNIPPET_CHARS = 320;

async function sha256Hex(input) {
  const enc = new TextEncoder().encode(input);
  if (globalThis.crypto?.subtle?.digest) {
    const buf = await globalThis.crypto.subtle.digest('SHA-256', enc);
    return Array.from(new Uint8Array(buf)).map(b => b.toString(16).padStart(2, '0')).join('');
  }
  let h = 2166136261 >>> 0;
  for (let i = 0; i < input.length; i++) {
    h ^= input.charCodeAt(i);
    h = Math.imul(h, 16777619);
  }
  return (h >>> 0).toString(16).padStart(8, '0');
}

/**
 * Build a retrieval-scoped KB context. Instead of dumping the whole graph,
 * we pick the top-K topics by TF-IDF over `userMessage`, plus any topic
 * whose id or label appears verbatim in the message. Relations are filtered
 * to those that touch the included set.
 *
 * A `[kb_hash: …]` suffix is appended so the Anthropic ephemeral prompt
 * cache automatically busts when topics are added/removed.
 *
 * Returns:
 *   { context, retrievedTopics, allTopics }
 * — `allTopics` is the full PocketBase list so callers can still run
 * `validateDelta` against the entire current graph.
 */
export async function buildKbContext(userMessage = '') {
  const [allTopics, allRelations] = await Promise.all([
    db.getTopics(),
    db.getRelations(),
  ]);

  const sortedIds = allTopics.map(t => t.id).sort().join('|');
  const fullHash = await sha256Hex(sortedIds);
  const kbHash = fullHash.slice(0, 8);

  if (allTopics.length === 0) {
    return {
      context: `KENNISGRAAF: (leeg — er zijn nog geen onderwerpen geëxtraheerd)\n[kb_hash: ${kbHash}]`,
      retrievedTopics: [],
      allTopics: [],
    };
  }

  const lowered = userMessage.toLowerCase();
  const mentionedIds = new Set();
  for (const t of allTopics) {
    const idHit = t.id && lowered.includes(t.id.toLowerCase());
    const labelHit = t.label && lowered.includes(t.label.toLowerCase());
    if (idHit || labelHit) mentionedIds.add(t.id);
  }

  const index = buildIndex(allTopics);
  const retrieved = retrieveTopK(index, userMessage, TOP_K);

  const includedById = new Map();
  for (const id of mentionedIds) {
    const t = allTopics.find(x => x.id === id);
    if (t) includedById.set(id, t);
  }
  for (const t of retrieved) {
    if (!includedById.has(t.id)) includedById.set(t.id, t);
  }
  const included = [...includedById.values()];

  const topicLines = included.map(t => {
    const desc = (t.description || '').replace(/\s+/g, ' ').trim().slice(0, DESC_SNIPPET_CHARS);
    return `- ${t.id} (${t.type || 'concept'}) "${t.label}": ${desc}`;
  });

  const includedIds = new Set(included.map(t => t.id));
  const relLines = [];
  for (const r of allRelations) {
    const src = typeof r.source === 'object' ? r.source.id : r.source;
    const tgt = typeof r.target === 'object' ? r.target.id : r.target;
    if (includedIds.has(src) && includedIds.has(tgt)) {
      relLines.push(`- ${src} --${r.type}--> ${tgt}`);
    }
  }

  // Pick which topics get their full article body: verbatim mentions first,
  // then the highest-ranked retrieved topics, capped at DEEP_CONTENT_LIMIT.
  const deepIds = [];
  for (const id of mentionedIds) deepIds.push(id);
  for (const t of retrieved) {
    if (deepIds.length >= DEEP_CONTENT_LIMIT) break;
    if (!mentionedIds.has(t.id)) deepIds.push(t.id);
  }
  const deepBlocks = await Promise.all(
    deepIds.slice(0, DEEP_CONTENT_LIMIT).map(async (id) => {
      const t = includedById.get(id);
      if (!t) return null;
      const content = await db.getContent(id).catch(() => null);
      if (!content) return null;
      let raw;
      if (typeof content === 'string') raw = content;
      else if (content.article) raw = typeof content.article === 'string' ? content.article : JSON.stringify(content.article);
      else raw = JSON.stringify(content);
      const snippet = raw.replace(/\s+/g, ' ').trim().slice(0, DEEP_SNIPPET_CHARS);
      if (!snippet) return null;
      return `### ${t.label}\n${snippet}`;
    }),
  );
  const mentionedDeepContent = deepBlocks.filter(Boolean);

  const context = [
    `KENNISGRAAF — RELEVANTE TOPICS (top ${included.length} van ${allTopics.length}):`,
    topicLines.join('\n'),
    ``,
    `KENNISGRAAF — RELATIES (binnen deze selectie):`,
    relLines.length ? relLines.join('\n') : '(geen relaties binnen deze selectie)',
    mentionedDeepContent.length
      ? `\n\nDIEPERE INHOUD (volledige leerinhoud van de meest relevante topics — gebruik álle feiten hieruit die de vraag beantwoorden):\n${mentionedDeepContent.join('\n\n')}`
      : '',
    ``,
    `Als de relevante context hierboven te beperkt is, gebruik dan de tool "lookup_topic" om de volledige beschrijving en eventuele leerinhoud van een specifiek topic op te halen.`,
    `[kb_hash: ${kbHash}]`,
  ].join('\n');

  return { context, retrievedTopics: included, allTopics };
}

/**
 * Validate a delta proposal against the current topic list (already fetched).
 * Drops:
 *   - topics whose id already exists (by id or case-folded label)
 *   - relations whose source/target isn't in the current graph + this delta
 *   - self-referencing relations
 * Caps to 3 topics + 5 relations.
 *
 * @param {object} delta - Raw input from the propose_graph_delta tool call
 * @param {Array}  existingTopics - Topics already fetched from PocketBase
 */
export function validateDelta(delta, existingTopics = []) {
  if (!delta || typeof delta !== 'object') return null;

  const existingIds = new Set(existingTopics.map(t => t.id));
  const existingLabels = new Set(existingTopics.map(t => (t.label || '').toLowerCase()));

  const safeTopics = [];
  for (const t of Array.isArray(delta.topics) ? delta.topics : []) {
    if (!t || typeof t.id !== 'string' || typeof t.label !== 'string') continue;
    if (existingIds.has(t.id)) continue;
    if (existingLabels.has(t.label.toLowerCase())) continue;
    if (!['concept', 'role', 'process'].includes(t.type)) continue;
    safeTopics.push({
      id: t.id.trim(),
      label: t.label.trim(),
      type: t.type,
      description: (t.description || '').trim(),
    });
    existingIds.add(t.id);
    existingLabels.add(t.label.toLowerCase());
    if (safeTopics.length >= 3) break;
  }

  const knownAfter = new Set([...existingIds, ...safeTopics.map(t => t.id)]);
  const safeRelations = [];
  for (const r of Array.isArray(delta.relations) ? delta.relations : []) {
    if (!r || typeof r.source !== 'string' || typeof r.target !== 'string') continue;
    if (r.source === r.target) continue;
    if (!knownAfter.has(r.source) || !knownAfter.has(r.target)) continue;
    if (!['related_to', 'depends_on', 'part_of', 'executed_by'].includes(r.type)) continue;
    safeRelations.push({ source: r.source, target: r.target, type: r.type });
    if (safeRelations.length >= 5) break;
  }

  if (safeTopics.length === 0 && safeRelations.length === 0) return null;

  return {
    reason: typeof delta.reason === 'string' ? delta.reason : '',
    topics: safeTopics,
    relations: safeRelations,
  };
}

/** Stable key for a delta (used to dedupe within a thread). */
export function deltaKey(delta) {
  const t = delta.topics.map(x => x.id).sort().join(',');
  const r = delta.relations.map(x => `${x.source}-${x.type}->${x.target}`).sort().join(',');
  return `t:${t}|r:${r}`;
}