/**
 * Lightweight, dependency-free TF-IDF retrieval over the knowledge graph.
 *
 * `buildIndex(topics)` tokenises the `label + description` of each topic and
 * computes document-frequency stats so queries can be scored with TF-IDF in
 * `retrieveTopK`. The index is cached against the `topics` array reference,
 * so repeated calls with the same array don't rebuild.
 *
 * Tokeniser: lowercase, split on `[^a-zA-Z0-9-]`, drop short tokens and a
 * small Dutch/English stopword list.
 */

const STOPWORDS = new Set([
  // English
  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from', 'has', 'have',
  'how', 'i', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'than', 'that', 'the',
  'this', 'to', 'was', 'were', 'what', 'when', 'where', 'which', 'who', 'why',
  'with', 'you', 'your', 'do', 'does', 'did',
  // Dutch
  'de', 'het', 'een', 'en', 'of', 'in', 'op', 'aan', 'bij', 'voor', 'naar',
  'met', 'uit', 'om', 'door', 'over', 'tegen', 'ook', 'er', 'is', 'zijn',
  'was', 'waren', 'wat', 'wie', 'hoe', 'waar', 'wanneer', 'welke', 'die',
  'dat', 'deze', 'dit', 'ik', 'jij', 'hij', 'zij', 'we', 'wij', 'jullie',
  'als', 'dan', 'maar', 'want', 'omdat', 'niet', 'wel', 'heeft', 'hebben',
  'word', 'wordt', 'worden', 'kan', 'kunnen', 'mag', 'moet', 'moeten',
  'zal', 'zou', 'zouden', 'al', 'ook', 'nog', 'naar',
]);

export function tokenize(text) {
  if (!text) return [];
  return String(text)
    .toLowerCase()
    .split(/[^a-z0-9-]+/i)
    .filter(t => t.length >= 2 && !STOPWORDS.has(t));
}

const indexCache = new WeakMap();

export function buildIndex(topics) {
  if (!Array.isArray(topics) || topics.length === 0) {
    return { topics: [], docFreq: new Map(), termsByDoc: [], N: 0 };
  }
  const cached = indexCache.get(topics);
  if (cached) return cached;

  const termsByDoc = topics.map(t => {
    const text = `${t.label || ''} ${t.description || ''}`;
    const tokens = tokenize(text);
    const tf = new Map();
    for (const tk of tokens) tf.set(tk, (tf.get(tk) || 0) + 1);
    return tf;
  });

  const docFreq = new Map();
  for (const tf of termsByDoc) {
    for (const term of tf.keys()) {
      docFreq.set(term, (docFreq.get(term) || 0) + 1);
    }
  }

  const index = { topics, docFreq, termsByDoc, N: topics.length };
  indexCache.set(topics, index);
  return index;
}

export function retrieveTopK(index, query, k = 10) {
  if (!index || !index.N || !query) return [];
  const qTokens = tokenize(query);
  if (qTokens.length === 0) return [];

  const idf = (term) => {
    const df = index.docFreq.get(term) || 0;
    if (df === 0) return 0;
    return Math.log((index.N + 1) / (df + 1)) + 1;
  };

  const scores = new Array(index.N);
  for (let i = 0; i < index.N; i++) {
    const tf = index.termsByDoc[i];
    let s = 0;
    for (const t of qTokens) {
      const f = tf.get(t);
      if (!f) continue;
      s += (1 + Math.log(f)) * idf(t);
    }
    scores[i] = s;
  }

  const ranked = [];
  for (let i = 0; i < index.N; i++) {
    if (scores[i] > 0) ranked.push({ i, s: scores[i] });
  }
  ranked.sort((a, b) => b.s - a.s);
  return ranked.slice(0, k).map(r => index.topics[r.i]);
}
