refactor: remove handbook sync state and related functionality

src/lib/extractionPipeline.js

@@ -1,24 +1,21 @@
 import * as db from './db';
 import { callLLM } from './llm';
 import { extractionLimiter } from './llmRetry';
-import { EMIT_KNOWLEDGE_GRAPH_TOOL, EMIT_HANDBOOK_DELTA_TOOL } from './llmTools';
-import { normalizeHandbookResult } from './llmSchemas';
+import { EMIT_KNOWLEDGE_GRAPH_TOOL } from './llmTools';
 
-const MAX_KNOWN_IDS_HINT = 200;
+const MAX_KNOWN_TOPICS_HINT = 200;
 
 /**
- * Build the "already-extracted topic IDs" hint that prepends every chunk
- * after the first. Capped at the most-recent `MAX_KNOWN_IDS_HINT` IDs so
- * the prompt stays a bounded size; the model uses this list to reuse IDs
- * rather than invent variants like `software-developer` for
- * `software-engineer`.
+ * Build the "already-extracted topics" hint included in every chunk prompt.
+ * Passes both ID and label so the model can match concepts by name and reuse
+ * the exact ID + label rather than inventing near-duplicate variants.
  */
-export function buildKnownIdsHint(ids) {
-  if (!ids || !ids.length) return '';
-  const recent = ids.slice(-MAX_KNOWN_IDS_HINT);
+export function buildKnownIdsHint(topics) {
+  if (!topics || !topics.length) return '';
+  const recent = topics.slice(-MAX_KNOWN_TOPICS_HINT);
   return [
-    'Already-extracted topic IDs (do NOT create new IDs for these — reuse them if the same concept appears here):',
-    ...recent.map((id) => `- ${id}`),
+    'Already-extracted topics (reuse their ID and label exactly if the same concept appears here):',
+    ...recent.map((t) => `- ${t.id}: "${t.label}"`),
     '',
   ].join('\n');
 }
@@ -45,41 +42,8 @@ Assign a learning_relevance to every topic:
 Relation types: related_to | depends_on | part_of | executed_by.
 `;
 
-const HANDBOOK_SYSTEM_PROMPT = `You are analysing an update to the Respellion Employee Handbook. Emit the extracted topics and relations through the emit_handbook_delta tool.
-
-CRITICAL INSTRUCTIONS:
-- Every process must have a role attached. Express this as: process --executed_by--> role.
-- Every concept must connect to a process or role.
-- Mark handbook topics with metadata.source = "github_handbook".
-- Assign learning_relevance using the same scale as extraction: core | standard | peripheral | exclude.
-
-Relation types: related_to | depends_on | part_of | executed_by.
-`;
-
 const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }];
 
-export async function analyzeHandbookDelta(fileContent, filePath, { signal } = {}) {
-  const result = await callLLM({
-    task: 'extract.handbook',
-    tier: 'standard',
-    system: cachedSystem(HANDBOOK_SYSTEM_PROMPT),
-    user: `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`,
-    tools: [EMIT_HANDBOOK_DELTA_TOOL],
-    toolChoice: { type: 'tool', name: EMIT_HANDBOOK_DELTA_TOOL.name },
-    maxTokens: 8192,
-    timeoutMs: 180_000,
-    limiter: extractionLimiter,
-    signal,
-  });
-
-  const raw = result.toolUses[0]?.input;
-  if (!raw) throw new Error('Handbook extraction did not emit a tool result.');
-  const extractedData = normalizeHandbookResult(raw);
-
-  await mergeKnowledgeGraph(extractedData);
-  return { success: true, data: extractedData };
-}
-
 /**
  * Sentence-aware chunker with overlap.
  *
@@ -169,7 +133,7 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
     console.log(`[Pipeline] Split "${sourceName}" into ${chunks.length} chunks for processing.`);
 
     const existingTopics = await db.getTopics();
-    const knownIds = existingTopics.map((t) => t.id);
+    const knownTopics = existingTopics.map((t) => ({ id: t.id, label: t.label }));
 
     const allExtractedTopics = [];
     const allExtractedRelations = [];
@@ -178,7 +142,7 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
       if (signal?.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError');
 
       console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`);
-      const hint = i > 0 ? buildKnownIdsHint(knownIds) : '';
+      const hint = buildKnownIdsHint(knownTopics);
       const result = await callLLM({
         task: 'extract.source',
         tier: 'standard',
@@ -197,7 +161,9 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
       if (Array.isArray(extractedData.topics)) {
         allExtractedTopics.push(...extractedData.topics);
         for (const t of extractedData.topics) {
-          if (t?.id && !knownIds.includes(t.id)) knownIds.push(t.id);
+          if (t?.id && !knownTopics.some((k) => k.id === t.id)) {
+            knownTopics.push({ id: t.id, label: t.label });
+          }
         }
       }
       if (Array.isArray(extractedData.relations)) {