From caaf2b9eba684201495f5068de3f29f71412f427 Mon Sep 17 00:00:00 2001
From: RaymondVerhoef <raymondverhoef@gmail.com>
Date: Wed, 20 May 2026 09:29:21 +0200
Subject: [PATCH] feat: implement extraction pipeline for knowledge graph
 generation from text and handbook updates

---
 src/lib/extractionPipeline.js | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/lib/extractionPipeline.js b/src/lib/extractionPipeline.js
index ea10f75..daae1e4 100644
--- a/src/lib/extractionPipeline.js
+++ b/src/lib/extractionPipeline.js
@@ -4,7 +4,13 @@ import * as db from './db';
 const SYSTEM_PROMPT = `You are an AI knowledge extractor for Respellion, an IT company built on radical transparency.
 You receive a source text. Your task is to extract all core concepts, roles, and processes from the text, and return them as a structured JSON Knowledge Graph.
 Facts should be integrated into the descriptions of the other labels and NOT be extracted as unique topics.
-CRITICAL: Extract all relevant topics and roles mentioned in the source text. Keep descriptions very concise.
+
+CRITICAL INSTRUCTIONS FOR COMPLETENESS:
+- You must extract EVERY SINGLE distinct role, process, and concept described or mentioned in the source text. 
+- DO NOT summarize, skip, truncate, or omit any items. 
+- If the document contains 29 roles, your JSON topics array must contain exactly 29 role topics. 
+- Completeness is of paramount importance. Failing to extract all topics will result in loss of critical company knowledge.
+- Keep descriptions concise (max 3 sentences) to ensure you have enough output tokens to list everything.
 
 You MUST assign a learning_relevance to each topic:
 - "core": Fundamental company knowledge.
@@ -16,7 +22,7 @@ ALWAYS return a valid JSON object in the following format:
 {
   "topics": [
     {
-      "id": "unique-slug",
+      "id": "a-unique-lowercase-kebab-case-slug-specific-to-this-topic (e.g., 'software-engineer' or 'data-quality-review'). DO NOT use generic IDs like 'role-1' or 'concept-2'.",
       "label": "Topic title",
       "type": "concept | role | process",
       "description": "A concise, clear explanation of max 3 sentences.",
@@ -31,7 +37,6 @@ ALWAYS return a valid JSON object in the following format:
     }
   ]
 }
-}
 Return JSON only. No markdown blocks or other text.`;
 
 const HANDBOOK_SYSTEM_PROMPT = `You are analyzing an update to the Respellion Employee Handbook.
@@ -94,6 +99,7 @@ export async function processSourceText(textContent, sourceName) {
 
   try {
     const responseText = await anthropicApi.generateContent(SYSTEM_PROMPT, `Analyze the following text:\n\n${textContent}`);
+    console.log('[Pipeline] Raw AI response:', responseText);
 
     let extractedData;
     try {