feat: implement AI-driven knowledge extraction pipeline for company documentation

src/lib/extractionPipeline.js

@@ -84,6 +84,23 @@ export async function analyzeHandbookDelta(fileContent, filePath) {
     throw error;
   }
 }
+function chunkText(text, maxChunkSize = 4000) {
+  const paragraphs = text.split(/\n+/);
+  const chunks = [];
+  let currentChunk = '';
+
+  for (const para of paragraphs) {
+    if ((currentChunk + '\n' + para).length > maxChunkSize) {
+      if (currentChunk) chunks.push(currentChunk.trim());
+      currentChunk = para;
+    } else {
+      currentChunk = currentChunk ? currentChunk + '\n' + para : para;
+    }
+  }
+  if (currentChunk) chunks.push(currentChunk.trim());
+  return chunks;
+}
+
 export async function processSourceText(textContent, sourceName) {
   // Deduplicate: skip if a source with the same name was already successfully processed
   const existing = await db.getSources();
@@ -98,8 +115,24 @@ export async function processSourceText(textContent, sourceName) {
   const sourceId = rec.id;
 
   try {
-    const responseText = await anthropicApi.generateContent(SYSTEM_PROMPT, `Analyze the following text:\n\n${textContent}`);
+    const chunks = chunkText(textContent, 4000);
-    console.log('[Pipeline] Raw AI response:', responseText);
+    console.log(`[Pipeline] Split "${sourceName}" into ${chunks.length} chunks for processing.`);
+
+    let allExtractedTopics = [];
+    let allExtractedRelations = [];
+
+    for (let i = 0; i < chunks.length; i++) {
+      if (i > 0) {
+        console.log(`[Pipeline] Pacing delay (12s) to prevent rate limits before chunk ${i + 1}/${chunks.length}...`);
+        await new Promise(r => setTimeout(r, 12000));
+      }
+
+      console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`);
+      const responseText = await anthropicApi.generateContent(
+        SYSTEM_PROMPT, 
+        `Analyze this part of the document (${i + 1}/${chunks.length}):\n\n${chunks[i]}`
+      );
+      console.log(`[Pipeline] Raw AI response for chunk ${i + 1}:`, responseText);
 
       let extractedData;
       try {
@@ -107,14 +140,23 @@ export async function processSourceText(textContent, sourceName) {
         const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
         extractedData = JSON.parse(jsonStr);
       } catch (e) {
-      console.error('[Pipeline] AI returned non-JSON response:', responseText?.substring(0, 500));
+        console.error(`[Pipeline] AI returned non-JSON response for chunk ${i + 1}:`, responseText?.substring(0, 500));
-      throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`);
+        throw new Error(`AI response for chunk ${i + 1} was not valid JSON.`);
       }
 
-    await mergeKnowledgeGraph(extractedData);
+      if (extractedData.topics && Array.isArray(extractedData.topics)) {
+        allExtractedTopics.push(...extractedData.topics);
+      }
+      if (extractedData.relations && Array.isArray(extractedData.relations)) {
+        allExtractedRelations.push(...extractedData.relations);
+      }
+    }
+
+    // Merge everything together
+    await mergeKnowledgeGraph({ topics: allExtractedTopics, relations: allExtractedRelations });
     await db.updateSourceStatus(sourceId, 'completed');
 
-    return { success: true, data: extractedData };
+    return { success: true, data: { topics: allExtractedTopics, relations: allExtractedRelations } };
 
   } catch (error) {
     await db.updateSourceStatus(sourceId, 'failed', error.message);