diff --git a/src/lib/extractionPipeline.js b/src/lib/extractionPipeline.js index daae1e4..7a33a30 100644 --- a/src/lib/extractionPipeline.js +++ b/src/lib/extractionPipeline.js @@ -84,6 +84,23 @@ export async function analyzeHandbookDelta(fileContent, filePath) { throw error; } } +function chunkText(text, maxChunkSize = 4000) { + const paragraphs = text.split(/\n+/); + const chunks = []; + let currentChunk = ''; + + for (const para of paragraphs) { + if ((currentChunk + '\n' + para).length > maxChunkSize) { + if (currentChunk) chunks.push(currentChunk.trim()); + currentChunk = para; + } else { + currentChunk = currentChunk ? currentChunk + '\n' + para : para; + } + } + if (currentChunk) chunks.push(currentChunk.trim()); + return chunks; +} + export async function processSourceText(textContent, sourceName) { // Deduplicate: skip if a source with the same name was already successfully processed const existing = await db.getSources(); @@ -98,23 +115,48 @@ export async function processSourceText(textContent, sourceName) { const sourceId = rec.id; try { - const responseText = await anthropicApi.generateContent(SYSTEM_PROMPT, `Analyze the following text:\n\n${textContent}`); - console.log('[Pipeline] Raw AI response:', responseText); + const chunks = chunkText(textContent, 4000); + console.log(`[Pipeline] Split "${sourceName}" into ${chunks.length} chunks for processing.`); - let extractedData; - try { - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - const jsonStr = jsonMatch ? jsonMatch[0] : responseText; - extractedData = JSON.parse(jsonStr); - } catch (e) { - console.error('[Pipeline] AI returned non-JSON response:', responseText?.substring(0, 500)); - throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`); + let allExtractedTopics = []; + let allExtractedRelations = []; + + for (let i = 0; i < chunks.length; i++) { + if (i > 0) { + console.log(`[Pipeline] Pacing delay (12s) to prevent rate limits before chunk ${i + 1}/${chunks.length}...`); + await new Promise(r => setTimeout(r, 12000)); + } + + console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`); + const responseText = await anthropicApi.generateContent( + SYSTEM_PROMPT, + `Analyze this part of the document (${i + 1}/${chunks.length}):\n\n${chunks[i]}` + ); + console.log(`[Pipeline] Raw AI response for chunk ${i + 1}:`, responseText); + + let extractedData; + try { + const jsonMatch = responseText.match(/\{[\s\S]*\}/); + const jsonStr = jsonMatch ? jsonMatch[0] : responseText; + extractedData = JSON.parse(jsonStr); + } catch (e) { + console.error(`[Pipeline] AI returned non-JSON response for chunk ${i + 1}:`, responseText?.substring(0, 500)); + throw new Error(`AI response for chunk ${i + 1} was not valid JSON.`); + } + + if (extractedData.topics && Array.isArray(extractedData.topics)) { + allExtractedTopics.push(...extractedData.topics); + } + if (extractedData.relations && Array.isArray(extractedData.relations)) { + allExtractedRelations.push(...extractedData.relations); + } } - await mergeKnowledgeGraph(extractedData); + // Merge everything together + await mergeKnowledgeGraph({ topics: allExtractedTopics, relations: allExtractedRelations }); await db.updateSourceStatus(sourceId, 'completed'); - return { success: true, data: extractedData }; + return { success: true, data: { topics: allExtractedTopics, relations: allExtractedRelations } }; } catch (error) { await db.updateSourceStatus(sourceId, 'failed', error.message);