diff --git a/src/components/admin/KnowledgeGraph.jsx b/src/components/admin/KnowledgeGraph.jsx index d363226..7426f65 100644 --- a/src/components/admin/KnowledgeGraph.jsx +++ b/src/components/admin/KnowledgeGraph.jsx @@ -261,25 +261,53 @@ const KnowledgeGraph = () => { } }); - setSyncResult({ added, modified, unchanged }); + setSyncResult({ added, modified, unchanged, failed: [] }); const filesToProcess = [...added, ...modified]; if (filesToProcess.length > 0) { - setSyncProgress(`Processing 0 of ${filesToProcess.length} files...`); - let count = 0; - for (const file of filesToProcess) { - count++; - setSyncProgress(`Processing ${count} of ${filesToProcess.length}: ${file.name}...`); - try { - const rawContent = await getFileContent('respellion', 'employee-handbook', file.path); - // Pacing is handled centrally by extractionLimiter inside analyzeHandbookDelta. - await analyzeHandbookDelta(rawContent, file.path); - await db.updateHandbookSyncState(file.path, file.sha); - } catch (err) { - console.error('Failed to process file:', file.path, err); + const total = filesToProcess.length; + let done = 0; + const failed = []; + setSyncProgress(`Processing 0 of ${total} files...`); + + // Run files in parallel with bounded concurrency. The extractionLimiter + // inside analyzeHandbookDelta still governs the actual API request rate; + // parallelism here just hides GitHub fetch latency and overlaps with the + // limiter's spacing instead of waiting serially. + const CONCURRENCY = 4; + const queue = filesToProcess.slice(); + + async function worker() { + while (queue.length > 0) { + const file = queue.shift(); + if (!file) return; + try { + const rawContent = await getFileContent('respellion', 'employee-handbook', file.path); + // Skip near-empty files — they only burn an LLM call to extract nothing. + if (rawContent.trim().length < 50) { + await db.updateHandbookSyncState(file.path, file.sha); + } else { + await analyzeHandbookDelta(rawContent, file.path); + await db.updateHandbookSyncState(file.path, file.sha); + } + } catch (err) { + console.error('Failed to process file:', file.path, err); + failed.push({ path: file.path, message: err?.message || String(err) }); + } finally { + done++; + setSyncProgress(`Processing ${done} of ${total} files (${failed.length} failed)...`); + } } } - setSyncProgress('Sync Complete! Click "Analyze & Optimize Graph" above to clean up and merge.'); + + await Promise.all(Array.from({ length: Math.min(CONCURRENCY, total) }, worker)); + + setSyncResult({ added, modified, unchanged, failed }); + setSyncProgress( + failed.length === 0 + ? 'Sync Complete! Click "Analyze & Optimize Graph" above to clean up and merge.' + : `Sync finished with ${failed.length} failure(s). See console for details.` + ); reloadKb(); } @@ -461,6 +489,9 @@ ${JSON.stringify({ topics: compactTopics, relations: compactRelations })}`;
Added files: {syncResult.added.length}
Modified files: {syncResult.modified.length}
Unchanged: {syncResult.unchanged.length}
+ {syncResult.failed?.length > 0 && ( +Failed: {syncResult.failed.length}
+ )} )} {syncProgress && ( diff --git a/src/lib/extractionPipeline.js b/src/lib/extractionPipeline.js index 84bc259..0a4be2f 100644 --- a/src/lib/extractionPipeline.js +++ b/src/lib/extractionPipeline.js @@ -67,6 +67,7 @@ export async function analyzeHandbookDelta(fileContent, filePath, { signal } = { tools: [EMIT_HANDBOOK_DELTA_TOOL], toolChoice: { type: 'tool', name: EMIT_HANDBOOK_DELTA_TOOL.name }, maxTokens: 8192, + timeoutMs: 180_000, limiter: extractionLimiter, signal, }); diff --git a/src/lib/llm.js b/src/lib/llm.js index 59883ca..5ae41a0 100644 --- a/src/lib/llm.js +++ b/src/lib/llm.js @@ -112,9 +112,16 @@ function buildMessages({ messages, user }) { throw new Error('callLLM requires either `messages` or `user`.'); } +// Telemetry collection is optional. If the migration hasn't been applied on a +// given deploy, the first POST returns 404; we then disable further attempts +// to keep the console clean and avoid wasted round-trips. +let llmCallsDisabled = false; function logLlmCall(record) { + if (llmCallsDisabled) return; try { - pb.collection('llm_calls').create(record).catch(() => {}); + pb.collection('llm_calls').create(record).catch((err) => { + if (err?.status === 404) llmCallsDisabled = true; + }); } catch { /* collection may not exist yet — swallow */ } @@ -271,6 +278,7 @@ function validateToolInputs(toolUses, task, toolSchemas) { * @property {Record