rve/learning-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: speed up handbook sync and stop llm_calls 404 noise

Browse Source 
Handbook sync ran files sequentially under a 5 req/min limiter with a
hardcoded 60s LLM timeout, causing long syncs and AbortError timeouts on
large files. Now: limiter at 20 req/min, files processed with concurrency
4, handbook extraction timeout raised to 180s, and near-empty files skip
the LLM call.

callLLM gains a timeoutMs option; passing a signal no longer silently
disables the per-request timeout.

llm_calls telemetry self-disables after the first 404 so deploys without
the migration applied don't spam the console.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
RaymondVerhoef
2026-05-22 16:35:56 +02:00
parent a38ad5d1e0
commit 25cae2fea9
4 changed files with 64 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
src/components/admin/KnowledgeGraph.jsx
View File

@@ -261,25 +261,53 @@ const KnowledgeGraph = () => {
}
});
setSyncResult({ added, modified, unchanged });
setSyncResult({ added, modified, unchanged, failed: [] });
const filesToProcess = [...added, ...modified];
if (filesToProcess.length > 0) {
setSyncProgress(`Processing 0 of ${filesToProcess.length} files...`);
let count = 0;
for (const file of filesToProcess) {
count++;
setSyncProgress(`Processing ${count} of ${filesToProcess.length}: ${file.name}...`);
try {
const rawContent = await getFileContent('respellion', 'employee-handbook', file.path);
// Pacing is handled centrally by extractionLimiter inside analyzeHandbookDelta.
await analyzeHandbookDelta(rawContent, file.path);
await db.updateHandbookSyncState(file.path, file.sha);
} catch (err) {
console.error('Failed to process file:', file.path, err);
const total = filesToProcess.length;
let done = 0;
const failed = [];
setSyncProgress(`Processing 0 of ${total} files...`);
// Run files in parallel with bounded concurrency. The extractionLimiter
// inside analyzeHandbookDelta still governs the actual API request rate;
// parallelism here just hides GitHub fetch latency and overlaps with the
// limiter's spacing instead of waiting serially.
const CONCURRENCY = 4;
const queue = filesToProcess.slice();
async function worker() {
while (queue.length > 0) {
const file = queue.shift();
if (!file) return;
try {
const rawContent = await getFileContent('respellion', 'employee-handbook', file.path);
// Skip near-empty files — they only burn an LLM call to extract nothing.
if (rawContent.trim().length < 50) {
await db.updateHandbookSyncState(file.path, file.sha);
} else {
await analyzeHandbookDelta(rawContent, file.path);
await db.updateHandbookSyncState(file.path, file.sha);
}
} catch (err) {
console.error('Failed to process file:', file.path, err);
failed.push({ path: file.path, message: err?.message || String(err) });
} finally {
done++;
setSyncProgress(`Processing ${done} of ${total} files (${failed.length} failed)...`);
}
}
}
setSyncProgress('Sync Complete! Click "Analyze & Optimize Graph" above to clean up and merge.');
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, total) }, worker));
setSyncResult({ added, modified, unchanged, failed });
setSyncProgress(
failed.length === 0
? 'Sync Complete! Click "Analyze & Optimize Graph" above to clean up and merge.'
: `Sync finished with ${failed.length} failure(s). See console for details.`
);
reloadKb();
}
@@ -461,6 +489,9 @@ ${JSON.stringify({ topics: compactTopics, relations: compactRelations })}`;
<p>Added files: {syncResult.added.length}</p>
<p>Modified files: {syncResult.modified.length}</p>
<p>Unchanged: {syncResult.unchanged.length}</p>
{syncResult.failed?.length > 0 && (
<p className="text-red-600">Failed: {syncResult.failed.length}</p>
)}
</div>
)}
{syncProgress && (