refactor: remove handbook sync state and related functionality
This commit is contained in:
RaymondVerhoef
@@ -4,8 +4,6 @@ import { Trash2, Edit2, Save, X, RefreshCw, AlertCircle, Plus, Link as LinkIcon
|
||||
import * as db from '../../lib/db';
|
||||
import { callLLM } from '../../lib/llm';
|
||||
import { EMIT_GRAPH_ACTIONS_TOOL } from '../../lib/llmTools';
|
||||
import { analyzeHandbookDelta } from '../../lib/extractionPipeline';
|
||||
import { getRepoFolder, getFileContent } from '../../lib/githubService';
|
||||
import Button from '../ui/Button';
|
||||
import SuggestionsQueue from './SuggestionsQueue';
|
||||
|
||||
@@ -20,10 +18,6 @@ const KnowledgeGraph = () => {
|
||||
const [analyzeError, setAnalyzeError] = useState(null);
|
||||
const [newRelData, setNewRelData] = useState({ target: '', type: 'related_to' });
|
||||
|
||||
const [isSyncing, setIsSyncing] = useState(false);
|
||||
const [syncResult, setSyncResult] = useState(null);
|
||||
const [syncError, setSyncError] = useState(null);
|
||||
const [syncProgress, setSyncProgress] = useState(null);
|
||||
|
||||
const [topics, setTopics] = useState([]);
|
||||
const [relations, setRelations] = useState([]);
|
||||
@@ -237,87 +231,6 @@ const KnowledgeGraph = () => {
|
||||
)));
|
||||
};
|
||||
|
||||
const syncHandbook = async () => {
|
||||
setIsSyncing(true);
|
||||
setSyncError(null);
|
||||
setSyncResult(null);
|
||||
|
||||
try {
|
||||
const files = await getRepoFolder('respellion', 'employee-handbook', 'docs');
|
||||
const syncStates = await db.getHandbookSyncStates();
|
||||
|
||||
const added = [];
|
||||
const modified = [];
|
||||
const unchanged = [];
|
||||
|
||||
files.forEach(file => {
|
||||
const state = syncStates.find(s => s.path === file.path);
|
||||
if (!state) {
|
||||
added.push(file);
|
||||
} else if (state.sha !== file.sha) {
|
||||
modified.push(file);
|
||||
} else {
|
||||
unchanged.push(file);
|
||||
}
|
||||
});
|
||||
|
||||
setSyncResult({ added, modified, unchanged, failed: [] });
|
||||
|
||||
const filesToProcess = [...added, ...modified];
|
||||
if (filesToProcess.length > 0) {
|
||||
const total = filesToProcess.length;
|
||||
let done = 0;
|
||||
const failed = [];
|
||||
setSyncProgress(`Processing 0 of ${total} files...`);
|
||||
|
||||
// Run files in parallel with bounded concurrency. The extractionLimiter
|
||||
// inside analyzeHandbookDelta still governs the actual API request rate;
|
||||
// parallelism here just hides GitHub fetch latency and overlaps with the
|
||||
// limiter's spacing instead of waiting serially.
|
||||
const CONCURRENCY = 4;
|
||||
const queue = filesToProcess.slice();
|
||||
|
||||
async function worker() {
|
||||
while (queue.length > 0) {
|
||||
const file = queue.shift();
|
||||
if (!file) return;
|
||||
try {
|
||||
const rawContent = await getFileContent('respellion', 'employee-handbook', file.path);
|
||||
// Skip near-empty files — they only burn an LLM call to extract nothing.
|
||||
if (rawContent.trim().length < 50) {
|
||||
await db.updateHandbookSyncState(file.path, file.sha);
|
||||
} else {
|
||||
await analyzeHandbookDelta(rawContent, file.path);
|
||||
await db.updateHandbookSyncState(file.path, file.sha);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to process file:', file.path, err);
|
||||
failed.push({ path: file.path, message: err?.message || String(err) });
|
||||
} finally {
|
||||
done++;
|
||||
setSyncProgress(`Processing ${done} of ${total} files (${failed.length} failed)...`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, total) }, worker));
|
||||
|
||||
setSyncResult({ added, modified, unchanged, failed });
|
||||
setSyncProgress(
|
||||
failed.length === 0
|
||||
? 'Sync Complete! Click "Analyze & Optimize Graph" above to clean up and merge.'
|
||||
: `Sync finished with ${failed.length} failure(s). See console for details.`
|
||||
);
|
||||
reloadKb();
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
setSyncError(e.message || 'Failed to check GitHub for updates.');
|
||||
} finally {
|
||||
setIsSyncing(false);
|
||||
setTimeout(() => setSyncProgress(null), 10000); // Clear progress after 10s
|
||||
}
|
||||
};
|
||||
|
||||
const analyzeGraph = async () => {
|
||||
if (!confirm('This will send the entire graph to the AI to evaluate content, merge duplicates, and create new logical relations. This may take a moment. Proceed?')) return;
|
||||
@@ -466,43 +379,6 @@ ${JSON.stringify({ topics: compactTopics, relations: compactRelations })}`;
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<Button
|
||||
onClick={syncHandbook}
|
||||
disabled={isSyncing}
|
||||
variant="outline"
|
||||
className="w-full flex justify-center items-center gap-2"
|
||||
>
|
||||
<RefreshCw size={16} className={isSyncing ? "animate-spin" : ""} />
|
||||
{isSyncing ? 'Checking for updates...' : 'Sync Employee Handbook'}
|
||||
</Button>
|
||||
{syncError && (
|
||||
<p className="mt-2 text-xs text-red-600 flex items-start gap-1">
|
||||
<AlertCircle size={14} className="shrink-0 mt-0.5" />
|
||||
{syncError}
|
||||
</p>
|
||||
)}
|
||||
{syncResult && (
|
||||
<div className="mt-2 text-xs text-fg-muted space-y-1 p-2 bg-bg rounded">
|
||||
<p className="font-medium text-fg">GitHub Sync Status:</p>
|
||||
<p>Added files: {syncResult.added.length}</p>
|
||||
<p>Modified files: {syncResult.modified.length}</p>
|
||||
<p>Unchanged: {syncResult.unchanged.length}</p>
|
||||
{syncResult.failed?.length > 0 && (
|
||||
<p className="text-red-600">Failed: {syncResult.failed.length}</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{syncProgress && (
|
||||
<div className="mt-2 text-xs text-teal p-2 bg-teal/10 border border-teal/20 rounded">
|
||||
<p className="font-medium flex items-center gap-2">
|
||||
{syncProgress !== 'Sync Complete!' && <RefreshCw size={12} className="animate-spin" />}
|
||||
{syncProgress}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* R42 chatbot suggestions queue */}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState, useRef } from 'react';
|
||||
import { UploadCloud, AlertCircle, CheckCircle, X } from 'lucide-react';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { UploadCloud, CheckCircle, AlertCircle, Loader2, X, Clock, Minus } from 'lucide-react';
|
||||
import { processSourceText } from '../../lib/extractionPipeline';
|
||||
import Card from '../ui/Card';
|
||||
import Button from '../ui/Button';
|
||||
@@ -7,122 +7,187 @@ import Button from '../ui/Button';
|
||||
// ── Main Component ────────────────────────────────────────────────────────────
|
||||
|
||||
const UploadZone = ({ onUploadComplete }) => {
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [status, setStatus] = useState(null);
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const [queue, setQueue] = useState([]);
|
||||
const [processingId, setProcessingId] = useState(null);
|
||||
const [rejectedNote, setRejectedNote] = useState(null);
|
||||
|
||||
const fileInputRef = useRef(null);
|
||||
const abortRef = useRef(null);
|
||||
|
||||
// ── File upload (drag & drop / browse) ────────────────────────────────────
|
||||
// ── Processing loop ────────────────────────────────────────────────────────
|
||||
|
||||
useEffect(() => {
|
||||
if (processingId !== null) return;
|
||||
const next = queue.find((q) => q.status === 'pending');
|
||||
if (!next) return;
|
||||
|
||||
setProcessingId(next.id);
|
||||
setQueue((q) => q.map((item) => item.id === next.id ? { ...item, status: 'processing' } : item));
|
||||
|
||||
const processFile = async (file) => {
|
||||
setIsProcessing(true);
|
||||
setStatus(null);
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
try {
|
||||
const text = await file.text();
|
||||
await processSourceText(text, file.name, { signal: controller.signal });
|
||||
setStatus({ type: 'success', msg: `Successfully processed: ${file.name}` });
|
||||
if (onUploadComplete) onUploadComplete();
|
||||
} catch (error) {
|
||||
if (error?.name === 'AbortError') {
|
||||
setStatus({ type: 'error', msg: `Cancelled: ${file.name}` });
|
||||
|
||||
next.file.text()
|
||||
.then((text) => processSourceText(text, next.name, { signal: controller.signal }))
|
||||
.then(() => {
|
||||
setQueue((q) => q.map((item) => item.id === next.id ? { ...item, status: 'done' } : item));
|
||||
if (onUploadComplete) onUploadComplete();
|
||||
})
|
||||
.catch((err) => {
|
||||
const isCancelled = err?.name === 'AbortError';
|
||||
setQueue((q) => q.map((item) =>
|
||||
item.id === next.id
|
||||
? { ...item, status: isCancelled ? 'cancelled' : 'failed', error: isCancelled ? null : err.message }
|
||||
: item
|
||||
));
|
||||
})
|
||||
.finally(() => {
|
||||
abortRef.current = null;
|
||||
setProcessingId(null);
|
||||
});
|
||||
}, [processingId, queue, onUploadComplete]);
|
||||
|
||||
// ── File ingestion ─────────────────────────────────────────────────────────
|
||||
|
||||
const addFiles = (fileList) => {
|
||||
const accepted = [];
|
||||
let rejected = 0;
|
||||
for (const file of fileList) {
|
||||
if (file.type === 'text/plain' || file.name.endsWith('.md')) {
|
||||
accepted.push({ id: crypto.randomUUID(), file, name: file.name, status: 'pending', error: null });
|
||||
} else {
|
||||
setStatus({ type: 'error', msg: `Error processing file: ${error.message}` });
|
||||
rejected++;
|
||||
}
|
||||
} finally {
|
||||
abortRef.current = null;
|
||||
setIsProcessing(false);
|
||||
}
|
||||
if (accepted.length > 0) setQueue((q) => [...q, ...accepted]);
|
||||
if (rejected > 0) {
|
||||
setRejectedNote(`${rejected} file${rejected > 1 ? 's' : ''} skipped — only .txt and .md are supported.`);
|
||||
setTimeout(() => setRejectedNote(null), 4000);
|
||||
}
|
||||
};
|
||||
|
||||
const cancelProcessing = () => {
|
||||
const cancelCurrent = () => {
|
||||
abortRef.current?.abort(new DOMException('Cancelled by user', 'AbortError'));
|
||||
};
|
||||
|
||||
const clearDone = () => {
|
||||
setQueue((q) => q.filter((item) => item.status === 'pending' || item.status === 'processing'));
|
||||
};
|
||||
|
||||
// ── Drag & drop / browse ───────────────────────────────────────────────────
|
||||
|
||||
const handleDragOver = (e) => { e.preventDefault(); setIsDragging(true); };
|
||||
const handleDragLeave = () => setIsDragging(false);
|
||||
|
||||
const handleDrop = (e) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
if (e.dataTransfer.files?.length > 0) {
|
||||
const file = e.dataTransfer.files[0];
|
||||
if (file.type === 'text/plain' || file.name.endsWith('.md')) {
|
||||
processFile(file);
|
||||
} else {
|
||||
setStatus({ type: 'error', msg: 'Only .txt and .md files are currently supported.' });
|
||||
}
|
||||
}
|
||||
if (e.dataTransfer.files?.length > 0) addFiles(e.dataTransfer.files);
|
||||
};
|
||||
|
||||
const handleFileSelect = (e) => {
|
||||
if (e.target.files?.length > 0) processFile(e.target.files[0]);
|
||||
if (e.target.files?.length > 0) {
|
||||
addFiles(e.target.files);
|
||||
e.target.value = '';
|
||||
}
|
||||
};
|
||||
|
||||
// ── Derived state ──────────────────────────────────────────────────────────
|
||||
|
||||
const hasDone = queue.some((q) => ['done', 'failed', 'cancelled'].includes(q.status));
|
||||
|
||||
// ── Render ─────────────────────────────────────────────────────────────────
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<div className="space-y-4">
|
||||
|
||||
{/* ─── Drag & Drop Upload ─── */}
|
||||
{/* ─── Drag & Drop Zone ─── */}
|
||||
<Card
|
||||
className={`border-2 border-dashed transition-colors flex flex-col items-center justify-center py-12 cursor-pointer ${
|
||||
className={`border-2 border-dashed transition-colors flex flex-col items-center justify-center py-10 cursor-pointer ${
|
||||
isDragging ? 'border-teal bg-teal/5' : 'border-bg-warm bg-bg-warm/20'
|
||||
} ${isProcessing ? 'opacity-50 pointer-events-none' : ''}`}
|
||||
}`}
|
||||
onDragOver={handleDragOver}
|
||||
onDragLeave={handleDragLeave}
|
||||
onDrop={handleDrop}
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
>
|
||||
<UploadCloud size={48} className="text-teal/40 mb-4" />
|
||||
<p className="font-medium text-lg">Drag files here</p>
|
||||
<p className="text-sm text-fg-muted mb-4">Supports .txt and .md (max 5MB)</p>
|
||||
<Button variant="outline" type="button" disabled={isProcessing}>
|
||||
{isProcessing ? 'AI extraction in progress...' : 'Browse files'}
|
||||
</Button>
|
||||
<UploadCloud size={40} className="text-teal/40 mb-3" />
|
||||
<p className="font-medium">Drag files here</p>
|
||||
<p className="text-sm text-fg-muted mb-4">Supports .txt and .md · max 5 MB each</p>
|
||||
<Button variant="outline" type="button">Browse files</Button>
|
||||
<input
|
||||
type="file"
|
||||
ref={fileInputRef}
|
||||
onChange={handleFileSelect}
|
||||
accept=".txt,.md"
|
||||
multiple
|
||||
className="hidden"
|
||||
/>
|
||||
</Card>
|
||||
|
||||
{/* Cancel sits outside the upload card so pointer-events-none doesn't disable it. */}
|
||||
{isProcessing && (
|
||||
<div className="flex justify-center">
|
||||
<button
|
||||
type="button"
|
||||
onClick={cancelProcessing}
|
||||
className="flex items-center gap-1 text-sm text-red-600 hover:text-red-700 underline"
|
||||
>
|
||||
<X size={14} /> Cancel extraction
|
||||
</button>
|
||||
</div>
|
||||
{/* ─── Rejected file note ─── */}
|
||||
{rejectedNote && (
|
||||
<p className="text-xs text-amber-600">{rejectedNote}</p>
|
||||
)}
|
||||
|
||||
{/* ─── Status messages ─── */}
|
||||
{status && (
|
||||
<div className={`p-4 rounded-[var(--r-sm)] flex items-start gap-3 ${
|
||||
status.type === 'error'
|
||||
? 'bg-red-50 text-red-900 border border-red-200'
|
||||
: 'bg-teal-50 text-teal-900 border border-teal-200'
|
||||
}`}>
|
||||
{status.type === 'error'
|
||||
? <AlertCircle className="text-red-500 flex-shrink-0" />
|
||||
: <CheckCircle className="text-teal-600 flex-shrink-0" />}
|
||||
<div>
|
||||
<p className="font-medium">{status.type === 'error' ? 'Error' : 'Success'}</p>
|
||||
<p className="text-sm">{status.msg}</p>
|
||||
</div>
|
||||
{/* ─── Queue ─── */}
|
||||
{queue.length > 0 && (
|
||||
<div className="space-y-1">
|
||||
{queue.map((item) => (
|
||||
<div
|
||||
key={item.id}
|
||||
className="flex items-center gap-3 px-3 py-2 rounded-[var(--r-sm)] bg-bg-warm/30 text-sm"
|
||||
>
|
||||
<StatusIcon status={item.status} />
|
||||
<span className="flex-1 truncate text-fg">{item.name}</span>
|
||||
<span className="text-xs text-fg-muted shrink-0">
|
||||
{item.status === 'processing' && 'Extracting…'}
|
||||
{item.status === 'pending' && 'Pending'}
|
||||
{item.status === 'done' && 'Done'}
|
||||
{item.status === 'cancelled' && 'Cancelled'}
|
||||
{item.status === 'failed' && (
|
||||
<span className="text-red-500" title={item.error}>Failed</span>
|
||||
)}
|
||||
</span>
|
||||
{item.status === 'processing' && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={cancelCurrent}
|
||||
className="text-red-500 hover:text-red-600 shrink-0"
|
||||
title="Cancel"
|
||||
>
|
||||
<X size={14} />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
|
||||
{hasDone && (
|
||||
<div className="flex justify-end pt-1">
|
||||
<button
|
||||
type="button"
|
||||
onClick={clearDone}
|
||||
className="text-xs text-fg-muted hover:text-fg underline"
|
||||
>
|
||||
Clear done / failed
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
// ── Status icon ───────────────────────────────────────────────────────────────
|
||||
|
||||
const StatusIcon = ({ status }) => {
|
||||
if (status === 'processing') return <Loader2 size={14} className="text-teal animate-spin shrink-0" />;
|
||||
if (status === 'done') return <CheckCircle size={14} className="text-teal shrink-0" />;
|
||||
if (status === 'failed') return <AlertCircle size={14} className="text-red-500 shrink-0" />;
|
||||
if (status === 'cancelled') return <Minus size={14} className="text-fg-muted shrink-0" />;
|
||||
return <Clock size={14} className="text-fg-muted shrink-0" />;
|
||||
};
|
||||
|
||||
export default UploadZone;
|
||||
|
||||
@@ -69,18 +69,21 @@ describe('buildKnownIdsHint', () => {
|
||||
expect(buildKnownIdsHint(null)).toBe('');
|
||||
});
|
||||
|
||||
it('formats the known IDs as a bulleted list with a leading instruction', () => {
|
||||
const hint = buildKnownIdsHint(['software-engineer', 'onboarding-buddy']);
|
||||
expect(hint).toContain('Already-extracted topic IDs');
|
||||
expect(hint).toContain('- software-engineer');
|
||||
expect(hint).toContain('- onboarding-buddy');
|
||||
it('formats topics as id: "label" pairs with a leading instruction', () => {
|
||||
const hint = buildKnownIdsHint([
|
||||
{ id: 'software-engineer', label: 'Software Engineer' },
|
||||
{ id: 'onboarding-buddy', label: 'Onboarding Buddy' },
|
||||
]);
|
||||
expect(hint).toContain('Already-extracted topics');
|
||||
expect(hint).toContain('- software-engineer: "Software Engineer"');
|
||||
expect(hint).toContain('- onboarding-buddy: "Onboarding Buddy"');
|
||||
expect(hint.endsWith('\n')).toBe(true);
|
||||
});
|
||||
|
||||
it('caps the hint at the 200 most recent IDs', () => {
|
||||
const ids = Array.from({ length: 250 }, (_, i) => `topic-${i}`);
|
||||
const hint = buildKnownIdsHint(ids);
|
||||
// The newest IDs must appear; the oldest must not.
|
||||
it('caps the hint at the 200 most recent topics', () => {
|
||||
const topics = Array.from({ length: 250 }, (_, i) => ({ id: `topic-${i}`, label: `Topic ${i}` }));
|
||||
const hint = buildKnownIdsHint(topics);
|
||||
// The newest topics must appear; the oldest must not.
|
||||
expect(hint).toContain('topic-249');
|
||||
expect(hint).toContain('topic-50');
|
||||
expect(hint).not.toContain('topic-49');
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
extractionResultSchema,
|
||||
handbookResultSchema,
|
||||
normalizeHandbookResult,
|
||||
learningArticleSchema,
|
||||
learningSlidesSchema,
|
||||
learningInfographicSchema,
|
||||
@@ -60,28 +58,6 @@ describe('extractionResultSchema', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('handbookResultSchema', () => {
|
||||
it('accepts the loose vocabulary including executes', () => {
|
||||
const parsed = handbookResultSchema.parse({
|
||||
topics: [{ ...sampleTopic, metadata: { source: 'github_handbook' } }],
|
||||
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
|
||||
});
|
||||
expect(parsed.relations[0].type).toBe('executes');
|
||||
});
|
||||
|
||||
it('normalises executes into executed_by with swapped source/target', () => {
|
||||
const parsed = handbookResultSchema.parse({
|
||||
topics: [sampleTopic],
|
||||
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
|
||||
});
|
||||
const normalised = normalizeHandbookResult(parsed);
|
||||
expect(normalised.relations[0]).toMatchObject({
|
||||
source: 'code-review',
|
||||
target: 'software-engineer',
|
||||
type: 'executed_by',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('learning schemas', () => {
|
||||
it('accepts an article payload', () => {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
EMIT_KNOWLEDGE_GRAPH_TOOL,
|
||||
EMIT_HANDBOOK_DELTA_TOOL,
|
||||
EMIT_LEARNING_ARTICLE_TOOL,
|
||||
EMIT_LEARNING_SLIDES_TOOL,
|
||||
EMIT_LEARNING_INFOGRAPHIC_TOOL,
|
||||
@@ -15,7 +14,6 @@ import { toolSchemaRegistry } from '../llmSchemas';
|
||||
|
||||
const allTools = [
|
||||
EMIT_KNOWLEDGE_GRAPH_TOOL,
|
||||
EMIT_HANDBOOK_DELTA_TOOL,
|
||||
EMIT_LEARNING_ARTICLE_TOOL,
|
||||
EMIT_LEARNING_SLIDES_TOOL,
|
||||
EMIT_LEARNING_INFOGRAPHIC_TOOL,
|
||||
|
||||
@@ -349,11 +349,10 @@ export async function getRecentLlmCalls(limit = 100) {
|
||||
*
|
||||
* `team_members`, `settings`, and `llm_calls` are intentionally preserved.
|
||||
*
|
||||
* @param {{ includeHandbookState?: boolean, includeProgress?: boolean }} [opts]
|
||||
* @param {{ includeProgress?: boolean }} [opts]
|
||||
* @returns {Promise<Array<{collection: string, deleted: number, error?: string}>>}
|
||||
*/
|
||||
export async function resetForSmokeTest({
|
||||
includeHandbookState = true,
|
||||
includeProgress = false,
|
||||
} = {}) {
|
||||
const collections = [
|
||||
@@ -366,7 +365,6 @@ export async function resetForSmokeTest({
|
||||
'sources',
|
||||
'topics',
|
||||
];
|
||||
if (includeHandbookState) collections.push('handbook_sync_state');
|
||||
if (includeProgress) collections.push('learn_progress', 'leaderboard');
|
||||
|
||||
const results = [];
|
||||
@@ -388,20 +386,4 @@ export async function resetForSmokeTest({
|
||||
return results;
|
||||
}
|
||||
|
||||
// ── Handbook Sync State ───────────────────────────────────────────────────────
|
||||
|
||||
export async function getHandbookSyncStates() {
|
||||
try {
|
||||
return await pb.collection('handbook_sync_state').getFullList();
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
export async function updateHandbookSyncState(path, sha) {
|
||||
try {
|
||||
const r = await pb.collection('handbook_sync_state').getFirstListItem(`path="${path}"`);
|
||||
return await pb.collection('handbook_sync_state').update(r.id, { sha });
|
||||
} catch {
|
||||
return await pb.collection('handbook_sync_state').create({ path, sha });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,24 +1,21 @@
|
||||
import * as db from './db';
|
||||
import { callLLM } from './llm';
|
||||
import { extractionLimiter } from './llmRetry';
|
||||
import { EMIT_KNOWLEDGE_GRAPH_TOOL, EMIT_HANDBOOK_DELTA_TOOL } from './llmTools';
|
||||
import { normalizeHandbookResult } from './llmSchemas';
|
||||
import { EMIT_KNOWLEDGE_GRAPH_TOOL } from './llmTools';
|
||||
|
||||
const MAX_KNOWN_IDS_HINT = 200;
|
||||
const MAX_KNOWN_TOPICS_HINT = 200;
|
||||
|
||||
/**
|
||||
* Build the "already-extracted topic IDs" hint that prepends every chunk
|
||||
* after the first. Capped at the most-recent `MAX_KNOWN_IDS_HINT` IDs so
|
||||
* the prompt stays a bounded size; the model uses this list to reuse IDs
|
||||
* rather than invent variants like `software-developer` for
|
||||
* `software-engineer`.
|
||||
* Build the "already-extracted topics" hint included in every chunk prompt.
|
||||
* Passes both ID and label so the model can match concepts by name and reuse
|
||||
* the exact ID + label rather than inventing near-duplicate variants.
|
||||
*/
|
||||
export function buildKnownIdsHint(ids) {
|
||||
if (!ids || !ids.length) return '';
|
||||
const recent = ids.slice(-MAX_KNOWN_IDS_HINT);
|
||||
export function buildKnownIdsHint(topics) {
|
||||
if (!topics || !topics.length) return '';
|
||||
const recent = topics.slice(-MAX_KNOWN_TOPICS_HINT);
|
||||
return [
|
||||
'Already-extracted topic IDs (do NOT create new IDs for these — reuse them if the same concept appears here):',
|
||||
...recent.map((id) => `- ${id}`),
|
||||
'Already-extracted topics (reuse their ID and label exactly if the same concept appears here):',
|
||||
...recent.map((t) => `- ${t.id}: "${t.label}"`),
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
@@ -45,41 +42,8 @@ Assign a learning_relevance to every topic:
|
||||
Relation types: related_to | depends_on | part_of | executed_by.
|
||||
`;
|
||||
|
||||
const HANDBOOK_SYSTEM_PROMPT = `You are analysing an update to the Respellion Employee Handbook. Emit the extracted topics and relations through the emit_handbook_delta tool.
|
||||
|
||||
CRITICAL INSTRUCTIONS:
|
||||
- Every process must have a role attached. Express this as: process --executed_by--> role.
|
||||
- Every concept must connect to a process or role.
|
||||
- Mark handbook topics with metadata.source = "github_handbook".
|
||||
- Assign learning_relevance using the same scale as extraction: core | standard | peripheral | exclude.
|
||||
|
||||
Relation types: related_to | depends_on | part_of | executed_by.
|
||||
`;
|
||||
|
||||
const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }];
|
||||
|
||||
export async function analyzeHandbookDelta(fileContent, filePath, { signal } = {}) {
|
||||
const result = await callLLM({
|
||||
task: 'extract.handbook',
|
||||
tier: 'standard',
|
||||
system: cachedSystem(HANDBOOK_SYSTEM_PROMPT),
|
||||
user: `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`,
|
||||
tools: [EMIT_HANDBOOK_DELTA_TOOL],
|
||||
toolChoice: { type: 'tool', name: EMIT_HANDBOOK_DELTA_TOOL.name },
|
||||
maxTokens: 8192,
|
||||
timeoutMs: 180_000,
|
||||
limiter: extractionLimiter,
|
||||
signal,
|
||||
});
|
||||
|
||||
const raw = result.toolUses[0]?.input;
|
||||
if (!raw) throw new Error('Handbook extraction did not emit a tool result.');
|
||||
const extractedData = normalizeHandbookResult(raw);
|
||||
|
||||
await mergeKnowledgeGraph(extractedData);
|
||||
return { success: true, data: extractedData };
|
||||
}
|
||||
|
||||
/**
|
||||
* Sentence-aware chunker with overlap.
|
||||
*
|
||||
@@ -169,7 +133,7 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
|
||||
console.log(`[Pipeline] Split "${sourceName}" into ${chunks.length} chunks for processing.`);
|
||||
|
||||
const existingTopics = await db.getTopics();
|
||||
const knownIds = existingTopics.map((t) => t.id);
|
||||
const knownTopics = existingTopics.map((t) => ({ id: t.id, label: t.label }));
|
||||
|
||||
const allExtractedTopics = [];
|
||||
const allExtractedRelations = [];
|
||||
@@ -178,7 +142,7 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
|
||||
if (signal?.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError');
|
||||
|
||||
console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`);
|
||||
const hint = i > 0 ? buildKnownIdsHint(knownIds) : '';
|
||||
const hint = buildKnownIdsHint(knownTopics);
|
||||
const result = await callLLM({
|
||||
task: 'extract.source',
|
||||
tier: 'standard',
|
||||
@@ -197,7 +161,9 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
|
||||
if (Array.isArray(extractedData.topics)) {
|
||||
allExtractedTopics.push(...extractedData.topics);
|
||||
for (const t of extractedData.topics) {
|
||||
if (t?.id && !knownIds.includes(t.id)) knownIds.push(t.id);
|
||||
if (t?.id && !knownTopics.some((k) => k.id === t.id)) {
|
||||
knownTopics.push({ id: t.id, label: t.label });
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Array.isArray(extractedData.relations)) {
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
/**
|
||||
* GitHub API Service
|
||||
* Fetches files from a public GitHub repository directly from the browser.
|
||||
* No proxy or token required for public repos.
|
||||
*/
|
||||
|
||||
const GITHUB_API = 'https://api.github.com';
|
||||
const GITHUB_RAW = 'https://raw.githubusercontent.com';
|
||||
|
||||
/**
|
||||
* List markdown/text files in a GitHub repository folder.
|
||||
* @param {string} owner - e.g. "respellion"
|
||||
* @param {string} repo - e.g. "employee-handbook"
|
||||
* @param {string} folder - e.g. "docs" or "" for root
|
||||
* @param {string} branch - e.g. "main"
|
||||
* @returns {Promise<Array<{name, sha, path}>>}
|
||||
*/
|
||||
export async function getRepoFolder(owner, repo, folder = '', branch = 'main') {
|
||||
// Use the GitHub Git Trees API to fetch all files recursively in one request
|
||||
const url = `${GITHUB_API}/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`;
|
||||
|
||||
const res = await fetch(url, {
|
||||
headers: { Accept: 'application/vnd.github+json' },
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({}));
|
||||
throw new Error(`GitHub API ${res.status}: ${err.message || 'Could not list folder'}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
// If folder is 'docs', we only want paths starting with 'docs/'
|
||||
const folderPrefix = folder ? folder.replace(/^\/|\/$/g, '') + '/' : '';
|
||||
|
||||
// Return only files (blobs) that match the folder prefix and file extensions
|
||||
return data.tree
|
||||
.filter(item =>
|
||||
item.type === 'blob' &&
|
||||
item.path.startsWith(folderPrefix) &&
|
||||
(item.path.endsWith('.md') || item.path.endsWith('.txt'))
|
||||
)
|
||||
.map(item => ({
|
||||
name: item.path.split('/').pop(),
|
||||
path: item.path,
|
||||
sha: item.sha
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch the raw text content of a file.
|
||||
* Uses raw.githubusercontent.com — no rate limits for public content.
|
||||
* @param {string} owner
|
||||
* @param {string} repo
|
||||
* @param {string} path - e.g. "docs/ROLES.md"
|
||||
* @param {string} branch
|
||||
* @returns {Promise<string>}
|
||||
*/
|
||||
export async function getFileContent(owner, repo, path, branch = 'main') {
|
||||
const url = `${GITHUB_RAW}/${owner}/${repo}/${branch}/${path}`;
|
||||
const res = await fetch(url);
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Could not fetch "${path}" from GitHub (${res.status})`);
|
||||
}
|
||||
|
||||
return res.text();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a GitHub tree URL into its parts.
|
||||
* e.g. "https://github.com/respellion/employee-handbook/tree/main/docs"
|
||||
* → { owner: 'respellion', repo: 'employee-handbook', branch: 'main', folder: 'docs' }
|
||||
*/
|
||||
export function parseGitHubUrl(url) {
|
||||
try {
|
||||
const match = url.match(/github\.com\/([^/]+)\/([^/]+)(?:\/tree\/([^/]+)(?:\/(.+))?)?/);
|
||||
if (!match) return null;
|
||||
return {
|
||||
owner: match[1],
|
||||
repo: match[2],
|
||||
branch: match[3] || 'main',
|
||||
folder: match[4] || '',
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import { z } from 'zod';
|
||||
|
||||
const topicTypeEnum = z.enum(['concept', 'role', 'process']);
|
||||
const relationTypeStrict = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by']);
|
||||
const relationTypeLoose = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by', 'executes']);
|
||||
const learningRelevanceEnum = z.enum(['core', 'standard', 'peripheral', 'exclude']);
|
||||
|
||||
const extractionTopicSchema = z.object({
|
||||
@@ -31,38 +30,6 @@ export const extractionResultSchema = z.object({
|
||||
relations: z.array(extractionRelationSchema),
|
||||
});
|
||||
|
||||
const handbookTopicSchema = extractionTopicSchema.extend({
|
||||
metadata: z.object({ source: z.string() }).optional(),
|
||||
});
|
||||
|
||||
const handbookRelationSchema = z.object({
|
||||
source: z.string().min(1),
|
||||
target: z.string().min(1),
|
||||
type: relationTypeLoose,
|
||||
description: z.string().optional(),
|
||||
});
|
||||
|
||||
export const handbookResultSchema = z.object({
|
||||
topics: z.array(handbookTopicSchema),
|
||||
relations: z.array(handbookRelationSchema),
|
||||
});
|
||||
|
||||
/**
|
||||
* Normalise legacy `executes` relations into the canonical `executed_by`
|
||||
* vocabulary by swapping source and target. The handbook prompt previously
|
||||
* emitted `role → executes → process`; the canonical form is
|
||||
* `process → executed_by → role`.
|
||||
*/
|
||||
export function normalizeHandbookResult(parsed) {
|
||||
return {
|
||||
...parsed,
|
||||
relations: parsed.relations.map((r) =>
|
||||
r.type === 'executes'
|
||||
? { ...r, type: 'executed_by', source: r.target, target: r.source }
|
||||
: r,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const articleSectionSchema = z.object({
|
||||
heading: z.string().min(1),
|
||||
@@ -218,7 +185,6 @@ export const replaceTakeawaysPatchSchema = z.object({
|
||||
*/
|
||||
export const toolSchemaRegistry = {
|
||||
emit_knowledge_graph: extractionResultSchema,
|
||||
emit_handbook_delta: handbookResultSchema,
|
||||
emit_learning_article: learningArticleSchema,
|
||||
emit_learning_slides: learningSlidesSchema,
|
||||
emit_learning_infographic: learningInfographicSchema,
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
const TOPIC_TYPES = ['concept', 'role', 'process'];
|
||||
const LEARNING_RELEVANCE = ['core', 'standard', 'peripheral', 'exclude'];
|
||||
const RELATION_TYPES_STRICT = ['related_to', 'depends_on', 'part_of', 'executed_by'];
|
||||
const RELATION_TYPES_LOOSE = ['related_to', 'depends_on', 'part_of', 'executed_by', 'executes'];
|
||||
|
||||
const extractionTopicSchema = {
|
||||
type: 'object',
|
||||
@@ -47,41 +46,6 @@ export const EMIT_KNOWLEDGE_GRAPH_TOOL = {
|
||||
},
|
||||
};
|
||||
|
||||
const handbookTopicSchema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
...extractionTopicSchema.properties,
|
||||
metadata: {
|
||||
type: 'object',
|
||||
properties: { source: { type: 'string' } },
|
||||
},
|
||||
},
|
||||
required: extractionTopicSchema.required,
|
||||
};
|
||||
|
||||
const handbookRelationSchema = {
|
||||
type: 'object',
|
||||
properties: {
|
||||
source: { type: 'string' },
|
||||
target: { type: 'string' },
|
||||
type: { type: 'string', enum: RELATION_TYPES_LOOSE },
|
||||
description: { type: 'string' },
|
||||
},
|
||||
required: ['source', 'target', 'type'],
|
||||
};
|
||||
|
||||
export const EMIT_HANDBOOK_DELTA_TOOL = {
|
||||
name: 'emit_handbook_delta',
|
||||
description: 'Return the topics and relations extracted from a handbook file update. Every process must have a role attached; every concept must connect to a process or role.',
|
||||
input_schema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
topics: { type: 'array', items: handbookTopicSchema },
|
||||
relations: { type: 'array', items: handbookRelationSchema },
|
||||
},
|
||||
required: ['topics', 'relations'],
|
||||
},
|
||||
};
|
||||
|
||||
const articleSectionSchema = {
|
||||
type: 'object',
|
||||
|
||||
@@ -31,7 +31,6 @@ const Admin = () => {
|
||||
const [saveStatus, setSaveStatus] = useState(null);
|
||||
|
||||
const [resetConfirmText, setResetConfirmText] = useState('');
|
||||
const [resetIncludeHandbook, setResetIncludeHandbook] = useState(true);
|
||||
const [resetIncludeProgress, setResetIncludeProgress] = useState(false);
|
||||
const [isResetting, setIsResetting] = useState(false);
|
||||
const [resetReport, setResetReport] = useState(null);
|
||||
@@ -69,7 +68,6 @@ const Admin = () => {
|
||||
setResetReport(null);
|
||||
try {
|
||||
const report = await db.resetForSmokeTest({
|
||||
includeHandbookState: resetIncludeHandbook,
|
||||
includeProgress: resetIncludeProgress,
|
||||
});
|
||||
setResetReport(report);
|
||||
@@ -317,15 +315,6 @@ const Admin = () => {
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<label className="flex items-center gap-2 text-sm cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={resetIncludeHandbook}
|
||||
onChange={(e) => setResetIncludeHandbook(e.target.checked)}
|
||||
className="rounded bg-bg-warm border-transparent focus:ring-0 text-teal"
|
||||
/>
|
||||
Also clear handbook sync state (so "Sync Employee Handbook" re-processes every file)
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-sm cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
|
||||
Reference in New Issue
Block a user