feat: phase 3 of AI pipeline hardening — extraction quality
Replace stateless one-shot extraction with a stateful, paced, cancellable pipeline. Six subtasks: - 3.1 Sentence-aware chunking with 800-char overlap (was paragraph-only at 4000 chars). Hard-split fallback for runaway sentences. - 3.2 Stateful extraction: chunks 2+ receive an "already-extracted topic IDs" hint capped at 200 IDs, so the model reuses IDs instead of inventing variants like software-developer vs software-engineer. - 3.3 Token-bucket limiter in llmRetry.js (extractionLimiter, 5 req/min). callLLM awaits the limiter before fetch; 429+Retry-After calls pauseUntil. Replaces hard setTimeout(12000) and setTimeout(15000). - 3.4 relevance_locked column on topics — admin edits to relevance are sticky across re-extraction. Migration + merge respects the flag + unlock checkbox in KnowledgeGraph edit form. - 3.5 Unify relation vocabulary — handbook prompt no longer mentions legacy "executes"; one-shot migration rewrites existing executes rows to executed_by with source/target swapped. - 3.6 Cancellation — Cancel button on UploadZone wired to an AbortController threaded into callLLM; aborted runs persist status = "cancelled" rather than "failed". Tests: 16 new unit tests for chunkText, buildKnownIdsHint, and createLimiter. All 61 tests pass, 0 lint errors, build clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { useState, useRef } from 'react';
|
||||
import { UploadCloud, AlertCircle, CheckCircle } from 'lucide-react';
|
||||
import { UploadCloud, AlertCircle, CheckCircle, X } from 'lucide-react';
|
||||
import { processSourceText } from '../../lib/extractionPipeline';
|
||||
import Card from '../ui/Card';
|
||||
import Button from '../ui/Button';
|
||||
@@ -12,24 +12,36 @@ const UploadZone = ({ onUploadComplete }) => {
|
||||
const [status, setStatus] = useState(null);
|
||||
|
||||
const fileInputRef = useRef(null);
|
||||
const abortRef = useRef(null);
|
||||
|
||||
// ── File upload (drag & drop / browse) ────────────────────────────────────
|
||||
|
||||
const processFile = async (file) => {
|
||||
setIsProcessing(true);
|
||||
setStatus(null);
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
try {
|
||||
const text = await file.text();
|
||||
await processSourceText(text, file.name);
|
||||
await processSourceText(text, file.name, { signal: controller.signal });
|
||||
setStatus({ type: 'success', msg: `Successfully processed: ${file.name}` });
|
||||
if (onUploadComplete) onUploadComplete();
|
||||
} catch (error) {
|
||||
setStatus({ type: 'error', msg: `Error processing file: ${error.message}` });
|
||||
if (error?.name === 'AbortError') {
|
||||
setStatus({ type: 'error', msg: `Cancelled: ${file.name}` });
|
||||
} else {
|
||||
setStatus({ type: 'error', msg: `Error processing file: ${error.message}` });
|
||||
}
|
||||
} finally {
|
||||
abortRef.current = null;
|
||||
setIsProcessing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const cancelProcessing = () => {
|
||||
abortRef.current?.abort(new DOMException('Cancelled by user', 'AbortError'));
|
||||
};
|
||||
|
||||
const handleDragOver = (e) => { e.preventDefault(); setIsDragging(true); };
|
||||
const handleDragLeave = () => setIsDragging(false);
|
||||
|
||||
@@ -80,6 +92,19 @@ const UploadZone = ({ onUploadComplete }) => {
|
||||
/>
|
||||
</Card>
|
||||
|
||||
{/* Cancel sits outside the upload card so pointer-events-none doesn't disable it. */}
|
||||
{isProcessing && (
|
||||
<div className="flex justify-center">
|
||||
<button
|
||||
type="button"
|
||||
onClick={cancelProcessing}
|
||||
className="flex items-center gap-1 text-sm text-red-600 hover:text-red-700 underline"
|
||||
>
|
||||
<X size={14} /> Cancel extraction
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* ─── Status messages ─── */}
|
||||
{status && (
|
||||
<div className={`p-4 rounded-[var(--r-sm)] flex items-start gap-3 ${
|
||||
|
||||
Reference in New Issue
Block a user