feat: phase 3 of AI pipeline hardening — extraction quality
Replace stateless one-shot extraction with a stateful, paced, cancellable pipeline. Six subtasks: - 3.1 Sentence-aware chunking with 800-char overlap (was paragraph-only at 4000 chars). Hard-split fallback for runaway sentences. - 3.2 Stateful extraction: chunks 2+ receive an "already-extracted topic IDs" hint capped at 200 IDs, so the model reuses IDs instead of inventing variants like software-developer vs software-engineer. - 3.3 Token-bucket limiter in llmRetry.js (extractionLimiter, 5 req/min). callLLM awaits the limiter before fetch; 429+Retry-After calls pauseUntil. Replaces hard setTimeout(12000) and setTimeout(15000). - 3.4 relevance_locked column on topics — admin edits to relevance are sticky across re-extraction. Migration + merge respects the flag + unlock checkbox in KnowledgeGraph edit form. - 3.5 Unify relation vocabulary — handbook prompt no longer mentions legacy "executes"; one-shot migration rewrites existing executes rows to executed_by with source/target swapped. - 3.6 Cancellation — Cancel button on UploadZone wired to an AbortController threaded into callLLM; aborted runs persist status = "cancelled" rather than "failed". Tests: 16 new unit tests for chunkText, buildKnownIdsHint, and createLimiter. All 61 tests pass, 0 lint errors, build clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -272,6 +272,7 @@ function validateToolInputs(toolUses, task, toolSchemas) {
|
||||
* @property {number} [maxTokens=4096]
|
||||
* @property {number} [temperature=0]
|
||||
* @property {AbortSignal} [signal]
|
||||
* @property {{ acquire: (opts?:{signal?:AbortSignal}) => Promise<void>, pauseUntil: (untilMs:number) => void }} [limiter]
|
||||
*/
|
||||
|
||||
/**
|
||||
@@ -291,6 +292,7 @@ export async function callLLM(options) {
|
||||
maxTokens = 4096,
|
||||
temperature = 0,
|
||||
signal,
|
||||
limiter,
|
||||
} = options;
|
||||
if (!task) throw new Error('callLLM requires a `task` label.');
|
||||
|
||||
@@ -318,6 +320,7 @@ export async function callLLM(options) {
|
||||
try {
|
||||
result = await withRetry(
|
||||
async () => {
|
||||
if (limiter) await limiter.acquire({ signal });
|
||||
const timeoutCtl = signal ? null : new AbortController();
|
||||
const timer = timeoutCtl ? setTimeout(() => timeoutCtl.abort(new DOMException('Timeout', 'AbortError')), DEFAULT_TIMEOUT_MS) : null;
|
||||
const fetchSignal = linkSignals(signal, timeoutCtl?.signal);
|
||||
@@ -337,6 +340,9 @@ export async function callLLM(options) {
|
||||
const errBody = await response.json().catch(() => ({}));
|
||||
if (isRetryableStatus(response.status)) {
|
||||
const retryAfterMs = parseRetryAfter(response.headers.get('Retry-After'));
|
||||
if (response.status === 429 && retryAfterMs != null && limiter) {
|
||||
limiter.pauseUntil(Date.now() + retryAfterMs);
|
||||
}
|
||||
throw new RetryableError(response.status, retryAfterMs, `HTTP ${response.status}`);
|
||||
}
|
||||
throw new LLMHttpError(response.status, response.statusText, errBody);
|
||||
|
||||
Reference in New Issue
Block a user