feat: phase 1 of AI pipeline hardening — single LLM client + tier-aware models
Implements phase 1 of AI_PIPELINE_HARDENING_PLAN.md. Every Anthropic call now goes through one module that owns retry, timeout, abort, structured- output parsing, schema validation, and best-effort call telemetry. * src/lib/llm.js — single callLLM entry point. Resolves model per tier (fast / standard / reasoning) with admin:model legacy fallback for the standard tier; 60s default timeout via AbortController; balanced-brace JSON extraction; LLMHttpError, LLMTruncatedError, LLMOutputError, and LLMValidationError surface clearly distinct failure modes. * src/lib/llmRetry.js — exponential backoff with full jitter, retries only on transient HTTP statuses, honours Retry-After up to 60s, never retries on AbortError. * src/lib/llmSchemas.js — Zod schemas for every structured task plus normalizeHandbookResult (collapses legacy "executes" relations into the canonical "executed_by" vocabulary). * src/lib/api.js — thin shim over callLLM so existing callers (extraction pipeline, learning, quiz, R42, knowledge graph) keep working unchanged. * src/lib/__tests__/ — 32 Vitest cases covering parse paths, error surfaces, simulation mode, model resolution, and schema validation. * src/pages/Admin/index.jsx — three model inputs (fast / standard / reasoning) replacing the single legacy field; legacy value falls back for the standard tier so existing overrides survive. Adds Zod and Vitest, plus an "npm run test" script. Also cleans up the pre-existing repo-wide ESLint failures so phase 1's "npm run lint passes" acceptance criterion can be checked: drops unused React imports across the JSX tree (React 19 JSX runtime auto-imports), attaches cause to rethrown errors in the service modules, ignores pb_migrations in the ESLint config (PocketBase JSVM globals), and removes one dead handleCreateCustom function in Leren.jsx. A real behaviour bug surfaced in Testen.jsx — the quiz timer captured a stale finishQuiz via setInterval closure; now updated via finishQuizRef so the timer always invokes the latest callback. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
RaymondVerhoef
232
src/lib/__tests__/llm.test.js
Normal file
232
src/lib/__tests__/llm.test.js
Normal file
@@ -0,0 +1,232 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { z } from 'zod';
|
||||
|
||||
vi.mock('../storage', () => ({
|
||||
storage: {
|
||||
_data: new Map(),
|
||||
get(key, fallback = null) {
|
||||
return this._data.has(key) ? this._data.get(key) : fallback;
|
||||
},
|
||||
set(key, value) { this._data.set(key, value); },
|
||||
remove(key) { this._data.delete(key); },
|
||||
getKeysByPrefix() { return []; },
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../pb', () => ({
|
||||
pb: { collection: () => ({ create: () => ({ catch: () => {} }) }) },
|
||||
}));
|
||||
|
||||
import {
|
||||
callLLM,
|
||||
LLMHttpError,
|
||||
LLMOutputError,
|
||||
LLMTruncatedError,
|
||||
LLMValidationError,
|
||||
parseStructuredText,
|
||||
resolveModel,
|
||||
} from '../llm';
|
||||
import { storage } from '../storage';
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
storage._data.clear();
|
||||
});
|
||||
|
||||
describe('parseStructuredText', () => {
|
||||
it('extracts an object from raw JSON', () => {
|
||||
expect(parseStructuredText('{"a":1}')).toEqual({ a: 1 });
|
||||
});
|
||||
|
||||
it('extracts an object from a json-fenced block', () => {
|
||||
const fenced = '```json\n{"hello":"world"}\n```';
|
||||
expect(parseStructuredText(fenced)).toEqual({ hello: 'world' });
|
||||
});
|
||||
|
||||
it('extracts an object surrounded by prose', () => {
|
||||
const messy = 'Sure! Here you go:\n{"id":"x","label":"X"}\nLet me know if you want changes.';
|
||||
expect(parseStructuredText(messy)).toEqual({ id: 'x', label: 'X' });
|
||||
});
|
||||
|
||||
it('extracts an array when it is the top-level value', () => {
|
||||
expect(parseStructuredText('[1,2,3]')).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it('ignores braces inside string literals', () => {
|
||||
const tricky = '{"text":"this { is not } a brace"}';
|
||||
expect(parseStructuredText(tricky)).toEqual({ text: 'this { is not } a brace' });
|
||||
});
|
||||
|
||||
it('throws LLMOutputError when no balanced JSON is present', () => {
|
||||
expect(() => parseStructuredText('no json here, just words')).toThrow(LLMOutputError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveModel', () => {
|
||||
it('falls back to tier defaults when no override is set', () => {
|
||||
expect(resolveModel('fast')).toBe('claude-haiku-4-5-20251001');
|
||||
expect(resolveModel('standard')).toBe('claude-sonnet-4-6');
|
||||
expect(resolveModel('reasoning')).toBe('claude-opus-4-7');
|
||||
});
|
||||
|
||||
it('honours an explicit tier override', () => {
|
||||
storage.set('admin:model:reasoning', 'claude-opus-9-future');
|
||||
expect(resolveModel('reasoning')).toBe('claude-opus-9-future');
|
||||
});
|
||||
|
||||
it('uses the legacy admin:model setting as a standard-tier fallback', () => {
|
||||
storage.set('admin:model', 'claude-some-legacy-id');
|
||||
expect(resolveModel('standard')).toBe('claude-some-legacy-id');
|
||||
});
|
||||
|
||||
it('prefers the tier-specific override over the legacy fallback', () => {
|
||||
storage.set('admin:model', 'claude-legacy');
|
||||
storage.set('admin:model:standard', 'claude-new');
|
||||
expect(resolveModel('standard')).toBe('claude-new');
|
||||
});
|
||||
});
|
||||
|
||||
function mockJsonResponse(body, { status = 200, headers = {} } = {}) {
|
||||
const h = new Headers({ 'content-type': 'application/json', ...headers });
|
||||
return new Response(JSON.stringify(body), { status, headers: h });
|
||||
}
|
||||
|
||||
describe('callLLM happy path', () => {
|
||||
it('returns parsed tool input when toolChoice forces a tool', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
id: 'msg_1',
|
||||
model: 'claude-sonnet-4-6',
|
||||
stop_reason: 'tool_use',
|
||||
usage: { input_tokens: 10, output_tokens: 20 },
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
name: 'emit_custom_topic',
|
||||
input: { label: 'Pair Programming', type: 'process', description: 'Two engineers, one keyboard.' },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await callLLM({
|
||||
task: 'learning.custom_topic',
|
||||
tier: 'standard',
|
||||
user: 'Pair programming',
|
||||
tools: [{ name: 'emit_custom_topic', description: 'x', input_schema: { type: 'object' } }],
|
||||
toolChoice: { type: 'tool', name: 'emit_custom_topic' },
|
||||
});
|
||||
|
||||
expect(result.toolUses).toHaveLength(1);
|
||||
expect(result.toolUses[0].input.label).toBe('Pair Programming');
|
||||
});
|
||||
|
||||
it('parses and validates plain text against a Zod schema', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
id: 'msg_2',
|
||||
model: 'claude-sonnet-4-6',
|
||||
stop_reason: 'end_turn',
|
||||
usage: { input_tokens: 5, output_tokens: 7 },
|
||||
content: [{ type: 'text', text: '```json\n{"value":42}\n```' }],
|
||||
}),
|
||||
);
|
||||
|
||||
const schema = z.object({ value: z.number() });
|
||||
const result = await callLLM({
|
||||
task: 'demo.json',
|
||||
user: 'give me a number',
|
||||
schema,
|
||||
});
|
||||
expect(result.parsed).toEqual({ value: 42 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('callLLM error paths', () => {
|
||||
it('throws LLMTruncatedError when stop_reason is max_tokens and a tool was requested', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
stop_reason: 'max_tokens',
|
||||
usage: { input_tokens: 1, output_tokens: 1 },
|
||||
content: [],
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
callLLM({
|
||||
task: 'extract.source',
|
||||
user: 'x',
|
||||
tools: [{ name: 'emit_knowledge_graph', description: 'x', input_schema: { type: 'object' } }],
|
||||
toolChoice: { type: 'tool', name: 'emit_knowledge_graph' },
|
||||
}),
|
||||
).rejects.toBeInstanceOf(LLMTruncatedError);
|
||||
});
|
||||
|
||||
it('throws LLMTruncatedError when stop_reason is max_tokens and a schema was requested', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
stop_reason: 'max_tokens',
|
||||
usage: { input_tokens: 1, output_tokens: 1 },
|
||||
content: [{ type: 'text', text: 'partial...' }],
|
||||
}),
|
||||
);
|
||||
const schema = z.object({ value: z.number() });
|
||||
await expect(
|
||||
callLLM({ task: 'demo.json', user: 'x', schema }),
|
||||
).rejects.toBeInstanceOf(LLMTruncatedError);
|
||||
});
|
||||
|
||||
it('throws LLMValidationError when tool input fails schema validation', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
stop_reason: 'tool_use',
|
||||
usage: {},
|
||||
content: [
|
||||
{ type: 'tool_use', name: 'emit_custom_topic', input: { label: 'X', type: 'concept' } },
|
||||
],
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
callLLM({
|
||||
task: 'learning.custom_topic',
|
||||
user: 'x',
|
||||
tools: [{ name: 'emit_custom_topic', description: 'x', input_schema: { type: 'object' } }],
|
||||
toolChoice: { type: 'tool', name: 'emit_custom_topic' },
|
||||
}),
|
||||
).rejects.toBeInstanceOf(LLMValidationError);
|
||||
});
|
||||
|
||||
it('surfaces a non-retryable HTTP error as LLMHttpError', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
new Response(JSON.stringify({ error: 'bad request' }), {
|
||||
status: 400,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
await expect(callLLM({ task: 'demo', user: 'x' })).rejects.toBeInstanceOf(LLMHttpError);
|
||||
});
|
||||
|
||||
it('detects an auth portal HTML response and raises a clear message', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
new Response('<html>login</html>', { status: 200, headers: { 'content-type': 'text/html' } }),
|
||||
);
|
||||
await expect(callLLM({ task: 'demo', user: 'x' })).rejects.toThrow(/session has expired/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('callLLM simulation mode', () => {
|
||||
it('returns the chat stub when admin:use_simulation is true and task is chat-like', async () => {
|
||||
storage.set('admin:use_simulation', true);
|
||||
const result = await callLLM({ task: 'chat.r42', user: 'hello' });
|
||||
expect(result.stopReason).toBe('end_turn');
|
||||
expect(result.text).toMatch(/Simulatiemodus/);
|
||||
});
|
||||
|
||||
it('returns the extraction stub for other tasks in simulation mode', async () => {
|
||||
storage.set('admin:use_simulation', true);
|
||||
const result = await callLLM({ task: 'extract.source', user: 'doc' });
|
||||
expect(() => JSON.parse(result.text)).not.toThrow();
|
||||
expect(JSON.parse(result.text)).toHaveProperty('topics');
|
||||
});
|
||||
});
|
||||
197
src/lib/__tests__/llmSchemas.test.js
Normal file
197
src/lib/__tests__/llmSchemas.test.js
Normal file
@@ -0,0 +1,197 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
extractionResultSchema,
|
||||
handbookResultSchema,
|
||||
normalizeHandbookResult,
|
||||
learningArticleSchema,
|
||||
learningSlidesSchema,
|
||||
learningInfographicSchema,
|
||||
learningAllSchema,
|
||||
quizQuestionsSchema,
|
||||
customTopicSchema,
|
||||
graphActionsSchema,
|
||||
proposeGraphDeltaSchema,
|
||||
} from '../llmSchemas';
|
||||
|
||||
const sampleTopic = {
|
||||
id: 'software-engineer',
|
||||
label: 'Software Engineer',
|
||||
type: 'role',
|
||||
description: 'Builds and maintains the platform.',
|
||||
learning_relevance: 'core',
|
||||
};
|
||||
|
||||
const sampleRelation = {
|
||||
source: 'software-engineer',
|
||||
target: 'onboarding',
|
||||
type: 'part_of',
|
||||
};
|
||||
|
||||
const sampleArticle = {
|
||||
title: 'Onboarding 101',
|
||||
intro: 'A short intro.',
|
||||
sections: [{ heading: 'Day one', body: 'Welcome to the team.' }],
|
||||
keyTakeaways: ['Show up', 'Ask questions'],
|
||||
};
|
||||
|
||||
const sampleSlide = {
|
||||
title: 'Welcome',
|
||||
bullets: ['Meet your buddy', 'Read the handbook'],
|
||||
speakerNote: 'Greet new joiners warmly.',
|
||||
};
|
||||
|
||||
const sampleInfographic = {
|
||||
headline: 'Onboarding flow',
|
||||
tagline: 'From hire to productive in 30 days',
|
||||
stats: [{ value: '30', label: 'days', icon: '📅' }],
|
||||
steps: [{ number: 1, title: 'Sign in', description: 'Use the welcome email.', icon: '🔑' }],
|
||||
quote: 'A great start beats a great recovery.',
|
||||
colorTheme: 'teal',
|
||||
};
|
||||
|
||||
describe('extractionResultSchema', () => {
|
||||
it('accepts a minimal extraction result', () => {
|
||||
const parsed = extractionResultSchema.parse({
|
||||
topics: [sampleTopic],
|
||||
relations: [sampleRelation],
|
||||
});
|
||||
expect(parsed.topics).toHaveLength(1);
|
||||
expect(parsed.relations[0].type).toBe('part_of');
|
||||
});
|
||||
});
|
||||
|
||||
describe('handbookResultSchema', () => {
|
||||
it('accepts the loose vocabulary including executes', () => {
|
||||
const parsed = handbookResultSchema.parse({
|
||||
topics: [{ ...sampleTopic, metadata: { source: 'github_handbook' } }],
|
||||
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
|
||||
});
|
||||
expect(parsed.relations[0].type).toBe('executes');
|
||||
});
|
||||
|
||||
it('normalises executes into executed_by with swapped source/target', () => {
|
||||
const parsed = handbookResultSchema.parse({
|
||||
topics: [sampleTopic],
|
||||
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
|
||||
});
|
||||
const normalised = normalizeHandbookResult(parsed);
|
||||
expect(normalised.relations[0]).toMatchObject({
|
||||
source: 'code-review',
|
||||
target: 'software-engineer',
|
||||
type: 'executed_by',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('learning schemas', () => {
|
||||
it('accepts an article payload', () => {
|
||||
expect(() => learningArticleSchema.parse({ article: sampleArticle })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a slides payload', () => {
|
||||
expect(() => learningSlidesSchema.parse({ slides: [sampleSlide] })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts an infographic payload', () => {
|
||||
expect(() => learningInfographicSchema.parse({ infographic: sampleInfographic })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a combined "all" payload', () => {
|
||||
expect(() =>
|
||||
learningAllSchema.parse({
|
||||
article: sampleArticle,
|
||||
slides: [sampleSlide],
|
||||
infographic: sampleInfographic,
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('quizQuestionsSchema', () => {
|
||||
it('accepts a quiz with four options and a valid correctIndex', () => {
|
||||
const parsed = quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q-1',
|
||||
question: 'What is the buddy system?',
|
||||
topicLabel: 'Onboarding',
|
||||
options: ['A', 'B', 'C', 'D'],
|
||||
correctIndex: 2,
|
||||
explanation: 'C describes the buddy system best.',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(parsed.questions[0].options).toHaveLength(4);
|
||||
});
|
||||
|
||||
it('rejects three options or an out-of-range correctIndex', () => {
|
||||
expect(() =>
|
||||
quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q',
|
||||
question: 'q',
|
||||
topicLabel: 't',
|
||||
options: ['A', 'B', 'C'],
|
||||
correctIndex: 0,
|
||||
explanation: 'e',
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q',
|
||||
question: 'q',
|
||||
topicLabel: 't',
|
||||
options: ['A', 'B', 'C', 'D'],
|
||||
correctIndex: 4,
|
||||
explanation: 'e',
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('customTopicSchema', () => {
|
||||
it('accepts a polished custom topic', () => {
|
||||
expect(() =>
|
||||
customTopicSchema.parse({
|
||||
label: 'Pair Programming',
|
||||
type: 'process',
|
||||
description: 'Two engineers, one keyboard.',
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('graphActionsSchema', () => {
|
||||
it('fills missing arrays with empty defaults', () => {
|
||||
const parsed = graphActionsSchema.parse({});
|
||||
expect(parsed.merges).toEqual([]);
|
||||
expect(parsed.deletions).toEqual([]);
|
||||
expect(parsed.newRelations).toEqual([]);
|
||||
expect(parsed.relevanceUpdates).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('proposeGraphDeltaSchema', () => {
|
||||
it('accepts a reason-only delta', () => {
|
||||
expect(() => proposeGraphDeltaSchema.parse({ reason: 'Nothing to add.' })).not.toThrow();
|
||||
});
|
||||
|
||||
it('caps topics at three and relations at five', () => {
|
||||
const bigTopics = Array.from({ length: 4 }, (_, i) => ({
|
||||
id: `t-${i}`,
|
||||
label: `Topic ${i}`,
|
||||
type: 'concept',
|
||||
description: 'desc',
|
||||
}));
|
||||
expect(() =>
|
||||
proposeGraphDeltaSchema.parse({ reason: 'too many', topics: bigTopics }),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user