feat: phase 1 of AI pipeline hardening — single LLM client + tier-aware models
Implements phase 1 of AI_PIPELINE_HARDENING_PLAN.md. Every Anthropic call now goes through one module that owns retry, timeout, abort, structured- output parsing, schema validation, and best-effort call telemetry. * src/lib/llm.js — single callLLM entry point. Resolves model per tier (fast / standard / reasoning) with admin:model legacy fallback for the standard tier; 60s default timeout via AbortController; balanced-brace JSON extraction; LLMHttpError, LLMTruncatedError, LLMOutputError, and LLMValidationError surface clearly distinct failure modes. * src/lib/llmRetry.js — exponential backoff with full jitter, retries only on transient HTTP statuses, honours Retry-After up to 60s, never retries on AbortError. * src/lib/llmSchemas.js — Zod schemas for every structured task plus normalizeHandbookResult (collapses legacy "executes" relations into the canonical "executed_by" vocabulary). * src/lib/api.js — thin shim over callLLM so existing callers (extraction pipeline, learning, quiz, R42, knowledge graph) keep working unchanged. * src/lib/__tests__/ — 32 Vitest cases covering parse paths, error surfaces, simulation mode, model resolution, and schema validation. * src/pages/Admin/index.jsx — three model inputs (fast / standard / reasoning) replacing the single legacy field; legacy value falls back for the standard tier so existing overrides survive. Adds Zod and Vitest, plus an "npm run test" script. Also cleans up the pre-existing repo-wide ESLint failures so phase 1's "npm run lint passes" acceptance criterion can be checked: drops unused React imports across the JSX tree (React 19 JSX runtime auto-imports), attaches cause to rethrown errors in the service modules, ignores pb_migrations in the ESLint config (PocketBase JSVM globals), and removes one dead handleCreateCustom function in Leren.jsx. A real behaviour bug surfaced in Testen.jsx — the quiz timer captured a stale finishQuiz via setInterval closure; now updated via finishQuizRef so the timer always invokes the latest callback. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
197
src/lib/__tests__/llmSchemas.test.js
Normal file
197
src/lib/__tests__/llmSchemas.test.js
Normal file
@@ -0,0 +1,197 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
extractionResultSchema,
|
||||
handbookResultSchema,
|
||||
normalizeHandbookResult,
|
||||
learningArticleSchema,
|
||||
learningSlidesSchema,
|
||||
learningInfographicSchema,
|
||||
learningAllSchema,
|
||||
quizQuestionsSchema,
|
||||
customTopicSchema,
|
||||
graphActionsSchema,
|
||||
proposeGraphDeltaSchema,
|
||||
} from '../llmSchemas';
|
||||
|
||||
const sampleTopic = {
|
||||
id: 'software-engineer',
|
||||
label: 'Software Engineer',
|
||||
type: 'role',
|
||||
description: 'Builds and maintains the platform.',
|
||||
learning_relevance: 'core',
|
||||
};
|
||||
|
||||
const sampleRelation = {
|
||||
source: 'software-engineer',
|
||||
target: 'onboarding',
|
||||
type: 'part_of',
|
||||
};
|
||||
|
||||
const sampleArticle = {
|
||||
title: 'Onboarding 101',
|
||||
intro: 'A short intro.',
|
||||
sections: [{ heading: 'Day one', body: 'Welcome to the team.' }],
|
||||
keyTakeaways: ['Show up', 'Ask questions'],
|
||||
};
|
||||
|
||||
const sampleSlide = {
|
||||
title: 'Welcome',
|
||||
bullets: ['Meet your buddy', 'Read the handbook'],
|
||||
speakerNote: 'Greet new joiners warmly.',
|
||||
};
|
||||
|
||||
const sampleInfographic = {
|
||||
headline: 'Onboarding flow',
|
||||
tagline: 'From hire to productive in 30 days',
|
||||
stats: [{ value: '30', label: 'days', icon: '📅' }],
|
||||
steps: [{ number: 1, title: 'Sign in', description: 'Use the welcome email.', icon: '🔑' }],
|
||||
quote: 'A great start beats a great recovery.',
|
||||
colorTheme: 'teal',
|
||||
};
|
||||
|
||||
describe('extractionResultSchema', () => {
|
||||
it('accepts a minimal extraction result', () => {
|
||||
const parsed = extractionResultSchema.parse({
|
||||
topics: [sampleTopic],
|
||||
relations: [sampleRelation],
|
||||
});
|
||||
expect(parsed.topics).toHaveLength(1);
|
||||
expect(parsed.relations[0].type).toBe('part_of');
|
||||
});
|
||||
});
|
||||
|
||||
describe('handbookResultSchema', () => {
|
||||
it('accepts the loose vocabulary including executes', () => {
|
||||
const parsed = handbookResultSchema.parse({
|
||||
topics: [{ ...sampleTopic, metadata: { source: 'github_handbook' } }],
|
||||
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
|
||||
});
|
||||
expect(parsed.relations[0].type).toBe('executes');
|
||||
});
|
||||
|
||||
it('normalises executes into executed_by with swapped source/target', () => {
|
||||
const parsed = handbookResultSchema.parse({
|
||||
topics: [sampleTopic],
|
||||
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
|
||||
});
|
||||
const normalised = normalizeHandbookResult(parsed);
|
||||
expect(normalised.relations[0]).toMatchObject({
|
||||
source: 'code-review',
|
||||
target: 'software-engineer',
|
||||
type: 'executed_by',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('learning schemas', () => {
|
||||
it('accepts an article payload', () => {
|
||||
expect(() => learningArticleSchema.parse({ article: sampleArticle })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a slides payload', () => {
|
||||
expect(() => learningSlidesSchema.parse({ slides: [sampleSlide] })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts an infographic payload', () => {
|
||||
expect(() => learningInfographicSchema.parse({ infographic: sampleInfographic })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a combined "all" payload', () => {
|
||||
expect(() =>
|
||||
learningAllSchema.parse({
|
||||
article: sampleArticle,
|
||||
slides: [sampleSlide],
|
||||
infographic: sampleInfographic,
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('quizQuestionsSchema', () => {
|
||||
it('accepts a quiz with four options and a valid correctIndex', () => {
|
||||
const parsed = quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q-1',
|
||||
question: 'What is the buddy system?',
|
||||
topicLabel: 'Onboarding',
|
||||
options: ['A', 'B', 'C', 'D'],
|
||||
correctIndex: 2,
|
||||
explanation: 'C describes the buddy system best.',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(parsed.questions[0].options).toHaveLength(4);
|
||||
});
|
||||
|
||||
it('rejects three options or an out-of-range correctIndex', () => {
|
||||
expect(() =>
|
||||
quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q',
|
||||
question: 'q',
|
||||
topicLabel: 't',
|
||||
options: ['A', 'B', 'C'],
|
||||
correctIndex: 0,
|
||||
explanation: 'e',
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q',
|
||||
question: 'q',
|
||||
topicLabel: 't',
|
||||
options: ['A', 'B', 'C', 'D'],
|
||||
correctIndex: 4,
|
||||
explanation: 'e',
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('customTopicSchema', () => {
|
||||
it('accepts a polished custom topic', () => {
|
||||
expect(() =>
|
||||
customTopicSchema.parse({
|
||||
label: 'Pair Programming',
|
||||
type: 'process',
|
||||
description: 'Two engineers, one keyboard.',
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('graphActionsSchema', () => {
|
||||
it('fills missing arrays with empty defaults', () => {
|
||||
const parsed = graphActionsSchema.parse({});
|
||||
expect(parsed.merges).toEqual([]);
|
||||
expect(parsed.deletions).toEqual([]);
|
||||
expect(parsed.newRelations).toEqual([]);
|
||||
expect(parsed.relevanceUpdates).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('proposeGraphDeltaSchema', () => {
|
||||
it('accepts a reason-only delta', () => {
|
||||
expect(() => proposeGraphDeltaSchema.parse({ reason: 'Nothing to add.' })).not.toThrow();
|
||||
});
|
||||
|
||||
it('caps topics at three and relations at five', () => {
|
||||
const bigTopics = Array.from({ length: 4 }, (_, i) => ({
|
||||
id: `t-${i}`,
|
||||
label: `Topic ${i}`,
|
||||
type: 'concept',
|
||||
description: 'desc',
|
||||
}));
|
||||
expect(() =>
|
||||
proposeGraphDeltaSchema.parse({ reason: 'too many', topics: bigTopics }),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user