Opening an onboarding theme could fail with "LLM output failed schema validation": the fast-tier model occasionally emits the key_points and topics_covered tool fields as JSON-encoded strings instead of real arrays. Same failure class the quiz schema already guards against with z.preprocess. - onboardingOverviewSchema: parse "[...]" strings back to arrays before validation; split a bullet/newline string of key points as a fallback; keep the first 5 key points on overage instead of failing the user. Genuinely bad output (too few points, non-JSON strings, wrong types) still fails validation. - emit_onboarding_overview tool: descriptions now state the fields must be JSON arrays, never strings (prompt-side nudge). - 5 regression tests reproducing the exact #32 payload shapes (npm test 139/139). Closes #32 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
249 lines
7.3 KiB
JavaScript
249 lines
7.3 KiB
JavaScript
import { describe, expect, it } from 'vitest';
|
|
import {
|
|
extractionResultSchema,
|
|
learningArticleSchema,
|
|
learningSlidesSchema,
|
|
learningInfographicSchema,
|
|
learningAllSchema,
|
|
quizQuestionsSchema,
|
|
customTopicSchema,
|
|
graphActionsSchema,
|
|
proposeGraphDeltaSchema,
|
|
onboardingOverviewSchema,
|
|
} from '../llmSchemas';
|
|
|
|
const sampleTopic = {
|
|
id: 'software-engineer',
|
|
label: 'Software Engineer',
|
|
type: 'role',
|
|
description: 'Builds and maintains the platform.',
|
|
learning_relevance: 'core',
|
|
};
|
|
|
|
const sampleRelation = {
|
|
source: 'software-engineer',
|
|
target: 'onboarding',
|
|
type: 'part_of',
|
|
};
|
|
|
|
const sampleArticle = {
|
|
title: 'Onboarding 101',
|
|
intro: 'A short intro.',
|
|
sections: [{ heading: 'Day one', body: 'Welcome to the team.' }],
|
|
keyTakeaways: ['Show up', 'Ask questions'],
|
|
};
|
|
|
|
const sampleSlide = {
|
|
title: 'Welcome',
|
|
bullets: ['Meet your buddy', 'Read the handbook'],
|
|
speakerNote: 'Greet new joiners warmly.',
|
|
};
|
|
|
|
const sampleInfographic = {
|
|
headline: 'Onboarding flow',
|
|
tagline: 'From hire to productive in 30 days',
|
|
stats: [{ value: '30', label: 'days', icon: '📅' }],
|
|
steps: [{ number: 1, title: 'Sign in', description: 'Use the welcome email.', icon: '🔑' }],
|
|
quote: 'A great start beats a great recovery.',
|
|
colorTheme: 'teal',
|
|
};
|
|
|
|
describe('extractionResultSchema', () => {
|
|
it('accepts a minimal extraction result', () => {
|
|
const parsed = extractionResultSchema.parse({
|
|
topics: [sampleTopic],
|
|
relations: [sampleRelation],
|
|
});
|
|
expect(parsed.topics).toHaveLength(1);
|
|
expect(parsed.relations[0].type).toBe('part_of');
|
|
});
|
|
});
|
|
|
|
|
|
describe('learning schemas', () => {
|
|
it('accepts an article payload', () => {
|
|
expect(() => learningArticleSchema.parse({ article: sampleArticle })).not.toThrow();
|
|
});
|
|
|
|
it('accepts a slides payload', () => {
|
|
expect(() => learningSlidesSchema.parse({ slides: [sampleSlide] })).not.toThrow();
|
|
});
|
|
|
|
it('accepts an infographic payload', () => {
|
|
expect(() => learningInfographicSchema.parse({ infographic: sampleInfographic })).not.toThrow();
|
|
});
|
|
|
|
it('accepts a combined "all" payload', () => {
|
|
expect(() =>
|
|
learningAllSchema.parse({
|
|
article: sampleArticle,
|
|
slides: [sampleSlide],
|
|
infographic: sampleInfographic,
|
|
}),
|
|
).not.toThrow();
|
|
});
|
|
});
|
|
|
|
describe('quizQuestionsSchema', () => {
|
|
it('accepts a quiz with four options, a valid correctIndex and a difficulty', () => {
|
|
const parsed = quizQuestionsSchema.parse({
|
|
questions: [
|
|
{
|
|
id: 'q-1',
|
|
question: 'What is the buddy system?',
|
|
topicLabel: 'Onboarding',
|
|
options: ['A', 'B', 'C', 'D'],
|
|
correctIndex: 2,
|
|
explanation: 'C describes the buddy system best.',
|
|
difficulty: 'easy',
|
|
},
|
|
],
|
|
});
|
|
expect(parsed.questions[0].options).toHaveLength(4);
|
|
expect(parsed.questions[0].difficulty).toBe('easy');
|
|
});
|
|
|
|
it('rejects three options or an out-of-range correctIndex', () => {
|
|
expect(() =>
|
|
quizQuestionsSchema.parse({
|
|
questions: [
|
|
{
|
|
id: 'q',
|
|
question: 'q',
|
|
topicLabel: 't',
|
|
options: ['A', 'B', 'C'],
|
|
correctIndex: 0,
|
|
explanation: 'e',
|
|
difficulty: 'medium',
|
|
},
|
|
],
|
|
}),
|
|
).toThrow();
|
|
expect(() =>
|
|
quizQuestionsSchema.parse({
|
|
questions: [
|
|
{
|
|
id: 'q',
|
|
question: 'q',
|
|
topicLabel: 't',
|
|
options: ['A', 'B', 'C', 'D'],
|
|
correctIndex: 4,
|
|
explanation: 'e',
|
|
difficulty: 'medium',
|
|
},
|
|
],
|
|
}),
|
|
).toThrow();
|
|
});
|
|
|
|
it('falls back to medium for a missing or unknown difficulty', () => {
|
|
const base = {
|
|
id: 'q',
|
|
question: 'q',
|
|
topicLabel: 't',
|
|
options: ['A', 'B', 'C', 'D'],
|
|
correctIndex: 0,
|
|
explanation: 'because',
|
|
};
|
|
// Missing difficulty should default to 'medium'
|
|
const parsed1 = quizQuestionsSchema.parse({ questions: [base] });
|
|
expect(parsed1.questions[0].difficulty).toBe('medium');
|
|
// Unknown difficulty should also default to 'medium'
|
|
const parsed2 = quizQuestionsSchema.parse({ questions: [{ ...base, difficulty: 'trivial' }] });
|
|
expect(parsed2.questions[0].difficulty).toBe('medium');
|
|
});
|
|
});
|
|
|
|
describe('customTopicSchema', () => {
|
|
it('accepts a polished custom topic', () => {
|
|
expect(() =>
|
|
customTopicSchema.parse({
|
|
label: 'Pair Programming',
|
|
type: 'process',
|
|
description: 'Two engineers, one keyboard.',
|
|
}),
|
|
).not.toThrow();
|
|
});
|
|
});
|
|
|
|
describe('graphActionsSchema', () => {
|
|
it('fills missing arrays with empty defaults', () => {
|
|
const parsed = graphActionsSchema.parse({});
|
|
expect(parsed.merges).toEqual([]);
|
|
expect(parsed.deletions).toEqual([]);
|
|
expect(parsed.newRelations).toEqual([]);
|
|
expect(parsed.relevanceUpdates).toEqual([]);
|
|
});
|
|
});
|
|
|
|
describe('proposeGraphDeltaSchema', () => {
|
|
it('accepts a reason-only delta', () => {
|
|
expect(() => proposeGraphDeltaSchema.parse({ reason: 'Nothing to add.' })).not.toThrow();
|
|
});
|
|
|
|
it('caps topics at three and relations at five', () => {
|
|
const bigTopics = Array.from({ length: 4 }, (_, i) => ({
|
|
id: `t-${i}`,
|
|
label: `Topic ${i}`,
|
|
type: 'concept',
|
|
description: 'desc',
|
|
}));
|
|
expect(() =>
|
|
proposeGraphDeltaSchema.parse({ reason: 'too many', topics: bigTopics }),
|
|
).toThrow();
|
|
});
|
|
});
|
|
|
|
describe('onboardingOverviewSchema (issue #32 — fast-tier stringified arrays)', () => {
|
|
const base = {
|
|
title: 'Privacy',
|
|
what_it_is: 'How we handle data.',
|
|
why_it_matters: 'You touch personal data weekly.',
|
|
};
|
|
const points = ['Point one', 'Point two', 'Point three'];
|
|
const topics = [{ topic_id: 'avg', label: 'AVG' }];
|
|
|
|
it('accepts well-formed output', () => {
|
|
const r = onboardingOverviewSchema.parse({ ...base, key_points: points, topics_covered: topics });
|
|
expect(r.key_points).toEqual(points);
|
|
expect(r.topics_covered).toEqual(topics);
|
|
});
|
|
|
|
it('coerces JSON-stringified arrays back to arrays (the exact #32 failure)', () => {
|
|
const r = onboardingOverviewSchema.parse({
|
|
...base,
|
|
key_points: JSON.stringify(points),
|
|
topics_covered: JSON.stringify(topics),
|
|
});
|
|
expect(r.key_points).toEqual(points);
|
|
expect(r.topics_covered).toEqual(topics);
|
|
});
|
|
|
|
it('splits a bullet/newline string of key points as a fallback', () => {
|
|
const r = onboardingOverviewSchema.parse({
|
|
...base,
|
|
key_points: '- Point one\n• Point two\n3. Point three',
|
|
topics_covered: topics,
|
|
});
|
|
expect(r.key_points).toEqual(points);
|
|
});
|
|
|
|
it('keeps the first 5 key points instead of failing on overage', () => {
|
|
const seven = Array.from({ length: 7 }, (_, i) => `P${i + 1}`);
|
|
const r = onboardingOverviewSchema.parse({ ...base, key_points: seven, topics_covered: topics });
|
|
expect(r.key_points).toEqual(['P1', 'P2', 'P3', 'P4', 'P5']);
|
|
});
|
|
|
|
it('still rejects genuinely bad output', () => {
|
|
expect(() =>
|
|
onboardingOverviewSchema.parse({ ...base, key_points: ['only', 'two'], topics_covered: topics }),
|
|
).toThrow();
|
|
expect(() =>
|
|
onboardingOverviewSchema.parse({ ...base, key_points: points, topics_covered: 'not json at all' }),
|
|
).toThrow();
|
|
expect(() =>
|
|
onboardingOverviewSchema.parse({ ...base, key_points: 42, topics_covered: topics }),
|
|
).toThrow();
|
|
});
|
|
});
|