Files
learning-platform/src/lib/__tests__/llmSchemas.test.js
RaymondVerhoef 182fb2357f
All checks were successful
On Pull Request to Main / test (pull_request) Successful in 41s
On Pull Request to Main / publish (pull_request) Successful in 1m15s
On Pull Request to Main / deploy-dev (pull_request) Successful in 3m17s
fix: coerce stringified arrays in onboarding overview output (#32)
Opening an onboarding theme could fail with "LLM output failed schema
validation": the fast-tier model occasionally emits the key_points and
topics_covered tool fields as JSON-encoded strings instead of real
arrays. Same failure class the quiz schema already guards against with
z.preprocess.

- onboardingOverviewSchema: parse "[...]" strings back to arrays before
  validation; split a bullet/newline string of key points as a fallback;
  keep the first 5 key points on overage instead of failing the user.
  Genuinely bad output (too few points, non-JSON strings, wrong types)
  still fails validation.
- emit_onboarding_overview tool: descriptions now state the fields must
  be JSON arrays, never strings (prompt-side nudge).
- 5 regression tests reproducing the exact #32 payload shapes
  (npm test 139/139).

Closes #32

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-13 09:38:46 +02:00

249 lines
7.3 KiB
JavaScript

import { describe, expect, it } from 'vitest';
import {
extractionResultSchema,
learningArticleSchema,
learningSlidesSchema,
learningInfographicSchema,
learningAllSchema,
quizQuestionsSchema,
customTopicSchema,
graphActionsSchema,
proposeGraphDeltaSchema,
onboardingOverviewSchema,
} from '../llmSchemas';
const sampleTopic = {
id: 'software-engineer',
label: 'Software Engineer',
type: 'role',
description: 'Builds and maintains the platform.',
learning_relevance: 'core',
};
const sampleRelation = {
source: 'software-engineer',
target: 'onboarding',
type: 'part_of',
};
const sampleArticle = {
title: 'Onboarding 101',
intro: 'A short intro.',
sections: [{ heading: 'Day one', body: 'Welcome to the team.' }],
keyTakeaways: ['Show up', 'Ask questions'],
};
const sampleSlide = {
title: 'Welcome',
bullets: ['Meet your buddy', 'Read the handbook'],
speakerNote: 'Greet new joiners warmly.',
};
const sampleInfographic = {
headline: 'Onboarding flow',
tagline: 'From hire to productive in 30 days',
stats: [{ value: '30', label: 'days', icon: '📅' }],
steps: [{ number: 1, title: 'Sign in', description: 'Use the welcome email.', icon: '🔑' }],
quote: 'A great start beats a great recovery.',
colorTheme: 'teal',
};
describe('extractionResultSchema', () => {
it('accepts a minimal extraction result', () => {
const parsed = extractionResultSchema.parse({
topics: [sampleTopic],
relations: [sampleRelation],
});
expect(parsed.topics).toHaveLength(1);
expect(parsed.relations[0].type).toBe('part_of');
});
});
describe('learning schemas', () => {
it('accepts an article payload', () => {
expect(() => learningArticleSchema.parse({ article: sampleArticle })).not.toThrow();
});
it('accepts a slides payload', () => {
expect(() => learningSlidesSchema.parse({ slides: [sampleSlide] })).not.toThrow();
});
it('accepts an infographic payload', () => {
expect(() => learningInfographicSchema.parse({ infographic: sampleInfographic })).not.toThrow();
});
it('accepts a combined "all" payload', () => {
expect(() =>
learningAllSchema.parse({
article: sampleArticle,
slides: [sampleSlide],
infographic: sampleInfographic,
}),
).not.toThrow();
});
});
describe('quizQuestionsSchema', () => {
it('accepts a quiz with four options, a valid correctIndex and a difficulty', () => {
const parsed = quizQuestionsSchema.parse({
questions: [
{
id: 'q-1',
question: 'What is the buddy system?',
topicLabel: 'Onboarding',
options: ['A', 'B', 'C', 'D'],
correctIndex: 2,
explanation: 'C describes the buddy system best.',
difficulty: 'easy',
},
],
});
expect(parsed.questions[0].options).toHaveLength(4);
expect(parsed.questions[0].difficulty).toBe('easy');
});
it('rejects three options or an out-of-range correctIndex', () => {
expect(() =>
quizQuestionsSchema.parse({
questions: [
{
id: 'q',
question: 'q',
topicLabel: 't',
options: ['A', 'B', 'C'],
correctIndex: 0,
explanation: 'e',
difficulty: 'medium',
},
],
}),
).toThrow();
expect(() =>
quizQuestionsSchema.parse({
questions: [
{
id: 'q',
question: 'q',
topicLabel: 't',
options: ['A', 'B', 'C', 'D'],
correctIndex: 4,
explanation: 'e',
difficulty: 'medium',
},
],
}),
).toThrow();
});
it('falls back to medium for a missing or unknown difficulty', () => {
const base = {
id: 'q',
question: 'q',
topicLabel: 't',
options: ['A', 'B', 'C', 'D'],
correctIndex: 0,
explanation: 'because',
};
// Missing difficulty should default to 'medium'
const parsed1 = quizQuestionsSchema.parse({ questions: [base] });
expect(parsed1.questions[0].difficulty).toBe('medium');
// Unknown difficulty should also default to 'medium'
const parsed2 = quizQuestionsSchema.parse({ questions: [{ ...base, difficulty: 'trivial' }] });
expect(parsed2.questions[0].difficulty).toBe('medium');
});
});
describe('customTopicSchema', () => {
it('accepts a polished custom topic', () => {
expect(() =>
customTopicSchema.parse({
label: 'Pair Programming',
type: 'process',
description: 'Two engineers, one keyboard.',
}),
).not.toThrow();
});
});
describe('graphActionsSchema', () => {
it('fills missing arrays with empty defaults', () => {
const parsed = graphActionsSchema.parse({});
expect(parsed.merges).toEqual([]);
expect(parsed.deletions).toEqual([]);
expect(parsed.newRelations).toEqual([]);
expect(parsed.relevanceUpdates).toEqual([]);
});
});
describe('proposeGraphDeltaSchema', () => {
it('accepts a reason-only delta', () => {
expect(() => proposeGraphDeltaSchema.parse({ reason: 'Nothing to add.' })).not.toThrow();
});
it('caps topics at three and relations at five', () => {
const bigTopics = Array.from({ length: 4 }, (_, i) => ({
id: `t-${i}`,
label: `Topic ${i}`,
type: 'concept',
description: 'desc',
}));
expect(() =>
proposeGraphDeltaSchema.parse({ reason: 'too many', topics: bigTopics }),
).toThrow();
});
});
describe('onboardingOverviewSchema (issue #32 — fast-tier stringified arrays)', () => {
const base = {
title: 'Privacy',
what_it_is: 'How we handle data.',
why_it_matters: 'You touch personal data weekly.',
};
const points = ['Point one', 'Point two', 'Point three'];
const topics = [{ topic_id: 'avg', label: 'AVG' }];
it('accepts well-formed output', () => {
const r = onboardingOverviewSchema.parse({ ...base, key_points: points, topics_covered: topics });
expect(r.key_points).toEqual(points);
expect(r.topics_covered).toEqual(topics);
});
it('coerces JSON-stringified arrays back to arrays (the exact #32 failure)', () => {
const r = onboardingOverviewSchema.parse({
...base,
key_points: JSON.stringify(points),
topics_covered: JSON.stringify(topics),
});
expect(r.key_points).toEqual(points);
expect(r.topics_covered).toEqual(topics);
});
it('splits a bullet/newline string of key points as a fallback', () => {
const r = onboardingOverviewSchema.parse({
...base,
key_points: '- Point one\n• Point two\n3. Point three',
topics_covered: topics,
});
expect(r.key_points).toEqual(points);
});
it('keeps the first 5 key points instead of failing on overage', () => {
const seven = Array.from({ length: 7 }, (_, i) => `P${i + 1}`);
const r = onboardingOverviewSchema.parse({ ...base, key_points: seven, topics_covered: topics });
expect(r.key_points).toEqual(['P1', 'P2', 'P3', 'P4', 'P5']);
});
it('still rejects genuinely bad output', () => {
expect(() =>
onboardingOverviewSchema.parse({ ...base, key_points: ['only', 'two'], topics_covered: topics }),
).toThrow();
expect(() =>
onboardingOverviewSchema.parse({ ...base, key_points: points, topics_covered: 'not json at all' }),
).toThrow();
expect(() =>
onboardingOverviewSchema.parse({ ...base, key_points: 42, topics_covered: topics }),
).toThrow();
});
});