fix: coerce stringified arrays in onboarding overview output (#32)
Opening an onboarding theme could fail with "LLM output failed schema validation": the fast-tier model occasionally emits the key_points and topics_covered tool fields as JSON-encoded strings instead of real arrays. Same failure class the quiz schema already guards against with z.preprocess. - onboardingOverviewSchema: parse "[...]" strings back to arrays before validation; split a bullet/newline string of key points as a fallback; keep the first 5 key points on overage instead of failing the user. Genuinely bad output (too few points, non-JSON strings, wrong types) still fails validation. - emit_onboarding_overview tool: descriptions now state the fields must be JSON arrays, never strings (prompt-side nudge). - 5 regression tests reproducing the exact #32 payload shapes (npm test 139/139). Closes #32 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import {
|
||||
customTopicSchema,
|
||||
graphActionsSchema,
|
||||
proposeGraphDeltaSchema,
|
||||
onboardingOverviewSchema,
|
||||
} from '../llmSchemas';
|
||||
|
||||
const sampleTopic = {
|
||||
@@ -192,3 +193,56 @@ describe('proposeGraphDeltaSchema', () => {
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('onboardingOverviewSchema (issue #32 — fast-tier stringified arrays)', () => {
|
||||
const base = {
|
||||
title: 'Privacy',
|
||||
what_it_is: 'How we handle data.',
|
||||
why_it_matters: 'You touch personal data weekly.',
|
||||
};
|
||||
const points = ['Point one', 'Point two', 'Point three'];
|
||||
const topics = [{ topic_id: 'avg', label: 'AVG' }];
|
||||
|
||||
it('accepts well-formed output', () => {
|
||||
const r = onboardingOverviewSchema.parse({ ...base, key_points: points, topics_covered: topics });
|
||||
expect(r.key_points).toEqual(points);
|
||||
expect(r.topics_covered).toEqual(topics);
|
||||
});
|
||||
|
||||
it('coerces JSON-stringified arrays back to arrays (the exact #32 failure)', () => {
|
||||
const r = onboardingOverviewSchema.parse({
|
||||
...base,
|
||||
key_points: JSON.stringify(points),
|
||||
topics_covered: JSON.stringify(topics),
|
||||
});
|
||||
expect(r.key_points).toEqual(points);
|
||||
expect(r.topics_covered).toEqual(topics);
|
||||
});
|
||||
|
||||
it('splits a bullet/newline string of key points as a fallback', () => {
|
||||
const r = onboardingOverviewSchema.parse({
|
||||
...base,
|
||||
key_points: '- Point one\n• Point two\n3. Point three',
|
||||
topics_covered: topics,
|
||||
});
|
||||
expect(r.key_points).toEqual(points);
|
||||
});
|
||||
|
||||
it('keeps the first 5 key points instead of failing on overage', () => {
|
||||
const seven = Array.from({ length: 7 }, (_, i) => `P${i + 1}`);
|
||||
const r = onboardingOverviewSchema.parse({ ...base, key_points: seven, topics_covered: topics });
|
||||
expect(r.key_points).toEqual(['P1', 'P2', 'P3', 'P4', 'P5']);
|
||||
});
|
||||
|
||||
it('still rejects genuinely bad output', () => {
|
||||
expect(() =>
|
||||
onboardingOverviewSchema.parse({ ...base, key_points: ['only', 'two'], topics_covered: topics }),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
onboardingOverviewSchema.parse({ ...base, key_points: points, topics_covered: 'not json at all' }),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
onboardingOverviewSchema.parse({ ...base, key_points: 42, topics_covered: topics }),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user