From 66e0c275da07bf4cab2943d9a46481c328d29c59 Mon Sep 17 00:00:00 2001 From: RaymondVerhoef Date: Wed, 20 May 2026 19:22:10 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20phase=204=20of=20AI=20pipeline=20harden?= =?UTF-8?q?ing=20=E2=80=94=20quiz=20&=20content=20quality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/lib/random.js: Fisher–Yates shuffle/sample/pickInt; replace every biased .sort(() => 0.5 - Math.random()) site in testService. - testService: debias correctIndex via prompt + runtime re-roll (up to 2x when one position holds >50%); quality gate rejecting <4 distinct options, banned filler ("all of the above" etc) and explanations shorter than 20 chars; dedup new questions against the existing bank via normalised question text. - Quiz schema/tool/prompt require difficulty ('easy'|'medium'|'hard'); db.getQuizBank defaults legacy records to 'medium' on read. - learningService.generateCustomTopic: kebab-case slug ID from the polished label with collision suffixes; default learning_relevance 'standard' when the model omits it. - Tests for random helpers, dedup/quality-gate behaviour and the extended quiz schema. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/lib/__tests__/llmSchemas.test.js | 21 ++- src/lib/__tests__/random.test.js | 48 ++++++ src/lib/__tests__/testService.test.js | 112 +++++++++++++ src/lib/db.js | 7 +- src/lib/learningService.js | 29 +++- src/lib/llmSchemas.js | 3 + src/lib/llmTools.js | 5 +- src/lib/random.js | 29 ++++ src/lib/testService.js | 227 ++++++++++++++++++-------- 9 files changed, 407 insertions(+), 74 deletions(-) create mode 100644 src/lib/__tests__/random.test.js create mode 100644 src/lib/__tests__/testService.test.js create mode 100644 src/lib/random.js diff --git a/src/lib/__tests__/llmSchemas.test.js b/src/lib/__tests__/llmSchemas.test.js index ab27d9c..b036138 100644 --- a/src/lib/__tests__/llmSchemas.test.js +++ b/src/lib/__tests__/llmSchemas.test.js @@ -108,7 +108,7 @@ describe('learning schemas', () => { }); describe('quizQuestionsSchema', () => { - it('accepts a quiz with four options and a valid correctIndex', () => { + it('accepts a quiz with four options, a valid correctIndex and a difficulty', () => { const parsed = quizQuestionsSchema.parse({ questions: [ { @@ -118,10 +118,12 @@ describe('quizQuestionsSchema', () => { options: ['A', 'B', 'C', 'D'], correctIndex: 2, explanation: 'C describes the buddy system best.', + difficulty: 'easy', }, ], }); expect(parsed.questions[0].options).toHaveLength(4); + expect(parsed.questions[0].difficulty).toBe('easy'); }); it('rejects three options or an out-of-range correctIndex', () => { @@ -135,6 +137,7 @@ describe('quizQuestionsSchema', () => { options: ['A', 'B', 'C'], correctIndex: 0, explanation: 'e', + difficulty: 'medium', }, ], }), @@ -149,11 +152,27 @@ describe('quizQuestionsSchema', () => { options: ['A', 'B', 'C', 'D'], correctIndex: 4, explanation: 'e', + difficulty: 'medium', }, ], }), ).toThrow(); }); + + it('rejects a missing or unknown difficulty', () => { + const base = { + id: 'q', + question: 'q', + topicLabel: 't', + options: ['A', 'B', 'C', 'D'], + correctIndex: 0, + explanation: 'because', + }; + expect(() => quizQuestionsSchema.parse({ questions: [base] })).toThrow(); + expect(() => + quizQuestionsSchema.parse({ questions: [{ ...base, difficulty: 'trivial' }] }), + ).toThrow(); + }); }); describe('customTopicSchema', () => { diff --git a/src/lib/__tests__/random.test.js b/src/lib/__tests__/random.test.js new file mode 100644 index 0000000..0033bd3 --- /dev/null +++ b/src/lib/__tests__/random.test.js @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import { shuffle, sample, pickInt } from '../random'; + +describe('shuffle', () => { + it('returns a new array containing the same elements', () => { + const input = [1, 2, 3, 4, 5]; + const out = shuffle(input); + expect(out).not.toBe(input); + expect([...out].sort()).toEqual([...input].sort()); + expect(input).toEqual([1, 2, 3, 4, 5]); + }); + + it('handles empty and single-element arrays', () => { + expect(shuffle([])).toEqual([]); + expect(shuffle([42])).toEqual([42]); + }); +}); + +describe('sample', () => { + it('returns up to n unique elements from the source array', () => { + const out = sample([1, 2, 3, 4, 5], 3); + expect(out).toHaveLength(3); + expect(new Set(out).size).toBe(3); + for (const v of out) expect([1, 2, 3, 4, 5]).toContain(v); + }); + + it('returns the full shuffled array when n exceeds length', () => { + const out = sample([1, 2, 3], 10); + expect(out).toHaveLength(3); + expect([...out].sort()).toEqual([1, 2, 3]); + }); + + it('returns an empty array when n is zero or negative', () => { + expect(sample([1, 2, 3], 0)).toEqual([]); + expect(sample([1, 2, 3], -2)).toEqual([]); + }); +}); + +describe('pickInt', () => { + it('returns an integer in the inclusive range', () => { + for (let i = 0; i < 100; i++) { + const v = pickInt(2, 5); + expect(Number.isInteger(v)).toBe(true); + expect(v).toBeGreaterThanOrEqual(2); + expect(v).toBeLessThanOrEqual(5); + } + }); +}); diff --git a/src/lib/__tests__/testService.test.js b/src/lib/__tests__/testService.test.js new file mode 100644 index 0000000..51e2447 --- /dev/null +++ b/src/lib/__tests__/testService.test.js @@ -0,0 +1,112 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest'; + +const bankStore = new Map(); +const callLLMMock = vi.fn(); + +vi.mock('../pb', () => ({ pb: { collection: () => ({}) } })); + +vi.mock('../db', () => ({ + getQuizBank: vi.fn(async (topicId) => bankStore.get(topicId) || []), + setQuizBank: vi.fn(async (topicId, qs) => { bankStore.set(topicId, qs); }), + getTopics: vi.fn(async () => []), + deleteQuestionFromBank: vi.fn(), + getCachedQuiz: vi.fn(), + setCachedQuiz: vi.fn(), + getQuizResult: vi.fn(), + saveQuizResult: vi.fn(), + getTeamMembers: vi.fn(async () => []), + upsertLeaderboardEntry: vi.fn(), + getCurriculum: vi.fn(), +})); + +vi.mock('../llm', () => ({ callLLM: (...args) => callLLMMock(...args) })); +vi.mock('../curriculumService', () => ({ + getCurriculumTopic: vi.fn(async () => ({ topic: null })), + getQuarterForWeek: vi.fn(() => 1), +})); + +import { forceGenerateTopicQuestions } from '../testService'; + +const topic = { id: 'onboarding', label: 'Onboarding', type: 'concept', description: 'Onboarding for new joiners.' }; + +function makeQuestion(i, overrides = {}) { + return { + id: `q-${i}`, + question: `Sample question ${i}?`, + topicLabel: 'Onboarding', + options: ['A) one', 'B) two', 'C) three', 'D) four'], + correctIndex: i % 4, + explanation: 'This is a substantive explanation for the correct answer.', + difficulty: 'medium', + ...overrides, + }; +} + +function llmEmits(questions) { + callLLMMock.mockResolvedValueOnce({ toolUses: [{ name: 'emit_quiz_questions', input: { questions } }] }); +} + +describe('forceGenerateTopicQuestions', () => { + let debugSpy, warnSpy; + beforeEach(() => { + bankStore.clear(); + callLLMMock.mockReset(); + debugSpy = vi.spyOn(console, 'debug').mockImplementation(() => {}); + warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + }); + afterEach(() => { debugSpy.mockRestore(); warnSpy.mockRestore(); }); + + it('persists a well-formed batch and assigns topic-scoped ids', async () => { + llmEmits([0, 1, 2, 3, 4].map((i) => makeQuestion(i))); + const out = await forceGenerateTopicQuestions(topic, 5); + expect(out).toHaveLength(5); + for (const q of out) expect(q.id.startsWith('onboarding-')).toBe(true); + expect(bankStore.get('onboarding')).toHaveLength(5); + }); + + it('re-rolls when one correctIndex dominates the batch, then accepts on the third try', async () => { + const allZero = [0, 1, 2, 3, 4].map((i) => makeQuestion(i, { correctIndex: 0 })); + llmEmits(allZero); + llmEmits(allZero); + llmEmits(allZero); + const out = await forceGenerateTopicQuestions(topic, 5); + expect(callLLMMock).toHaveBeenCalledTimes(3); + expect(out).toHaveLength(5); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('correctIndex dominated')); + }); + + it('rejects a batch containing a banned "all of the above" option', async () => { + const bad = [0, 1, 2, 3, 4].map((i) => + makeQuestion(i, { options: ['A) x', 'B) y', 'C) z', 'D) All of the above'] }), + ); + llmEmits(bad); + llmEmits(bad); + llmEmits(bad); + await expect(forceGenerateTopicQuestions(topic, 5)).rejects.toThrow(/banned filler|rejected/i); + }); + + it('rejects a batch where an explanation is too short', async () => { + const bad = [0, 1, 2, 3, 4].map((i) => makeQuestion(i, { explanation: 'Because.' })); + llmEmits(bad); + llmEmits(bad); + llmEmits(bad); + await expect(forceGenerateTopicQuestions(topic, 5)).rejects.toThrow(/too short|rejected/i); + }); + + it('drops duplicates whose normalized text matches an existing bank entry', async () => { + bankStore.set('onboarding', [ + { ...makeQuestion(99), id: 'old-1', question: 'What is the BUDDY system???' }, + ]); + llmEmits([ + makeQuestion(0, { question: 'what is the buddy system!' }), + makeQuestion(1, { question: 'Brand new question one?' }), + makeQuestion(2, { question: 'Brand new question two?' }), + makeQuestion(3, { question: 'Brand new question three?' }), + makeQuestion(4, { question: 'Brand new question four?' }), + ]); + const out = await forceGenerateTopicQuestions(topic, 5); + expect(out).toHaveLength(4); + expect(out.find((q) => q.question.toLowerCase().includes('buddy'))).toBeUndefined(); + expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('dropped duplicate'), expect.any(String)); + }); +}); diff --git a/src/lib/db.js b/src/lib/db.js index 331566b..b489c20 100644 --- a/src/lib/db.js +++ b/src/lib/db.js @@ -90,10 +90,15 @@ export async function deleteContent(topicId) { // ── Quiz Banks ─────────────────────────────────────────────────────────────── +function normalizeQuizQuestion(q) { + if (!q || typeof q !== 'object') return q; + return q.difficulty ? q : { ...q, difficulty: 'medium' }; +} + export async function getQuizBank(topicId) { try { const r = await pb.collection('quiz_banks').getFirstListItem(`topic_id="${topicId}"`); - return r.questions || []; + return (r.questions || []).map(normalizeQuizQuestion); } catch { return []; } } diff --git a/src/lib/learningService.js b/src/lib/learningService.js index 788846b..f47ac1c 100644 --- a/src/lib/learningService.js +++ b/src/lib/learningService.js @@ -154,6 +154,28 @@ export async function deleteCachedContent(topicId) { return db.deleteContent(topicId); } +function slugify(label) { + const base = String(label || '') + .toLowerCase() + .normalize('NFKD') + .replace(/\p{Diacritic}/gu, '') + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + return base || 'topic'; +} + +async function pickUniqueTopicId(label) { + const existing = await db.getTopics(); + const used = new Set(existing.map((t) => t.id)); + const base = slugify(label); + if (!used.has(base)) return base; + for (let i = 2; i < 1000; i++) { + const candidate = `${base}-${i}`; + if (!used.has(candidate)) return candidate; + } + return `${base}-${Date.now().toString(36)}`; +} + export async function generateCustomTopic(label) { const result = await callLLM({ task: 'topic.custom', @@ -168,7 +190,12 @@ export async function generateCustomTopic(label) { const emitted = result.toolUses[0]?.input; if (!emitted) throw new Error('Could not process custom topic. Please try again.'); - const newTopic = { ...emitted, id: 'custom_' + Date.now().toString(36) }; + const id = await pickUniqueTopicId(emitted.label); + const newTopic = { + ...emitted, + id, + learning_relevance: emitted.learning_relevance || 'standard', + }; await db.upsertTopic(newTopic); return newTopic; } diff --git a/src/lib/llmSchemas.js b/src/lib/llmSchemas.js index 052fd85..e3f5932 100644 --- a/src/lib/llmSchemas.js +++ b/src/lib/llmSchemas.js @@ -122,6 +122,8 @@ export const learningAllSchema = z.object({ infographic: infographicBodySchema, }); +const quizDifficultyEnum = z.enum(['easy', 'medium', 'hard']); + const quizQuestionSchema = z.object({ id: z.string().min(1), question: z.string().min(1), @@ -129,6 +131,7 @@ const quizQuestionSchema = z.object({ options: z.array(z.string().min(1)).length(4), correctIndex: z.number().int().min(0).max(3), explanation: z.string().min(1), + difficulty: quizDifficultyEnum, }); export const quizQuestionsSchema = z.object({ diff --git a/src/lib/llmTools.js b/src/lib/llmTools.js index 630f38d..000e9a5 100644 --- a/src/lib/llmTools.js +++ b/src/lib/llmTools.js @@ -205,6 +205,8 @@ export const EMIT_CUSTOM_TOPIC_TOOL = { }, }; +const QUIZ_DIFFICULTIES = ['easy', 'medium', 'hard']; + const quizQuestionSchema = { type: 'object', properties: { @@ -214,8 +216,9 @@ const quizQuestionSchema = { options: { type: 'array', items: { type: 'string' }, minItems: 4, maxItems: 4 }, correctIndex: { type: 'integer', minimum: 0, maximum: 3 }, explanation: { type: 'string', description: 'Why the correct answer is correct (1–2 sentences).' }, + difficulty: { type: 'string', enum: QUIZ_DIFFICULTIES, description: 'Per-question difficulty tag.' }, }, - required: ['id', 'question', 'topicLabel', 'options', 'correctIndex', 'explanation'], + required: ['id', 'question', 'topicLabel', 'options', 'correctIndex', 'explanation', 'difficulty'], }; export const EMIT_QUIZ_QUESTIONS_TOOL = { diff --git a/src/lib/random.js b/src/lib/random.js new file mode 100644 index 0000000..02cf02f --- /dev/null +++ b/src/lib/random.js @@ -0,0 +1,29 @@ +/** + * Shared randomness helpers. + * + * `Array.prototype.sort(() => 0.5 - Math.random())` is biased — modern V8 + * sorts use Timsort, which compares each element more than once and skews + * the resulting permutation. Use `shuffle` for anything user-visible + * (quiz options, review topic selection, leaderboards). + */ + +export function shuffle(arr) { + const out = [...arr]; + for (let i = out.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [out[i], out[j]] = [out[j], out[i]]; + } + return out; +} + +export function sample(arr, n) { + if (n <= 0) return []; + if (n >= arr.length) return shuffle(arr); + return shuffle(arr).slice(0, n); +} + +export function pickInt(min, maxInclusive) { + const lo = Math.ceil(min); + const hi = Math.floor(maxInclusive); + return lo + Math.floor(Math.random() * (hi - lo + 1)); +} diff --git a/src/lib/testService.js b/src/lib/testService.js index 1329e98..6391af3 100644 --- a/src/lib/testService.js +++ b/src/lib/testService.js @@ -2,81 +2,73 @@ import * as db from './db'; import { callLLM } from './llm'; import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools'; import { getCurriculumTopic, getQuarterForWeek } from './curriculumService'; +import { shuffle, sample } from './random'; const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform. You generate multiple-choice questions to test employee knowledge on specific topics. Always write in clear, professional English. -Emit questions through the emit_quiz_questions tool. Each question has exactly four options; correctIndex is 0-based. For a typical 5-question batch, mix difficulty roughly 2 easy / 2 medium / 1 hard; scale the ratio proportionally for larger batches.`; +Emit questions through the emit_quiz_questions tool. Each question has exactly four options; correctIndex is 0-based. Tag every question with difficulty ('easy', 'medium' or 'hard'). For a typical 5-question batch, mix difficulty roughly 2 easy / 2 medium / 1 hard; scale the ratio proportionally for larger batches. + +Distribute correctIndex roughly evenly across 0, 1, 2, and 3. Do not place the correct answer at the same position more than 4 out of 10 times. + +Never use filler options such as "all of the above", "none of the above", or "both A and B". Every explanation must be a substantive sentence (≥ 20 characters) describing why the correct answer is correct.`; + +const BANNED_OPTION_PATTERNS = [ + /all of the above/i, + /none of the above/i, + /both a and b/i, + /both b and c/i, + /both c and d/i, + /both a and c/i, + /both b and d/i, + /both a and d/i, +]; const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }]; -async function selectTestTopics(userId, weekNumber) { - const allTopics = await db.getTopics(); - const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude'); - if (!topics || topics.length === 0) return { primaryTopic: null, reviewTopics: [], isReviewWeek: false }; - - // Try curriculum-based selection first - try { - const { topic, curriculumEntry } = await getCurriculumTopic(weekNumber); - - if (curriculumEntry?.is_review_week) { - // Review week: pull topics from the whole quarter - const quarter = getQuarterForWeek(weekNumber); - const curriculum = await db.getCurriculum(new Date().getFullYear()); - const quarterTopicIds = curriculum - .filter(w => w.quarter === quarter && w.topic_id && !w.is_review_week) - .map(w => w.topic_id); - const quarterTopics = topics.filter(t => quarterTopicIds.includes(t.id)); - // Use all quarter topics as review topics (no single primary) - return { - primaryTopic: quarterTopics[0] || topics[0], - reviewTopics: quarterTopics.slice(1), - isReviewWeek: true, - }; - } - - if (topic) { - const others = topics.filter(t => t.id !== topic.id); - const shuffled = others.sort(() => 0.5 - Math.random()); - const reviewTopics = shuffled.slice(0, Math.min(5, shuffled.length)); - return { primaryTopic: topic, reviewTopics, isReviewWeek: false }; - } - } catch (e) { - console.warn('[Test] Curriculum lookup failed, falling back to hash:', e.message); - } - - // Fallback: hash-based selection - const str = `${userId}:${weekNumber}`; - let hash = 0; - for (let i = 0; i < str.length; i++) { - hash = (hash << 5) - hash + str.charCodeAt(i); - hash |= 0; - } - const primaryIndex = Math.abs(hash) % topics.length; - const primaryTopic = topics[primaryIndex]; - - const others = topics.filter((_, i) => i !== primaryIndex); - const shuffled = others.sort(() => 0.5 - Math.random()); - const reviewTopics = shuffled.slice(0, Math.min(5, shuffled.length)); - - return { primaryTopic, reviewTopics, isReviewWeek: false }; +function normalizeQuestionText(text) { + return String(text || '') + .toLowerCase() + .replace(/[\p{P}\p{S}]/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); } -export async function getCachedQuiz(userId, weekNumber) { - return db.getCachedQuiz(userId, weekNumber); +function dominantCorrectIndex(questions) { + if (!questions.length) return null; + const counts = [0, 0, 0, 0]; + for (const q of questions) counts[q.correctIndex] = (counts[q.correctIndex] || 0) + 1; + const max = Math.max(...counts); + return max / questions.length > 0.5 ? { index: counts.indexOf(max), ratio: max / questions.length } : null; } -export async function forceGenerateTopicQuestions(topic, count = 5) { - let bank = await db.getQuizBank(topic.id); +function validateBatchQuality(questions) { + for (const q of questions) { + const distinct = new Set(q.options.map((o) => o.trim().toLowerCase())); + if (distinct.size < 4) { + return `Question "${q.question}" has duplicate options.`; + } + for (const opt of q.options) { + if (BANNED_OPTION_PATTERNS.some((re) => re.test(opt))) { + return `Question "${q.question}" uses a banned filler option ("${opt}").`; + } + } + if (!q.explanation || q.explanation.trim().length < 20) { + return `Question "${q.question}" has an explanation that is too short.`; + } + } + return null; +} +async function callQuizModel(topic, count) { const prompt = `Generate exactly ${count} multiple-choice quiz questions for this knowledge topic and emit them via the emit_quiz_questions tool: Topic: ${topic.label} Type: ${topic.type} Description: ${topic.description} -Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial.`; +Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial. Example: a question whose correct answer is option C uses "correctIndex": 2.`; const result = await callLLM({ task: 'quiz.generate', @@ -89,30 +81,125 @@ Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and }); const emitted = result.toolUses[0]?.input; - if (!emitted) throw new Error(`Could not generate questions for ${topic.label}`); + if (!emitted?.questions?.length) { + throw new Error(`Could not generate questions for ${topic.label}`); + } + return emitted.questions; +} - const newQuestions = (emitted.questions || []).map(q => ({ - ...q, - id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`, - })); +async function selectTestTopics(userId, weekNumber) { + const allTopics = await db.getTopics(); + const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude'); + if (!topics || topics.length === 0) return { primaryTopic: null, reviewTopics: [], isReviewWeek: false }; - bank = [...bank, ...newQuestions]; - await db.setQuizBank(topic.id, bank); - return newQuestions; + try { + const { topic, curriculumEntry } = await getCurriculumTopic(weekNumber); + + if (curriculumEntry?.is_review_week) { + const quarter = getQuarterForWeek(weekNumber); + const curriculum = await db.getCurriculum(new Date().getFullYear()); + const quarterTopicIds = curriculum + .filter(w => w.quarter === quarter && w.topic_id && !w.is_review_week) + .map(w => w.topic_id); + const quarterTopics = topics.filter(t => quarterTopicIds.includes(t.id)); + return { + primaryTopic: quarterTopics[0] || topics[0], + reviewTopics: quarterTopics.slice(1), + isReviewWeek: true, + }; + } + + if (topic) { + const others = topics.filter(t => t.id !== topic.id); + const reviewTopics = sample(others, Math.min(5, others.length)); + return { primaryTopic: topic, reviewTopics, isReviewWeek: false }; + } + } catch (e) { + console.warn('[Test] Curriculum lookup failed, falling back to hash:', e.message); + } + + const str = `${userId}:${weekNumber}`; + let hash = 0; + for (let i = 0; i < str.length; i++) { + hash = (hash << 5) - hash + str.charCodeAt(i); + hash |= 0; + } + const primaryIndex = Math.abs(hash) % topics.length; + const primaryTopic = topics[primaryIndex]; + + const others = topics.filter((_, i) => i !== primaryIndex); + const reviewTopics = sample(others, Math.min(5, others.length)); + + return { primaryTopic, reviewTopics, isReviewWeek: false }; +} + +export async function getCachedQuiz(userId, weekNumber) { + return db.getCachedQuiz(userId, weekNumber); +} + +export async function forceGenerateTopicQuestions(topic, count = 5) { + const existingBank = await db.getQuizBank(topic.id); + const existingKeys = new Set(existingBank.map((q) => normalizeQuestionText(q.question))); + + let lastQualityError = null; + let candidates = null; + + for (let attempt = 0; attempt < 3; attempt++) { + const questions = await callQuizModel(topic, count); + + const qualityError = validateBatchQuality(questions); + if (qualityError) { + lastQualityError = qualityError; + console.warn(`[quiz] batch rejected (attempt ${attempt + 1}): ${qualityError}`); + continue; + } + + const dominant = dominantCorrectIndex(questions); + if (dominant && attempt < 2) { + console.warn(`[quiz] correctIndex dominated by ${dominant.index} (${Math.round(dominant.ratio * 100)}%) — re-rolling`); + continue; + } + + candidates = questions; + break; + } + + if (!candidates) { + throw new Error(`Quality gate rejected the generated batch for ${topic.label}: ${lastQualityError || 'unbalanced answer distribution'}. Click "Generate" to try again.`); + } + + const accepted = []; + for (const q of candidates) { + const key = normalizeQuestionText(q.question); + if (existingKeys.has(key)) { + console.debug('[quiz] dropped duplicate:', q.question); + continue; + } + existingKeys.add(key); + accepted.push({ + ...q, + id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`, + }); + } + + if (!accepted.length) { + throw new Error(`All generated questions for ${topic.label} were duplicates of existing ones.`); + } + + const merged = [...existingBank, ...accepted]; + await db.setQuizBank(topic.id, merged); + return accepted; } async function getOrGenerateTopicQuestions(topic, count) { let bank = await db.getQuizBank(topic.id); if (bank.length < count) { - // Seed an empty/low bank with a small initial batch; admins can grow it - // explicitly via TestManager when they want more depth per topic. await forceGenerateTopicQuestions(topic, 5); bank = await db.getQuizBank(topic.id); } - const shuffled = [...bank].sort(() => 0.5 - Math.random()); - return shuffled.slice(0, Math.min(count, shuffled.length)); + return sample(bank, Math.min(count, bank.length)); } export async function getTopicQuestionBank(topicId) { @@ -153,10 +240,10 @@ export async function generateWeeklyQuiz(userId, weekNumber, force = false) { } } - questions.sort(() => 0.5 - Math.random()); + const shuffled = shuffle(questions); const quiz = { - questions, + questions: shuffled, meta: { userId, weekNumber,