import * as db from './db'; import { callLLM, cachedSystem } from './llm'; import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools'; import { getCurrentWeekContent } from './curriculumService'; import { shuffle, sample } from './random'; const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform. You generate multiple-choice questions to test employee knowledge on specific topics. Always write in clear, professional English. Emit questions through the emit_quiz_questions tool. Each question has exactly four options; correctIndex is 0-based. Tag every question with difficulty ('easy', 'medium' or 'hard'). For a typical 5-question batch, mix difficulty roughly 2 easy / 2 medium / 1 hard; scale the ratio proportionally for larger batches. Distribute correctIndex roughly evenly across 0, 1, 2, and 3. Do not place the correct answer at the same position more than 4 out of 10 times. Never use filler options such as "all of the above", "none of the above", or "both A and B". Every explanation must be a substantive sentence (≥ 20 characters) describing why the correct answer is correct.`; const BANNED_OPTION_PATTERNS = [ /all of the above/i, /none of the above/i, /both a and b/i, /both b and c/i, /both c and d/i, /both a and c/i, /both b and d/i, /both a and d/i, ]; function normalizeQuestionText(text) { return String(text || '') .toLowerCase() .replace(/[\p{P}\p{S}]/gu, ' ') .replace(/\s+/g, ' ') .trim(); } function dominantCorrectIndex(questions) { if (!questions.length) return null; const counts = [0, 0, 0, 0]; for (const q of questions) counts[q.correctIndex] = (counts[q.correctIndex] || 0) + 1; const max = Math.max(...counts); return max / questions.length > 0.8 ? { index: counts.indexOf(max), ratio: max / questions.length } : null; } function validateBatchQuality(questions) { for (const q of questions) { const distinct = new Set(q.options.map((o) => o.trim().toLowerCase())); if (distinct.size < 4) { return `Question "${q.question}" has duplicate options.`; } for (const opt of q.options) { if (BANNED_OPTION_PATTERNS.some((re) => re.test(opt))) { return `Question "${q.question}" uses a banned filler option ("${opt}").`; } } if (!q.explanation || q.explanation.trim().length < 20) { return `Question "${q.question}" has an explanation that is too short.`; } } return null; } async function callQuizModel(topic, count) { const prompt = `Generate exactly ${count} multiple-choice quiz questions for this knowledge topic and emit them via the emit_quiz_questions tool: Topic: ${topic.label} Type: ${topic.type} Description: ${topic.description} Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial. Ensure correct answers are evenly distributed.`; const result = await callLLM({ task: 'quiz.generate', tier: 'standard', system: cachedSystem(QUIZ_SYSTEM), user: prompt, tools: [EMIT_QUIZ_QUESTIONS_TOOL], toolChoice: { type: 'tool', name: EMIT_QUIZ_QUESTIONS_TOOL.name }, maxTokens: 4096, }); const emitted = result.toolUses[0]?.input; if (!emitted) { throw new Error(`LLM did not emit tool output for ${topic.label}`); } // Handle different shapes the LLM might return: // 1. { questions: [...] } — expected // 2. [...] — model returned array directly // 3. { questions: { ... } } — model wrapped single question in object let questions; if (Array.isArray(emitted.questions)) { questions = emitted.questions; } else if (Array.isArray(emitted)) { questions = emitted; } else if (emitted.questions && typeof emitted.questions === 'object') { // Single question wrapped as object instead of array questions = [emitted.questions]; } else { throw new Error(`Unexpected quiz output shape for ${topic.label}`); } if (!questions.length) { throw new Error(`No questions generated for ${topic.label}`); } // Ensure each question has required fields (fill in defaults for missing ones) return questions.map((q, i) => ({ id: q.id || `gen-${i}-${Math.random().toString(36).slice(2, 8)}`, question: q.question || '', topicLabel: q.topicLabel || topic.label, options: Array.isArray(q.options) ? q.options.slice(0, 4) : [], correctIndex: typeof q.correctIndex === 'number' ? q.correctIndex : 0, explanation: q.explanation || 'No explanation provided.', difficulty: q.difficulty || 'medium', })).filter(q => q.question && q.options.length === 4); } async function selectTestTopics(userId, isoWeekNumber) { const allTopics = await db.getTopics(); const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude'); if (!topics || topics.length === 0) return { primaryTopic: null, reviewTopics: [], isReviewWeek: false }; try { const weekContent = await getCurrentWeekContent(isoWeekNumber); if (weekContent && weekContent.topics && weekContent.topics.length > 0) { const primaryTopic = weekContent.topics[0]; if (!primaryTopic?.id) { console.warn('[Test] First curriculum topic has no id — falling back to hash'); } else { const others = topics.filter(t => t.id !== primaryTopic.id); const reviewTopics = sample(others, Math.min(5, others.length)); return { primaryTopic, reviewTopics, isReviewWeek: false }; } } } catch (e) { console.warn('[Test] Curriculum lookup failed, falling back to hash:', e.message); } // Fallback hash-based assignment const str = `${userId}:${isoWeekNumber}`; let hash = 0; for (let i = 0; i < str.length; i++) { hash = (hash << 5) - hash + str.charCodeAt(i); hash |= 0; } const primaryIndex = Math.abs(hash) % topics.length; const primaryTopic = topics[primaryIndex]; const others = topics.filter((_, i) => i !== primaryIndex); const reviewTopics = sample(others, Math.min(5, others.length)); return { primaryTopic, reviewTopics, isReviewWeek: false }; } export async function getCachedQuiz(userId, weekNumber) { return db.getCachedQuiz(userId, weekNumber); } export async function forceGenerateTopicQuestions(topic, count = 5) { const existingBank = await db.getQuizBank(topic.id); const existingKeys = new Set(existingBank.map((q) => normalizeQuestionText(q.question))); let lastQualityError = null; let candidates = null; for (let attempt = 0; attempt < 3; attempt++) { let questions; try { questions = await callQuizModel(topic, count); } catch (err) { console.warn(`[quiz] LLM failed (attempt ${attempt + 1}):`, err.message); lastQualityError = err.message; continue; } const qualityError = validateBatchQuality(questions); if (qualityError) { lastQualityError = qualityError; console.warn(`[quiz] batch rejected (attempt ${attempt + 1}): ${qualityError}`); continue; } const dominant = dominantCorrectIndex(questions); if (dominant && attempt < 2) { console.warn(`[quiz] correctIndex dominated by ${dominant.index} (${Math.round(dominant.ratio * 100)}%) — re-rolling`); continue; } candidates = questions; break; } if (!candidates) { throw new Error(`Quality gate rejected the generated batch for ${topic.label}: ${lastQualityError || 'unbalanced answer distribution'}. Click "Generate" to try again.`); } const accepted = []; for (const q of candidates) { const key = normalizeQuestionText(q.question); if (existingKeys.has(key)) { console.debug('[quiz] dropped duplicate:', q.question); continue; } existingKeys.add(key); accepted.push({ ...q, id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`, }); } if (!accepted.length) { throw new Error(`All generated questions for ${topic.label} were duplicates of existing ones.`); } const merged = [...existingBank, ...accepted]; await db.setQuizBank(topic.id, merged); return accepted; } async function getOrGenerateTopicQuestions(topic, count) { let bank = await db.getQuizBank(topic.id); if (bank.length < count) { try { await forceGenerateTopicQuestions(topic, 5); bank = await db.getQuizBank(topic.id); } catch (err) { console.warn(`[quiz] Failed to generate questions for ${topic.label}:`, err.message); } } return sample(bank, Math.min(count, bank.length)); } export async function getTopicQuestionBank(topicId) { return db.getQuizBank(topicId); } export async function deleteQuestion(topicId, questionId) { return db.deleteQuestionFromBank(topicId, questionId); } export async function generateWeeklyQuiz(userId, weekNumber, force = false) { if (!force) { const cached = await db.getCachedQuiz(userId, weekNumber); if (cached?.questions?.length > 0) return cached; } const { primaryTopic, reviewTopics } = await selectTestTopics(userId, weekNumber); if (!primaryTopic) throw new Error('No topics available to generate a quiz.'); const questions = []; const primaryQs = await getOrGenerateTopicQuestions(primaryTopic, 3); questions.push(...primaryQs); for (const rt of reviewTopics) { if (questions.length >= 5) break; const rtQs = await getOrGenerateTopicQuestions(rt, 1); questions.push(...rtQs); } if (questions.length < 5) { const needed = 5 - questions.length; const extraQs = await getOrGenerateTopicQuestions(primaryTopic, needed + 5); const existingIds = new Set(questions.map(q => q.id)); for (const eq of extraQs) { if (!existingIds.has(eq.id) && questions.length < 5) { questions.push(eq); } } } const shuffled = shuffle(questions).slice(0, 5); if (shuffled.length === 0) { throw new Error('Could not assemble enough questions for this week\'s test. Please try again.'); } const quiz = { questions: shuffled, meta: { userId, weekNumber, generatedAt: new Date().toISOString(), primaryTopic: primaryTopic.label, }, }; await db.setCachedQuiz(userId, weekNumber, quiz); return quiz; } export async function saveTestResult(userId, weekNumber, result) { await db.saveQuizResult(userId, weekNumber, result); const pointsEarned = result.score * 2; const members = await db.getTeamMembers(); const member = members.find(m => m.id === userId); await db.upsertLeaderboardEntry(userId, member?.name || 'Unknown', pointsEarned, 1); return { pointsEarned }; } export async function getTestResult(userId, weekNumber) { return db.getQuizResult(userId, weekNumber); }