fix: generate weekly quiz in a single batch LLM call
The weekly quiz path made up to six sequential LLM calls (one per topic) with three retries each, blowing past the 30s budget. Worse, the quiz_banks collection has been dropped, so getQuizBank/setQuizBank are no-ops and every generated batch was thrown away — the assembled quiz ended up empty and surfaced as "Could not assemble enough questions." Replace the per-topic loop with a single batched call on the fast tier that emits all five questions in one round-trip, using the review topics as prompt context instead of separate calls. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,8 @@ import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools';
|
||||
import { getCurrentWeekContent } from './curriculumService';
|
||||
import { shuffle, sample } from './random';
|
||||
|
||||
const QUIZ_BATCH_TIMEOUT_MS = 25_000;
|
||||
|
||||
const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform.
|
||||
You generate multiple-choice questions to test employee knowledge on specific topics.
|
||||
Always write in clear, professional English.
|
||||
@@ -218,19 +220,61 @@ export async function forceGenerateTopicQuestions(topic, count = 5) {
|
||||
return accepted;
|
||||
}
|
||||
|
||||
async function getOrGenerateTopicQuestions(topic, count) {
|
||||
let bank = await db.getQuizBank(topic.id);
|
||||
// Single-call batch generator used by the weekly quiz path. Generates the
|
||||
// entire 5-question test in one LLM round-trip instead of looping topic by
|
||||
// topic — the persisted quiz_banks collection is deprecated so caching can't
|
||||
// amortise multiple calls, and 6 sequential calls blew through the <30s budget.
|
||||
async function callQuizBatchModel(primaryTopic, reviewTopics, count) {
|
||||
const reviewBlock = reviewTopics.length
|
||||
? `\n\nReview topics — draw ${Math.max(0, count - 3)} questions from these to broaden the test:\n${reviewTopics
|
||||
.map((t, i) => `${i + 1}. ${t.label} (${t.type}) — ${t.description}`)
|
||||
.join('\n')}`
|
||||
: '';
|
||||
|
||||
if (bank.length < count) {
|
||||
try {
|
||||
await forceGenerateTopicQuestions(topic, 5);
|
||||
bank = await db.getQuizBank(topic.id);
|
||||
} catch (err) {
|
||||
console.warn(`[quiz] Failed to generate questions for ${topic.label}:`, err.message);
|
||||
}
|
||||
}
|
||||
const prompt = `Generate exactly ${count} multiple-choice quiz questions for this week's test and emit them via the emit_quiz_questions tool.
|
||||
|
||||
return sample(bank, Math.min(count, bank.length));
|
||||
Primary topic — focus 3 questions here:
|
||||
- Label: ${primaryTopic.label}
|
||||
- Type: ${primaryTopic.type}
|
||||
- Description: ${primaryTopic.description}${reviewBlock}
|
||||
|
||||
Rules:
|
||||
- Set "topicLabel" to the label of the topic the question covers.
|
||||
- Prefix the four options with "A) ", "B) ", "C) ", "D) ".
|
||||
- Make questions specific and practical, not trivial recall.
|
||||
- Distribute correctIndex roughly evenly across 0, 1, 2, 3 within the batch.`;
|
||||
|
||||
const result = await callLLM({
|
||||
task: 'quiz.batch_generate',
|
||||
tier: 'fast',
|
||||
system: cachedSystem(QUIZ_SYSTEM),
|
||||
user: prompt,
|
||||
tools: [EMIT_QUIZ_QUESTIONS_TOOL],
|
||||
toolChoice: { type: 'tool', name: EMIT_QUIZ_QUESTIONS_TOOL.name },
|
||||
maxTokens: 4096,
|
||||
timeoutMs: QUIZ_BATCH_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
const emitted = result.toolUses[0]?.input;
|
||||
if (!emitted) throw new Error('LLM did not emit tool output.');
|
||||
|
||||
let questions;
|
||||
if (Array.isArray(emitted.questions)) questions = emitted.questions;
|
||||
else if (Array.isArray(emitted)) questions = emitted;
|
||||
else if (emitted.questions && typeof emitted.questions === 'object') questions = [emitted.questions];
|
||||
else throw new Error('Unexpected quiz output shape.');
|
||||
|
||||
if (!questions.length) throw new Error('LLM emitted zero questions.');
|
||||
|
||||
return questions.map((q, i) => ({
|
||||
id: `${primaryTopic.id}-${Date.now().toString(36)}-${i}`,
|
||||
question: q.question || '',
|
||||
topicLabel: q.topicLabel || primaryTopic.label,
|
||||
options: Array.isArray(q.options) ? q.options.slice(0, 4) : [],
|
||||
correctIndex: typeof q.correctIndex === 'number' ? q.correctIndex : 0,
|
||||
explanation: q.explanation || 'No explanation provided.',
|
||||
difficulty: q.difficulty || 'medium',
|
||||
})).filter((q) => q.question && q.options.length === 4);
|
||||
}
|
||||
|
||||
export async function getTopicQuestionBank(topicId) {
|
||||
@@ -250,34 +294,41 @@ export async function generateWeeklyQuiz(userId, weekNumber, force = false) {
|
||||
const { primaryTopic, reviewTopics } = await selectTestTopics(userId, weekNumber);
|
||||
if (!primaryTopic) throw new Error('No topics available to generate a quiz.');
|
||||
|
||||
const questions = [];
|
||||
// Cap review-topic context to keep the prompt focused and the call fast.
|
||||
const reviewContext = (reviewTopics || []).slice(0, 3);
|
||||
|
||||
const primaryQs = await getOrGenerateTopicQuestions(primaryTopic, 3);
|
||||
questions.push(...primaryQs);
|
||||
let accepted = [];
|
||||
let lastError = null;
|
||||
|
||||
for (const rt of reviewTopics) {
|
||||
if (questions.length >= 5) break;
|
||||
const rtQs = await getOrGenerateTopicQuestions(rt, 1);
|
||||
questions.push(...rtQs);
|
||||
}
|
||||
|
||||
if (questions.length < 5) {
|
||||
const needed = 5 - questions.length;
|
||||
const extraQs = await getOrGenerateTopicQuestions(primaryTopic, needed + 5);
|
||||
const existingIds = new Set(questions.map(q => q.id));
|
||||
for (const eq of extraQs) {
|
||||
if (!existingIds.has(eq.id) && questions.length < 5) {
|
||||
questions.push(eq);
|
||||
}
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
let candidates;
|
||||
try {
|
||||
candidates = await callQuizBatchModel(primaryTopic, reviewContext, 5);
|
||||
} catch (err) {
|
||||
lastError = err.message;
|
||||
console.warn(`[quiz] batch attempt ${attempt + 1} failed:`, err.message);
|
||||
continue;
|
||||
}
|
||||
|
||||
const qualityError = validateBatchQuality(candidates);
|
||||
if (qualityError) {
|
||||
lastError = qualityError;
|
||||
console.warn(`[quiz] batch quality rejected (attempt ${attempt + 1}): ${qualityError}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
accepted = candidates;
|
||||
break;
|
||||
}
|
||||
|
||||
const shuffled = shuffle(questions).slice(0, 5);
|
||||
|
||||
if (shuffled.length === 0) {
|
||||
throw new Error('Could not assemble enough questions for this week\'s test. Please try again.');
|
||||
if (accepted.length < 5) {
|
||||
throw new Error(
|
||||
`Could not generate a 5-question test for ${primaryTopic.label}${lastError ? `: ${lastError}` : ''}. Please try again.`,
|
||||
);
|
||||
}
|
||||
|
||||
const shuffled = shuffle(accepted).slice(0, 5);
|
||||
|
||||
const quiz = {
|
||||
questions: shuffled,
|
||||
meta: {
|
||||
|
||||
Reference in New Issue
Block a user