fix: generate weekly quiz in a single batch LLM call

The weekly quiz path made up to six sequential LLM calls (one per topic)
with three retries each, blowing past the 30s budget. Worse, the
quiz_banks collection has been dropped, so getQuizBank/setQuizBank are
no-ops and every generated batch was thrown away — the assembled quiz
ended up empty and surfaced as "Could not assemble enough questions."

Replace the per-topic loop with a single batched call on the fast tier
that emits all five questions in one round-trip, using the review topics
as prompt context instead of separate calls.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
RaymondVerhoef
2026-05-26 13:22:44 +02:00
parent a653812cd8
commit 9ea5d5444d

View File

@@ -4,6 +4,8 @@ import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools';
import { getCurrentWeekContent } from './curriculumService';
import { shuffle, sample } from './random';
const QUIZ_BATCH_TIMEOUT_MS = 25_000;
const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform.
You generate multiple-choice questions to test employee knowledge on specific topics.
Always write in clear, professional English.
@@ -218,19 +220,61 @@ export async function forceGenerateTopicQuestions(topic, count = 5) {
return accepted;
}
async function getOrGenerateTopicQuestions(topic, count) {
let bank = await db.getQuizBank(topic.id);
// Single-call batch generator used by the weekly quiz path. Generates the
// entire 5-question test in one LLM round-trip instead of looping topic by
// topic — the persisted quiz_banks collection is deprecated so caching can't
// amortise multiple calls, and 6 sequential calls blew through the <30s budget.
async function callQuizBatchModel(primaryTopic, reviewTopics, count) {
const reviewBlock = reviewTopics.length
? `\n\nReview topics — draw ${Math.max(0, count - 3)} questions from these to broaden the test:\n${reviewTopics
.map((t, i) => `${i + 1}. ${t.label} (${t.type}) — ${t.description}`)
.join('\n')}`
: '';
if (bank.length < count) {
try {
await forceGenerateTopicQuestions(topic, 5);
bank = await db.getQuizBank(topic.id);
} catch (err) {
console.warn(`[quiz] Failed to generate questions for ${topic.label}:`, err.message);
}
}
const prompt = `Generate exactly ${count} multiple-choice quiz questions for this week's test and emit them via the emit_quiz_questions tool.
return sample(bank, Math.min(count, bank.length));
Primary topic — focus 3 questions here:
- Label: ${primaryTopic.label}
- Type: ${primaryTopic.type}
- Description: ${primaryTopic.description}${reviewBlock}
Rules:
- Set "topicLabel" to the label of the topic the question covers.
- Prefix the four options with "A) ", "B) ", "C) ", "D) ".
- Make questions specific and practical, not trivial recall.
- Distribute correctIndex roughly evenly across 0, 1, 2, 3 within the batch.`;
const result = await callLLM({
task: 'quiz.batch_generate',
tier: 'fast',
system: cachedSystem(QUIZ_SYSTEM),
user: prompt,
tools: [EMIT_QUIZ_QUESTIONS_TOOL],
toolChoice: { type: 'tool', name: EMIT_QUIZ_QUESTIONS_TOOL.name },
maxTokens: 4096,
timeoutMs: QUIZ_BATCH_TIMEOUT_MS,
});
const emitted = result.toolUses[0]?.input;
if (!emitted) throw new Error('LLM did not emit tool output.');
let questions;
if (Array.isArray(emitted.questions)) questions = emitted.questions;
else if (Array.isArray(emitted)) questions = emitted;
else if (emitted.questions && typeof emitted.questions === 'object') questions = [emitted.questions];
else throw new Error('Unexpected quiz output shape.');
if (!questions.length) throw new Error('LLM emitted zero questions.');
return questions.map((q, i) => ({
id: `${primaryTopic.id}-${Date.now().toString(36)}-${i}`,
question: q.question || '',
topicLabel: q.topicLabel || primaryTopic.label,
options: Array.isArray(q.options) ? q.options.slice(0, 4) : [],
correctIndex: typeof q.correctIndex === 'number' ? q.correctIndex : 0,
explanation: q.explanation || 'No explanation provided.',
difficulty: q.difficulty || 'medium',
})).filter((q) => q.question && q.options.length === 4);
}
export async function getTopicQuestionBank(topicId) {
@@ -250,34 +294,41 @@ export async function generateWeeklyQuiz(userId, weekNumber, force = false) {
const { primaryTopic, reviewTopics } = await selectTestTopics(userId, weekNumber);
if (!primaryTopic) throw new Error('No topics available to generate a quiz.');
const questions = [];
// Cap review-topic context to keep the prompt focused and the call fast.
const reviewContext = (reviewTopics || []).slice(0, 3);
const primaryQs = await getOrGenerateTopicQuestions(primaryTopic, 3);
questions.push(...primaryQs);
let accepted = [];
let lastError = null;
for (const rt of reviewTopics) {
if (questions.length >= 5) break;
const rtQs = await getOrGenerateTopicQuestions(rt, 1);
questions.push(...rtQs);
for (let attempt = 0; attempt < 2; attempt++) {
let candidates;
try {
candidates = await callQuizBatchModel(primaryTopic, reviewContext, 5);
} catch (err) {
lastError = err.message;
console.warn(`[quiz] batch attempt ${attempt + 1} failed:`, err.message);
continue;
}
if (questions.length < 5) {
const needed = 5 - questions.length;
const extraQs = await getOrGenerateTopicQuestions(primaryTopic, needed + 5);
const existingIds = new Set(questions.map(q => q.id));
for (const eq of extraQs) {
if (!existingIds.has(eq.id) && questions.length < 5) {
questions.push(eq);
}
}
const qualityError = validateBatchQuality(candidates);
if (qualityError) {
lastError = qualityError;
console.warn(`[quiz] batch quality rejected (attempt ${attempt + 1}): ${qualityError}`);
continue;
}
const shuffled = shuffle(questions).slice(0, 5);
if (shuffled.length === 0) {
throw new Error('Could not assemble enough questions for this week\'s test. Please try again.');
accepted = candidates;
break;
}
if (accepted.length < 5) {
throw new Error(
`Could not generate a 5-question test for ${primaryTopic.label}${lastError ? `: ${lastError}` : ''}. Please try again.`,
);
}
const shuffled = shuffle(accepted).slice(0, 5);
const quiz = {
questions: shuffled,
meta: {