fix: generate weekly quiz in a single batch LLM call
The weekly quiz path made up to six sequential LLM calls (one per topic) with three retries each, blowing past the 30s budget. Worse, the quiz_banks collection has been dropped, so getQuizBank/setQuizBank are no-ops and every generated batch was thrown away — the assembled quiz ended up empty and surfaced as "Could not assemble enough questions." Replace the per-topic loop with a single batched call on the fast tier that emits all five questions in one round-trip, using the review topics as prompt context instead of separate calls. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -4,6 +4,8 @@ import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools';
|
|||||||
import { getCurrentWeekContent } from './curriculumService';
|
import { getCurrentWeekContent } from './curriculumService';
|
||||||
import { shuffle, sample } from './random';
|
import { shuffle, sample } from './random';
|
||||||
|
|
||||||
|
const QUIZ_BATCH_TIMEOUT_MS = 25_000;
|
||||||
|
|
||||||
const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform.
|
const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform.
|
||||||
You generate multiple-choice questions to test employee knowledge on specific topics.
|
You generate multiple-choice questions to test employee knowledge on specific topics.
|
||||||
Always write in clear, professional English.
|
Always write in clear, professional English.
|
||||||
@@ -218,19 +220,61 @@ export async function forceGenerateTopicQuestions(topic, count = 5) {
|
|||||||
return accepted;
|
return accepted;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function getOrGenerateTopicQuestions(topic, count) {
|
// Single-call batch generator used by the weekly quiz path. Generates the
|
||||||
let bank = await db.getQuizBank(topic.id);
|
// entire 5-question test in one LLM round-trip instead of looping topic by
|
||||||
|
// topic — the persisted quiz_banks collection is deprecated so caching can't
|
||||||
|
// amortise multiple calls, and 6 sequential calls blew through the <30s budget.
|
||||||
|
async function callQuizBatchModel(primaryTopic, reviewTopics, count) {
|
||||||
|
const reviewBlock = reviewTopics.length
|
||||||
|
? `\n\nReview topics — draw ${Math.max(0, count - 3)} questions from these to broaden the test:\n${reviewTopics
|
||||||
|
.map((t, i) => `${i + 1}. ${t.label} (${t.type}) — ${t.description}`)
|
||||||
|
.join('\n')}`
|
||||||
|
: '';
|
||||||
|
|
||||||
if (bank.length < count) {
|
const prompt = `Generate exactly ${count} multiple-choice quiz questions for this week's test and emit them via the emit_quiz_questions tool.
|
||||||
try {
|
|
||||||
await forceGenerateTopicQuestions(topic, 5);
|
|
||||||
bank = await db.getQuizBank(topic.id);
|
|
||||||
} catch (err) {
|
|
||||||
console.warn(`[quiz] Failed to generate questions for ${topic.label}:`, err.message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return sample(bank, Math.min(count, bank.length));
|
Primary topic — focus 3 questions here:
|
||||||
|
- Label: ${primaryTopic.label}
|
||||||
|
- Type: ${primaryTopic.type}
|
||||||
|
- Description: ${primaryTopic.description}${reviewBlock}
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- Set "topicLabel" to the label of the topic the question covers.
|
||||||
|
- Prefix the four options with "A) ", "B) ", "C) ", "D) ".
|
||||||
|
- Make questions specific and practical, not trivial recall.
|
||||||
|
- Distribute correctIndex roughly evenly across 0, 1, 2, 3 within the batch.`;
|
||||||
|
|
||||||
|
const result = await callLLM({
|
||||||
|
task: 'quiz.batch_generate',
|
||||||
|
tier: 'fast',
|
||||||
|
system: cachedSystem(QUIZ_SYSTEM),
|
||||||
|
user: prompt,
|
||||||
|
tools: [EMIT_QUIZ_QUESTIONS_TOOL],
|
||||||
|
toolChoice: { type: 'tool', name: EMIT_QUIZ_QUESTIONS_TOOL.name },
|
||||||
|
maxTokens: 4096,
|
||||||
|
timeoutMs: QUIZ_BATCH_TIMEOUT_MS,
|
||||||
|
});
|
||||||
|
|
||||||
|
const emitted = result.toolUses[0]?.input;
|
||||||
|
if (!emitted) throw new Error('LLM did not emit tool output.');
|
||||||
|
|
||||||
|
let questions;
|
||||||
|
if (Array.isArray(emitted.questions)) questions = emitted.questions;
|
||||||
|
else if (Array.isArray(emitted)) questions = emitted;
|
||||||
|
else if (emitted.questions && typeof emitted.questions === 'object') questions = [emitted.questions];
|
||||||
|
else throw new Error('Unexpected quiz output shape.');
|
||||||
|
|
||||||
|
if (!questions.length) throw new Error('LLM emitted zero questions.');
|
||||||
|
|
||||||
|
return questions.map((q, i) => ({
|
||||||
|
id: `${primaryTopic.id}-${Date.now().toString(36)}-${i}`,
|
||||||
|
question: q.question || '',
|
||||||
|
topicLabel: q.topicLabel || primaryTopic.label,
|
||||||
|
options: Array.isArray(q.options) ? q.options.slice(0, 4) : [],
|
||||||
|
correctIndex: typeof q.correctIndex === 'number' ? q.correctIndex : 0,
|
||||||
|
explanation: q.explanation || 'No explanation provided.',
|
||||||
|
difficulty: q.difficulty || 'medium',
|
||||||
|
})).filter((q) => q.question && q.options.length === 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getTopicQuestionBank(topicId) {
|
export async function getTopicQuestionBank(topicId) {
|
||||||
@@ -250,34 +294,41 @@ export async function generateWeeklyQuiz(userId, weekNumber, force = false) {
|
|||||||
const { primaryTopic, reviewTopics } = await selectTestTopics(userId, weekNumber);
|
const { primaryTopic, reviewTopics } = await selectTestTopics(userId, weekNumber);
|
||||||
if (!primaryTopic) throw new Error('No topics available to generate a quiz.');
|
if (!primaryTopic) throw new Error('No topics available to generate a quiz.');
|
||||||
|
|
||||||
const questions = [];
|
// Cap review-topic context to keep the prompt focused and the call fast.
|
||||||
|
const reviewContext = (reviewTopics || []).slice(0, 3);
|
||||||
|
|
||||||
const primaryQs = await getOrGenerateTopicQuestions(primaryTopic, 3);
|
let accepted = [];
|
||||||
questions.push(...primaryQs);
|
let lastError = null;
|
||||||
|
|
||||||
for (const rt of reviewTopics) {
|
for (let attempt = 0; attempt < 2; attempt++) {
|
||||||
if (questions.length >= 5) break;
|
let candidates;
|
||||||
const rtQs = await getOrGenerateTopicQuestions(rt, 1);
|
try {
|
||||||
questions.push(...rtQs);
|
candidates = await callQuizBatchModel(primaryTopic, reviewContext, 5);
|
||||||
}
|
} catch (err) {
|
||||||
|
lastError = err.message;
|
||||||
if (questions.length < 5) {
|
console.warn(`[quiz] batch attempt ${attempt + 1} failed:`, err.message);
|
||||||
const needed = 5 - questions.length;
|
continue;
|
||||||
const extraQs = await getOrGenerateTopicQuestions(primaryTopic, needed + 5);
|
|
||||||
const existingIds = new Set(questions.map(q => q.id));
|
|
||||||
for (const eq of extraQs) {
|
|
||||||
if (!existingIds.has(eq.id) && questions.length < 5) {
|
|
||||||
questions.push(eq);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const qualityError = validateBatchQuality(candidates);
|
||||||
|
if (qualityError) {
|
||||||
|
lastError = qualityError;
|
||||||
|
console.warn(`[quiz] batch quality rejected (attempt ${attempt + 1}): ${qualityError}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
accepted = candidates;
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
const shuffled = shuffle(questions).slice(0, 5);
|
if (accepted.length < 5) {
|
||||||
|
throw new Error(
|
||||||
if (shuffled.length === 0) {
|
`Could not generate a 5-question test for ${primaryTopic.label}${lastError ? `: ${lastError}` : ''}. Please try again.`,
|
||||||
throw new Error('Could not assemble enough questions for this week\'s test. Please try again.');
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const shuffled = shuffle(accepted).slice(0, 5);
|
||||||
|
|
||||||
const quiz = {
|
const quiz = {
|
||||||
questions: shuffled,
|
questions: shuffled,
|
||||||
meta: {
|
meta: {
|
||||||
|
|||||||
Reference in New Issue
Block a user