Add comprehensive documentation for employee learning platform

- Created handover document outlining design decisions and application functionality.
- Developed implementation plan detailing phased approach for service development.
- Specified ingestion service responsibilities, API surface, and processing pipeline.

app/services/ingestion/.gitignore

@@ -0,0 +1,4 @@
+node_modules/
+dist/
+.env
+*.log

app/services/ingestion/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "ingestion",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "tsx watch src/index.ts",
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "migrate": "tsx src/migrations/001_initial_schema.ts",
+    "migrate:qdrant": "tsx src/migrations/002_qdrant_setup.ts"
+  },
+  "dependencies": {
+    "fastify": "^4",
+    "@anthropic-ai/sdk": "^0.24",
+    "openai": "^4",
+    "@qdrant/js-client-rest": "^1.9",
+    "pocketbase": "^0.21",
+    "pdf-parse": "^1.1",
+    "uuid": "^9",
+    "zod": "^3"
+  },
+  "devDependencies": {
+    "typescript": "^5",
+    "tsx": "^4",
+    "dotenv": "^16",
+    "@types/node": "^20",
+    "@types/pdf-parse": "^1.1",
+    "@types/uuid": "^9"
+  }
+}

app/services/ingestion/src/index.ts

@@ -0,0 +1,18 @@
+import 'dotenv/config';
+import Fastify from 'fastify';
+import documentRoutes from './routes/documents.js';
+
+const PORT = parseInt(process.env['INGESTION_PORT'] ?? '3001', 10);
+
+async function start(): Promise<void> {
+  const app = Fastify({ logger: true });
+
+  await app.register(documentRoutes);
+
+  await app.listen({ port: PORT, host: '0.0.0.0' });
+}
+
+start().catch((err: unknown) => {
+  console.error('Failed to start ingestion service:', err);
+  process.exit(1);
+});

app/services/ingestion/src/jobs/queue.ts

@@ -0,0 +1,102 @@
+import { v4 as uuid } from 'uuid';
+import { extract } from '../pipeline/extract.js';
+import { chunk } from '../pipeline/chunk.js';
+import { clean } from '../pipeline/clean.js';
+import { extractStructure } from '../pipeline/structure.js';
+import { writeToPocketBase, markDocumentFailed } from '../pipeline/write.js';
+import { embedAndStore } from '../pipeline/embed.js';
+import type { Job, JobProgress, IngestBody } from '../types.js';
+
+// ---------------------------------------------------------------------------
+// In-memory store
+// ---------------------------------------------------------------------------
+
+const jobs = new Map<string, Job>();
+
+const DEFAULT_PROGRESS: JobProgress = {
+  chunksTotal: 0,
+  chunksEmbedded: 0,
+  themesFound: 0,
+  topicsFound: 0,
+};
+
+export function createJob(params: IngestBody): Job {
+  const job: Job = {
+    id: uuid(),
+    documentId: params.documentId,
+    filename: params.filename,
+    format: params.format,
+    filePath: params.filePath,
+    status: 'queued',
+    progress: { ...DEFAULT_PROGRESS },
+    error: null,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  };
+  jobs.set(job.id, job);
+  void runPipeline(job.id);
+  return job;
+}
+
+export function getJob(id: string): Job | undefined {
+  return jobs.get(id);
+}
+
+function updateJob(id: string, updates: Partial<Omit<Job, 'id' | 'createdAt'>>): void {
+  const job = jobs.get(id);
+  if (!job) return;
+  jobs.set(id, { ...job, ...updates, updatedAt: new Date() });
+}
+
+function mergeProgress(id: string, partial: Partial<JobProgress>): void {
+  const job = jobs.get(id);
+  if (!job) return;
+  updateJob(id, { progress: { ...job.progress, ...partial } });
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline orchestration
+// ---------------------------------------------------------------------------
+
+async function runPipeline(jobId: string): Promise<void> {
+  const job = jobs.get(jobId);
+  if (!job) return;
+
+  try {
+    // Stage 1: text extraction
+    updateJob(jobId, { status: 'extracting' });
+    const text = await extract(job.filePath, job.format);
+
+    // Stages 2–3: chunking + cleaning
+    updateJob(jobId, { status: 'chunking' });
+    const rawChunks = chunk(text, job.format, job.documentId);
+    const cleanChunks = clean(rawChunks);
+    mergeProgress(jobId, { chunksTotal: cleanChunks.length });
+
+    // Stage 4: structure extraction (AI)
+    updateJob(jobId, { status: 'structuring' });
+    const draftKB = await extractStructure(cleanChunks, job.filename, job.format);
+    const topicsFound = draftKB.themes.reduce((n, t) => n + t.topics.length, 0);
+    mergeProgress(jobId, { themesFound: draftKB.themes.length, topicsFound });
+
+    // Stage 5: PocketBase write
+    updateJob(jobId, { status: 'writing' });
+    const writtenTopics = await writeToPocketBase(draftKB, job.documentId, cleanChunks.length);
+
+    // Stage 6: embeddings + Qdrant write
+    updateJob(jobId, { status: 'embedding' });
+    await embedAndStore(cleanChunks, writtenTopics, embedded => {
+      mergeProgress(jobId, { chunksEmbedded: embedded });
+    });
+
+    updateJob(jobId, { status: 'done' });
+  } catch (err: unknown) {
+    const reason = err instanceof Error ? err.message : String(err);
+    updateJob(jobId, { status: 'failed', error: reason });
+
+    const j = jobs.get(jobId);
+    if (j) {
+      await markDocumentFailed(j.documentId, reason).catch(() => undefined);
+    }
+  }
+}

app/services/ingestion/src/lib/anthropic.ts

@@ -0,0 +1,9 @@
+import Anthropic from '@anthropic-ai/sdk';
+
+export const anthropic = new Anthropic({
+  apiKey: process.env['ANTHROPIC_API_KEY'],
+});
+
+export const MODELS = {
+  SONNET: 'claude-sonnet-4-20250514',
+} as const;

app/services/ingestion/src/lib/openai.ts

@@ -0,0 +1,9 @@
+import OpenAI from 'openai';
+
+export const openai = new OpenAI({
+  apiKey: process.env['OPENAI_API_KEY'],
+});
+
+export const EMBEDDING_MODEL = 'text-embedding-3-small';
+export const EMBEDDING_DIMENSIONS = 1536;
+export const EMBEDDING_BATCH_SIZE = 100;

app/services/ingestion/src/lib/pocketbase.ts

@@ -0,0 +1,14 @@
+import PocketBase from 'pocketbase';
+
+const POCKETBASE_URL = process.env['POCKETBASE_URL'] ?? '';
+const POCKETBASE_ADMIN_EMAIL = process.env['POCKETBASE_ADMIN_EMAIL'] ?? '';
+const POCKETBASE_ADMIN_PASSWORD = process.env['POCKETBASE_ADMIN_PASSWORD'] ?? '';
+
+const pb = new PocketBase(POCKETBASE_URL);
+
+export async function getPocketBase(): Promise<PocketBase> {
+  if (!pb.authStore.isValid) {
+    await pb.admins.authWithPassword(POCKETBASE_ADMIN_EMAIL, POCKETBASE_ADMIN_PASSWORD);
+  }
+  return pb;
+}

app/services/ingestion/src/lib/qdrant.ts

@@ -0,0 +1,14 @@
+import { QdrantClient } from '@qdrant/js-client-rest';
+
+const QDRANT_URL = process.env['QDRANT_URL'] ?? '';
+const QDRANT_API_KEY = process.env['QDRANT_API_KEY'] ?? '';
+
+export const qdrant = new QdrantClient({
+  url: QDRANT_URL,
+  ...(QDRANT_API_KEY ? { apiKey: QDRANT_API_KEY } : {}),
+});
+
+export const QDRANT_COLLECTIONS = {
+  SOURCE_CHUNKS: 'source_chunks',
+  TOPIC_SUMMARIES: 'topic_summaries',
+} as const;

app/services/ingestion/src/migrations/001_initial_schema.ts

@@ -0,0 +1,402 @@
+import 'dotenv/config';
+import PocketBase, { type CollectionModel } from 'pocketbase';
+
+// ---------------------------------------------------------------------------
+// Env validation
+// ---------------------------------------------------------------------------
+
+const POCKETBASE_URL = process.env['POCKETBASE_URL'] ?? '';
+const POCKETBASE_ADMIN_EMAIL = process.env['POCKETBASE_ADMIN_EMAIL'] ?? '';
+const POCKETBASE_ADMIN_PASSWORD = process.env['POCKETBASE_ADMIN_PASSWORD'] ?? '';
+
+if (!POCKETBASE_URL || !POCKETBASE_ADMIN_EMAIL || !POCKETBASE_ADMIN_PASSWORD) {
+  console.error('Missing env vars: POCKETBASE_URL, POCKETBASE_ADMIN_EMAIL, POCKETBASE_ADMIN_PASSWORD');
+  process.exit(1);
+}
+
+// ---------------------------------------------------------------------------
+// Field types
+// ---------------------------------------------------------------------------
+
+interface FieldDef {
+  name: string;
+  type: string;
+  required: boolean;
+  options: Record<string, unknown>;
+}
+
+// ---------------------------------------------------------------------------
+// Field helpers
+// ---------------------------------------------------------------------------
+
+const field = {
+  text: (name: string, required = false): FieldDef => ({
+    name, type: 'text', required,
+    options: { min: null, max: null, pattern: '' },
+  }),
+
+  number: (name: string, required = false): FieldDef => ({
+    name, type: 'number', required,
+    options: { min: null, max: null, noDecimal: false },
+  }),
+
+  select: (name: string, values: string[], required = false, maxSelect = 1): FieldDef => ({
+    name, type: 'select', required,
+    options: { maxSelect, values },
+  }),
+
+  json: (name: string): FieldDef => ({
+    name, type: 'json', required: false, options: {},
+  }),
+
+  date: (name: string): FieldDef => ({
+    name, type: 'date', required: false, options: { min: '', max: '' },
+  }),
+
+  editor: (name: string): FieldDef => ({
+    name, type: 'editor', required: false, options: {},
+  }),
+
+  file: (name: string, mimeTypes: string[] = [], maxSelect = 1): FieldDef => ({
+    name, type: 'file', required: false,
+    options: { maxSelect, maxSize: 52428800, mimeTypes, thumbs: [], protected: false },
+  }),
+
+  relation: (name: string, collectionId: string, maxSelect: number | null = 1, required = false): FieldDef => ({
+    name, type: 'relation', required,
+    options: { collectionId, cascadeDelete: false, minSelect: null, maxSelect, displayFields: null },
+  }),
+};
+
+// ---------------------------------------------------------------------------
+// Badge seed data (from data-model.md + handover.md)
+// ---------------------------------------------------------------------------
+
+const BADGES = [
+  // Bronze — beginner milestones
+  { key: 'first_commit',    tier: 'bronze',    label: 'First Commit',    description: 'Complete your first topic',                          icon: '🔰' },
+  { key: 'week_shipped',    tier: 'bronze',    label: 'Week Shipped',    description: 'Complete your first full week',                      icon: '📦' },
+  { key: 'streak_3',        tier: 'bronze',    label: '3-Week Streak',   description: 'Maintain a 3-week learning streak',                  icon: '🔥' },
+  { key: 'quiz_taker',      tier: 'bronze',    label: 'Quiz Taker',      description: 'Complete your first scenario quiz',                  icon: '❓' },
+  { key: 'flashcard_fan',   tier: 'bronze',    label: 'Flashcard Fan',   description: 'Complete your first flashcard set',                  icon: '🃏' },
+  { key: 'reached_junior',  tier: 'bronze',    label: 'Junior Dev',      description: 'Reach Junior level',                                 icon: '👶' },
+  // Silver — intermediate
+  { key: 'streak_13',       tier: 'silver',    label: '13-Week Streak',  description: 'Maintain a 13-week learning streak',                 icon: '💥' },
+  { key: 'half_cycle',      tier: 'silver',    label: 'Half Cycle',      description: 'Complete week 13 — halfway through the cycle',       icon: '🔄' },
+  { key: 'case_student',    tier: 'silver',    label: 'Case Student',    description: 'Complete 5 case studies',                            icon: '📋' },
+  { key: 'reached_medior',  tier: 'silver',    label: 'Medior Dev',      description: 'Reach Medior level',                                 icon: '💼' },
+  { key: 'reached_senior',  tier: 'silver',    label: 'Senior Dev',      description: 'Reach Senior level',                                 icon: '🏅' },
+  // Gold — advanced
+  { key: 'full_cycle',      tier: 'gold',      label: 'Full Cycle',      description: 'Complete a full 26-week cycle',                      icon: '🏆' },
+  { key: 'type_explorer',   tier: 'gold',      label: 'Type Explorer',   description: 'Use all 10 micro learning types at least once',      icon: '🗺️' },
+  { key: 'reached_staff',   tier: 'gold',      label: 'Staff Eng',       description: 'Reach Staff level',                                  icon: '⭐' },
+  // Legendary — exceptional
+  { key: 'streak_26',       tier: 'legendary', label: 'Unbroken',        description: 'Complete all 26 weeks without breaking a streak',    icon: '⚡' },
+  { key: 'second_cycle',    tier: 'legendary', label: 'Second Cycle',    description: 'Complete two full cycles',                           icon: '🌀' },
+  { key: 'reached_principal', tier: 'legendary', label: 'Principal',     description: 'Reach Principal level — maximum rank',               icon: '👑' },
+  // Content — domain mastery
+  { key: 'governance_nerd',   tier: 'content', label: 'Governance Nerd',   description: 'Complete all published governance topics',         icon: '⚖️' },
+  { key: 'process_architect', tier: 'content', label: 'Process Architect', description: 'Complete all published process topics',            icon: '🏗️' },
+  { key: 'deep_reader',       tier: 'content', label: 'Deep Reader',       description: 'Complete 50 or more unique topics',                icon: '📚' },
+] as const;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function requireId(ids: Map<string, string>, name: string): string {
+  const id = ids.get(name);
+  if (id === undefined) throw new Error(`Collection ID not found: ${name}`);
+  return id;
+}
+
+// ---------------------------------------------------------------------------
+// Migration
+// ---------------------------------------------------------------------------
+
+async function run(): Promise<void> {
+  console.log('Connecting to PocketBase...');
+  const pb = new PocketBase(POCKETBASE_URL);
+  await pb.admins.authWithPassword(POCKETBASE_ADMIN_EMAIL, POCKETBASE_ADMIN_PASSWORD);
+  console.log('Authenticated.\n');
+
+  // Snapshot of existing collections before we begin
+  const existingCollections = await pb.collections.getFullList();
+  const ids = new Map<string, string>();
+  for (const col of existingCollections) {
+    ids.set(col.name, col.id);
+  }
+  const existingNames = new Set(ids.keys());
+
+  // Create a collection only if it doesn't already exist
+  async function ensureCollection(params: { name: string; type: string; schema: FieldDef[] }): Promise<void> {
+    if (ids.has(params.name)) {
+      console.log(`  skip   ${params.name}`);
+      return;
+    }
+    const created = await pb.collections.create(params as Record<string, unknown>);
+    ids.set(created.name, created.id);
+    console.log(`  create ${created.name}`);
+  }
+
+  // ---------------------------------------------------------------------------
+  // Extend users (auth collection — already exists in PocketBase)
+  // ---------------------------------------------------------------------------
+  console.log('users:');
+  const usersCol = await pb.collections.getFirstListItem<CollectionModel>('name="users"');
+  ids.set('users', usersCol.id);
+
+  const hasRole = usersCol.schema.some(s => s.name === 'role');
+  if (!hasRole) {
+    const updateBody: Record<string, unknown> = {
+      schema: [
+        ...usersCol.schema,
+        field.select('role', ['admin', 'employee'], true),
+        field.text('display_name'),
+        field.file('avatar', ['image/jpeg', 'image/png', 'image/webp']),
+      ],
+    };
+    await pb.collections.update(usersCol.id, updateBody);
+    console.log('  extended with role, display_name, avatar');
+  } else {
+    console.log('  skip   users (already extended)');
+  }
+
+  // ---------------------------------------------------------------------------
+  // source_documents
+  // ---------------------------------------------------------------------------
+  console.log('\ncollections:');
+  await ensureCollection({
+    name: 'source_documents',
+    type: 'base',
+    schema: [
+      field.text('filename', true),
+      field.file('file', ['application/pdf', 'text/markdown', 'text/x-markdown', 'text/plain']),
+      field.select('format', ['pdf', 'md', 'txt'], true),
+      field.select('status', ['processing', 'processed', 'failed'], true),
+      field.date('ingested_at'),
+      field.number('chunk_count'),
+      field.relation('created_by', requireId(ids, 'users')),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // themes
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'themes',
+    type: 'base',
+    schema: [
+      field.text('title', true),
+      field.text('description'),
+      field.select('status', ['draft', 'published'], true),
+      field.relation('source_documents', requireId(ids, 'source_documents'), null),
+      field.relation('approved_by', requireId(ids, 'users')),
+      field.date('approved_at'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // topics — create without self-referential fields first
+  // ---------------------------------------------------------------------------
+  const topicsBaseSchema: FieldDef[] = [
+    field.relation('theme', requireId(ids, 'themes'), 1, true),
+    field.text('title', true),
+    field.editor('body'),
+    field.select('difficulty', ['introductory', 'intermediate', 'advanced'], true),
+    field.number('complexity_weight'),
+    field.select('status', ['draft', 'published'], true),
+    field.json('key_terms'),
+    field.json('qdrant_chunk_ids'),
+  ];
+  const topicsIsNew = !existingNames.has('topics');
+  await ensureCollection({ name: 'topics', type: 'base', schema: topicsBaseSchema });
+
+  // ---------------------------------------------------------------------------
+  // micro_learnings
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'micro_learnings',
+    type: 'base',
+    schema: [
+      field.relation('topic', requireId(ids, 'topics'), 1, true),
+      field.select('type', [
+        'concept_explainer', 'scenario_quiz', 'misconceptions', 'how_to',
+        'comparison_card', 'reflection_prompt', 'flashcard_set', 'case_study',
+        'glossary_anchor', 'myth_vs_evidence',
+      ], true),
+      field.json('content'),
+      field.select('status', ['queued', 'generated', 'published', 'rejected'], true),
+      field.text('generation_model'),
+      field.date('generated_at'),
+      field.date('published_at'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // curriculum_versions
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'curriculum_versions',
+    type: 'base',
+    schema: [
+      field.number('version', true),
+      field.select('status', ['draft', 'active', 'superseded'], true),
+      field.date('generated_at'),
+      field.relation('approved_by', requireId(ids, 'users')),
+      field.date('approved_at'),
+      field.text('generation_notes'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // curriculum_weeks
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'curriculum_weeks',
+    type: 'base',
+    schema: [
+      field.relation('curriculum_version', requireId(ids, 'curriculum_versions'), 1, true),
+      field.number('week_number', true),
+      field.relation('theme', requireId(ids, 'themes'), 1, true),
+      field.relation('topics', requireId(ids, 'topics'), null),
+      field.json('topic_order'),
+      field.number('estimated_duration_minutes'),
+      field.text('admin_notes'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // employee_curriculum_state
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'employee_curriculum_state',
+    type: 'base',
+    schema: [
+      field.relation('user', requireId(ids, 'users'), 1, true),
+      field.number('current_cycle', true),
+      field.number('current_week', true),
+      field.date('start_date'),
+      field.relation('active_version', requireId(ids, 'curriculum_versions')),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // session_completions
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'session_completions',
+    type: 'base',
+    schema: [
+      field.relation('user', requireId(ids, 'users'), 1, true),
+      field.relation('topic', requireId(ids, 'topics'), 1, true),
+      field.relation('micro_learning', requireId(ids, 'micro_learnings'), 1, true),
+      field.number('week_number', true),
+      field.number('cycle', true),
+      field.date('completed_at'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // gamification_profiles
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'gamification_profiles',
+    type: 'base',
+    schema: [
+      field.relation('user', requireId(ids, 'users'), 1, true),
+      field.number('total_commits'),
+      field.select('current_level', ['intern', 'junior', 'medior', 'senior', 'staff', 'principal']),
+      field.number('current_streak_weeks'),
+      field.number('longest_streak_weeks'),
+      field.json('types_used'),
+      field.number('last_active_week'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // badges
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'badges',
+    type: 'base',
+    schema: [
+      field.text('key', true),
+      field.select('tier', ['bronze', 'silver', 'gold', 'legendary', 'content'], true),
+      field.text('label', true),
+      field.text('description'),
+      field.text('icon'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // employee_badges
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'employee_badges',
+    type: 'base',
+    schema: [
+      field.relation('user', requireId(ids, 'users'), 1, true),
+      field.relation('badge', requireId(ids, 'badges'), 1, true),
+      field.date('earned_at'),
+      field.number('cycle'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // milestone_cards
+  // ---------------------------------------------------------------------------
+  await ensureCollection({
+    name: 'milestone_cards',
+    type: 'base',
+    schema: [
+      field.relation('user', requireId(ids, 'users'), 1, true),
+      field.number('cycle', true),
+      field.number('week', true),
+      field.number('total_commits'),
+      field.number('streak_weeks'),
+      field.json('badge_keys'),
+    ],
+  });
+
+  // ---------------------------------------------------------------------------
+  // Add self-referential relations to topics (requires topics ID to exist first)
+  // ---------------------------------------------------------------------------
+  if (topicsIsNew) {
+    console.log('\ntopics self-refs:');
+    const topicsId = requireId(ids, 'topics');
+    const updateBody: Record<string, unknown> = {
+      schema: [
+        ...topicsBaseSchema,
+        field.relation('related_topics', topicsId, null),
+        field.relation('prerequisite_topics', topicsId, null),
+        field.relation('contrast_topics', topicsId, null),
+      ],
+    };
+    await pb.collections.update(topicsId, updateBody);
+    console.log('  updated topics with related_topics, prerequisite_topics, contrast_topics');
+  }
+
+  // ---------------------------------------------------------------------------
+  // Seed badges
+  // ---------------------------------------------------------------------------
+  console.log('\nbadges:');
+  const existingBadges = await pb.collection('badges').getFullList<{ key: string }>();
+  const existingBadgeKeys = new Set(existingBadges.map(b => b.key));
+
+  for (const badge of BADGES) {
+    if (existingBadgeKeys.has(badge.key)) {
+      console.log(`  skip   ${badge.key}`);
+      continue;
+    }
+    await pb.collection('badges').create(badge);
+    console.log(`  seed   ${badge.key}`);
+  }
+
+  console.log('\nDone.');
+}
+
+run().catch((err: unknown) => {
+  console.error('Migration failed:', err);
+  process.exit(1);
+});

app/services/ingestion/src/migrations/002_qdrant_setup.ts

@@ -0,0 +1,76 @@
+import 'dotenv/config';
+import { QdrantClient } from '@qdrant/js-client-rest';
+
+// ---------------------------------------------------------------------------
+// Env validation
+// ---------------------------------------------------------------------------
+
+const QDRANT_URL = process.env['QDRANT_URL'] ?? '';
+const QDRANT_API_KEY = process.env['QDRANT_API_KEY'] ?? '';
+
+if (!QDRANT_URL) {
+  console.error('Missing env var: QDRANT_URL');
+  process.exit(1);
+}
+
+// ---------------------------------------------------------------------------
+// Collection definitions
+// ---------------------------------------------------------------------------
+
+const VECTOR_SIZE = 1536; // text-embedding-3-small
+const DISTANCE = 'Cosine' as const;
+
+const COLLECTIONS = [
+  {
+    name: 'source_chunks',
+    // Payload indices for R42 context-weighted retrieval (boost by theme_id)
+    payloadIndices: ['theme_id', 'topic_id', 'source_document_id', 'format'],
+  },
+  {
+    name: 'topic_summaries',
+    payloadIndices: ['theme_id', 'topic_id'],
+  },
+] as const;
+
+// ---------------------------------------------------------------------------
+// Setup
+// ---------------------------------------------------------------------------
+
+async function run(): Promise<void> {
+  console.log('Connecting to Qdrant...');
+  const client = new QdrantClient({
+    url: QDRANT_URL,
+    ...(QDRANT_API_KEY ? { apiKey: QDRANT_API_KEY } : {}),
+  });
+
+  const { collections } = await client.getCollections();
+  const existingNames = new Set(collections.map(c => c.name));
+
+  for (const col of COLLECTIONS) {
+    if (existingNames.has(col.name)) {
+      console.log(`  skip   ${col.name}`);
+      continue;
+    }
+
+    await client.createCollection(col.name, {
+      vectors: { size: VECTOR_SIZE, distance: DISTANCE },
+    });
+    console.log(`  create ${col.name}`);
+
+    // Keyword payload indices for efficient filtering
+    for (const field of col.payloadIndices) {
+      await client.createPayloadIndex(col.name, {
+        field_name: field,
+        field_schema: 'keyword',
+      });
+      console.log(`  index  ${col.name}.${field}`);
+    }
+  }
+
+  console.log('\nDone.');
+}
+
+run().catch((err: unknown) => {
+  console.error('Qdrant setup failed:', err);
+  process.exit(1);
+});

app/services/ingestion/src/pipeline/chunk.ts

@@ -0,0 +1,246 @@
+import { v4 as uuid } from 'uuid';
+import type { Chunk, DocumentFormat } from '../types.js';
+
+const MD_MIN = 100;
+const MD_MAX = 1500;
+const TXT_WINDOW = 800;
+const TXT_OVERLAP = 150;
+const PDF_MIN = 100;
+const PDF_MAX = 1200;
+
+// ---------------------------------------------------------------------------
+// MD chunking — heading-based
+// ---------------------------------------------------------------------------
+
+interface HeadingSection {
+  level: number;
+  heading: string;
+  parent: string | null;
+  content: string;
+}
+
+function parseMdSections(text: string): HeadingSection[] {
+  const lines = text.split('\n');
+  const sections: HeadingSection[] = [];
+  let current: HeadingSection | null = null;
+  const parentStack: { level: number; heading: string }[] = [];
+
+  for (const line of lines) {
+    const h3 = line.match(/^### (.+)/);
+    const h2 = line.match(/^## (.+)/);
+    const h1 = line.match(/^# (.+)/);
+    const match = h3 ?? h2 ?? h1;
+
+    if (match) {
+      if (current) sections.push(current);
+      const level = h1 ? 1 : h2 ? 2 : 3;
+      const heading = match[1] ?? line;
+
+      // Maintain parent stack
+      while (parentStack.length > 0 && (parentStack[parentStack.length - 1]?.level ?? 0) >= level) {
+        parentStack.pop();
+      }
+      const parent = parentStack[parentStack.length - 1]?.heading ?? null;
+      parentStack.push({ level, heading });
+
+      current = { level, heading, parent, content: '' };
+    } else if (current) {
+      current.content += line + '\n';
+    }
+  }
+  if (current) sections.push(current);
+  return sections;
+}
+
+function splitOnParagraphs(text: string, maxSize: number): string[] {
+  const paragraphs = text.split(/\n\n+/);
+  const parts: string[] = [];
+  let current = '';
+
+  for (const para of paragraphs) {
+    if ((current + para).length > maxSize && current.length > 0) {
+      parts.push(current.trim());
+      current = para;
+    } else {
+      current = current ? current + '\n\n' + para : para;
+    }
+  }
+  if (current.trim()) parts.push(current.trim());
+  return parts;
+}
+
+function chunkMd(text: string, documentId: string): Chunk[] {
+  const sections = parseMdSections(text);
+  const merged: HeadingSection[] = [];
+
+  for (let i = 0; i < sections.length; i++) {
+    const sec = sections[i];
+    if (sec === undefined) continue;
+    const fullText = `${'#'.repeat(sec.level)} ${sec.heading}\n\n${sec.content}`.trim();
+
+    if (fullText.length < MD_MIN && i + 1 < sections.length) {
+      // Merge with next sibling by appending to next's content prefix
+      const next = sections[i + 1];
+      if (next !== undefined) {
+        next.content = fullText + '\n\n' + next.content;
+        continue;
+      }
+    }
+    merged.push(sec);
+  }
+
+  const chunks: Chunk[] = [];
+  let globalIndex = 0;
+
+  for (const sec of merged) {
+    const fullText = `${'#'.repeat(sec.level)} ${sec.heading}\n\n${sec.content}`.trim();
+    const parts = fullText.length > MD_MAX ? splitOnParagraphs(fullText, MD_MAX) : [fullText];
+
+    for (const part of parts) {
+      if (part.length < 1) continue;
+      chunks.push({
+        id: uuid(),
+        documentId,
+        text: part,
+        format: 'md',
+        index: globalIndex++,
+        metadata: {
+          headingLevel: sec.level,
+          headingText: sec.heading,
+          parentHeading: sec.parent ?? undefined,
+        },
+      });
+    }
+  }
+
+  return chunks;
+}
+
+// ---------------------------------------------------------------------------
+// TXT chunking — sliding window
+// ---------------------------------------------------------------------------
+
+function splitSentences(text: string): string[] {
+  return text.match(/[^.!?]+[.!?]+\s*/g) ?? [text];
+}
+
+function chunkTxt(text: string, documentId: string): Chunk[] {
+  const paragraphs = text.split(/\n\n+/).filter(p => p.trim().length > 0);
+  const chunks: Chunk[] = [];
+  let current = '';
+  let index = 0;
+  const total = text.length;
+
+  const pushChunk = (t: string) => {
+    const pos = index * TXT_WINDOW;
+    chunks.push({
+      id: uuid(),
+      documentId,
+      text: t.trim(),
+      format: 'txt',
+      index: index++,
+      metadata: {
+        approximatePosition:
+          pos < total * 0.25 ? 'start' : pos > total * 0.75 ? 'end' : 'middle',
+      },
+    });
+  };
+
+  for (const para of paragraphs) {
+    if ((current + ' ' + para).trim().length <= TXT_WINDOW) {
+      current = (current + ' ' + para).trim();
+    } else {
+      if (current.length >= MD_MIN) pushChunk(current);
+      // Para may itself exceed window — split on sentences
+      if (para.length > TXT_WINDOW) {
+        const sentences = splitSentences(para);
+        let buf = '';
+        for (const sent of sentences) {
+          if ((buf + sent).length > TXT_WINDOW && buf.length >= MD_MIN) {
+            pushChunk(buf);
+            // Keep overlap
+            const words = buf.split(' ');
+            buf = words.slice(-Math.floor(TXT_OVERLAP / 5)).join(' ') + ' ' + sent;
+          } else {
+            buf += sent;
+          }
+        }
+        current = buf;
+      } else {
+        current = para;
+      }
+    }
+  }
+  if (current.trim().length >= MD_MIN) pushChunk(current);
+
+  return chunks;
+}
+
+// ---------------------------------------------------------------------------
+// PDF chunking — page + paragraph
+// ---------------------------------------------------------------------------
+
+function chunkPdf(text: string, documentId: string): Chunk[] {
+  const pages = text.split('---PAGE---');
+  const chunks: Chunk[] = [];
+  let globalIndex = 0;
+
+  for (let pageIdx = 0; pageIdx < pages.length; pageIdx++) {
+    const page = pages[pageIdx];
+    if (page === undefined || !page.trim()) continue;
+
+    const paragraphs = page.split(/\n\n+/).filter(p => p.trim().length > 0);
+    let chunkOnPage = 0;
+    let accumulator = '';
+
+    const flushAccumulator = () => {
+      const t = accumulator.trim();
+      if (t.length >= PDF_MIN) {
+        chunks.push({
+          id: uuid(),
+          documentId,
+          text: t,
+          format: 'pdf',
+          index: globalIndex++,
+          metadata: { pageNumber: pageIdx + 1, chunkIndexOnPage: chunkOnPage++ },
+        });
+      }
+      accumulator = '';
+    };
+
+    for (const para of paragraphs) {
+      if ((accumulator + '\n\n' + para).trim().length > PDF_MAX) {
+        flushAccumulator();
+        // Para may exceed max on its own — hard split at sentence boundary
+        if (para.length > PDF_MAX) {
+          const sentences = splitSentences(para);
+          for (const sent of sentences) {
+            if ((accumulator + sent).length > PDF_MAX && accumulator.length >= PDF_MIN) {
+              flushAccumulator();
+            }
+            accumulator += sent;
+          }
+        } else {
+          accumulator = para;
+        }
+      } else {
+        accumulator = accumulator ? accumulator + '\n\n' + para : para;
+      }
+    }
+    flushAccumulator();
+  }
+
+  return chunks;
+}
+
+// ---------------------------------------------------------------------------
+// Public entry
+// ---------------------------------------------------------------------------
+
+export function chunk(text: string, format: DocumentFormat, documentId: string): Chunk[] {
+  switch (format) {
+    case 'md':  return chunkMd(text, documentId);
+    case 'txt': return chunkTxt(text, documentId);
+    case 'pdf': return chunkPdf(text, documentId);
+  }
+}

app/services/ingestion/src/pipeline/clean.ts

@@ -0,0 +1,20 @@
+import type { Chunk } from '../types.js';
+
+const MIN_CLEAN_LENGTH = 80;
+
+export function clean(chunks: Chunk[]): Chunk[] {
+  return chunks
+    .map(c => ({ ...c, text: cleanText(c.text) }))
+    .filter(c => c.text.length >= MIN_CLEAN_LENGTH);
+}
+
+function cleanText(text: string): string {
+  return text
+    .normalize('NFC')
+    // Remove null bytes and non-printable characters (keep tabs + newlines)
+    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
+    // Collapse 3+ consecutive newlines to 2
+    .replace(/\n{3,}/g, '\n\n')
+    // Trim leading/trailing whitespace
+    .trim();
+}

app/services/ingestion/src/pipeline/embed.ts

@@ -0,0 +1,102 @@
+import { v4 as uuid } from 'uuid';
+import { openai, EMBEDDING_MODEL, EMBEDDING_BATCH_SIZE } from '../lib/openai.js';
+import { qdrant, QDRANT_COLLECTIONS } from '../lib/qdrant.js';
+import { updateTopicQdrantIds } from './write.js';
+import type { Chunk, WrittenTopic, SourceChunkPayload, TopicSummaryPayload } from '../types.js';
+
+async function embedTexts(texts: string[]): Promise<number[][]> {
+  const vectors: number[][] = [];
+
+  for (let i = 0; i < texts.length; i += EMBEDDING_BATCH_SIZE) {
+    const batch = texts.slice(i, i + EMBEDDING_BATCH_SIZE);
+    const response = await openai.embeddings.create({
+      model: EMBEDDING_MODEL,
+      input: batch,
+    });
+    for (const item of response.data) {
+      vectors.push(item.embedding);
+    }
+  }
+
+  return vectors;
+}
+
+export async function embedAndStore(
+  chunks: Chunk[],
+  writtenTopics: WrittenTopic[],
+  onProgress: (embedded: number) => void,
+): Promise<void> {
+  // Build chunk → topic mapping
+  const chunkTopicMap = new Map<string, string>();
+  const chunkThemeMap = new Map<string, string>();
+  for (const topic of writtenTopics) {
+    for (const chunkId of topic.sourceChunkIds) {
+      chunkTopicMap.set(chunkId, topic.id);
+      chunkThemeMap.set(chunkId, topic.themeId);
+    }
+  }
+
+  // -------------------------------------------------------------------------
+  // Source chunks
+  // -------------------------------------------------------------------------
+  const chunkTexts = chunks.map(c => c.text);
+  const chunkVectors = await embedTexts(chunkTexts);
+
+  const sourcePoints = chunks.map((c, i) => {
+    const vector = chunkVectors[i];
+    if (!vector) throw new Error(`Missing embedding for chunk index ${i}`);
+
+    const payload: SourceChunkPayload = {
+      source_document_id: c.documentId,
+      chunk_index: c.index,
+      text: c.text,
+      theme_id: chunkThemeMap.get(c.id) ?? null,
+      topic_id: chunkTopicMap.get(c.id) ?? null,
+      format: c.format,
+    };
+
+    return { id: c.id, vector, payload };
+  });
+
+  // Upsert in batches
+  for (let i = 0; i < sourcePoints.length; i += EMBEDDING_BATCH_SIZE) {
+    const batch = sourcePoints.slice(i, i + EMBEDDING_BATCH_SIZE);
+    await qdrant.upsert(QDRANT_COLLECTIONS.SOURCE_CHUNKS, { points: batch });
+    onProgress(i + batch.length);
+  }
+
+  // -------------------------------------------------------------------------
+  // Topic summaries
+  // -------------------------------------------------------------------------
+  const topicTexts = writtenTopics.map(t => t.body);
+  const topicVectors = await embedTexts(topicTexts);
+
+  const summaryPoints = writtenTopics.map((topic, i) => {
+    const vector = topicVectors[i];
+    if (!vector) throw new Error(`Missing embedding for topic index ${i}`);
+
+    const payload: TopicSummaryPayload = {
+      topic_id: topic.id,
+      theme_id: topic.themeId,
+      title: topic.title,
+      text: topic.body,
+    };
+
+    return { id: uuid(), vector, payload };
+  });
+
+  for (let i = 0; i < summaryPoints.length; i += EMBEDDING_BATCH_SIZE) {
+    const batch = summaryPoints.slice(i, i + EMBEDDING_BATCH_SIZE);
+    await qdrant.upsert(QDRANT_COLLECTIONS.TOPIC_SUMMARIES, { points: batch });
+  }
+
+  // -------------------------------------------------------------------------
+  // Update topics.qdrant_chunk_ids in PocketBase
+  // -------------------------------------------------------------------------
+  for (const topic of writtenTopics) {
+    const qdrantIds = topic.sourceChunkIds.filter(id => chunkTopicMap.get(id) === topic.id);
+    if (qdrantIds.length > 0) {
+      await updateTopicQdrantIds(topic.id, qdrantIds);
+    }
+  }
+}

app/services/ingestion/src/pipeline/extract.ts

@@ -0,0 +1,40 @@
+import fs from 'node:fs/promises';
+import type { DocumentFormat } from '../types.js';
+
+// Lazy import — pdf-parse has side effects on module load
+async function getPdfParse() {
+  const mod = await import('pdf-parse');
+  return mod.default ?? mod;
+}
+
+export async function extract(filePath: string, format: DocumentFormat): Promise<string> {
+  let fileBuffer: Buffer;
+  try {
+    fileBuffer = await fs.readFile(filePath);
+  } catch {
+    throw new Error('file_not_found');
+  }
+
+  if (format === 'txt' || format === 'md') {
+    return fileBuffer.toString('utf-8');
+  }
+
+  // PDF — collect one text string per page via pagerender callback
+  const pdfParse = await getPdfParse();
+  const pageTexts: string[] = [];
+
+  await pdfParse(fileBuffer, {
+    pagerender: (pageData: { getTextContent: () => Promise<{ items: Array<{ str: string }> }> }) =>
+      pageData.getTextContent().then(tc => {
+        const text = tc.items.map(i => i.str).join(' ').trim();
+        if (text) pageTexts.push(text);
+        return text;
+      }),
+  });
+
+  if (pageTexts.length === 0) {
+    throw new Error('pdf_extraction_empty');
+  }
+
+  return pageTexts.join('\n\n---PAGE---\n\n');
+}

app/services/ingestion/src/pipeline/structure.ts

@@ -0,0 +1,181 @@
+import { anthropic, MODELS } from '../lib/anthropic.js';
+import { DraftKBSchema, type Chunk, type DraftKB, type DraftTheme, type DraftTopic, type DocumentFormat } from '../types.js';
+
+const BATCH_SIZE = 40;
+const BATCH_OVERLAP = 5;
+const LARGE_DOC_THRESHOLD = 60;
+
+const SYSTEM_PROMPT = `You are a knowledge architect. Your task is to analyse a set of text chunks from a source document and extract a structured knowledge base.
+
+Output ONLY valid JSON matching the schema provided. No preamble, no explanation, no markdown fences.
+
+Rules:
+- Group related content into Themes. A Theme is a broad subject area.
+- Under each Theme, identify discrete Topics. A Topic covers one specific concept.
+- Identify relationships between Topics: related, prerequisite, or contrast.
+  - related: Topics that complement each other
+  - prerequisite: Topic A must be understood before Topic B
+  - contrast: Topics that represent opposing approaches or concepts
+- For each Topic, extract key terms suitable for a glossary.
+- Assign a complexity weight (1–5) to each Topic.
+  1 = introductory, 5 = advanced
+- Draft a body for each Topic (2–4 paragraphs) based on the source chunks.
+- Draft a description for each Theme (1–2 sentences).
+- Every Topic must reference the chunk IDs that contributed to it.
+
+Output schema:
+{
+  "themes": [
+    {
+      "title": "string",
+      "description": "string",
+      "topics": [
+        {
+          "title": "string",
+          "body": "string",
+          "difficulty": "introductory" | "intermediate" | "advanced",
+          "complexityWeight": 1-5,
+          "keyTerms": ["string"],
+          "sourceChunkIds": ["chunk-id"],
+          "relationships": {
+            "related": ["topic title"],
+            "prerequisites": ["topic title"],
+            "contrasts": ["topic title"]
+          }
+        }
+      ]
+    }
+  ]
+}`;
+
+const STRICT_SUFFIX = '\n\nCRITICAL: Your entire response must be valid JSON only. No text before or after.';
+
+function buildUserPrompt(chunks: Chunk[], filename: string, format: DocumentFormat, strict: boolean): string {
+  const chunkText = chunks
+    .map(c => `[CHUNK-${c.id}]\n${c.text}`)
+    .join('\n\n');
+
+  return `Source document: ${filename}\nFormat: ${format}\n\nChunks:\n${chunkText}\n\nExtract the knowledge base structure from these chunks.${strict ? STRICT_SUFFIX : ''}`;
+}
+
+async function callClaude(chunks: Chunk[], filename: string, format: DocumentFormat, strict: boolean): Promise<DraftKB> {
+  const response = await anthropic.messages.create({
+    model: MODELS.SONNET,
+    max_tokens: 8000,
+    temperature: 0,
+    system: SYSTEM_PROMPT,
+    messages: [{ role: 'user', content: buildUserPrompt(chunks, filename, format, strict) }],
+  });
+
+  const textBlock = response.content.find(b => b.type === 'text');
+  if (!textBlock || textBlock.type !== 'text') {
+    throw new Error('structure_extraction_failed: no text block in response');
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(textBlock.text);
+  } catch {
+    if (strict) throw new Error('structure_extraction_failed');
+    return callClaude(chunks, filename, format, true);
+  }
+
+  const result = DraftKBSchema.safeParse(parsed);
+  if (!result.success) {
+    if (strict) throw new Error('structure_extraction_failed');
+    return callClaude(chunks, filename, format, true);
+  }
+
+  if (result.data.themes.length === 0) {
+    throw new Error('no_structure_found');
+  }
+
+  return result.data;
+}
+
+// ---------------------------------------------------------------------------
+// DraftKB merge (for large documents processed in batches)
+// ---------------------------------------------------------------------------
+
+interface MergedTheme {
+  title: string;
+  description: string;
+  topicMap: Map<string, DraftTopic>;
+}
+
+function mergeDraftKBs(batches: DraftKB[]): DraftKB {
+  const themeMap = new Map<string, MergedTheme>();
+
+  for (const kb of batches) {
+    for (const theme of kb.themes) {
+      const key = theme.title.toLowerCase().trim();
+      const existing = themeMap.get(key);
+
+      if (!existing) {
+        themeMap.set(key, {
+          title: theme.title,
+          description: theme.description,
+          topicMap: new Map(theme.topics.map(t => [t.title.toLowerCase().trim(), { ...t }])),
+        });
+      } else {
+        if (theme.description.length > existing.description.length) {
+          existing.description = theme.description;
+        }
+        for (const topic of theme.topics) {
+          const tKey = topic.title.toLowerCase().trim();
+          const existingTopic = existing.topicMap.get(tKey);
+          if (!existingTopic) {
+            existing.topicMap.set(tKey, { ...topic });
+          } else {
+            existingTopic.body =
+              existingTopic.body.length >= topic.body.length ? existingTopic.body : topic.body;
+            existingTopic.sourceChunkIds = [
+              ...new Set([...existingTopic.sourceChunkIds, ...topic.sourceChunkIds]),
+            ];
+            existingTopic.relationships = {
+              related: [...new Set([...existingTopic.relationships.related, ...topic.relationships.related])],
+              prerequisites: [...new Set([...existingTopic.relationships.prerequisites, ...topic.relationships.prerequisites])],
+              contrasts: [...new Set([...existingTopic.relationships.contrasts, ...topic.relationships.contrasts])],
+            };
+          }
+        }
+      }
+    }
+  }
+
+  const themes: DraftTheme[] = [...themeMap.values()].map(t => ({
+    title: t.title,
+    description: t.description,
+    topics: [...t.topicMap.values()],
+  }));
+
+  return { themes };
+}
+
+// ---------------------------------------------------------------------------
+// Public entry
+// ---------------------------------------------------------------------------
+
+export async function extractStructure(
+  chunks: Chunk[],
+  filename: string,
+  format: DocumentFormat,
+): Promise<DraftKB> {
+  if (chunks.length <= LARGE_DOC_THRESHOLD) {
+    return callClaude(chunks, filename, format, false);
+  }
+
+  const batches: DraftKB[] = [];
+  let start = 0;
+
+  while (start < chunks.length) {
+    const end = Math.min(start + BATCH_SIZE, chunks.length);
+    const batchChunks = chunks.slice(start, end);
+    const batchKB = await callClaude(batchChunks, filename, format, false);
+    batches.push(batchKB);
+    if (end >= chunks.length) break;
+    start = end - BATCH_OVERLAP;
+  }
+
+  return mergeDraftKBs(batches);
+}

app/services/ingestion/src/pipeline/write.ts

@@ -0,0 +1,91 @@
+import { getPocketBase } from '../lib/pocketbase.js';
+import type { Chunk, DraftKB, WrittenTopic } from '../types.js';
+
+export async function writeToPocketBase(
+  draftKB: DraftKB,
+  documentId: string,
+  chunkCount: number,
+): Promise<WrittenTopic[]> {
+  const pb = await getPocketBase();
+
+  await pb.collection('source_documents').update(documentId, {
+    status: 'processed',
+    chunk_count: chunkCount,
+    ingested_at: new Date().toISOString(),
+  });
+
+  const writtenTopics: WrittenTopic[] = [];
+  // title → PocketBase topic ID (for relationship resolution)
+  const topicIdByTitle = new Map<string, string>();
+
+  // Pass 1: create all themes and topics (without relationships)
+  for (const theme of draftKB.themes) {
+    const themeRecord = await pb.collection('themes').create({
+      title: theme.title,
+      description: theme.description,
+      status: 'draft',
+      source_documents: [documentId],
+    });
+
+    for (const topic of theme.topics) {
+      const topicRecord = await pb.collection('topics').create({
+        theme: themeRecord.id,
+        title: topic.title,
+        body: topic.body,
+        difficulty: topic.difficulty,
+        complexity_weight: topic.complexityWeight,
+        status: 'draft',
+        key_terms: topic.keyTerms,
+        qdrant_chunk_ids: [],
+        related_topics: [],
+        prerequisite_topics: [],
+        contrast_topics: [],
+      });
+
+      topicIdByTitle.set(topic.title.toLowerCase().trim(), topicRecord.id);
+
+      writtenTopics.push({
+        id: topicRecord.id,
+        title: topic.title,
+        themeId: themeRecord.id,
+        body: topic.body,
+        sourceChunkIds: topic.sourceChunkIds,
+      });
+    }
+  }
+
+  // Pass 2: resolve relationships (title → ID lookup)
+  for (const theme of draftKB.themes) {
+    for (const topic of theme.topics) {
+      const topicId = topicIdByTitle.get(topic.title.toLowerCase().trim());
+      if (!topicId) continue;
+
+      const resolve = (titles: string[]): string[] =>
+        titles
+          .map(t => topicIdByTitle.get(t.toLowerCase().trim()))
+          .filter((id): id is string => id !== undefined);
+
+      await pb.collection('topics').update(topicId, {
+        related_topics: resolve(topic.relationships.related),
+        prerequisite_topics: resolve(topic.relationships.prerequisites),
+        contrast_topics: resolve(topic.relationships.contrasts),
+      });
+    }
+  }
+
+  return writtenTopics;
+}
+
+export async function updateTopicQdrantIds(
+  topicId: string,
+  qdrantChunkIds: string[],
+): Promise<void> {
+  const pb = await getPocketBase();
+  await pb.collection('topics').update(topicId, { qdrant_chunk_ids: qdrantChunkIds });
+}
+
+export async function markDocumentFailed(documentId: string, reason: string): Promise<void> {
+  console.error(`[write] document ${documentId} failed: ${reason}`);
+  const pb = await getPocketBase();
+  await pb.collection('source_documents').update(documentId, { status: 'failed' });
+}

app/services/ingestion/src/routes/documents.ts

@@ -0,0 +1,29 @@
+import type { FastifyPluginAsync } from 'fastify';
+import { IngestBodySchema } from '../types.js';
+import { createJob, getJob } from '../jobs/queue.js';
+
+const documentRoutes: FastifyPluginAsync = async (app) => {
+  app.post('/ingest', async (request, reply) => {
+    const parsed = IngestBodySchema.safeParse(request.body);
+    if (!parsed.success) {
+      return reply.status(400).send({ error: 'Invalid request body', details: parsed.error.format() });
+    }
+    const job = createJob(parsed.data);
+    return reply.status(202).send({ jobId: job.id, status: job.status });
+  });
+
+  app.get<{ Params: { jobId: string } }>('/status/:jobId', async (request, reply) => {
+    const job = getJob(request.params.jobId);
+    if (!job) {
+      return reply.status(404).send({ error: 'Job not found' });
+    }
+    return reply.send({
+      jobId: job.id,
+      status: job.status,
+      progress: job.progress,
+      error: job.error,
+    });
+  });
+};
+
+export default documentRoutes;

app/services/ingestion/src/types.ts

@@ -0,0 +1,145 @@
+import { z } from 'zod';
+
+// ---------------------------------------------------------------------------
+// Shared primitives
+// ---------------------------------------------------------------------------
+
+export type DocumentFormat = 'pdf' | 'md' | 'txt';
+
+// ---------------------------------------------------------------------------
+// Pipeline: Chunk
+// ---------------------------------------------------------------------------
+
+export interface ChunkMetadata {
+  // MD-specific
+  headingLevel?: number;
+  headingText?: string;
+  parentHeading?: string;
+  // TXT-specific
+  approximatePosition?: 'start' | 'middle' | 'end';
+  // PDF-specific
+  pageNumber?: number;
+  chunkIndexOnPage?: number;
+}
+
+export interface Chunk {
+  id: string;
+  documentId: string;
+  text: string;
+  format: DocumentFormat;
+  index: number;
+  metadata: ChunkMetadata;
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline: DraftKB (AI output — validated with Zod before use)
+// ---------------------------------------------------------------------------
+
+const DraftTopicRelationshipsSchema = z.object({
+  related: z.array(z.string()),
+  prerequisites: z.array(z.string()),
+  contrasts: z.array(z.string()),
+});
+
+export const DraftTopicSchema = z.object({
+  title: z.string().min(1),
+  body: z.string().min(1),
+  difficulty: z.enum(['introductory', 'intermediate', 'advanced']),
+  complexityWeight: z.number().int().min(1).max(5),
+  keyTerms: z.array(z.string()),
+  sourceChunkIds: z.array(z.string()),
+  relationships: DraftTopicRelationshipsSchema,
+});
+
+export const DraftThemeSchema = z.object({
+  title: z.string().min(1),
+  description: z.string().min(1),
+  topics: z.array(DraftTopicSchema).min(1),
+});
+
+export const DraftKBSchema = z.object({
+  themes: z.array(DraftThemeSchema).min(1),
+});
+
+export type DraftTopic = z.infer<typeof DraftTopicSchema>;
+export type DraftTheme = z.infer<typeof DraftThemeSchema>;
+export type DraftKB = z.infer<typeof DraftKBSchema>;
+
+// ---------------------------------------------------------------------------
+// Pipeline: PocketBase write result
+// ---------------------------------------------------------------------------
+
+export interface WrittenTopic {
+  id: string;
+  title: string;
+  themeId: string;
+  body: string;
+  sourceChunkIds: string[];
+}
+
+// ---------------------------------------------------------------------------
+// Qdrant payload types
+// ---------------------------------------------------------------------------
+
+export interface SourceChunkPayload {
+  source_document_id: string;
+  chunk_index: number;
+  text: string;
+  theme_id: string | null;
+  topic_id: string | null;
+  format: DocumentFormat;
+}
+
+export interface TopicSummaryPayload {
+  topic_id: string;
+  theme_id: string;
+  title: string;
+  text: string;
+}
+
+// ---------------------------------------------------------------------------
+// Job system
+// ---------------------------------------------------------------------------
+
+export type JobStatus =
+  | 'queued'
+  | 'extracting'
+  | 'chunking'
+  | 'structuring'
+  | 'writing'
+  | 'embedding'
+  | 'done'
+  | 'failed';
+
+export interface JobProgress {
+  chunksTotal: number;
+  chunksEmbedded: number;
+  themesFound: number;
+  topicsFound: number;
+}
+
+export interface Job {
+  id: string;
+  documentId: string;
+  filename: string;
+  format: DocumentFormat;
+  filePath: string;
+  status: JobStatus;
+  progress: JobProgress;
+  error: string | null;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+// ---------------------------------------------------------------------------
+// API schemas (Zod — validates external input)
+// ---------------------------------------------------------------------------
+
+export const IngestBodySchema = z.object({
+  documentId: z.string().min(1),
+  filename: z.string().min(1),
+  format: z.enum(['pdf', 'md', 'txt']),
+  filePath: z.string().min(1),
+});
+
+export type IngestBody = z.infer<typeof IngestBodySchema>;

app/services/ingestion/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "Node16",
+    "moduleResolution": "Node16",
+    "rootDir": "src",
+    "outDir": "dist",
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "forceConsistentCasingInFileNames": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["dist", "node_modules"]
+}