learning-platform/src/lib/extractionPipeline.js

import { anthropicApi } from './api';
import * as db from './db';

const SYSTEM_PROMPT = `You are an AI knowledge extractor for Respellion, an IT company built on radical transparency.
You receive a source text. Your task is to extract all core concepts, roles, and processes from the text, and return them as a structured JSON Knowledge Graph.
Facts should be integrated into the descriptions of the other labels and NOT be extracted as unique topics.

CRITICAL INSTRUCTIONS FOR COMPLETENESS:
- You must extract EVERY SINGLE distinct role, process, and concept described or mentioned in the source text. 
- DO NOT summarize, skip, truncate, or omit any items. 
- If the document contains 29 roles, your JSON topics array must contain exactly 29 role topics. 
- Completeness is of paramount importance. Failing to extract all topics will result in loss of critical company knowledge.
- Keep descriptions concise (max 3 sentences) to ensure you have enough output tokens to list everything.

You MUST assign a learning_relevance to each topic:
- "core": Fundamental company knowledge.
- "standard": Normal learning topics.
- "peripheral": Good to know, but low priority.
- "exclude": Pure operational reference material (e.g., printer guides, wifi passwords) that should NEVER be tested.

ALWAYS return a valid JSON object in the following format:
{
  "topics": [
    {
      "id": "a-unique-lowercase-kebab-case-slug-specific-to-this-topic (e.g., 'software-engineer' or 'data-quality-review'). DO NOT use generic IDs like 'role-1' or 'concept-2'.",
      "label": "Topic title",
      "type": "concept | role | process",
      "description": "A concise, clear explanation of max 3 sentences.",
      "learning_relevance": "core | standard | peripheral | exclude"
    }
  ],
  "relations": [
    {
      "source": "topic-id-1",
      "target": "topic-id-2",
      "type": "related_to | depends_on | part_of | executed_by"
    }
  ]
}
Return JSON only. No markdown blocks or other text.`;

const HANDBOOK_SYSTEM_PROMPT = `You are analyzing an update to the Respellion Employee Handbook.
Your task is to identify changes and extract structural knowledge.

CRITICAL INSTRUCTION:
You must explicitly identify and create relations between Roles, Processes, and Concepts.
Every Process must have a Role attached (who does it).
Every Concept must have a relation to a Process or Role.

You MUST assign a learning_relevance to each topic:
- "core": Fundamental company knowledge.
- "standard": Normal learning topics.
- "peripheral": Good to know, but low priority.
- "exclude": Pure operational reference material (e.g., printer guides, wifi passwords) that should NEVER be tested.

Return a JSON object:
{
  "topics": [
    { "id": "...", "label": "...", "type": "role | process | concept", "description": "...", "learning_relevance": "standard", "metadata": { "source": "github_handbook" } }
  ],
  "relations": [
    { "source": "role-id", "target": "process-id", "type": "executes | related_to | depends_on | part_of", "description": "Brief metadata about this specific relation" }
  ]
}
Return JSON only. No markdown blocks or other text.`;

export async function analyzeHandbookDelta(fileContent, filePath) {
  const responseText = await anthropicApi.generateContent(HANDBOOK_SYSTEM_PROMPT, `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`);

  let extractedData;
  try {
    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
    const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
    extractedData = JSON.parse(jsonStr);
  } catch (e) {
    console.error('[Pipeline] AI returned non-JSON response for handbook delta:', responseText?.substring(0, 500));
    throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`, { cause: e });
  }

  await mergeKnowledgeGraph(extractedData);
  return { success: true, data: extractedData };
}
function chunkText(text, maxChunkSize = 4000) {
  const paragraphs = text.split(/\n+/);
  const chunks = [];
  let currentChunk = '';

  for (const para of paragraphs) {
    if ((currentChunk + '\n' + para).length > maxChunkSize) {
      if (currentChunk) chunks.push(currentChunk.trim());
      currentChunk = para;
    } else {
      currentChunk = currentChunk ? currentChunk + '\n' + para : para;
    }
  }
  if (currentChunk) chunks.push(currentChunk.trim());
  return chunks;
}

export async function processSourceText(textContent, sourceName) {
  // Deduplicate: skip if a source with the same name was already successfully processed
  const existing = await db.getSources();
  const alreadyDone = existing.find(
    s => s.name === sourceName && s.status === 'completed'
  );
  if (alreadyDone) {
    throw new Error(`"${sourceName}" has already been processed. Delete the existing source first if you want to re-analyse it.`);
  }

  const rec = await db.addSource({ name: sourceName, status: 'processing' });
  const sourceId = rec.id;

  try {
    const chunks = chunkText(textContent, 4000);
    console.log(`[Pipeline] Split "${sourceName}" into ${chunks.length} chunks for processing.`);

    let allExtractedTopics = [];
    let allExtractedRelations = [];

    for (let i = 0; i < chunks.length; i++) {
      if (i > 0) {
        console.log(`[Pipeline] Pacing delay (12s) to prevent rate limits before chunk ${i + 1}/${chunks.length}...`);
        await new Promise(r => setTimeout(r, 12000));
      }

      console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`);
      const responseText = await anthropicApi.generateContent(
        SYSTEM_PROMPT, 
        `Analyze this part of the document (${i + 1}/${chunks.length}):\n\n${chunks[i]}`
      );
      console.log(`[Pipeline] Raw AI response for chunk ${i + 1}:`, responseText);

      let extractedData;
      try {
        const jsonMatch = responseText.match(/\{[\s\S]*\}/);
        const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
        extractedData = JSON.parse(jsonStr);
      } catch (e) {
        console.error(`[Pipeline] AI returned non-JSON response for chunk ${i + 1}:`, responseText?.substring(0, 500));
        throw new Error(`AI response for chunk ${i + 1} was not valid JSON.`, { cause: e });
      }

      if (extractedData.topics && Array.isArray(extractedData.topics)) {
        allExtractedTopics.push(...extractedData.topics);
      }
      if (extractedData.relations && Array.isArray(extractedData.relations)) {
        allExtractedRelations.push(...extractedData.relations);
      }
    }

    // Merge everything together
    await mergeKnowledgeGraph({ topics: allExtractedTopics, relations: allExtractedRelations });
    await db.updateSourceStatus(sourceId, 'completed');

    return { success: true, data: { topics: allExtractedTopics, relations: allExtractedRelations } };

  } catch (error) {
    await db.updateSourceStatus(sourceId, 'failed', error.message);
    throw error;
  }
}

async function mergeKnowledgeGraph(newData) {
  const existingTopics = await db.getTopics();
  const existingRelations = await db.getRelations();

  const topicsMap = new Map(existingTopics.map(t => [t.id, t]));

  if (newData.topics && Array.isArray(newData.topics)) {
    for (const t of newData.topics) {
      if (topicsMap.has(t.id)) {
        // Upsert: merge new data into existing topic
        const existing = topicsMap.get(t.id);
        topicsMap.set(t.id, { 
          ...existing, 
          ...t, 
          // Keep existing description if new one is empty, or combine them if needed. Here we prefer the new one.
          description: t.description || existing.description 
        });
      } else {
        topicsMap.set(t.id, t);
      }
    }
  }

  const newRelations = [...existingRelations];
  if (newData.relations && Array.isArray(newData.relations)) {
    for (const r of newData.relations) {
      const isDup = newRelations.some(ex => ex.source === r.source && ex.target === r.target && ex.type === r.type);
      if (!isDup) {
        newRelations.push(r);
      }
    }
  }

  await db.saveTopics(Array.from(topicsMap.values()));
  await db.saveRelations(newRelations);
}