feat: implement automated knowledge graph extraction pipeline and visualization component

src/components/admin/KnowledgeGraph.jsx

@@ -311,11 +311,12 @@ Rules:
 1. Identify topics that mean exactly the same thing. Choose one to keep, and one to delete.
 2. Identify topics that are too vague, irrelevant, or malformed to delete.
 3. Identify missing logical relations (depends_on, part_of, related_to) if two topics are conceptually linked but missing a relation.
-4. Return ONLY a valid JSON object describing the ACTIONS to take. Do not return the entire graph. Do not wrap in markdown blocks.`;
+4. Evaluate the learning_relevance of each topic. If a topic is purely operational/mundane (like a printer guide), mark it as "exclude". If it's low priority, mark "peripheral".
+5. Return ONLY a valid JSON object describing the ACTIONS to take. Do not return the entire graph. Do not wrap in markdown blocks.`;
 
       // Send a compact representation to minimize token usage and avoid rate limits.
-      // The AI only needs id, label, and type to identify duplicates/merges.
-      const compactTopics = currentTopics.map(({ id, label, type }) => ({ id, label, type }));
+      // The AI only needs id, label, type, and relevance to identify duplicates/merges and adjust relevance.
+      const compactTopics = currentTopics.map(({ id, label, type, learning_relevance }) => ({ id, label, type, learning_relevance }));
       const compactRelations = currentRelations.map(r => ({
         source: r.source?.id || r.source,
         target: r.target?.id || r.target,
@@ -329,7 +330,8 @@ Analyze this graph and return ONLY the optimized JSON object with this EXACT str
 {
   "merges": [ { "keepId": "id_to_keep", "deleteId": "id_to_delete" } ],
   "deletions": [ "id_to_delete_completely" ],
-  "newRelations": [ { "source": "id1", "target": "id2", "type": "depends_on" } ]
+  "newRelations": [ { "source": "id1", "target": "id2", "type": "depends_on" } ],
+  "relevanceUpdates": [ { "id": "topic_id", "learning_relevance": "exclude" } ]
 }`;
 
       const responseText = await anthropicApi.generateContent(systemPrompt, userPrompt);
@@ -364,6 +366,15 @@ Analyze this graph and return ONLY the optimized JSON object with this EXACT str
         }
       }
 
+      if (actions.relevanceUpdates && Array.isArray(actions.relevanceUpdates)) {
+        for (const update of actions.relevanceUpdates) {
+          const topicIndex = updatedTopics.findIndex(t => t.id === update.id);
+          if (topicIndex !== -1) {
+            updatedTopics[topicIndex] = { ...updatedTopics[topicIndex], learning_relevance: update.learning_relevance };
+          }
+        }
+      }
+
       // Ensure all relations only reference existing nodes and are normalized to string IDs
       const finalTopicIds = new Set(updatedTopics.map(t => t.id));
       updatedRelations = updatedRelations

src/lib/db.js

@@ -27,7 +27,11 @@ export async function upsertTopic(topic) {
     await pb.collection('topics').getOne(topic.id);
     return await pb.collection('topics').update(topic.id, topic);
   } catch {
-    return await pb.collection('topics').create({ id: topic.id, ...topic });
+    return await pb.collection('topics').create({ 
+      id: topic.id, 
+      learning_relevance: 'standard',
+      ...topic 
+    });
   }
 }
 

src/lib/extractionPipeline.js

@@ -6,6 +6,12 @@ You receive a source text. Your task is to extract core concepts, roles, and pro
 Facts should be integrated into the descriptions of the other labels and NOT be extracted as unique topics.
 CRITICAL: To ensure the response fits within limits, extract a maximum of 15 key topics and their most important relations. Keep descriptions very concise.
 
+You MUST assign a learning_relevance to each topic:
+- "core": Fundamental company knowledge.
+- "standard": Normal learning topics.
+- "peripheral": Good to know, but low priority.
+- "exclude": Pure operational reference material (e.g., printer guides, wifi passwords) that should NEVER be tested.
+
 ALWAYS return a valid JSON object in the following format:
 {
   "topics": [
@@ -13,7 +19,8 @@ ALWAYS return a valid JSON object in the following format:
       "id": "unique-slug",
       "label": "Topic title",
       "type": "concept | role | process",
-      "description": "A concise, clear explanation of max 3 sentences."
+      "description": "A concise, clear explanation of max 3 sentences.",
+      "learning_relevance": "core | standard | peripheral | exclude"
     }
   ],
   "relations": [
@@ -35,10 +42,16 @@ You must explicitly identify and create relations between Roles, Processes, and
 Every Process must have a Role attached (who does it).
 Every Concept must have a relation to a Process or Role.
 
+You MUST assign a learning_relevance to each topic:
+- "core": Fundamental company knowledge.
+- "standard": Normal learning topics.
+- "peripheral": Good to know, but low priority.
+- "exclude": Pure operational reference material (e.g., printer guides, wifi passwords) that should NEVER be tested.
+
 Return a JSON object:
 {
   "topics": [
-    { "id": "...", "label": "...", "type": "role | process | concept", "description": "...", "metadata": { "source": "github_handbook" } }
+    { "id": "...", "label": "...", "type": "role | process | concept", "description": "...", "learning_relevance": "standard", "metadata": { "source": "github_handbook" } }
   ],
   "relations": [
     { "source": "role-id", "target": "process-id", "type": "executes | related_to | depends_on | part_of", "description": "Brief metadata about this specific relation" }