feat: add GitHub repository synchronization functionality with document extraction pipeline and update security policy to permit GitHub API access.
This commit is contained in:
@@ -26,6 +26,15 @@ ALWAYS return a valid JSON object in the following format:
|
||||
Return JSON only. No markdown blocks or other text.`;
|
||||
|
||||
export async function processSourceText(textContent, sourceName) {
|
||||
// Deduplicate: skip if a source with the same name was already successfully processed
|
||||
const existing = await db.getSources();
|
||||
const alreadyDone = existing.find(
|
||||
s => s.name === sourceName && s.status === 'completed'
|
||||
);
|
||||
if (alreadyDone) {
|
||||
throw new Error(`"${sourceName}" has already been processed. Delete the existing source first if you want to re-analyse it.`);
|
||||
}
|
||||
|
||||
const rec = await db.addSource({ name: sourceName, status: 'processing' });
|
||||
const sourceId = rec.id;
|
||||
|
||||
@@ -38,7 +47,8 @@ export async function processSourceText(textContent, sourceName) {
|
||||
const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
|
||||
extractedData = JSON.parse(jsonStr);
|
||||
} catch (e) {
|
||||
throw new Error('AI response was not valid JSON.');
|
||||
console.error('[Pipeline] AI returned non-JSON response:', responseText?.substring(0, 500));
|
||||
throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`);
|
||||
}
|
||||
|
||||
await mergeKnowledgeGraph(extractedData);
|
||||
|
||||
Reference in New Issue
Block a user