feat: add GitHub repository synchronization functionality with document extraction pipeline and update security policy to permit GitHub API access.
All checks were successful
On Push to Main / test (push) Successful in 29s
On Push to Main / publish (push) Successful in 58s
On Push to Main / deploy-dev (push) Successful in 1m32s

This commit is contained in:
RaymondVerhoef
2026-05-17 15:15:51 +02:00
parent 2374413282
commit e2de7f0729
5 changed files with 296 additions and 41 deletions

View File

@@ -26,6 +26,15 @@ ALWAYS return a valid JSON object in the following format:
Return JSON only. No markdown blocks or other text.`;
export async function processSourceText(textContent, sourceName) {
// Deduplicate: skip if a source with the same name was already successfully processed
const existing = await db.getSources();
const alreadyDone = existing.find(
s => s.name === sourceName && s.status === 'completed'
);
if (alreadyDone) {
throw new Error(`"${sourceName}" has already been processed. Delete the existing source first if you want to re-analyse it.`);
}
const rec = await db.addSource({ name: sourceName, status: 'processing' });
const sourceId = rec.id;
@@ -38,7 +47,8 @@ export async function processSourceText(textContent, sourceName) {
const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
extractedData = JSON.parse(jsonStr);
} catch (e) {
throw new Error('AI response was not valid JSON.');
console.error('[Pipeline] AI returned non-JSON response:', responseText?.substring(0, 500));
throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`);
}
await mergeKnowledgeGraph(extractedData);