feat: add GitHub repository synchronization functionality with document extraction pipeline and update security policy to permit GitHub API access.

src/lib/extractionPipeline.js

@@ -26,6 +26,15 @@ ALWAYS return a valid JSON object in the following format:
 Return JSON only. No markdown blocks or other text.`;
 
 export async function processSourceText(textContent, sourceName) {
+  // Deduplicate: skip if a source with the same name was already successfully processed
+  const existing = await db.getSources();
+  const alreadyDone = existing.find(
+    s => s.name === sourceName && s.status === 'completed'
+  );
+  if (alreadyDone) {
+    throw new Error(`"${sourceName}" has already been processed. Delete the existing source first if you want to re-analyse it.`);
+  }
+
   const rec = await db.addSource({ name: sourceName, status: 'processing' });
   const sourceId = rec.id;
 
@@ -38,7 +47,8 @@ export async function processSourceText(textContent, sourceName) {
       const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
       extractedData = JSON.parse(jsonStr);
     } catch (e) {
-      throw new Error('AI response was not valid JSON.');
+      console.error('[Pipeline] AI returned non-JSON response:', responseText?.substring(0, 500));
+      throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`);
     }
 
     await mergeKnowledgeGraph(extractedData);

src/lib/giteaService.js

@@ -0,0 +1,76 @@
+/**
+ * GitHub API Service
+ * Fetches files from a public GitHub repository directly from the browser.
+ * No proxy or token required for public repos.
+ */
+
+const GITHUB_API  = 'https://api.github.com';
+const GITHUB_RAW  = 'https://raw.githubusercontent.com';
+
+/**
+ * List markdown/text files in a GitHub repository folder.
+ * @param {string} owner  - e.g. "respellion"
+ * @param {string} repo   - e.g. "employee-handbook"
+ * @param {string} folder - e.g. "docs" or "" for root
+ * @param {string} branch - e.g. "main"
+ * @returns {Promise<Array<{name, sha, path}>>}
+ */
+export async function getRepoFolder(owner, repo, folder = '', branch = 'main') {
+  const path = folder ? folder.replace(/^\/|\/$/g, '') : '';
+  const url  = `${GITHUB_API}/repos/${owner}/${repo}/contents/${path}?ref=${branch}`;
+
+  const res = await fetch(url, {
+    headers: { Accept: 'application/vnd.github+json' },
+  });
+
+  if (!res.ok) {
+    const err = await res.json().catch(() => ({}));
+    throw new Error(`GitHub API ${res.status}: ${err.message || 'Could not list folder'}`);
+  }
+
+  const items = await res.json();
+  // Return only files (.md / .txt), not sub-directories
+  return items.filter(
+    item => item.type === 'file' && (item.name.endsWith('.md') || item.name.endsWith('.txt'))
+  );
+}
+
+/**
+ * Fetch the raw text content of a file.
+ * Uses raw.githubusercontent.com — no rate limits for public content.
+ * @param {string} owner
+ * @param {string} repo
+ * @param {string} path   - e.g. "docs/ROLES.md"
+ * @param {string} branch
+ * @returns {Promise<string>}
+ */
+export async function getFileContent(owner, repo, path, branch = 'main') {
+  const url = `${GITHUB_RAW}/${owner}/${repo}/${branch}/${path}`;
+  const res = await fetch(url);
+
+  if (!res.ok) {
+    throw new Error(`Could not fetch "${path}" from GitHub (${res.status})`);
+  }
+
+  return res.text();
+}
+
+/**
+ * Parse a GitHub tree URL into its parts.
+ * e.g. "https://github.com/respellion/employee-handbook/tree/main/docs"
+ * → { owner: 'respellion', repo: 'employee-handbook', branch: 'main', folder: 'docs' }
+ */
+export function parseGitHubUrl(url) {
+  try {
+    const match = url.match(/github\.com\/([^/]+)\/([^/]+)(?:\/tree\/([^/]+)(?:\/(.+))?)?/);
+    if (!match) return null;
+    return {
+      owner:  match[1],
+      repo:   match[2],
+      branch: match[3] || 'main',
+      folder: match[4] || '',
+    };
+  } catch {
+    return null;
+  }
+}