feat: add GitHub repository synchronization functionality with document extraction pipeline and update security policy to permit GitHub API access.

2026-05-17 15:15:51 +02:00
parent 2374413282
commit e2de7f0729
5 changed files with 296 additions and 41 deletions
--- a/3
+++ b/3
@@ -6,7 +6,7 @@
        X-Content-Type-Options "nosniff"
        X-XSS-Protection "1; mode=block"
        Referrer-Policy "strict-origin-when-cross-origin"
-        Content-Security-Policy "default-src 'self'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; script-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' https://fonts.gstatic.com"
+        Content-Security-Policy "default-src 'self'; connect-src 'self' https://api.github.com https://raw.githubusercontent.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; script-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' https://fonts.gstatic.com"
        Permissions-Policy "geolocation=(), microphone=(), camera=()"
    }

@@ -19,6 +19,7 @@
        }
    }

+
    handle /api/* {
        reverse_proxy pocketbase-learning:8090
    }
--- a/src/components/admin/UploadZone.jsx
+++ b/src/components/admin/UploadZone.jsx
@@ -1,24 +1,46 @@
-import React, { useState, useRef } from 'react';
-import { UploadCloud, Link as LinkIcon, AlertCircle, CheckCircle } from 'lucide-react';
+import React, { useState, useRef, useEffect } from 'react';
+import { UploadCloud, AlertCircle, CheckCircle, RefreshCw, GitBranch, FileText, ArrowUpCircle, MinusCircle } from 'lucide-react';
 import { processSourceText } from '../../lib/extractionPipeline';
+import { getRepoFolder, getFileContent, parseGitHubUrl } from '../../lib/giteaService';
+import * as db from '../../lib/db';
 import Card from '../ui/Card';
 import Button from '../ui/Button';
-import Input from '../ui/Input';
+
+// ── File status badge ─────────────────────────────────────────────────────────
+
+const StatusBadge = ({ status }) => {
+  if (status === 'new')      return <span className="text-xs font-medium text-teal bg-teal/10 px-2 py-0.5 rounded-full flex items-center gap-1"><ArrowUpCircle size={11}/> New</span>;
+  if (status === 'updated')  return <span className="text-xs font-medium text-purple-700 bg-purple-100 px-2 py-0.5 rounded-full flex items-center gap-1"><RefreshCw size={11}/> Updated</span>;
+  if (status === 'current')  return <span className="text-xs font-medium text-fg-muted bg-bg-warm px-2 py-0.5 rounded-full flex items-center gap-1"><MinusCircle size={11}/> Up to date</span>;
+  if (status === 'done')     return <span className="text-xs font-medium text-teal bg-teal/10 px-2 py-0.5 rounded-full flex items-center gap-1"><CheckCircle size={11}/> Processed</span>;
+  if (status === 'error')    return <span className="text-xs font-medium text-red-700 bg-red-100 px-2 py-0.5 rounded-full flex items-center gap-1"><AlertCircle size={11}/> Failed</span>;
+  if (status === 'skipped')  return <span className="text-xs font-medium text-fg-muted bg-bg-warm px-2 py-0.5 rounded-full flex items-center gap-1"><MinusCircle size={11}/> Skipped</span>;
+  return null;
+};
+
+// ── Main Component ────────────────────────────────────────────────────────────

 const UploadZone = ({ onUploadComplete }) => {
  const [isDragging, setIsDragging]         = useState(false);
-  const [url, setUrl] = useState('');
  const [isProcessing, setIsProcessing]     = useState(false);
-  const [status, setStatus] = useState(null); // { type: 'error' | 'success', msg: string }
+  const [status, setStatus]                 = useState(null);
+
+  // GitHub sync state
+  const [githubUrl, setGithubUrl]           = useState('https://github.com/respellion/employee-handbook/tree/main/docs');
+  const [isFetching, setIsFetching]         = useState(false);
+  const [fileList, setFileList]             = useState(null);
+  const [syncProgress, setSyncProgress]     = useState(null);

  const fileInputRef = useRef(null);

-  const handleDragOver = (e) => {
-    e.preventDefault();
-    setIsDragging(true);
-  };
+  // Load GitHub URL from PocketBase settings on mount
+  useEffect(() => {
+    db.getSetting('github:url', '').then(url => {
+      if (url) setGithubUrl(url);
+    });
+  }, []);

-  const handleDragLeave = () => setIsDragging(false);
+  // ── File upload (drag & drop / browse) ────────────────────────────────────

  const processFile = async (file) => {
    setIsProcessing(true);
@@ -35,6 +57,9 @@ const UploadZone = ({ onUploadComplete }) => {
    }
  };

+  const handleDragOver  = (e) => { e.preventDefault(); setIsDragging(true); };
+  const handleDragLeave = ()  => setIsDragging(false);
+
  const handleDrop = (e) => {
    e.preventDefault();
    setIsDragging(false);
@@ -52,13 +77,94 @@ const UploadZone = ({ onUploadComplete }) => {
    if (e.target.files?.length > 0) processFile(e.target.files[0]);
  };

-  const handleUrlSubmit = async (e) => {
-    e.preventDefault();
-    setStatus({ type: 'error', msg: 'URL import is disabled in the browser due to CORS restrictions.' });
+  // ── Gitea Sync ─────────────────────────────────────────────────────────────
+
+  const handleFetchFiles = async () => {
+    const parsed = parseGitHubUrl(githubUrl);
+    if (!parsed) {
+      setStatus({ type: 'error', msg: 'Enter a valid GitHub folder URL (e.g. https://github.com/owner/repo/tree/main/docs).' });
+      return;
+    }
+    setIsFetching(true);
+    setFileList(null);
+    setStatus(null);
+    try {
+      const files = await getRepoFolder(parsed.owner, parsed.repo, parsed.folder, parsed.branch);
+      const enriched = await Promise.all(files.map(async (f) => {
+        const storedSha = await db.getSetting(`github:sha:${f.name}`, null);
+        let syncStatus = 'new';
+        if (storedSha === f.sha) syncStatus = 'current';
+        else if (storedSha && storedSha !== f.sha) syncStatus = 'updated';
+        return { ...f, syncStatus, _parsed: parsed };
+      }));
+      setFileList(enriched);
+      // Persist URL for next visit
+      await db.setSetting('github:url', githubUrl.trim());
+    } catch (err) {
+      setStatus({ type: 'error', msg: `Could not fetch from GitHub: ${err.message}` });
+    } finally {
+      setIsFetching(false);
+    }
  };

+  const handleSync = async () => {
+    const toProcess = fileList.filter(f => f.syncStatus === 'new' || f.syncStatus === 'updated');
+    if (toProcess.length === 0) {
+      setStatus({ type: 'success', msg: 'Everything is already up to date.' });
+      return;
+    }
+
+    setIsProcessing(true);
+    setSyncProgress({ current: 0, total: toProcess.length });
+    let processed = 0, failed = 0;
+
+    // Update a single file's status in the list
+    const updateFileStatus = (filename, newStatus) => {
+      setFileList(prev => prev.map(f => f.name === filename ? { ...f, syncStatus: newStatus } : f));
+    };
+
+    for (const file of toProcess) {
+      const { _parsed: p } = file;
+      try {
+        const content = await getFileContent(p.owner, p.repo, file.path, p.branch);
+
+        // If this is an update, delete the old source so deduplication allows re-processing
+        if (file.syncStatus === 'updated') {
+          const existing = await db.getSources();
+          const old = existing.find(s => s.name === file.name && s.status === 'completed');
+          if (old) await db.deleteSource(old.id);
+        }
+
+        await processSourceText(content, file.name);
+        await db.setSetting(`github:sha:${file.name}`, file.sha);
+        updateFileStatus(file.name, 'done');
+        processed++;
+      } catch (err) {
+        console.error(`[GitHub Sync] Failed: ${file.name}`, err);
+        updateFileStatus(file.name, 'error');
+        failed++;
+      }
+      setSyncProgress(p => ({ ...p, current: p.current + 1 }));
+    }
+
+    setSyncProgress(null);
+    setIsProcessing(false);
+    if (onUploadComplete) onUploadComplete();
+
+    const parts = [];
+    if (processed > 0) parts.push(`${processed} file${processed > 1 ? 's' : ''} processed`);
+    if (failed > 0)    parts.push(`${failed} failed`);
+    setStatus({ type: failed > 0 ? 'error' : 'success', msg: `Sync complete: ${parts.join(', ')}.` });
+  };
+
+  const toProcess = fileList ? fileList.filter(f => f.syncStatus === 'new' || f.syncStatus === 'updated') : [];
+
+  // ── Render ─────────────────────────────────────────────────────────────────
+
  return (
    <div className="space-y-6">
+
+      {/* ─── Drag & Drop Upload ─── */}
      <Card
        className={`border-2 border-dashed transition-colors flex flex-col items-center justify-center py-12 cursor-pointer ${
          isDragging ? 'border-teal bg-teal/5' : 'border-bg-warm bg-bg-warm/20'
@@ -72,7 +178,7 @@ const UploadZone = ({ onUploadComplete }) => {
        <p className="font-medium text-lg">Drag files here</p>
        <p className="text-sm text-fg-muted mb-4">Supports .txt and .md (max 5MB)</p>
        <Button variant="outline" type="button" disabled={isProcessing}>
-          {isProcessing ? 'AI extraction in progress...' : 'Browse files'}
+          {isProcessing && !syncProgress ? 'AI extraction in progress...' : 'Browse files'}
        </Button>
        <input
          type="file"
@@ -83,32 +189,94 @@ const UploadZone = ({ onUploadComplete }) => {
        />
      </Card>

+      {/* ─── Divider ─── */}
      <div className="flex items-center gap-4">
        <div className="flex-1 border-t border-bg-warm"></div>
        <span className="text-sm text-fg-muted uppercase tracking-wider">OR</span>
        <div className="flex-1 border-t border-bg-warm"></div>
      </div>

-      <form onSubmit={handleUrlSubmit} className="flex gap-2 items-end">
-        <div className="flex-1">
-          <Input 
-            label="Import from URL"
-            placeholder="https://wiki.respellion.com/article"
-            value={url}
-            onChange={(e) => setUrl(e.target.value)}
-            disabled={isProcessing}
+      {/* ─── GitHub Sync Panel ─── */}
+      <Card className="border border-bg-warm">
+        <div className="flex items-start justify-between mb-4 gap-4">
+          <div className="flex items-center gap-3 flex-shrink-0">
+            <div className="w-9 h-9 rounded-[var(--r-sm)] bg-teal/10 flex items-center justify-center">
+              <GitBranch size={18} className="text-teal" />
+            </div>
+            <h3 className="font-semibold">GitHub Sync</h3>
+          </div>
+          <input
+            type="url"
+            value={githubUrl}
+            onChange={e => setGithubUrl(e.target.value)}
+            placeholder="https://github.com/owner/repo/tree/main/folder"
+            className="flex-1 h-9 px-3 rounded-[var(--r-sm)] border border-bg-warm bg-bg text-sm focus:outline-none focus:ring-2 focus:ring-teal/30 focus:border-teal"
          />
        </div>
-        <Button type="submit" disabled={isProcessing || !url}>
-          <LinkIcon size={18} className="mr-2" /> Import
+        <div className="flex justify-end gap-2 mb-4">
+          <div className="flex items-center gap-2">
+            {fileList && toProcess.length > 0 && (
+              <Button onClick={handleSync} disabled={isProcessing}>
+                {isProcessing && syncProgress
+                  ? `Processing ${syncProgress.current}/${syncProgress.total}...`
+                  : `Sync ${toProcess.length} file${toProcess.length > 1 ? 's' : ''}`}
              </Button>
-      </form>
+            )}
+            <Button
+              variant="outline"
+              onClick={handleFetchFiles}
+              disabled={isFetching || isProcessing}
+            >
+              <RefreshCw size={15} className={`mr-1.5 ${isFetching ? 'animate-spin' : ''}`} />
+              {isFetching ? 'Checking...' : fileList ? 'Refresh' : 'Check for updates'}
+            </Button>
+          </div>
+        </div>

+
+        {/* File list */}
+        {fileList && (
+          <div className="divide-y divide-bg-warm border border-bg-warm rounded-[var(--r-sm)] overflow-hidden">
+            {fileList.length === 0 ? (
+              <p className="text-sm text-fg-muted p-4 text-center">No .md or .txt files found in this folder.</p>
+            ) : fileList.map(file => (
+              <div key={file.name} className="flex items-center justify-between px-4 py-3 hover:bg-bg-warm/30 transition-colors">
+                <div className="flex items-center gap-3">
+                  <FileText size={15} className="text-fg-muted flex-shrink-0" />
+                  <span className="text-sm font-medium">{file.name}</span>
+                </div>
+                <StatusBadge status={file.syncStatus} />
+              </div>
+            ))}
+          </div>
+        )}
+
+        {/* Progress bar */}
+        {syncProgress && (
+          <div className="mt-4">
+            <div className="h-1.5 bg-bg-warm rounded-full overflow-hidden">
+              <div
+                className="h-full bg-teal transition-all duration-300"
+                style={{ width: `${(syncProgress.current / syncProgress.total) * 100}%` }}
+              />
+            </div>
+            <p className="text-xs text-fg-muted mt-1.5">
+              Processing {syncProgress.current} of {syncProgress.total} files…
+            </p>
+          </div>
+        )}
+      </Card>
+
+      {/* ─── Status messages ─── */}
      {status && (
        <div className={`p-4 rounded-[var(--r-sm)] flex items-start gap-3 ${
-          status.type === 'error' ? 'bg-red-50 text-red-900 border border-red-200' : 'bg-teal-50 text-teal-900 border border-teal-200'
+          status.type === 'error'
+            ? 'bg-red-50 text-red-900 border border-red-200'
+            : 'bg-teal-50 text-teal-900 border border-teal-200'
        }`}>
-          {status.type === 'error' ? <AlertCircle className="text-red-500 flex-shrink-0" /> : <CheckCircle className="text-teal-600 flex-shrink-0" />}
+          {status.type === 'error'
+            ? <AlertCircle className="text-red-500 flex-shrink-0" />
+            : <CheckCircle className="text-teal-600 flex-shrink-0" />}
          <div>
            <p className="font-medium">{status.type === 'error' ? 'Error' : 'Success'}</p>
            <p className="text-sm">{status.msg}</p>
--- a/src/lib/extractionPipeline.js
+++ b/src/lib/extractionPipeline.js
@@ -26,6 +26,15 @@ ALWAYS return a valid JSON object in the following format:
 Return JSON only. No markdown blocks or other text.`;

 export async function processSourceText(textContent, sourceName) {
+  // Deduplicate: skip if a source with the same name was already successfully processed
+  const existing = await db.getSources();
+  const alreadyDone = existing.find(
+    s => s.name === sourceName && s.status === 'completed'
+  );
+  if (alreadyDone) {
+    throw new Error(`"${sourceName}" has already been processed. Delete the existing source first if you want to re-analyse it.`);
+  }
+
  const rec = await db.addSource({ name: sourceName, status: 'processing' });
  const sourceId = rec.id;

@@ -38,7 +47,8 @@ export async function processSourceText(textContent, sourceName) {
      const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
      extractedData = JSON.parse(jsonStr);
    } catch (e) {
-      throw new Error('AI response was not valid JSON.');
+      console.error('[Pipeline] AI returned non-JSON response:', responseText?.substring(0, 500));
+      throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`);
    }

    await mergeKnowledgeGraph(extractedData);
--- a/src/lib/giteaService.js
+++ b/src/lib/giteaService.js
@@ -0,0 +1,76 @@
+/**
+ * GitHub API Service
+ * Fetches files from a public GitHub repository directly from the browser.
+ * No proxy or token required for public repos.
+ */
+
+const GITHUB_API  = 'https://api.github.com';
+const GITHUB_RAW  = 'https://raw.githubusercontent.com';
+
+/**
+ * List markdown/text files in a GitHub repository folder.
+ * @param {string} owner  - e.g. "respellion"
+ * @param {string} repo   - e.g. "employee-handbook"
+ * @param {string} folder - e.g. "docs" or "" for root
+ * @param {string} branch - e.g. "main"
+ * @returns {Promise<Array<{name, sha, path}>>}
+ */
+export async function getRepoFolder(owner, repo, folder = '', branch = 'main') {
+  const path = folder ? folder.replace(/^\/|\/$/g, '') : '';
+  const url  = `${GITHUB_API}/repos/${owner}/${repo}/contents/${path}?ref=${branch}`;
+
+  const res = await fetch(url, {
+    headers: { Accept: 'application/vnd.github+json' },
+  });
+
+  if (!res.ok) {
+    const err = await res.json().catch(() => ({}));
+    throw new Error(`GitHub API ${res.status}: ${err.message || 'Could not list folder'}`);
+  }
+
+  const items = await res.json();
+  // Return only files (.md / .txt), not sub-directories
+  return items.filter(
+    item => item.type === 'file' && (item.name.endsWith('.md') || item.name.endsWith('.txt'))
+  );
+}
+
+/**
+ * Fetch the raw text content of a file.
+ * Uses raw.githubusercontent.com — no rate limits for public content.
+ * @param {string} owner
+ * @param {string} repo
+ * @param {string} path   - e.g. "docs/ROLES.md"
+ * @param {string} branch
+ * @returns {Promise<string>}
+ */
+export async function getFileContent(owner, repo, path, branch = 'main') {
+  const url = `${GITHUB_RAW}/${owner}/${repo}/${branch}/${path}`;
+  const res = await fetch(url);
+
+  if (!res.ok) {
+    throw new Error(`Could not fetch "${path}" from GitHub (${res.status})`);
+  }
+
+  return res.text();
+}
+
+/**
+ * Parse a GitHub tree URL into its parts.
+ * e.g. "https://github.com/respellion/employee-handbook/tree/main/docs"
+ * → { owner: 'respellion', repo: 'employee-handbook', branch: 'main', folder: 'docs' }
+ */
+export function parseGitHubUrl(url) {
+  try {
+    const match = url.match(/github\.com\/([^/]+)\/([^/]+)(?:\/tree\/([^/]+)(?:\/(.+))?)?/);
+    if (!match) return null;
+    return {
+      owner:  match[1],
+      repo:   match[2],
+      branch: match[3] || 'main',
+      folder: match[4] || '',
+    };
+  } catch {
+    return null;
+  }
+}
--- a/src/pages/Admin/index.jsx
+++ b/src/pages/Admin/index.jsx
@@ -35,7 +35,7 @@ const Admin = () => {
    }
  }, [activeTab]);

-  const saveSettings = (e) => {
+  const saveSettings = async (e) => {
    e.preventDefault();
    storage.set('admin:model', model.trim());
    storage.set('admin:use_simulation', useSimulation);