diff --git a/Caddyfile b/Caddyfile index 157cb61..65def0a 100644 --- a/Caddyfile +++ b/Caddyfile @@ -6,7 +6,7 @@ X-Content-Type-Options "nosniff" X-XSS-Protection "1; mode=block" Referrer-Policy "strict-origin-when-cross-origin" - Content-Security-Policy "default-src 'self'; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; script-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' https://fonts.gstatic.com" + Content-Security-Policy "default-src 'self'; connect-src 'self' https://api.github.com https://raw.githubusercontent.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; script-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self' https://fonts.gstatic.com" Permissions-Policy "geolocation=(), microphone=(), camera=()" } @@ -19,6 +19,7 @@ } } + handle /api/* { reverse_proxy pocketbase-learning:8090 } diff --git a/src/components/admin/UploadZone.jsx b/src/components/admin/UploadZone.jsx index b8abd3e..5c6ed02 100644 --- a/src/components/admin/UploadZone.jsx +++ b/src/components/admin/UploadZone.jsx @@ -1,24 +1,46 @@ -import React, { useState, useRef } from 'react'; -import { UploadCloud, Link as LinkIcon, AlertCircle, CheckCircle } from 'lucide-react'; +import React, { useState, useRef, useEffect } from 'react'; +import { UploadCloud, AlertCircle, CheckCircle, RefreshCw, GitBranch, FileText, ArrowUpCircle, MinusCircle } from 'lucide-react'; import { processSourceText } from '../../lib/extractionPipeline'; +import { getRepoFolder, getFileContent, parseGitHubUrl } from '../../lib/giteaService'; +import * as db from '../../lib/db'; import Card from '../ui/Card'; import Button from '../ui/Button'; -import Input from '../ui/Input'; + +// ── File status badge ───────────────────────────────────────────────────────── + +const StatusBadge = ({ status }) => { + if (status === 'new') return New; + if (status === 'updated') return Updated; + if (status === 'current') return Up to date; + if (status === 'done') return Processed; + if (status === 'error') return Failed; + if (status === 'skipped') return Skipped; + return null; +}; + +// ── Main Component ──────────────────────────────────────────────────────────── const UploadZone = ({ onUploadComplete }) => { - const [isDragging, setIsDragging] = useState(false); - const [url, setUrl] = useState(''); - const [isProcessing, setIsProcessing] = useState(false); - const [status, setStatus] = useState(null); // { type: 'error' | 'success', msg: string } - + const [isDragging, setIsDragging] = useState(false); + const [isProcessing, setIsProcessing] = useState(false); + const [status, setStatus] = useState(null); + + // GitHub sync state + const [githubUrl, setGithubUrl] = useState('https://github.com/respellion/employee-handbook/tree/main/docs'); + const [isFetching, setIsFetching] = useState(false); + const [fileList, setFileList] = useState(null); + const [syncProgress, setSyncProgress] = useState(null); + const fileInputRef = useRef(null); - const handleDragOver = (e) => { - e.preventDefault(); - setIsDragging(true); - }; + // Load GitHub URL from PocketBase settings on mount + useEffect(() => { + db.getSetting('github:url', '').then(url => { + if (url) setGithubUrl(url); + }); + }, []); - const handleDragLeave = () => setIsDragging(false); + // ── File upload (drag & drop / browse) ──────────────────────────────────── const processFile = async (file) => { setIsProcessing(true); @@ -35,6 +57,9 @@ const UploadZone = ({ onUploadComplete }) => { } }; + const handleDragOver = (e) => { e.preventDefault(); setIsDragging(true); }; + const handleDragLeave = () => setIsDragging(false); + const handleDrop = (e) => { e.preventDefault(); setIsDragging(false); @@ -52,14 +77,95 @@ const UploadZone = ({ onUploadComplete }) => { if (e.target.files?.length > 0) processFile(e.target.files[0]); }; - const handleUrlSubmit = async (e) => { - e.preventDefault(); - setStatus({ type: 'error', msg: 'URL import is disabled in the browser due to CORS restrictions.' }); + // ── Gitea Sync ───────────────────────────────────────────────────────────── + + const handleFetchFiles = async () => { + const parsed = parseGitHubUrl(githubUrl); + if (!parsed) { + setStatus({ type: 'error', msg: 'Enter a valid GitHub folder URL (e.g. https://github.com/owner/repo/tree/main/docs).' }); + return; + } + setIsFetching(true); + setFileList(null); + setStatus(null); + try { + const files = await getRepoFolder(parsed.owner, parsed.repo, parsed.folder, parsed.branch); + const enriched = await Promise.all(files.map(async (f) => { + const storedSha = await db.getSetting(`github:sha:${f.name}`, null); + let syncStatus = 'new'; + if (storedSha === f.sha) syncStatus = 'current'; + else if (storedSha && storedSha !== f.sha) syncStatus = 'updated'; + return { ...f, syncStatus, _parsed: parsed }; + })); + setFileList(enriched); + // Persist URL for next visit + await db.setSetting('github:url', githubUrl.trim()); + } catch (err) { + setStatus({ type: 'error', msg: `Could not fetch from GitHub: ${err.message}` }); + } finally { + setIsFetching(false); + } }; + const handleSync = async () => { + const toProcess = fileList.filter(f => f.syncStatus === 'new' || f.syncStatus === 'updated'); + if (toProcess.length === 0) { + setStatus({ type: 'success', msg: 'Everything is already up to date.' }); + return; + } + + setIsProcessing(true); + setSyncProgress({ current: 0, total: toProcess.length }); + let processed = 0, failed = 0; + + // Update a single file's status in the list + const updateFileStatus = (filename, newStatus) => { + setFileList(prev => prev.map(f => f.name === filename ? { ...f, syncStatus: newStatus } : f)); + }; + + for (const file of toProcess) { + const { _parsed: p } = file; + try { + const content = await getFileContent(p.owner, p.repo, file.path, p.branch); + + // If this is an update, delete the old source so deduplication allows re-processing + if (file.syncStatus === 'updated') { + const existing = await db.getSources(); + const old = existing.find(s => s.name === file.name && s.status === 'completed'); + if (old) await db.deleteSource(old.id); + } + + await processSourceText(content, file.name); + await db.setSetting(`github:sha:${file.name}`, file.sha); + updateFileStatus(file.name, 'done'); + processed++; + } catch (err) { + console.error(`[GitHub Sync] Failed: ${file.name}`, err); + updateFileStatus(file.name, 'error'); + failed++; + } + setSyncProgress(p => ({ ...p, current: p.current + 1 })); + } + + setSyncProgress(null); + setIsProcessing(false); + if (onUploadComplete) onUploadComplete(); + + const parts = []; + if (processed > 0) parts.push(`${processed} file${processed > 1 ? 's' : ''} processed`); + if (failed > 0) parts.push(`${failed} failed`); + setStatus({ type: failed > 0 ? 'error' : 'success', msg: `Sync complete: ${parts.join(', ')}.` }); + }; + + const toProcess = fileList ? fileList.filter(f => f.syncStatus === 'new' || f.syncStatus === 'updated') : []; + + // ── Render ───────────────────────────────────────────────────────────────── + return (
- {

Drag files here

Supports .txt and .md (max 5MB)

- + {/* ─── Divider ─── */}
OR
-
-
- setUrl(e.target.value)} - disabled={isProcessing} + {/* ─── GitHub Sync Panel ─── */} + +
+
+
+ +
+

GitHub Sync

+
+ setGithubUrl(e.target.value)} + placeholder="https://github.com/owner/repo/tree/main/folder" + className="flex-1 h-9 px-3 rounded-[var(--r-sm)] border border-bg-warm bg-bg text-sm focus:outline-none focus:ring-2 focus:ring-teal/30 focus:border-teal" />
- - +
+
+ {fileList && toProcess.length > 0 && ( + + )} + +
+
+ + {/* File list */} + {fileList && ( +
+ {fileList.length === 0 ? ( +

No .md or .txt files found in this folder.

+ ) : fileList.map(file => ( +
+
+ + {file.name} +
+ +
+ ))} +
+ )} + + {/* Progress bar */} + {syncProgress && ( +
+
+
+
+

+ Processing {syncProgress.current} of {syncProgress.total} files… +

+
+ )} + + + {/* ─── Status messages ─── */} {status && (
- {status.type === 'error' ? : } + {status.type === 'error' + ? + : }

{status.type === 'error' ? 'Error' : 'Success'}

{status.msg}

diff --git a/src/lib/extractionPipeline.js b/src/lib/extractionPipeline.js index b109979..0801a62 100644 --- a/src/lib/extractionPipeline.js +++ b/src/lib/extractionPipeline.js @@ -26,6 +26,15 @@ ALWAYS return a valid JSON object in the following format: Return JSON only. No markdown blocks or other text.`; export async function processSourceText(textContent, sourceName) { + // Deduplicate: skip if a source with the same name was already successfully processed + const existing = await db.getSources(); + const alreadyDone = existing.find( + s => s.name === sourceName && s.status === 'completed' + ); + if (alreadyDone) { + throw new Error(`"${sourceName}" has already been processed. Delete the existing source first if you want to re-analyse it.`); + } + const rec = await db.addSource({ name: sourceName, status: 'processing' }); const sourceId = rec.id; @@ -38,7 +47,8 @@ export async function processSourceText(textContent, sourceName) { const jsonStr = jsonMatch ? jsonMatch[0] : responseText; extractedData = JSON.parse(jsonStr); } catch (e) { - throw new Error('AI response was not valid JSON.'); + console.error('[Pipeline] AI returned non-JSON response:', responseText?.substring(0, 500)); + throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`); } await mergeKnowledgeGraph(extractedData); diff --git a/src/lib/giteaService.js b/src/lib/giteaService.js new file mode 100644 index 0000000..1b60767 --- /dev/null +++ b/src/lib/giteaService.js @@ -0,0 +1,76 @@ +/** + * GitHub API Service + * Fetches files from a public GitHub repository directly from the browser. + * No proxy or token required for public repos. + */ + +const GITHUB_API = 'https://api.github.com'; +const GITHUB_RAW = 'https://raw.githubusercontent.com'; + +/** + * List markdown/text files in a GitHub repository folder. + * @param {string} owner - e.g. "respellion" + * @param {string} repo - e.g. "employee-handbook" + * @param {string} folder - e.g. "docs" or "" for root + * @param {string} branch - e.g. "main" + * @returns {Promise>} + */ +export async function getRepoFolder(owner, repo, folder = '', branch = 'main') { + const path = folder ? folder.replace(/^\/|\/$/g, '') : ''; + const url = `${GITHUB_API}/repos/${owner}/${repo}/contents/${path}?ref=${branch}`; + + const res = await fetch(url, { + headers: { Accept: 'application/vnd.github+json' }, + }); + + if (!res.ok) { + const err = await res.json().catch(() => ({})); + throw new Error(`GitHub API ${res.status}: ${err.message || 'Could not list folder'}`); + } + + const items = await res.json(); + // Return only files (.md / .txt), not sub-directories + return items.filter( + item => item.type === 'file' && (item.name.endsWith('.md') || item.name.endsWith('.txt')) + ); +} + +/** + * Fetch the raw text content of a file. + * Uses raw.githubusercontent.com — no rate limits for public content. + * @param {string} owner + * @param {string} repo + * @param {string} path - e.g. "docs/ROLES.md" + * @param {string} branch + * @returns {Promise} + */ +export async function getFileContent(owner, repo, path, branch = 'main') { + const url = `${GITHUB_RAW}/${owner}/${repo}/${branch}/${path}`; + const res = await fetch(url); + + if (!res.ok) { + throw new Error(`Could not fetch "${path}" from GitHub (${res.status})`); + } + + return res.text(); +} + +/** + * Parse a GitHub tree URL into its parts. + * e.g. "https://github.com/respellion/employee-handbook/tree/main/docs" + * → { owner: 'respellion', repo: 'employee-handbook', branch: 'main', folder: 'docs' } + */ +export function parseGitHubUrl(url) { + try { + const match = url.match(/github\.com\/([^/]+)\/([^/]+)(?:\/tree\/([^/]+)(?:\/(.+))?)?/); + if (!match) return null; + return { + owner: match[1], + repo: match[2], + branch: match[3] || 'main', + folder: match[4] || '', + }; + } catch { + return null; + } +} diff --git a/src/pages/Admin/index.jsx b/src/pages/Admin/index.jsx index 23f7cfe..e4d71cf 100644 --- a/src/pages/Admin/index.jsx +++ b/src/pages/Admin/index.jsx @@ -35,7 +35,7 @@ const Admin = () => { } }, [activeTab]); - const saveSettings = (e) => { + const saveSettings = async (e) => { e.preventDefault(); storage.set('admin:model', model.trim()); storage.set('admin:use_simulation', useSimulation);