feat: add new page and migration scripts for access rules and collections

- Introduced new page component for library topics with type checks. - Added migration scripts to update access rules for various collections including badges, curriculum versions, and themes. - Implemented PocketBase integration for managing collection access rules dynamically. - Ensured proper type validation for page props and metadata generation functions.
2026-05-23 22:26:40 +02:00
parent 14286d6cb1
commit 8684ffa35b
109 changed files with 2065 additions and 114 deletions
--- a/app/services/ingestion/package-lock.json
+++ b/app/services/ingestion/package-lock.json
@@ -9,6 +9,7 @@
      "version": "0.1.0",
      "dependencies": {
        "@anthropic-ai/sdk": "^0.24",
+        "@fastify/cors": "^9.0.1",
        "@qdrant/js-client-rest": "^1.9",
        "fastify": "^4",
        "openai": "^4",
@@ -511,6 +512,16 @@
        "fast-uri": "^2.0.0"
      }
    },
+    "node_modules/@fastify/cors": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/@fastify/cors/-/cors-9.0.1.tgz",
+      "integrity": "sha512-YY9Ho3ovI+QHIL2hW+9X4XqQjXLjJqsU+sMV/xFsxZkE8p3GNnYVFpoOxF7SsP5ZL76gwvbo3V9L+FIekBGU4Q==",
+      "license": "MIT",
+      "dependencies": {
+        "fastify-plugin": "^4.0.0",
+        "mnemonist": "0.39.6"
+      }
+    },
    "node_modules/@fastify/error": {
      "version": "3.4.1",
      "resolved": "https://registry.npmjs.org/@fastify/error/-/error-3.4.1.tgz",
@@ -1067,6 +1078,12 @@
        "toad-cache": "^3.3.0"
      }
    },
+    "node_modules/fastify-plugin": {
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/fastify-plugin/-/fastify-plugin-4.5.1.tgz",
+      "integrity": "sha512-stRHYGeuqpEZTL1Ef0Ovr2ltazUT9g844X5z/zEBFLG8RYlpDiOCIG+ATvYEp+/zmc7sN29mcIMp8gvYplYPIQ==",
+      "license": "MIT"
+    },
    "node_modules/fastq": {
      "version": "1.20.1",
      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
@@ -1375,6 +1392,15 @@
        "node": ">= 0.6"
      }
    },
+    "node_modules/mnemonist": {
+      "version": "0.39.6",
+      "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.6.tgz",
+      "integrity": "sha512-A/0v5Z59y63US00cRSLiloEIw3t5G+MiKz4BhX21FI+YBJXBOGW0ohFxTxO08dsOYlzxo87T7vGfZKYp2bcAWA==",
+      "license": "MIT",
+      "dependencies": {
+        "obliterator": "^2.0.1"
+      }
+    },
    "node_modules/ms": {
      "version": "2.1.3",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -1427,6 +1453,12 @@
        }
      }
    },
+    "node_modules/obliterator": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz",
+      "integrity": "sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==",
+      "license": "MIT"
+    },
    "node_modules/on-exit-leak-free": {
      "version": "2.1.2",
      "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
--- a/app/services/ingestion/package.json
+++ b/app/services/ingestion/package.json
@@ -8,10 +8,12 @@
    "build": "tsc",
    "start": "node dist/index.js",
    "migrate": "tsx src/migrations/001_initial_schema.ts",
-    "migrate:qdrant": "tsx src/migrations/002_qdrant_setup.ts"
+    "migrate:qdrant": "tsx src/migrations/002_qdrant_setup.ts",
+    "migrate:rules": "tsx src/migrations/003_access_rules.ts"
  },
  "dependencies": {
    "@anthropic-ai/sdk": "^0.24",
+    "@fastify/cors": "^9.0.1",
    "@qdrant/js-client-rest": "^1.9",
    "fastify": "^4",
    "openai": "^4",
--- a/app/services/ingestion/src/index.ts
+++ b/app/services/ingestion/src/index.ts
@@ -1,5 +1,6 @@
 import 'dotenv/config';
 import Fastify from 'fastify';
+import cors from '@fastify/cors';
 import documentRoutes from './routes/documents.js';

 const PORT = parseInt(process.env['INGESTION_PORT'] ?? '3001', 10);
@@ -7,6 +8,7 @@ const PORT = parseInt(process.env['INGESTION_PORT'] ?? '3001', 10);
 async function start(): Promise<void> {
  const app = Fastify({ logger: true });

+  await app.register(cors, { origin: true });
  await app.register(documentRoutes);

  await app.listen({ port: PORT, host: '0.0.0.0' });
--- a/app/services/ingestion/src/jobs/queue.ts
+++ b/app/services/ingestion/src/jobs/queue.ts
@@ -1,4 +1,7 @@
 import { v4 as uuid } from 'uuid';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { writeFile, unlink } from 'fs/promises';
 import { extract } from '../pipeline/extract.js';
 import { chunk } from '../pipeline/chunk.js';
 import { clean } from '../pipeline/clean.js';
@@ -26,7 +29,8 @@ export function createJob(params: IngestBody): Job {
    documentId: params.documentId,
    filename: params.filename,
    format: params.format,
-    filePath: params.filePath,
+    fileUrl: params.fileUrl,
+    filePath: '',
    status: 'queued',
    progress: { ...DEFAULT_PROGRESS },
    error: null,
@@ -62,10 +66,20 @@ async function runPipeline(jobId: string): Promise<void> {
  const job = jobs.get(jobId);
  if (!job) return;

+  let tempPath: string | null = null;
+
  try {
+    // Stage 0: download file from PocketBase to temp dir
+    const res = await fetch(job.fileUrl);
+    if (!res.ok) throw new Error(`Failed to download file: ${res.status}`);
+    const buffer = Buffer.from(await res.arrayBuffer());
+    tempPath = join(tmpdir(), `ingest-${jobId}-${job.filename}`);
+    await writeFile(tempPath, buffer);
+    updateJob(jobId, { filePath: tempPath });
+
    // Stage 1: text extraction
    updateJob(jobId, { status: 'extracting' });
-    const text = await extract(job.filePath, job.format);
+    const text = await extract(tempPath, job.format);

    // Stages 2–3: chunking + cleaning
    updateJob(jobId, { status: 'chunking' });
@@ -98,5 +112,9 @@ async function runPipeline(jobId: string): Promise<void> {
    if (j) {
      await markDocumentFailed(j.documentId, reason).catch(() => undefined);
    }
+  } finally {
+    if (tempPath) {
+      await unlink(tempPath).catch(() => undefined);
+    }
  }
 }
--- a/app/services/ingestion/src/migrations/003_access_rules.ts
+++ b/app/services/ingestion/src/migrations/003_access_rules.ts
@@ -0,0 +1,60 @@
+import 'dotenv/config'
+import PocketBase from 'pocketbase'
+
+const POCKETBASE_URL = process.env['POCKETBASE_URL'] ?? ''
+const POCKETBASE_ADMIN_EMAIL = process.env['POCKETBASE_ADMIN_EMAIL'] ?? ''
+const POCKETBASE_ADMIN_PASSWORD = process.env['POCKETBASE_ADMIN_PASSWORD'] ?? ''
+
+if (!POCKETBASE_URL || !POCKETBASE_ADMIN_EMAIL || !POCKETBASE_ADMIN_PASSWORD) {
+  console.error('Missing env vars')
+  process.exit(1)
+}
+
+const AUTH_ANY = '@request.auth.id != ""'
+const AUTH_ADMIN = '@request.auth.role = "admin"'
+const AUTH_OWN = '@request.auth.id = user'
+
+// [listRule, viewRule, createRule, updateRule, deleteRule]
+const RULES: Record<string, [string, string, string, string, string]> = {
+  source_documents:          [AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  themes:                    [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  topics:                    [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  micro_learnings:           [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  curriculum_versions:       [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  curriculum_weeks:          [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  employee_curriculum_state: [AUTH_ANY,    AUTH_ANY,    AUTH_ANY,    AUTH_OWN,    AUTH_ADMIN],
+  session_completions:       [AUTH_ANY,    AUTH_ANY,    AUTH_ANY,    '',          ''],
+  gamification_profiles:     [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  badges:                    [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  employee_badges:           [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  milestone_cards:           [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+}
+
+async function main() {
+  const pb = new PocketBase(POCKETBASE_URL)
+  await pb.admins.authWithPassword(POCKETBASE_ADMIN_EMAIL, POCKETBASE_ADMIN_PASSWORD)
+
+  const collections = await pb.collections.getFullList()
+
+  for (const col of collections) {
+    const rules = RULES[col.name]
+    if (!rules) continue
+
+    const [listRule, viewRule, createRule, updateRule, deleteRule] = rules
+    await pb.collections.update(col.id, {
+      listRule,
+      viewRule,
+      createRule,
+      updateRule,
+      deleteRule,
+    })
+    console.log(`✓ ${col.name}`)
+  }
+
+  console.log('\nAccess rules applied.')
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})
--- a/app/services/ingestion/src/routes/documents.ts
+++ b/app/services/ingestion/src/routes/documents.ts
@@ -12,7 +12,7 @@ const documentRoutes: FastifyPluginAsync = async (app) => {
    return reply.status(202).send({ jobId: job.id, status: job.status });
  });

-  app.get<{ Params: { jobId: string } }>('/status/:jobId', async (request, reply) => {
+  app.get<{ Params: { jobId: string } }>('/ingest/status/:jobId', async (request, reply) => {
    const job = getJob(request.params.jobId);
    if (!job) {
      return reply.status(404).send({ error: 'Job not found' });
@@ -20,8 +20,11 @@ const documentRoutes: FastifyPluginAsync = async (app) => {
    return reply.send({
      jobId: job.id,
      status: job.status,
-      progress: job.progress,
-      error: job.error,
+      chunksTotal: job.progress.chunksTotal,
+      chunksEmbedded: job.progress.chunksEmbedded,
+      themesFound: job.progress.themesFound,
+      topicsFound: job.progress.topicsFound,
+      reason: job.error ?? undefined,
    });
  });
 };
--- a/app/services/ingestion/src/types.ts
+++ b/app/services/ingestion/src/types.ts
@@ -125,6 +125,7 @@ export interface Job {
  documentId: string;
  filename: string;
  format: DocumentFormat;
+  fileUrl: string;
  filePath: string;
  status: JobStatus;
  progress: JobProgress;
@@ -141,7 +142,7 @@ export const IngestBodySchema = z.object({
  documentId: z.string().min(1),
  filename: z.string().min(1),
  format: z.enum(['pdf', 'md', 'txt']),
-  filePath: z.string().min(1),
+  fileUrl: z.string().url(),
 });

 export type IngestBody = z.infer<typeof IngestBodySchema>;