import type { Chunk } from '../types.js';

const MIN_CLEAN_LENGTH = 80;

export function clean(chunks: Chunk[]): Chunk[] {
  return chunks
    .map(c => ({ ...c, text: cleanText(c.text) }))
    .filter(c => c.text.length >= MIN_CLEAN_LENGTH);
}

function cleanText(text: string): string {
  return text
    .normalize('NFC')
    // Remove null bytes and non-printable characters (keep tabs + newlines)
    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '')
    // Collapse 3+ consecutive newlines to 2
    .replace(/\n{3,}/g, '\n\n')
    // Trim leading/trailing whitespace
    .trim();
}
