import type { PlaceholderMapping } from "../types"; import { whitelistKey } from "./project-whitelist"; import type { ProjectWhitelistEntry } from "./project-whitelist"; const SEPARATOR = "\n\n\n\n"; export const MAX_BATCH_CODEPOINTS = 24000; export type BatchItem = { key: string; text: string }; export type BatchSegment = BatchItem & { start: number; end: number }; export type TextBatch = { text: string; segments: BatchSegment[] }; /** Filename separators look like spaces to NER, except inside known literal terms. */ export function filenameScanText(stem: string, knownTerms: Iterable): string { const protectedOffsets = new Set(); for (const term of knownTerms) { if (!term.includes("_")) continue; const escaped = term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); const pattern = new RegExp(`(? protectedOffsets.has(offset) ? "_" : " "); } /** Keep file boundaries while reducing many small text/path API calls to a few batches. */ export function makeTextBatches(items: BatchItem[], limit = MAX_BATCH_CODEPOINTS): TextBatch[] { const batches: TextBatch[] = []; let segments: BatchSegment[] = []; let text = ""; let length = 0; const separatorLength = [...SEPARATOR].length; const flush = () => { if (segments.length) batches.push({ text, segments }); segments = []; text = ""; length = 0; }; for (const item of items) { if (!item.text) continue; const itemLength = [...item.text].length; if (segments.length && length + separatorLength + itemLength > limit) flush(); if (segments.length) { text += SEPARATOR; length += separatorLength; } segments.push({ ...item, start: length, end: length + itemLength }); text += item.text; length += itemLength; } flush(); return batches; } export function spansBySegment( batch: TextBatch, spans: T[], ): Map { const result = new Map(batch.segments.map((segment) => [segment.key, []])); let segmentIndex = 0; for (const span of [...spans].sort((a, b) => a.start - b.start)) { while (segmentIndex < batch.segments.length && span.start >= batch.segments[segmentIndex].end) segmentIndex++; const segment = batch.segments[segmentIndex]; if (!segment || span.start < segment.start || span.end > segment.end) continue; result.get(segment.key)!.push({ ...span, start: span.start - segment.start, end: span.end - segment.start }); } return result; } export type GroupableHit = { id: string; type: string; original: string; placeholder: string }; export type ReviewGroup = { key: string; type: string; original: string; placeholder: string; hits: T[]; }; export function groupReviewHits(hits: T[]): ReviewGroup[] { const groups = new Map>(); for (const hit of hits) { const key = JSON.stringify([hit.type, hit.original, hit.placeholder]); let group = groups.get(key); if (!group) { group = { key, type: hit.type, original: hit.original, placeholder: hit.placeholder, hits: [] }; groups.set(key, group); } group.hits.push(hit); } return [...groups.values()].sort((a, b) => a.type.localeCompare(b.type) || a.original.localeCompare(b.original)); } export function withoutWhitelistedHits(hits: T[], whitelist: ProjectWhitelistEntry[]): T[] { const excluded = new Set(whitelist.map(whitelistKey)); return hits.filter((hit) => !excluded.has(whitelistKey(hit))); } /** A later run must not turn an existing Velum placeholder plus adjacent text into a new entity. */ export function withoutPlaceholderHits( hits: T[], knownPlaceholders: Iterable = [], ): T[] { const placeholder = /(? hit.type === "PASSWORT" || ( !placeholder.test(hit.original) && !known.some((key) => hit.original.includes(key)) )); } export function withoutWhitelistedRules( rules: T[], whitelist: ProjectWhitelistEntry[], ): T[] { const excluded = new Set(whitelist.map(whitelistKey)); return rules.filter((rule) => !excluded.has(whitelistKey({ type: rule.type, original: rule.value }))); } export function canonicalPerson(placeholder: string, merges: Record): string { const seen = new Set(); let current = placeholder; while (merges[current] && !seen.has(current)) { seen.add(current); current = merges[current]; } return current; } export function mergePeople(merges: Record, source: string, target: string): Record { const from = canonicalPerson(source, merges); const into = canonicalPerson(target, merges); if (from === into) return merges; const next = { ...merges }; for (const person of new Set([...Object.keys(next), ...Object.values(next), from])) { if (canonicalPerson(person, merges) === from && person !== into) next[person] = into; } delete next[into]; return next; } export function mappingForSelection( preview: PlaceholderMapping, previous: PlaceholderMapping, accepted: GroupableHit[], merges: Record, ): PlaceholderMapping { const keys = new Set([...Object.keys(previous), ...accepted.map((hit) => hit.placeholder)]); const mapping: PlaceholderMapping = {}; for (const key of keys) { const entry = preview[key] ?? previous[key]; if (entry) mapping[key] = { ...entry }; } for (const [source, target] of Object.entries(merges)) { const root = canonicalPerson(target, merges); if (source === root || mapping[source]?.type !== "PERSON" || mapping[root]?.type !== "PERSON") { throw new Error("Ungültige Personen-Zusammenführung."); } mapping[source] = { ...mapping[source], merged_into: root }; } return mapping; }