From e7e21e9281a7e333914ed17cb2f54f477514074f Mon Sep 17 00:00:00 2001 From: muena Date: Fri, 25 Sep 2026 08:08:20 +0200 Subject: [PATCH] feat: remember project exclusions and merge selected people --- README.md | 4 +- src/lib/project-review.ts | 58 ++++++++ src/lib/project-whitelist.ts | 5 + src/lib/velum-project.ts | 50 +++++++ src/pseudonymize-project-files.tsx | 221 ++++++++++++++++++++++++----- tests/project-review.test.ts | 40 +++++- tests/velum-project.test.ts | 26 ++++ 7 files changed, 366 insertions(+), 38 deletions(-) create mode 100644 src/lib/project-whitelist.ts diff --git a/README.md b/README.md index c681749..c175769 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ Raycast-Extension für PII-sichere Text- und Projekt-Workflows mit Velum: Inhalt ## Projektdateien -`Projektdateien pseudonymisieren` übernimmt zuerst den geöffneten Finder-Ordner; bei Bedarf kann er im Formular erneut übernommen oder manuell gewählt werden. Im Formular werden die zu erkennenden Kategorien ausgewählt (Orte sind wegen häufiger Fehlalarme zunächst abgewählt). Danach analysiert Velum alle geeigneten Text-, DOCX-, XLSX-, PPTX- und PDF-Dateien sowie ihre Namen. Kleine Textdateien und Namen werden in Paketen analysiert, um die Anzahl der API-Aufrufe zu verringern. Die Prüfung zeigt jede eindeutige Zuordnung einmal, nach Kategorie gruppiert, mit der Anzahl ihrer Vorkommen. Eine Zuordnung an- oder abzuwählen gilt für alle betroffenen Dateien; einzelne Vorkommen lassen sich bei Bedarf ansehen. `Pseudonymisierte Kopien erstellen` schreibt Dateien in den Raycast-Support-Bereich außerhalb des Projektordners. Alternativ ersetzt `Originaldateien ersetzen` die Dateien und gegebenenfalls ihre Namen vor Ort; zuvor legt Velum eine private Sicherung der Originale im Raycast-Support-Bereich an. PDF wird dabei als DOCX ausgegeben. +`Projektdateien pseudonymisieren` übernimmt zuerst den geöffneten Finder-Ordner; bei Bedarf kann er im Formular erneut übernommen oder manuell gewählt werden. Im Formular werden die zu erkennenden Kategorien ausgewählt (Orte sind wegen häufiger Fehlalarme zunächst abgewählt). Danach analysiert Velum alle geeigneten Text-, DOCX-, XLSX-, PPTX- und PDF-Dateien sowie ihre Namen. Kleine Textdateien und Namen werden in Paketen analysiert, um die Anzahl der API-Aufrufe zu verringern. Die Prüfung zeigt jede eindeutige Zuordnung einmal, nach Kategorie gruppiert, mit der Anzahl ihrer Vorkommen. Eine Zuordnung an- oder abzuwählen gilt für alle betroffenen Dateien; einzelne Vorkommen lassen sich bei Bedarf ansehen. Nach der Auswahl lassen sich erkannte Personen optional zusammenfassen, sodass ihre Erwähnungen denselben Platzhalter erhalten. `Pseudonymisierte Kopien erstellen` schreibt Dateien in den Raycast-Support-Bereich außerhalb des Projektordners. Alternativ ersetzt `Originaldateien ersetzen` die Dateien und gegebenenfalls ihre Namen vor Ort; zuvor legt Velum eine private Sicherung der Originale im Raycast-Support-Bereich an. PDF wird dabei als DOCX ausgegeben. -Projektbegriffe werden zeilenweise als `BEGRIFF: KundenSuite` oder `PERSON: Max Mustermann` eingegeben, nur für dieses Projekt gespeichert und mit `force_masks` pro API-Aufruf übertragen. Die globale Velum-Regelliste bleibt unverändert. Eine neue Projektbearbeitung übernimmt die Zuordnung des letzten Projektstands, sodass Platzhalter konsistent bleiben. +Projektbegriffe werden zeilenweise als `BEGRIFF: KundenSuite` oder `PERSON: Max Mustermann` eingegeben, nur für dieses Projekt gespeichert und mit `force_masks` pro API-Aufruf übertragen. Die globale Velum-Regelliste bleibt unverändert. Beim Erstellen eines Projektstands werden abgewählte Treffer als typgebundene Projekt-Whitelist außerhalb des Quellordners gespeichert und bei späteren Läufen nicht erneut angeboten. Die Whitelist lässt sich im Startformular bearbeiten. Eine neue Projektbearbeitung übernimmt die bestätigten Zuordnungen des letzten Projektstands, sodass Platzhalter konsistent bleiben. Nach einem Lauf mit Kopien bleibt der Klartext-Quellordner erhalten und kann nach Dateiänderungen erneut bearbeitet werden. Nach dem Ersetzen vor Ort werden unveränderte bereits maskierte Dateien in den nächsten Stand übernommen; neue oder geänderte Dateien werden analysiert. Ein neuer Projektstand enthält auch die übernommenen Dateien für die Rückübersetzung. `Projektdateien rückübersetzen` wählt einen Projektstand und einzelne Dateien und erstellt Klartext-Kopien unter `~/Downloads/Velum`. Der Quellordner bleibt unverändert. Die neue Projektfunktion benötigt eine Velum-API mit `/api/project-capabilities`, Trefferprüfung für Dokumente, projektbezogenen `force_masks` und `/api/depseudonymize-document`. diff --git a/src/lib/project-review.ts b/src/lib/project-review.ts index 1239af3..cd5eaee 100644 --- a/src/lib/project-review.ts +++ b/src/lib/project-review.ts @@ -1,3 +1,7 @@ +import type { PlaceholderMapping } from "../types"; +import { whitelistKey } from "./project-whitelist"; +import type { ProjectWhitelistEntry } from "./project-whitelist"; + const SEPARATOR = "\n\n\n\n"; export const MAX_BATCH_CODEPOINTS = 24000; @@ -64,3 +68,57 @@ export function groupReviewHits(hits: T[]): ReviewGroup< } return [...groups.values()].sort((a, b) => a.type.localeCompare(b.type) || a.original.localeCompare(b.original)); } + +export function withoutWhitelistedHits(hits: T[], whitelist: ProjectWhitelistEntry[]): T[] { + const excluded = new Set(whitelist.map(whitelistKey)); + return hits.filter((hit) => !excluded.has(whitelistKey(hit))); +} + +export function withoutWhitelistedRules( + rules: T[], whitelist: ProjectWhitelistEntry[], +): T[] { + const excluded = new Set(whitelist.map(whitelistKey)); + return rules.filter((rule) => !excluded.has(whitelistKey({ type: rule.type, original: rule.value }))); +} + +export function canonicalPerson(placeholder: string, merges: Record): string { + const seen = new Set(); + let current = placeholder; + while (merges[current] && !seen.has(current)) { + seen.add(current); + current = merges[current]; + } + return current; +} + +export function mergePeople(merges: Record, source: string, target: string): Record { + const from = canonicalPerson(source, merges); + const into = canonicalPerson(target, merges); + if (from === into) return merges; + const next = { ...merges }; + for (const person of new Set([...Object.keys(next), ...Object.values(next), from])) { + if (canonicalPerson(person, merges) === from && person !== into) next[person] = into; + } + delete next[into]; + return next; +} + +export function mappingForSelection( + preview: PlaceholderMapping, previous: PlaceholderMapping, accepted: GroupableHit[], + merges: Record, +): PlaceholderMapping { + const keys = new Set([...Object.keys(previous), ...accepted.map((hit) => hit.placeholder)]); + const mapping: PlaceholderMapping = {}; + for (const key of keys) { + const entry = preview[key] ?? previous[key]; + if (entry) mapping[key] = { ...entry }; + } + for (const [source, target] of Object.entries(merges)) { + const root = canonicalPerson(target, merges); + if (source === root || mapping[source]?.type !== "PERSON" || mapping[root]?.type !== "PERSON") { + throw new Error("Ungültige Personen-Zusammenführung."); + } + mapping[source] = { ...mapping[source], merged_into: root }; + } + return mapping; +} diff --git a/src/lib/project-whitelist.ts b/src/lib/project-whitelist.ts new file mode 100644 index 0000000..a0f2e90 --- /dev/null +++ b/src/lib/project-whitelist.ts @@ -0,0 +1,5 @@ +export type ProjectWhitelistEntry = { type: string; original: string }; + +export function whitelistKey(entry: ProjectWhitelistEntry): string { + return JSON.stringify([entry.type.toUpperCase(), entry.original.trim().toLocaleLowerCase("de")]); +} diff --git a/src/lib/velum-project.ts b/src/lib/velum-project.ts index 2bc9dee..e9d9c4e 100644 --- a/src/lib/velum-project.ts +++ b/src/lib/velum-project.ts @@ -3,6 +3,11 @@ import { constants as fsConstants, promises as fs } from "node:fs"; import { basename, dirname, extname, join, relative, resolve, sep } from "node:path"; import { environment } from "@raycast/api"; import type { PlaceholderMapping } from "../types"; +import { whitelistKey } from "./project-whitelist"; +import type { ProjectWhitelistEntry } from "./project-whitelist"; + +export { whitelistKey } from "./project-whitelist"; +export type { ProjectWhitelistEntry } from "./project-whitelist"; export type ProjectRule = { type: "BEGRIFF" | "PERSON" | "ORG" | "ORT" | "FIRMA" | "KUNDE"; value: string }; export type ProjectFile = { relPath: string; absPath: string; kind: "text" | "document"; size: number }; @@ -19,6 +24,8 @@ export type ProjectRun = { matchModes?: Record; files: RunFile[]; inPlace?: boolean; + /** The project source already contains outputs of an earlier in-place run. */ + sourceInPlace?: boolean; }; const DOCUMENTS = new Set([".docx", ".xlsx", ".pptx", ".pdf"]); @@ -49,6 +56,38 @@ export function rulesPath(projectPath: string, support = environment.supportPath return join(projectDataRoot(projectPath, support), "rules.json"); } +export function whitelistPath(projectPath: string, support = environment.supportPath): string { + return join(projectDataRoot(projectPath, support), "whitelist.json"); +} + +export async function loadProjectWhitelist(projectPath: string, support?: string): Promise { + try { + const value = JSON.parse(await fs.readFile(whitelistPath(projectPath, support), "utf8")) as unknown; + if (!Array.isArray(value)) throw new Error("Ungültige Projekt-Whitelist."); + return value.filter((entry): entry is ProjectWhitelistEntry => Boolean(entry) && typeof entry === "object" + && typeof entry.type === "string" && typeof entry.original === "string" + && entry.type.length > 0 && entry.original.trim().length > 0); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } +} + +export async function saveProjectWhitelist(projectPath: string, entries: ProjectWhitelistEntry[], support?: string): Promise { + const unique = [...new Map(entries.map((entry) => [whitelistKey(entry), + { type: entry.type.toUpperCase(), original: entry.original.trim() }])).values()]; + const path = whitelistPath(projectPath, support); + await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const temp = `${path}.${randomUUID()}.tmp`; + try { + await fs.writeFile(temp, JSON.stringify(unique), { encoding: "utf8", mode: 0o600 }); + await fs.rename(temp, path); + } catch (error) { + await fs.rm(temp, { force: true }); + throw error; + } +} + export async function loadProjectRules(projectPath: string, support?: string): Promise { try { const value = JSON.parse(await fs.readFile(rulesPath(projectPath, support), "utf8")) as unknown; @@ -241,6 +280,17 @@ export async function listRuns(projectPath: string, support = environment.suppor return runs.sort((a, b) => b.run.createdAt.localeCompare(a.run.createdAt)); } +/** An unchanged output from an in-place run can be carried into the next snapshot. */ +export async function matchesPreviousOutput(directory: string, relPath: string, currentHash: string): Promise { + try { + const previous = await fs.readFile(resolveWithin(join(directory, "files"), relPath)); + return digest(previous) === currentHash; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } +} + export async function listAllRuns(support = environment.supportPath): Promise> { diff --git a/src/pseudonymize-project-files.tsx b/src/pseudonymize-project-files.tsx index 439eddf..365fc9a 100644 --- a/src/pseudonymize-project-files.tsx +++ b/src/pseudonymize-project-files.tsx @@ -9,10 +9,14 @@ import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapab import type { DocumentHit } from "./velum"; import { createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans, - replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, saveRun, scanProjectFiles, toUtf16Offsets, + loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, + saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey, } from "./lib/velum-project"; -import type { ProjectFile, ProjectRule } from "./lib/velum-project"; -import { groupReviewHits, makeTextBatches, spansBySegment } from "./lib/project-review"; +import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project"; +import { + canonicalPerson, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment, + withoutWhitelistedHits, withoutWhitelistedRules, +} from "./lib/project-review"; const run = promisify(execFile); const MAX_REVIEW_HITS = 50000; @@ -23,8 +27,10 @@ type Hit = { }; type Draft = { file: ProjectFile; hash: string; hits: Hit[] }; type PathDraft = { key: string; value: string; hits: Hit[] }; -type Preview = { drafts: Draft[]; paths: PathDraft[]; mapping: PlaceholderMapping; - matchModes?: Record; hits: Hit[]; entityTypes: EntityType[]; skipped: number }; +type Preview = { drafts: Draft[]; retained: Array<{ file: ProjectFile; hash: string }>; + paths: PathDraft[]; mapping: PlaceholderMapping; previousMapping: PlaceholderMapping; + matchModes?: Record; hits: Hit[]; entityTypes: EntityType[]; + skipped: number; sourceInPlace: boolean }; const typeLabels: Record = { PERSON: "Personen", ORT: "Orte", ORG: "Organisationen", FIRMA: "Firmen", @@ -89,29 +95,41 @@ function pathSegments(files: ProjectFile[]): PathDraft[] { return [...map.values()]; } -async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], entityTypes: EntityType[], +async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], whitelist: ProjectWhitelistEntry[], entityTypes: EntityType[], skipped: number, onProgress: (message: string) => void): Promise { await requireProjectCapabilities(); - const masks = forceMasks(rules); - const latest = (await listRuns(root))[0]?.run; + const masks = forceMasks(withoutWhitelistedRules(rules, whitelist)); + const latestEntry = (await listRuns(root))[0]; + const latest = latestEntry?.run; if (latest?.origin === "imported") { throw new Error("Dieser Quellordner enthält bereits pseudonymisierte Dateien. Für einen neuen Lauf bitte einen Klartext-Quellordner wählen."); } - let mapping: PlaceholderMapping = latest?.mapping ?? {}; + const previousMapping: PlaceholderMapping = latest?.mapping ?? {}; + let mapping: PlaceholderMapping = previousMapping; const drafts: Draft[] = []; + const retained: Preview["retained"] = []; const draftsByPath = new Map(); const hits: Hit[] = []; const texts = new Map(); + const sourceInPlace = Boolean(latest?.inPlace || latest?.sourceInPlace); + const previousOutputs = new Set(sourceInPlace ? latest?.files.map((file) => file.output) : []); + const previousDirectory = latestEntry?.directory; for (const file of files) { const data = await fs.readFile(file.absPath); - const draft: Draft = { file, hash: digest(data), hits: [] }; + const hash = digest(data); + if (previousDirectory && previousOutputs.has(file.relPath) + && await matchesPreviousOutput(previousDirectory, file.relPath, hash)) { + retained.push({ file, hash }); + continue; + } + const draft: Draft = { file, hash, hits: [] }; drafts.push(draft); draftsByPath.set(file.relPath, draft); if (file.kind === "text") texts.set(file.relPath, textFromBytes(data)); } const addHits = (draft: Draft, found: Hit[]) => { - draft.hits = found; - hits.push(...found); + draft.hits = withoutWhitelistedHits(found, whitelist); + hits.push(...draft.hits); if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); }; const batches = makeTextBatches([...texts].map(([key, text]) => ({ key, text }))); @@ -133,7 +151,7 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], mapping = result.mapping; addHits(draft, documentHits(draft.file, result.hits)); } - const paths = pathSegments(files); + const paths = pathSegments(drafts.map((draft) => draft.file)); const pathBatches = makeTextBatches(paths.map((path) => ({ key: path.key, text: path.value }))); const byPath = new Map(paths.map((path) => [path.key, path])); for (const [index, batch] of pathBatches.entries()) { @@ -143,19 +161,21 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], const spans = spansBySegment(batch, result.spans); for (const segment of batch.segments) { const path = byPath.get(segment.key)!; - path.hits = toUtf16Offsets(path.value, spans.get(path.key) ?? []).map((span, hitIndex) => ({ + path.hits = withoutWhitelistedHits(toUtf16Offsets(path.value, spans.get(path.key) ?? []).map((span, hitIndex) => ({ ...span, id: `${path.key}:path:${hitIndex}`, file: path.key, source: "Datei- oder Ordnername", kind: "path" as const, context: path.value, - })); + })), whitelist); hits.push(...path.hits); if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); } } - return { drafts, paths, mapping, matchModes: latest?.matchModes, hits, entityTypes, skipped }; + return { drafts, retained, paths, mapping, previousMapping, + matchModes: latest?.matchModes, hits, entityTypes, skipped, sourceInPlace }; } -function outputPath(relPath: string, paths: PathDraft[], accepted: Set, document: boolean): string { +function outputPath(relPath: string, paths: PathDraft[], accepted: Set, document: boolean, + merges: Record): string { const lookup = new Map(paths.map((item) => [item.key, item])); const parts = relPath.split("/"); const output = parts.map((part, index) => { @@ -164,11 +184,16 @@ function outputPath(relPath: string, paths: PathDraft[], accepted: Set, if (!path) return part; const isFile = index === parts.length - 1; const extension = isFile ? (document && extname(part).toLowerCase() === ".pdf" ? ".docx" : extname(part)) : ""; - return replaceSpans(path.value, path.hits, accepted) + extension; + return replaceSpans(path.value, withPersonMerges(path.hits, merges), accepted) + extension; }); return safeRelativePath(output.join("/")); } +function withPersonMerges(hits: Hit[], merges: Record): Hit[] { + return hits.map((hit) => hit.type === "PERSON" + ? { ...hit, placeholder: canonicalPerson(hit.placeholder, merges) } : hit); +} + function OccurrenceReview(props: { original: string; placeholder: string; hits: Hit[] }) { return {props.hits.map((hit) => ; } -function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview }) { +function HitReview(props: { root: string; rules: ProjectRule[]; whitelist: ProjectWhitelistEntry[]; + preview: Preview; onWhitelistChanged: (entries: ProjectWhitelistEntry[]) => void }) { const groups = useMemo(() => groupReviewHits(props.preview.hits), [props.preview.hits]); const [selectedGroups, setSelectedGroups] = useState>(() => new Set(groups.map((group) => group.key))); + const [stage, setStage] = useState<"selection" | "merge">("selection"); + const [personMerges, setPersonMerges] = useState>({}); const [busy, setBusy] = useState(false); const accepted = useMemo(() => new Set(groups.filter((group) => selectedGroups.has(group.key)) .flatMap((group) => group.hits.map((hit) => hit.id))), [groups, selectedGroups]); @@ -189,6 +217,21 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview for (const group of groups) (result.get(group.type) ?? (result.set(group.type, []), result.get(group.type)!)).push(group); return result; }, [groups]); + const people = useMemo(() => { + const result = new Map(); + for (const group of groups) { + if (group.type !== "PERSON" || !selectedGroups.has(group.key)) continue; + const originals = result.get(group.placeholder) ?? []; + if (!originals.includes(group.original)) originals.push(group.original); + result.set(group.placeholder, originals); + } + return [...result].map(([placeholder, originals]) => ({ placeholder, originals })); + }, [groups, selectedGroups]); + const earlierPeople = useMemo(() => Object.entries(props.preview.previousMapping) + .filter(([placeholder, entry]) => entry.type === "PERSON" && !entry.merged_into + && !people.some((person) => person.placeholder === placeholder)) + .map(([placeholder, entry]) => ({ placeholder, originals: [entry.original] })), + [people, props.preview.previousMapping]); function toggle(key: string) { setSelectedGroups((current) => { const next = new Set(current); @@ -212,13 +255,34 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview const created = await createRun(props.root); directory = created.directory; const { run } = created; - run.mapping = props.preview.mapping; + const acceptedHits = groups.filter((group) => selectedGroups.has(group.key)) + .flatMap((group) => group.hits); + const selectedPeople = new Set(people.map((person) => person.placeholder)); + const availableTargets = new Set([...selectedPeople, ...earlierPeople.map((person) => person.placeholder)]); + const activeMerges = Object.fromEntries(Object.entries(personMerges).filter(([source, target]) => + selectedPeople.has(source) && availableTargets.has(canonicalPerson(target, personMerges)))); + run.mapping = mappingForSelection(props.preview.mapping, props.preview.previousMapping, + acceptedHits, activeMerges); run.matchModes = props.preview.matchModes; + run.sourceInPlace = props.preview.sourceInPlace || inPlace; const usedPaths = new Set(); + const replacements: RunFile[] = []; + for (const retained of props.preview.retained) { + const data = await fs.readFile(retained.file.absPath); + if (digest(data) !== retained.hash) throw new Error(`Datei wurde seit der Vorschau geändert: ${retained.file.relPath}`); + const relPath = safeRelativePath(retained.file.relPath); + if (usedPaths.has(relPath)) throw new Error(`Zwei Dateien hätten denselben Ausgabenamen: ${relPath}`); + usedPaths.add(relPath); + const target = resolveWithin(join(directory, "files"), relPath); + await fs.mkdir(dirname(target), { recursive: true, mode: 0o700 }); + await fs.writeFile(target, data, { mode: 0o600 }); + run.files.push({ source: relPath, output: relPath, kind: retained.file.kind }); + } for (const draft of props.preview.drafts) { const data = await fs.readFile(draft.file.absPath); if (digest(data) !== draft.hash) throw new Error(`Datei wurde seit der Vorschau geändert: ${draft.file.relPath}`); - const relPath = outputPath(draft.file.relPath, props.preview.paths, accepted, draft.file.kind === "document"); + const relPath = outputPath(draft.file.relPath, props.preview.paths, accepted, + draft.file.kind === "document", activeMerges); if (usedPaths.has(relPath)) throw new Error(`Zwei Dateien hätten denselben Ausgabenamen: ${relPath}`); usedPaths.add(relPath); const target = resolveWithin(join(directory, "files"), relPath); @@ -227,24 +291,36 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview if (draft.file.kind === "document") { const docIds = draft.hits.filter((hit) => accepted.has(hit.id)).map((hit) => hit.id.split(":document:")[1]); const result = await pseudonymizeDocument( - basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds, props.preview.entityTypes, + basename(draft.file.relPath), data, run.mapping, + forceMasks(withoutWhitelistedRules(props.rules, props.whitelist)), docIds, props.preview.entityTypes, ); output = Buffer.from(result.file_base64, "base64"); } else { - output = Buffer.from(replaceSpans(textFromBytes(data), draft.hits, accepted), "utf8"); + output = Buffer.from(replaceSpans(textFromBytes(data), withPersonMerges(draft.hits, activeMerges), accepted), "utf8"); } await fs.writeFile(target, output, { mode: 0o600 }); - run.files.push({ source: draft.file.relPath, output: relPath, kind: draft.file.kind }); + const file: RunFile = { source: draft.file.relPath, output: relPath, kind: draft.file.kind }; + run.files.push(file); + replacements.push(file); } await saveRun(directory, run); persisted = true; if (inPlace) { - await replaceProjectFilesInPlace(props.root, directory, run.files); + await replaceProjectFilesInPlace(props.root, directory, replacements); run.inPlace = true; await saveRun(directory, run); } + const selectedTerms = new Set(groups.filter((group) => selectedGroups.has(group.key)).map(whitelistKey)); + const rejected = groups.filter((group) => !selectedGroups.has(group.key) + && !selectedTerms.has(whitelistKey(group))) + .map((group) => ({ type: group.type, original: group.original })); + if (rejected.length) { + const nextWhitelist = [...await loadProjectWhitelist(props.root), ...rejected]; + await saveProjectWhitelist(props.root, nextWhitelist); + props.onWhitelistChanged(await loadProjectWhitelist(props.root)); + } toast.style = Toast.Style.Success; - toast.title = inPlace ? `${run.files.length} Originaldateien ersetzt` : `${run.files.length} Kopien erstellt`; + toast.title = inPlace ? `${replacements.length} Dateien ersetzt` : `${run.files.length} Kopien erstellt`; await showInFinder(inPlace ? props.root : join(directory, "files")).catch(() => undefined); } catch (error) { if (directory && !persisted) await fs.rm(directory, { recursive: true, force: true }); @@ -254,15 +330,53 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview } finally { setBusy(false); } } + if (stage === "merge") return + + + apply()} /> + apply(true)} /> + setStage("selection")} /> + } /> + + + {people.map((person) => { + const canonical = canonicalPerson(person.placeholder, personMerges); + const targets = [...people, ...earlierPeople] + .filter((other) => canonicalPerson(other.placeholder, personMerges) !== canonical); + return + {targets.length > 0 && + {targets.map((target) => setPersonMerges((current) => mergePeople(current, + person.placeholder, target.placeholder))} />)} + } + {canonical !== person.placeholder && setPersonMerges((current) => { + const next = { ...current }; delete next[person.placeholder]; return next; + })} />} + apply()} /> + apply(true)} /> + } />; + })} + + ; + return - apply()} /> - apply(true)} /> + setStage("merge")} /> } /> {[...byType].map(([type, entries]) => setSelectedGroups((current) => new Set([...current, ...entries.map((entry) => entry.key)]))} /> } /> - apply()} /> - apply(true)} /> + setStage("merge")} /> } />)} )} ; } +function ProjectWhitelist(props: { entries: ProjectWhitelistEntry[]; onRemove: (entry: ProjectWhitelistEntry) => Promise }) { + return + + {props.entries.map((entry) => + props.onRemove(entry)} /> + } />)} + + ; +} + export default function Command() { const { push } = useNavigation(); const [root, setRoot] = useState(""); const [rules, setRules] = useState(""); const [availableTypes, setAvailableTypes] = useState([]); const [selectedTypes, setSelectedTypes] = useState([]); + const [whitelist, setWhitelist] = useState([]); const [busy, setBusy] = useState(false); useEffect(() => { finderFolder().then((path) => { if (path) setRoot((current) => current || path); }); }, []); useEffect(() => { @@ -316,6 +443,25 @@ export default function Command() { }).catch(() => { if (active) setRules(""); }); return () => { active = false; }; }, [root]); + useEffect(() => { + if (!root) { setWhitelist([]); return; } + let active = true; + loadProjectWhitelist(root).then((found) => { if (active) setWhitelist(found); }) + .catch((error: Error) => showToast({ style: Toast.Style.Failure, + title: "Projekt-Whitelist nicht lesbar", message: error.message })); + return () => { active = false; }; + }, [root]); + async function removeFromWhitelist(entry: ProjectWhitelistEntry) { + try { + const next = (await loadProjectWhitelist(root)).filter((item) => whitelistKey(item) !== whitelistKey(entry)); + await saveProjectWhitelist(root, next); + setWhitelist(next); + await showToast({ style: Toast.Style.Success, title: "Begriff wird wieder vorgeschlagen" }); + } catch (error) { + await showToast({ style: Toast.Style.Failure, title: "Entfernen fehlgeschlagen", + message: (error as Error).message }); + } + } async function next() { setBusy(true); const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" }); @@ -327,11 +473,13 @@ export default function Command() { toast.title = "Dateien werden erfasst"; const scan = await scanProjectFiles(root); if (!scan.files.length) throw new Error("Keine geeigneten Dateien im Projektordner gefunden."); - const preview = await prepare(root, scan.files, parsed, selectedTypes, scan.skipped.length, + const currentWhitelist = await loadProjectWhitelist(root); + const preview = await prepare(root, scan.files, parsed, currentWhitelist, selectedTypes, scan.skipped.length, (message) => { toast.title = message; }); toast.style = Toast.Style.Success; toast.title = `${groupReviewHits(preview.hits).length} Zuordnungen zur Prüfung`; - push(); + push(); } catch (error) { toast.style = Toast.Style.Failure; toast.title = "Analyse fehlgeschlagen"; @@ -347,6 +495,8 @@ export default function Command() { return
+ } /> }> + onChange={setSelectedTypes} info="Orte sind wegen häufiger Fehlalarme zunächst abgewählt. Abgewählte Treffer kommen nach dem Erstellen in die Projekt-Whitelist."> {availableTypes.map((type) => )} - + + ; } diff --git a/tests/project-review.test.ts b/tests/project-review.test.ts index a2755ce..47ad20a 100644 --- a/tests/project-review.test.ts +++ b/tests/project-review.test.ts @@ -1,5 +1,8 @@ import { expect, it } from "bun:test"; -import { groupReviewHits, makeTextBatches, spansBySegment } from "../src/lib/project-review"; +import { + canonicalPerson, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, + spansBySegment, withoutWhitelistedHits, withoutWhitelistedRules, +} from "../src/lib/project-review"; it("ordnet gebündelte API-Treffer nach Unicode-Positionen den richtigen Dateien zu", () => { const [batch] = makeTextBatches([ @@ -30,3 +33,38 @@ it("zeigt gleiche Zuordnungen einmal mit der Anzahl ihrer Vorkommen", () => { expect(groups.find((group) => group.placeholder === "PERSON_01")?.hits).toHaveLength(2); expect(groups.find((group) => group.placeholder === "PERSON_01")?.hits.map((hit) => hit.id)).toEqual(["a", "b"]); }); + +it("unterdrückt projektweit nur exakt abgewählte Begriffe samt erzwungener Regel", () => { + const whitelist = [{ type: "PERSON", original: " Max Muster " }]; + const hits = [ + { id: "1", type: "PERSON", original: "max muster", placeholder: "PERSON_01" }, + { id: "2", type: "ORG", original: "Max Muster", placeholder: "ORG_01" }, + { id: "3", type: "PERSON", original: "Max Mustermann", placeholder: "PERSON_02" }, + ]; + expect(withoutWhitelistedHits(hits, whitelist).map((hit) => hit.id)).toEqual(["2", "3"]); + expect(withoutWhitelistedRules([ + { type: "PERSON", value: "Max Muster" }, { type: "ORG", value: "Max Muster" }, + ], whitelist)).toEqual([{ type: "ORG", value: "Max Muster" }]); +}); + +it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => { + let merges = mergePeople({}, "PERSON_02", "PERSON_01"); + merges = mergePeople(merges, "PERSON_03", "PERSON_02"); + expect(canonicalPerson("PERSON_03", merges)).toBe("PERSON_01"); + const previous = { PERSON_09: { original: "Alt", type: "PERSON" } }; + const preview = { + ...previous, + PERSON_01: { original: "Franz Fuchs", type: "PERSON" }, + PERSON_02: { original: "Franz", type: "PERSON" }, + PERSON_03: { original: "Hr. Fuchs", type: "PERSON" }, + PERSON_04: { original: "Fehlalarm", type: "PERSON" }, + }; + const accepted = ["PERSON_01", "PERSON_02", "PERSON_03"].map((placeholder) => ({ + id: placeholder, type: "PERSON", original: preview[placeholder as keyof typeof preview].original, placeholder, + })); + const mapping = mappingForSelection(preview, previous, accepted, merges); + expect(mapping.PERSON_02.merged_into).toBe("PERSON_01"); + expect(mapping.PERSON_03.merged_into).toBe("PERSON_01"); + expect(mapping.PERSON_09.original).toBe("Alt"); + expect(mapping.PERSON_04).toBeUndefined(); +}); diff --git a/tests/velum-project.test.ts b/tests/velum-project.test.ts index 9a4c684..94a22f4 100644 --- a/tests/velum-project.test.ts +++ b/tests/velum-project.test.ts @@ -20,6 +20,14 @@ describe("Velum-Projektstände", () => { const rules = project.parseProjectRules("PERSON: Max Mustermann\nKundenSuite"); await project.saveProjectRules(source, rules, support); expect(await project.loadProjectRules(source, support)).toEqual(rules); + await project.saveProjectWhitelist(source, [ + { type: "PERSON", original: "Falscher Treffer" }, + { type: "PERSON", original: "falscher treffer" }, + ], support); + expect(await project.loadProjectWhitelist(source, support)).toEqual([ + { type: "PERSON", original: "falscher treffer" }, + ]); + expect(await fs.readFile(join(source, "Notiz.txt"), "utf8")).toBe("Max Mustermann"); const created = await project.createRun(source, support); created.run.mapping = { PERSON_01: { original: "Max Mustermann", type: "PERSON" } }; created.run.files.push({ source: "Notiz.txt", output: "PERSON_01.txt", kind: "text" }); @@ -82,4 +90,22 @@ describe("Velum-Projektstände", () => { expect(await fs.readFile(join(source, "Bericht.txt"), "utf8")).toBe("Original"); expect(await fs.readFile(join(source, "PERSON_01.txt"), "utf8")).toBe("Andere Datei"); }); + + it("erkennt unveränderte Ausgaben für einen erneuten Lauf desselben Ordners", async () => { + const root = await fs.mkdtemp(join(tmpdir(), "velum-repeat-test-")); + roots.push(root); + const source = join(root, "Quelle"); + const support = join(root, "Raycast"); + await fs.mkdir(source); + const created = await project.createRun(source, support); + const filename = "PERSON_01.txt"; + await fs.writeFile(join(created.directory, "files", filename), "PERSON_01"); + const current = join(source, filename); + await fs.writeFile(current, "PERSON_01"); + const hash = project.digest(await fs.readFile(current)); + expect(await project.matchesPreviousOutput(created.directory, filename, hash)).toBe(true); + await fs.writeFile(current, "PERSON_01 und neuer Text"); + expect(await project.matchesPreviousOutput(created.directory, filename, + project.digest(await fs.readFile(current)))).toBe(false); + }); });