feat: review project pseudonyms by category

This commit is contained in:
muena
2026-09-24 20:11:10 +02:00
parent 8b254eb87d
commit 226d230b6d
6 changed files with 242 additions and 127 deletions

66
src/lib/project-review.ts Normal file
View File

@@ -0,0 +1,66 @@
const SEPARATOR = "\n\n\n\n";
export const MAX_BATCH_CODEPOINTS = 24000;
export type BatchItem = { key: string; text: string };
export type BatchSegment = BatchItem & { start: number; end: number };
export type TextBatch = { text: string; segments: BatchSegment[] };
/** Keep file boundaries while reducing many small text/path API calls to a few batches. */
export function makeTextBatches(items: BatchItem[], limit = MAX_BATCH_CODEPOINTS): TextBatch[] {
const batches: TextBatch[] = [];
let segments: BatchSegment[] = [];
let text = "";
let length = 0;
const separatorLength = [...SEPARATOR].length;
const flush = () => {
if (segments.length) batches.push({ text, segments });
segments = [];
text = "";
length = 0;
};
for (const item of items) {
if (!item.text) continue;
const itemLength = [...item.text].length;
if (segments.length && length + separatorLength + itemLength > limit) flush();
if (segments.length) { text += SEPARATOR; length += separatorLength; }
segments.push({ ...item, start: length, end: length + itemLength });
text += item.text;
length += itemLength;
}
flush();
return batches;
}
export function spansBySegment<T extends { start: number; end: number }>(
batch: TextBatch, spans: T[],
): Map<string, T[]> {
const result = new Map<string, T[]>(batch.segments.map((segment) => [segment.key, []]));
let segmentIndex = 0;
for (const span of [...spans].sort((a, b) => a.start - b.start)) {
while (segmentIndex < batch.segments.length && span.start >= batch.segments[segmentIndex].end) segmentIndex++;
const segment = batch.segments[segmentIndex];
if (!segment || span.start < segment.start || span.end > segment.end) continue;
result.get(segment.key)!.push({ ...span, start: span.start - segment.start, end: span.end - segment.start });
}
return result;
}
export type GroupableHit = { id: string; type: string; original: string; placeholder: string };
export type ReviewGroup<T extends GroupableHit> = {
key: string; type: string; original: string; placeholder: string; hits: T[];
};
export function groupReviewHits<T extends GroupableHit>(hits: T[]): ReviewGroup<T>[] {
const groups = new Map<string, ReviewGroup<T>>();
for (const hit of hits) {
const key = JSON.stringify([hit.type, hit.original, hit.placeholder]);
let group = groups.get(key);
if (!group) {
group = { key, type: hit.type, original: hit.original, placeholder: hit.placeholder,
hits: [] };
groups.set(key, group);
}
group.hits.push(hit);
}
return [...groups.values()].sort((a, b) => a.type.localeCompare(b.type) || a.original.localeCompare(b.original));
}

View File

@@ -3,17 +3,19 @@ import { promises as fs } from "node:fs";
import { basename, dirname, extname, join } from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { useEffect, useState } from "react";
import type { PlaceholderMapping } from "./types";
import { pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
import { useEffect, useMemo, useState } from "react";
import type { EntityType, PlaceholderMapping } from "./types";
import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
import type { DocumentHit } from "./velum";
import {
createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans,
replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, saveRun, scanProjectFiles, toUtf16Offsets,
} from "./lib/velum-project";
import type { ProjectFile, ProjectRule } from "./lib/velum-project";
import { groupReviewHits, makeTextBatches, spansBySegment } from "./lib/project-review";
const run = promisify(execFile);
const MAX_REVIEW_HITS = 50000;
type Hit = {
id: string; file: string; source: string; original: string; placeholder: string;
type: string; score: number; context: string; start: number; end: number;
@@ -22,12 +24,14 @@ type Hit = {
type Draft = { file: ProjectFile; hash: string; hits: Hit[] };
type PathDraft = { key: string; value: string; hits: Hit[] };
type Preview = { drafts: Draft[]; paths: PathDraft[]; mapping: PlaceholderMapping;
matchModes?: Record<string, "token" | "anywhere">; hits: Hit[] };
matchModes?: Record<string, "token" | "anywhere">; hits: Hit[]; entityTypes: EntityType[]; skipped: number };
function defaultAccepted(hit: Hit, rules: ProjectRule[]): boolean {
if (rules.some((rule) => rule.type === hit.type && rule.value.toLocaleLowerCase() === hit.original.toLocaleLowerCase())) return true;
return !["PERSON", "ORT", "ORG"].includes(hit.type);
}
const typeLabels: Record<string, string> = {
PERSON: "Personen", ORT: "Orte", ORG: "Organisationen", FIRMA: "Firmen",
KUNDE: "Kunden", ADRESSE: "Adressen", BEGRIFF: "Eigene Begriffe",
EMAIL: "E-Mail-Adressen", TELEFON: "Telefonnummern", IBAN: "IBAN",
};
const labelForType = (type: string) => typeLabels[type] ?? type;
async function finderFolder(): Promise<string | undefined> {
try {
@@ -81,7 +85,8 @@ function pathSegments(files: ProjectFile[]): PathDraft[] {
return [...map.values()];
}
async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[]): Promise<Preview> {
async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], entityTypes: EntityType[],
skipped: number, onProgress: (message: string) => void): Promise<Preview> {
await requireProjectCapabilities();
const masks = forceMasks(rules);
const latest = (await listRuns(root))[0]?.run;
@@ -90,37 +95,60 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[])
}
let mapping: PlaceholderMapping = latest?.mapping ?? {};
const drafts: Draft[] = [];
const draftsByPath = new Map<string, Draft>();
const hits: Hit[] = [];
const texts = new Map<string, string>();
for (const file of files) {
const data = await fs.readFile(file.absPath);
let found: Hit[];
if (file.kind === "document") {
const result = await pseudonymizeDocument(basename(file.relPath), data, mapping, masks);
mapping = result.mapping;
found = documentHits(file, result.hits);
} else {
const original = textFromBytes(data);
const result = await pseudonymize(original, mapping, undefined, masks);
mapping = result.mapping;
found = contentHits(file, result.spans, original);
}
drafts.push({ file, hash: digest(data), hits: found });
const draft: Draft = { file, hash: digest(data), hits: [] };
drafts.push(draft);
draftsByPath.set(file.relPath, draft);
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
}
const addHits = (draft: Draft, found: Hit[]) => {
draft.hits = found;
hits.push(...found);
if (hits.length > 5000) throw new Error("Mehr als 5.000 Treffer. Bitte weniger Dateien auswählen.");
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
};
const batches = makeTextBatches([...texts].map(([key, text]) => ({ key, text })));
for (const [index, batch] of batches.entries()) {
onProgress(`Textdateien analysieren: Paket ${index + 1}/${batches.length}`);
const result = await pseudonymize(batch.text, mapping, entityTypes, masks);
mapping = result.mapping;
const byFile = spansBySegment(batch, result.spans);
for (const segment of batch.segments) {
const draft = draftsByPath.get(segment.key)!;
addHits(draft, contentHits(draft.file, byFile.get(segment.key) ?? [], segment.text));
}
}
const documents = drafts.filter((draft) => draft.file.kind === "document");
for (const [index, draft] of documents.entries()) {
onProgress(`Dokumente analysieren: ${index + 1}/${documents.length}`);
const data = await fs.readFile(draft.file.absPath);
const result = await pseudonymizeDocument(basename(draft.file.relPath), data, mapping, masks, undefined, entityTypes);
mapping = result.mapping;
addHits(draft, documentHits(draft.file, result.hits));
}
const paths = pathSegments(files);
for (const path of paths) {
const result = await pseudonymize(path.value, mapping, undefined, masks);
const pathBatches = makeTextBatches(paths.map((path) => ({ key: path.key, text: path.value })));
const byPath = new Map(paths.map((path) => [path.key, path]));
for (const [index, batch] of pathBatches.entries()) {
onProgress(`Dateinamen analysieren: Paket ${index + 1}/${pathBatches.length}`);
const result = await pseudonymize(batch.text, mapping, entityTypes, masks);
mapping = result.mapping;
path.hits = toUtf16Offsets(path.value, result.spans).map((span, index) => ({
...span, id: `${path.key}:path:${index}`, file: path.key,
source: "Datei- oder Ordnername", kind: "path" as const,
context: path.value,
}));
hits.push(...path.hits);
if (hits.length > 5000) throw new Error("Mehr als 5.000 Treffer. Bitte weniger Dateien auswählen.");
const spans = spansBySegment(batch, result.spans);
for (const segment of batch.segments) {
const path = byPath.get(segment.key)!;
path.hits = toUtf16Offsets(path.value, spans.get(path.key) ?? []).map((span, hitIndex) => ({
...span, id: `${path.key}:path:${hitIndex}`, file: path.key,
source: "Datei- oder Ordnername", kind: "path" as const,
context: path.value,
}));
hits.push(...path.hits);
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
}
}
return { drafts, paths, mapping, matchModes: latest?.matchModes, hits };
return { drafts, paths, mapping, matchModes: latest?.matchModes, hits, entityTypes, skipped };
}
function outputPath(relPath: string, paths: PathDraft[], accepted: Set<string>, document: boolean): string {
@@ -137,79 +165,31 @@ function outputPath(relPath: string, paths: PathDraft[], accepted: Set<string>,
return safeRelativePath(output.join("/"));
}
function FileSelection(props: { root: string; rules: ProjectRule[] }) {
const { push } = useNavigation();
const [files, setFiles] = useState<ProjectFile[]>([]);
const [skipped, setSkipped] = useState<string[]>([]);
const [selected, setSelected] = useState<Set<string>>(new Set());
const [busy, setBusy] = useState(true);
useEffect(() => {
scanProjectFiles(props.root).then((scan) => {
setFiles(scan.files);
setSkipped(scan.skipped);
setSelected(new Set(scan.files.map((file) => file.relPath)));
}).catch((error: Error) => showToast({ style: Toast.Style.Failure, title: "Projekt nicht lesbar", message: error.message }))
.finally(() => setBusy(false));
}, [props.root]);
function toggle(path: string) {
setSelected((current) => {
const next = new Set(current);
if (next.has(path)) next.delete(path);
else next.add(path);
return next;
});
}
async function review() {
const chosen = files.filter((file) => selected.has(file.relPath));
if (!chosen.length) {
await showToast({ style: Toast.Style.Failure, title: "Keine Dateien ausgewählt" });
return;
}
setBusy(true);
const toast = await showToast({ style: Toast.Style.Animated, title: "Treffer werden analysiert" });
try {
const preview = await prepare(props.root, chosen, props.rules);
toast.style = Toast.Style.Success;
toast.title = `${preview.hits.length} Treffer zur Prüfung`;
push(<HitReview root={props.root} rules={props.rules} preview={preview} />);
} catch (error) {
toast.style = Toast.Style.Failure;
toast.title = "Analyse fehlgeschlagen";
toast.message = (error as Error).message;
} finally { setBusy(false); }
}
return <List isLoading={busy} navigationTitle="Dateien auswählen" searchBarPlaceholder="Dateien filtern">
<List.EmptyView title="Keine geeigneten Dateien" description="Bitte einen anderen Projektordner wählen oder die ausgelassenen Dateien prüfen." />
<List.Section title={`${selected.size} von ${files.length} Dateien ausgewählt · ${skipped.length} übersprungen`}>
{files.map((file) => <List.Item key={file.relPath}
icon={selected.has(file.relPath) ? Icon.CheckCircle : Icon.Circle}
title={file.relPath} subtitle={file.kind === "document" ? "Office/PDF" : "Text"}
actions={<ActionPanel>
<Action title="Treffer prüfen" icon={Icon.Eye} onAction={review} />
<Action title={selected.has(file.relPath) ? "Datei abwählen" : "Datei auswählen"} onAction={() => toggle(file.relPath)} />
<Action title="Alle auswählen" onAction={() => setSelected(new Set(files.map((item) => item.relPath)))} />
<Action title="Keine auswählen" onAction={() => setSelected(new Set())} />
</ActionPanel>} />)}
</List.Section>
{skipped.length > 0 && <List.Section title="Übersprungen (binär, zu groß oder ausgeschlossen)">
{skipped.slice(0, 100).map((path) => <List.Item key={path} icon={Icon.XMarkCircle} title={path} />)}
</List.Section>}
function OccurrenceReview(props: { original: string; placeholder: string; hits: Hit[] }) {
return <List isShowingDetail navigationTitle={`${props.original} → ${props.placeholder}`}>
{props.hits.map((hit) => <List.Item key={hit.id} title={hit.source}
subtitle={hit.kind === "path" ? "Datei- oder Ordnername" : "Dateiinhalt"}
detail={<List.Item.Detail markdown={hit.context.replace(/</g, "&lt;")} />} />)}
</List>;
}
function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview }) {
const [accepted, setAccepted] = useState<Set<string>>(
new Set(props.preview.hits.filter((hit) => defaultAccepted(hit, props.rules)).map((hit) => hit.id)),
);
const groups = useMemo(() => groupReviewHits(props.preview.hits), [props.preview.hits]);
const [selectedGroups, setSelectedGroups] = useState<Set<string>>(() => new Set(groups.map((group) => group.key)));
const [busy, setBusy] = useState(false);
function toggle(id: string) {
setAccepted((current) => {
const accepted = useMemo(() => new Set(groups.filter((group) => selectedGroups.has(group.key))
.flatMap((group) => group.hits.map((hit) => hit.id))), [groups, selectedGroups]);
const selectedOccurrences = accepted.size;
const byType = useMemo(() => {
const result = new Map<string, typeof groups>();
for (const group of groups) (result.get(group.type) ?? (result.set(group.type, []), result.get(group.type)!)).push(group);
return result;
}, [groups]);
function toggle(key: string) {
setSelectedGroups((current) => {
const next = new Set(current);
if (next.has(id)) next.delete(id);
else next.add(id);
if (next.has(key)) next.delete(key);
else next.add(key);
return next;
});
}
@@ -243,7 +223,7 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
if (draft.file.kind === "document") {
const docIds = draft.hits.filter((hit) => accepted.has(hit.id)).map((hit) => hit.id.split(":document:")[1]);
const result = await pseudonymizeDocument(
basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds,
basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds, props.preview.entityTypes,
);
output = Buffer.from(result.file_base64, "base64");
} else {
@@ -270,28 +250,40 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
} finally { setBusy(false); }
}
const groups = new Map<string, Hit[]>();
for (const hit of props.preview.hits) (groups.get(hit.file) ?? (groups.set(hit.file, []), groups.get(hit.file)!)).push(hit);
return <List isLoading={busy} isShowingDetail navigationTitle="Treffer prüfen"
searchBarPlaceholder="Treffer, Typ oder Datei suchen">
<List.EmptyView title="Keine Treffer" description="Die gewählten Dateien können unverändert kopiert werden."
actions={<ActionPanel>
<Action title="Pseudonymisierte Kopien erstellen" onAction={() => apply()} />
<Action title="Originaldateien ersetzen" onAction={() => apply(true)} />
</ActionPanel>} />
{[...groups].map(([file, hits]) => <List.Section key={file} title={file}>
{hits.map((hit) => <List.Item key={hit.id}
icon={{ source: accepted.has(hit.id) ? Icon.CheckCircle : Icon.Circle, tintColor: accepted.has(hit.id) ? Color.Green : Color.SecondaryText }}
title={`${hit.original} → ${hit.placeholder}`}
subtitle={`${hit.type} · ${hit.source}`}
detail={<List.Item.Detail markdown={`**${hit.type}** · ${hit.source}\n\n${hit.context.replace(/</g, "&lt;")}`} />}
return <List isLoading={busy} navigationTitle="Zuordnungen prüfen"
searchBarPlaceholder="Begriff oder Platzhalter suchen">
<List.Section title="Abschluss">
<List.Item key="apply" icon={Icon.Play}
title={`${selectedGroups.size} von ${groups.length} Zuordnungen ausgewählt`}
subtitle={`${selectedOccurrences} Vorkommen · ${props.preview.drafts.length} Dateien · ${props.preview.skipped} übersprungen`}
actions={<ActionPanel>
<Action title="Pseudonymisierte Kopien erstellen" icon={Icon.Document} onAction={() => apply()} />
<Action title="Originaldateien ersetzen" icon={Icon.Pencil} onAction={() => apply(true)} />
<Action title={accepted.has(hit.id) ? "Treffer ausnehmen" : "Treffer übernehmen"} onAction={() => toggle(hit.id)} />
<Action title="Gleichen Begriff übernehmen" onAction={() => setAccepted((current) => new Set([
...current, ...props.preview.hits.filter((other) => other.type === hit.type && other.original === hit.original).map((other) => other.id),
]))} />
</ActionPanel>} />
</List.Section>
{[...byType].map(([type, entries]) => <List.Section key={type}
title={`${labelForType(type)} · ${entries.filter((entry) => selectedGroups.has(entry.key)).length}/${entries.length}`}>
{entries.map((group) => <List.Item key={group.key}
icon={{ source: selectedGroups.has(group.key) ? Icon.CheckCircle : Icon.Circle,
tintColor: selectedGroups.has(group.key) ? Color.Green : Color.SecondaryText }}
title={group.original} subtitle={`→ ${group.placeholder}`}
accessories={[{ text: `${group.hits.length} Vorkommen` }]}
keywords={[type, labelForType(type)]}
actions={<ActionPanel>
<Action title={selectedGroups.has(group.key) ? "Zuordnung abwählen" : "Zuordnung auswählen"}
onAction={() => toggle(group.key)} />
<Action title={`${labelForType(type)} komplett abwählen`}
onAction={() => setSelectedGroups((current) => {
const next = new Set(current);
for (const entry of entries) next.delete(entry.key);
return next;
})} />
<Action title={`${labelForType(type)} komplett auswählen`}
onAction={() => setSelectedGroups((current) => new Set([...current, ...entries.map((entry) => entry.key)]))} />
<Action.Push title="Vorkommen ansehen" icon={Icon.Eye}
target={<OccurrenceReview original={group.original} placeholder={group.placeholder} hits={group.hits} />} />
<Action title="Pseudonymisierte Kopien erstellen" icon={Icon.Document} onAction={() => apply()} />
<Action title="Originaldateien ersetzen" icon={Icon.Pencil} onAction={() => apply(true)} />
</ActionPanel>} />)}
</List.Section>)}
</List>;
@@ -301,8 +293,17 @@ export default function Command() {
const { push } = useNavigation();
const [root, setRoot] = useState("");
const [rules, setRules] = useState("");
const [availableTypes, setAvailableTypes] = useState<EntityType[]>([]);
const [selectedTypes, setSelectedTypes] = useState<EntityType[]>([]);
const [busy, setBusy] = useState(false);
useEffect(() => { finderFolder().then((path) => { if (path) setRoot((current) => current || path); }); }, []);
useEffect(() => {
getEntityTypes().then((types) => {
setAvailableTypes(types);
setSelectedTypes(types.filter((type) => type !== "ORT"));
}).catch((error: Error) => showToast({ style: Toast.Style.Failure,
title: "Kategorien nicht verfügbar", message: error.message }));
}, []);
useEffect(() => {
if (!root) return;
let active = true;
@@ -313,13 +314,24 @@ export default function Command() {
}, [root]);
async function next() {
setBusy(true);
const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" });
try {
if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen.");
if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen.");
const parsed = parseProjectRules(rules);
await saveProjectRules(root, parsed);
push(<FileSelection root={root} rules={parsed} />);
toast.title = "Dateien werden erfasst";
const scan = await scanProjectFiles(root);
if (!scan.files.length) throw new Error("Keine geeigneten Dateien im Projektordner gefunden.");
const preview = await prepare(root, scan.files, parsed, selectedTypes, scan.skipped.length,
(message) => { toast.title = message; });
toast.style = Toast.Style.Success;
toast.title = `${groupReviewHits(preview.hits).length} Zuordnungen zur Prüfung`;
push(<HitReview root={root} rules={parsed} preview={preview} />);
} catch (error) {
await showToast({ style: Toast.Style.Failure, title: "Projekt nicht bereit", message: (error as Error).message });
toast.style = Toast.Style.Failure;
toast.title = "Analyse fehlgeschlagen";
toast.message = (error as Error).message;
} finally { setBusy(false); }
}
async function useFinderFolder() {
@@ -329,7 +341,7 @@ export default function Command() {
message: "Bitte den Ordner im Finder öffnen oder unten manuell auswählen." });
}
return <Form isLoading={busy} actions={<ActionPanel>
<Action title="Dateien auswählen" onAction={next} />
<Action title="Zuordnungen prüfen" onAction={next} />
<Action title="Geöffneten Finder-Ordner übernehmen" icon={Icon.Folder} onAction={useFinderFolder} />
</ActionPanel>}>
<Form.FilePicker id="projectPath" title="Projektordner" allowMultipleSelection={false}
@@ -338,6 +350,10 @@ export default function Command() {
<Form.TextArea id="projectRules" title="Begriffe nur für dieses Projekt" value={rules}
onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"}
info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." />
<Form.Description title="Ablauf" text="Dateien und Treffer auswählen; erst danach werden Kopien außerhalb des Projektordners erstellt." />
<Form.TagPicker id="entityTypes" title="Kategorien pseudonymisieren" value={selectedTypes}
onChange={setSelectedTypes} info="Alle geeigneten Dateien werden analysiert. Orte sind wegen häufiger Fehlalarme zunächst abgewählt; alle gefundenen Zuordnungen können anschließend einzeln abgewählt werden.">
{availableTypes.map((type) => <Form.TagPicker.Item key={type} value={type} title={labelForType(type)} />)}
</Form.TagPicker>
<Form.Description title="Ablauf" text="Alle geeigneten Dateien erfassen, Zuordnungen je Kategorie prüfen und dann Kopien erstellen oder Originale ersetzen." />
</Form>;
}

View File

@@ -118,8 +118,7 @@ export async function pseudonymize(
body: JSON.stringify({
text,
mapping,
entity_types:
entityTypes && entityTypes.length > 0 ? entityTypes : undefined,
entity_types: entityTypes,
force_masks: forceMasks,
}),
});
@@ -163,11 +162,13 @@ export async function pseudonymizeDocument(
mapping: PlaceholderMapping,
forceMasks: Record<string, string[]>,
acceptedHitIds?: string[],
entityTypes?: EntityType[],
): Promise<DocumentResult> {
const form = new FormData();
form.set("file", new Blob([new Uint8Array(data)]), filename);
form.set("mapping", JSON.stringify(mapping));
form.set("force_masks", JSON.stringify(forceMasks));
if (entityTypes !== undefined) form.set("entity_types", JSON.stringify(entityTypes));
if (acceptedHitIds) form.set("accepted_hit_ids", JSON.stringify(acceptedHitIds));
const response = await fetchWithAuth(apiUrl("/api/pseudonymize-document"), {
method: "POST", body: form,