diff --git a/README.md b/README.md index 8e7d64b..c681749 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Raycast-Extension für PII-sichere Text- und Projekt-Workflows mit Velum: Inhalt ## Projektdateien -`Projektdateien pseudonymisieren` übernimmt zuerst den geöffneten Finder-Ordner; bei Bedarf kann er im Formular erneut übernommen oder manuell gewählt werden. Danach werden geeignete Text-, DOCX-, XLSX-, PPTX- und PDF-Dateien ausgewählt. Enter führt in der Dateiliste zur Trefferprüfung. Alle erkannten Treffer einschließlich Datei- und Ordnernamen erscheinen mit Kontext und müssen vor dem Abschluss geprüft werden. Personen, Orte und Organisationen sind zunächst abgewählt; explizite Projektbegriffe und strukturierte Treffer sind vorausgewählt. `Kopien erstellen` schreibt Dateien in den Raycast-Support-Bereich außerhalb des Projektordners. Alternativ ersetzt `Originaldateien ersetzen` die gewählten Dateien und gegebenenfalls ihre Namen vor Ort; zuvor legt Velum eine private Sicherung der Originale im Raycast-Support-Bereich an. PDF wird dabei als DOCX ausgegeben. +`Projektdateien pseudonymisieren` übernimmt zuerst den geöffneten Finder-Ordner; bei Bedarf kann er im Formular erneut übernommen oder manuell gewählt werden. Im Formular werden die zu erkennenden Kategorien ausgewählt (Orte sind wegen häufiger Fehlalarme zunächst abgewählt). Danach analysiert Velum alle geeigneten Text-, DOCX-, XLSX-, PPTX- und PDF-Dateien sowie ihre Namen. Kleine Textdateien und Namen werden in Paketen analysiert, um die Anzahl der API-Aufrufe zu verringern. Die Prüfung zeigt jede eindeutige Zuordnung einmal, nach Kategorie gruppiert, mit der Anzahl ihrer Vorkommen. Eine Zuordnung an- oder abzuwählen gilt für alle betroffenen Dateien; einzelne Vorkommen lassen sich bei Bedarf ansehen. `Pseudonymisierte Kopien erstellen` schreibt Dateien in den Raycast-Support-Bereich außerhalb des Projektordners. Alternativ ersetzt `Originaldateien ersetzen` die Dateien und gegebenenfalls ihre Namen vor Ort; zuvor legt Velum eine private Sicherung der Originale im Raycast-Support-Bereich an. PDF wird dabei als DOCX ausgegeben. Projektbegriffe werden zeilenweise als `BEGRIFF: KundenSuite` oder `PERSON: Max Mustermann` eingegeben, nur für dieses Projekt gespeichert und mit `force_masks` pro API-Aufruf übertragen. Die globale Velum-Regelliste bleibt unverändert. Eine neue Projektbearbeitung übernimmt die Zuordnung des letzten Projektstands, sodass Platzhalter konsistent bleiben. diff --git a/package.json b/package.json index 9a405e0..187906a 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,7 @@ { "name": "pseudonymize-project-files", "title": "Projektdateien pseudonymisieren", - "description": "Finder-Projektordner wählen, Dateien und Treffer prüfen und Kopien erstellen.", + "description": "Finder-Projektordner und Kategorien wählen, Zuordnungen prüfen und Dateien pseudonymisieren.", "mode": "view", "icon": "extension-icon.png" }, diff --git a/src/lib/project-review.ts b/src/lib/project-review.ts new file mode 100644 index 0000000..1239af3 --- /dev/null +++ b/src/lib/project-review.ts @@ -0,0 +1,66 @@ +const SEPARATOR = "\n\n\n\n"; +export const MAX_BATCH_CODEPOINTS = 24000; + +export type BatchItem = { key: string; text: string }; +export type BatchSegment = BatchItem & { start: number; end: number }; +export type TextBatch = { text: string; segments: BatchSegment[] }; + +/** Keep file boundaries while reducing many small text/path API calls to a few batches. */ +export function makeTextBatches(items: BatchItem[], limit = MAX_BATCH_CODEPOINTS): TextBatch[] { + const batches: TextBatch[] = []; + let segments: BatchSegment[] = []; + let text = ""; + let length = 0; + const separatorLength = [...SEPARATOR].length; + const flush = () => { + if (segments.length) batches.push({ text, segments }); + segments = []; + text = ""; + length = 0; + }; + for (const item of items) { + if (!item.text) continue; + const itemLength = [...item.text].length; + if (segments.length && length + separatorLength + itemLength > limit) flush(); + if (segments.length) { text += SEPARATOR; length += separatorLength; } + segments.push({ ...item, start: length, end: length + itemLength }); + text += item.text; + length += itemLength; + } + flush(); + return batches; +} + +export function spansBySegment( + batch: TextBatch, spans: T[], +): Map { + const result = new Map(batch.segments.map((segment) => [segment.key, []])); + let segmentIndex = 0; + for (const span of [...spans].sort((a, b) => a.start - b.start)) { + while (segmentIndex < batch.segments.length && span.start >= batch.segments[segmentIndex].end) segmentIndex++; + const segment = batch.segments[segmentIndex]; + if (!segment || span.start < segment.start || span.end > segment.end) continue; + result.get(segment.key)!.push({ ...span, start: span.start - segment.start, end: span.end - segment.start }); + } + return result; +} + +export type GroupableHit = { id: string; type: string; original: string; placeholder: string }; +export type ReviewGroup = { + key: string; type: string; original: string; placeholder: string; hits: T[]; +}; + +export function groupReviewHits(hits: T[]): ReviewGroup[] { + const groups = new Map>(); + for (const hit of hits) { + const key = JSON.stringify([hit.type, hit.original, hit.placeholder]); + let group = groups.get(key); + if (!group) { + group = { key, type: hit.type, original: hit.original, placeholder: hit.placeholder, + hits: [] }; + groups.set(key, group); + } + group.hits.push(hit); + } + return [...groups.values()].sort((a, b) => a.type.localeCompare(b.type) || a.original.localeCompare(b.original)); +} diff --git a/src/pseudonymize-project-files.tsx b/src/pseudonymize-project-files.tsx index 27b34e7..c6ed9be 100644 --- a/src/pseudonymize-project-files.tsx +++ b/src/pseudonymize-project-files.tsx @@ -3,17 +3,19 @@ import { promises as fs } from "node:fs"; import { basename, dirname, extname, join } from "node:path"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { useEffect, useState } from "react"; -import type { PlaceholderMapping } from "./types"; -import { pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum"; +import { useEffect, useMemo, useState } from "react"; +import type { EntityType, PlaceholderMapping } from "./types"; +import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum"; import type { DocumentHit } from "./velum"; import { createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, saveRun, scanProjectFiles, toUtf16Offsets, } from "./lib/velum-project"; import type { ProjectFile, ProjectRule } from "./lib/velum-project"; +import { groupReviewHits, makeTextBatches, spansBySegment } from "./lib/project-review"; const run = promisify(execFile); +const MAX_REVIEW_HITS = 50000; type Hit = { id: string; file: string; source: string; original: string; placeholder: string; type: string; score: number; context: string; start: number; end: number; @@ -22,12 +24,14 @@ type Hit = { type Draft = { file: ProjectFile; hash: string; hits: Hit[] }; type PathDraft = { key: string; value: string; hits: Hit[] }; type Preview = { drafts: Draft[]; paths: PathDraft[]; mapping: PlaceholderMapping; - matchModes?: Record; hits: Hit[] }; + matchModes?: Record; hits: Hit[]; entityTypes: EntityType[]; skipped: number }; -function defaultAccepted(hit: Hit, rules: ProjectRule[]): boolean { - if (rules.some((rule) => rule.type === hit.type && rule.value.toLocaleLowerCase() === hit.original.toLocaleLowerCase())) return true; - return !["PERSON", "ORT", "ORG"].includes(hit.type); -} +const typeLabels: Record = { + PERSON: "Personen", ORT: "Orte", ORG: "Organisationen", FIRMA: "Firmen", + KUNDE: "Kunden", ADRESSE: "Adressen", BEGRIFF: "Eigene Begriffe", + EMAIL: "E-Mail-Adressen", TELEFON: "Telefonnummern", IBAN: "IBAN", +}; +const labelForType = (type: string) => typeLabels[type] ?? type; async function finderFolder(): Promise { try { @@ -81,7 +85,8 @@ function pathSegments(files: ProjectFile[]): PathDraft[] { return [...map.values()]; } -async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[]): Promise { +async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], entityTypes: EntityType[], + skipped: number, onProgress: (message: string) => void): Promise { await requireProjectCapabilities(); const masks = forceMasks(rules); const latest = (await listRuns(root))[0]?.run; @@ -90,37 +95,60 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[]) } let mapping: PlaceholderMapping = latest?.mapping ?? {}; const drafts: Draft[] = []; + const draftsByPath = new Map(); const hits: Hit[] = []; + const texts = new Map(); for (const file of files) { const data = await fs.readFile(file.absPath); - let found: Hit[]; - if (file.kind === "document") { - const result = await pseudonymizeDocument(basename(file.relPath), data, mapping, masks); - mapping = result.mapping; - found = documentHits(file, result.hits); - } else { - const original = textFromBytes(data); - const result = await pseudonymize(original, mapping, undefined, masks); - mapping = result.mapping; - found = contentHits(file, result.spans, original); - } - drafts.push({ file, hash: digest(data), hits: found }); + const draft: Draft = { file, hash: digest(data), hits: [] }; + drafts.push(draft); + draftsByPath.set(file.relPath, draft); + if (file.kind === "text") texts.set(file.relPath, textFromBytes(data)); + } + const addHits = (draft: Draft, found: Hit[]) => { + draft.hits = found; hits.push(...found); - if (hits.length > 5000) throw new Error("Mehr als 5.000 Treffer. Bitte weniger Dateien auswählen."); + if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); + }; + const batches = makeTextBatches([...texts].map(([key, text]) => ({ key, text }))); + for (const [index, batch] of batches.entries()) { + onProgress(`Textdateien analysieren: Paket ${index + 1}/${batches.length}`); + const result = await pseudonymize(batch.text, mapping, entityTypes, masks); + mapping = result.mapping; + const byFile = spansBySegment(batch, result.spans); + for (const segment of batch.segments) { + const draft = draftsByPath.get(segment.key)!; + addHits(draft, contentHits(draft.file, byFile.get(segment.key) ?? [], segment.text)); + } + } + const documents = drafts.filter((draft) => draft.file.kind === "document"); + for (const [index, draft] of documents.entries()) { + onProgress(`Dokumente analysieren: ${index + 1}/${documents.length}`); + const data = await fs.readFile(draft.file.absPath); + const result = await pseudonymizeDocument(basename(draft.file.relPath), data, mapping, masks, undefined, entityTypes); + mapping = result.mapping; + addHits(draft, documentHits(draft.file, result.hits)); } const paths = pathSegments(files); - for (const path of paths) { - const result = await pseudonymize(path.value, mapping, undefined, masks); + const pathBatches = makeTextBatches(paths.map((path) => ({ key: path.key, text: path.value }))); + const byPath = new Map(paths.map((path) => [path.key, path])); + for (const [index, batch] of pathBatches.entries()) { + onProgress(`Dateinamen analysieren: Paket ${index + 1}/${pathBatches.length}`); + const result = await pseudonymize(batch.text, mapping, entityTypes, masks); mapping = result.mapping; - path.hits = toUtf16Offsets(path.value, result.spans).map((span, index) => ({ - ...span, id: `${path.key}:path:${index}`, file: path.key, - source: "Datei- oder Ordnername", kind: "path" as const, - context: path.value, - })); - hits.push(...path.hits); - if (hits.length > 5000) throw new Error("Mehr als 5.000 Treffer. Bitte weniger Dateien auswählen."); + const spans = spansBySegment(batch, result.spans); + for (const segment of batch.segments) { + const path = byPath.get(segment.key)!; + path.hits = toUtf16Offsets(path.value, spans.get(path.key) ?? []).map((span, hitIndex) => ({ + ...span, id: `${path.key}:path:${hitIndex}`, file: path.key, + source: "Datei- oder Ordnername", kind: "path" as const, + context: path.value, + })); + hits.push(...path.hits); + if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); + } } - return { drafts, paths, mapping, matchModes: latest?.matchModes, hits }; + return { drafts, paths, mapping, matchModes: latest?.matchModes, hits, entityTypes, skipped }; } function outputPath(relPath: string, paths: PathDraft[], accepted: Set, document: boolean): string { @@ -137,79 +165,31 @@ function outputPath(relPath: string, paths: PathDraft[], accepted: Set, return safeRelativePath(output.join("/")); } -function FileSelection(props: { root: string; rules: ProjectRule[] }) { - const { push } = useNavigation(); - const [files, setFiles] = useState([]); - const [skipped, setSkipped] = useState([]); - const [selected, setSelected] = useState>(new Set()); - const [busy, setBusy] = useState(true); - useEffect(() => { - scanProjectFiles(props.root).then((scan) => { - setFiles(scan.files); - setSkipped(scan.skipped); - setSelected(new Set(scan.files.map((file) => file.relPath))); - }).catch((error: Error) => showToast({ style: Toast.Style.Failure, title: "Projekt nicht lesbar", message: error.message })) - .finally(() => setBusy(false)); - }, [props.root]); - - function toggle(path: string) { - setSelected((current) => { - const next = new Set(current); - if (next.has(path)) next.delete(path); - else next.add(path); - return next; - }); - } - - async function review() { - const chosen = files.filter((file) => selected.has(file.relPath)); - if (!chosen.length) { - await showToast({ style: Toast.Style.Failure, title: "Keine Dateien ausgewählt" }); - return; - } - setBusy(true); - const toast = await showToast({ style: Toast.Style.Animated, title: "Treffer werden analysiert" }); - try { - const preview = await prepare(props.root, chosen, props.rules); - toast.style = Toast.Style.Success; - toast.title = `${preview.hits.length} Treffer zur Prüfung`; - push(); - } catch (error) { - toast.style = Toast.Style.Failure; - toast.title = "Analyse fehlgeschlagen"; - toast.message = (error as Error).message; - } finally { setBusy(false); } - } - - return - - - {files.map((file) => - - toggle(file.relPath)} /> - setSelected(new Set(files.map((item) => item.relPath)))} /> - setSelected(new Set())} /> - } />)} - - {skipped.length > 0 && - {skipped.slice(0, 100).map((path) => )} - } +function OccurrenceReview(props: { original: string; placeholder: string; hits: Hit[] }) { + return + {props.hits.map((hit) => } />)} ; } function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview }) { - const [accepted, setAccepted] = useState>( - new Set(props.preview.hits.filter((hit) => defaultAccepted(hit, props.rules)).map((hit) => hit.id)), - ); + const groups = useMemo(() => groupReviewHits(props.preview.hits), [props.preview.hits]); + const [selectedGroups, setSelectedGroups] = useState>(() => new Set(groups.map((group) => group.key))); const [busy, setBusy] = useState(false); - function toggle(id: string) { - setAccepted((current) => { + const accepted = useMemo(() => new Set(groups.filter((group) => selectedGroups.has(group.key)) + .flatMap((group) => group.hits.map((hit) => hit.id))), [groups, selectedGroups]); + const selectedOccurrences = accepted.size; + const byType = useMemo(() => { + const result = new Map(); + for (const group of groups) (result.get(group.type) ?? (result.set(group.type, []), result.get(group.type)!)).push(group); + return result; + }, [groups]); + function toggle(key: string) { + setSelectedGroups((current) => { const next = new Set(current); - if (next.has(id)) next.delete(id); - else next.add(id); + if (next.has(key)) next.delete(key); + else next.add(key); return next; }); } @@ -243,7 +223,7 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview if (draft.file.kind === "document") { const docIds = draft.hits.filter((hit) => accepted.has(hit.id)).map((hit) => hit.id.split(":document:")[1]); const result = await pseudonymizeDocument( - basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds, + basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds, props.preview.entityTypes, ); output = Buffer.from(result.file_base64, "base64"); } else { @@ -270,28 +250,40 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview } finally { setBusy(false); } } - const groups = new Map(); - for (const hit of props.preview.hits) (groups.get(hit.file) ?? (groups.set(hit.file, []), groups.get(hit.file)!)).push(hit); - return - - apply()} /> - apply(true)} /> - } /> - {[...groups].map(([file, hits]) => - {hits.map((hit) => } + return + + apply()} /> apply(true)} /> - toggle(hit.id)} /> - setAccepted((current) => new Set([ - ...current, ...props.preview.hits.filter((other) => other.type === hit.type && other.original === hit.original).map((other) => other.id), - ]))} /> + } /> + + {[...byType].map(([type, entries]) => selectedGroups.has(entry.key)).length}/${entries.length}`}> + {entries.map((group) => + toggle(group.key)} /> + setSelectedGroups((current) => { + const next = new Set(current); + for (const entry of entries) next.delete(entry.key); + return next; + })} /> + setSelectedGroups((current) => new Set([...current, ...entries.map((entry) => entry.key)]))} /> + } /> + apply()} /> + apply(true)} /> } />)} )} ; @@ -301,8 +293,17 @@ export default function Command() { const { push } = useNavigation(); const [root, setRoot] = useState(""); const [rules, setRules] = useState(""); + const [availableTypes, setAvailableTypes] = useState([]); + const [selectedTypes, setSelectedTypes] = useState([]); const [busy, setBusy] = useState(false); useEffect(() => { finderFolder().then((path) => { if (path) setRoot((current) => current || path); }); }, []); + useEffect(() => { + getEntityTypes().then((types) => { + setAvailableTypes(types); + setSelectedTypes(types.filter((type) => type !== "ORT")); + }).catch((error: Error) => showToast({ style: Toast.Style.Failure, + title: "Kategorien nicht verfügbar", message: error.message })); + }, []); useEffect(() => { if (!root) return; let active = true; @@ -313,13 +314,24 @@ export default function Command() { }, [root]); async function next() { setBusy(true); + const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" }); try { if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen."); + if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen."); const parsed = parseProjectRules(rules); await saveProjectRules(root, parsed); - push(); + toast.title = "Dateien werden erfasst"; + const scan = await scanProjectFiles(root); + if (!scan.files.length) throw new Error("Keine geeigneten Dateien im Projektordner gefunden."); + const preview = await prepare(root, scan.files, parsed, selectedTypes, scan.skipped.length, + (message) => { toast.title = message; }); + toast.style = Toast.Style.Success; + toast.title = `${groupReviewHits(preview.hits).length} Zuordnungen zur Prüfung`; + push(); } catch (error) { - await showToast({ style: Toast.Style.Failure, title: "Projekt nicht bereit", message: (error as Error).message }); + toast.style = Toast.Style.Failure; + toast.title = "Analyse fehlgeschlagen"; + toast.message = (error as Error).message; } finally { setBusy(false); } } async function useFinderFolder() { @@ -329,7 +341,7 @@ export default function Command() { message: "Bitte den Ordner im Finder öffnen oder unten manuell auswählen." }); } return
- + }> - + + {availableTypes.map((type) => )} + + ; } diff --git a/src/velum.ts b/src/velum.ts index 674ca1c..253277f 100644 --- a/src/velum.ts +++ b/src/velum.ts @@ -118,8 +118,7 @@ export async function pseudonymize( body: JSON.stringify({ text, mapping, - entity_types: - entityTypes && entityTypes.length > 0 ? entityTypes : undefined, + entity_types: entityTypes, force_masks: forceMasks, }), }); @@ -163,11 +162,13 @@ export async function pseudonymizeDocument( mapping: PlaceholderMapping, forceMasks: Record, acceptedHitIds?: string[], + entityTypes?: EntityType[], ): Promise { const form = new FormData(); form.set("file", new Blob([new Uint8Array(data)]), filename); form.set("mapping", JSON.stringify(mapping)); form.set("force_masks", JSON.stringify(forceMasks)); + if (entityTypes !== undefined) form.set("entity_types", JSON.stringify(entityTypes)); if (acceptedHitIds) form.set("accepted_hit_ids", JSON.stringify(acceptedHitIds)); const response = await fetchWithAuth(apiUrl("/api/pseudonymize-document"), { method: "POST", body: form, diff --git a/tests/project-review.test.ts b/tests/project-review.test.ts new file mode 100644 index 0000000..a2755ce --- /dev/null +++ b/tests/project-review.test.ts @@ -0,0 +1,32 @@ +import { expect, it } from "bun:test"; +import { groupReviewHits, makeTextBatches, spansBySegment } from "../src/lib/project-review"; + +it("ordnet gebündelte API-Treffer nach Unicode-Positionen den richtigen Dateien zu", () => { + const [batch] = makeTextBatches([ + { key: "eins.txt", text: "😀 Max" }, + { key: "zwei.txt", text: "Max und Max" }, + ]); + expect(batch.segments).toHaveLength(2); + const second = batch.segments[1]; + const spans = spansBySegment(batch, [ + { start: 2, end: 5, original: "Max" }, + { start: second.start, end: second.start + 3, original: "Max" }, + { start: batch.segments[0].end, end: second.start + 1, original: "Grenztreffer" }, + ]); + expect(spans.get("eins.txt")).toEqual([{ start: 2, end: 5, original: "Max" }]); + expect(spans.get("zwei.txt")).toEqual([{ start: 0, end: 3, original: "Max" }]); + expect(makeTextBatches([{ key: "a", text: "abc" }, { key: "b", text: "def" }], 5)).toHaveLength(2); +}); + +it("zeigt gleiche Zuordnungen einmal mit der Anzahl ihrer Vorkommen", () => { + const hits = [ + { id: "a", type: "PERSON", original: "Max", placeholder: "PERSON_01", file: "a.txt" }, + { id: "b", type: "PERSON", original: "Max", placeholder: "PERSON_01", file: "b.txt" }, + { id: "c", type: "PERSON", original: "Max", placeholder: "PERSON_02", file: "c.txt" }, + { id: "d", type: "ORT", original: "Max", placeholder: "ORT_01", file: "a.txt" }, + ]; + const groups = groupReviewHits(hits); + expect(groups).toHaveLength(3); + expect(groups.find((group) => group.placeholder === "PERSON_01")?.hits).toHaveLength(2); + expect(groups.find((group) => group.placeholder === "PERSON_01")?.hits.map((hit) => hit.id)).toEqual(["a", "b"]); +});