import { Action, ActionPanel, Alert, Color, confirmAlert, Form, getSelectedFinderItems, Icon, List, showInFinder, showToast, Toast, useNavigation } from "@raycast/api"; import { promises as fs } from "node:fs"; import { basename, dirname, extname, join } from "node:path"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { useEffect, useMemo, useState } from "react"; import type { EntityType, PlaceholderMapping } from "./types"; import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum"; import type { DocumentHit } from "./velum"; import { createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans, loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey, } from "./lib/velum-project"; import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project"; import { canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment, withoutWhitelistedHits, withoutWhitelistedRules, } from "./lib/project-review"; const run = promisify(execFile); const MAX_REVIEW_HITS = 50000; type Hit = { id: string; file: string; source: string; original: string; placeholder: string; type: string; score: number; context: string; start: number; end: number; kind: "content" | "path"; }; type Draft = { file: ProjectFile; hash: string; hits: Hit[] }; type PathDraft = { key: string; value: string; hits: Hit[] }; type Preview = { drafts: Draft[]; retained: Array<{ file: ProjectFile; hash: string }>; paths: PathDraft[]; mapping: PlaceholderMapping; previousMapping: PlaceholderMapping; matchModes?: Record; hits: Hit[]; entityTypes: EntityType[]; skipped: number; sourceInPlace: boolean }; const typeLabels: Record = { PERSON: "Personen", ORT: "Orte", ORG: "Organisationen", FIRMA: "Firmen", KUNDE: "Kunden", ADRESSE: "Adressen", BEGRIFF: "Eigene Begriffe", EMAIL: "E-Mail-Adressen", TELEFON: "Telefonnummern", IBAN: "IBAN", UID: "UID-Nummern", FIRMENBUCH: "Firmenbuchnummern", SVNR: "Sozialversicherungsnummern", PASSWORT: "Passwörter", BENUTZER: "Benutzerkennungen", PROJEKT: "Projektbezeichnungen", DATUM: "Datumsangaben", REFERENZ: "Referenzen", KARTE: "Kreditkarten", IP: "IP-Adressen", URL: "Internetadressen", DVR: "DVR-Nummern", GISA: "GISA-Nummern", }; const labelForType = (type: string) => typeLabels[type] ?? type; async function finderFolder(): Promise { try { // The user asked for the open Finder folder, even when a file inside it is selected. const { stdout } = await run("/usr/bin/osascript", ["-e", 'tell application "Finder" to get POSIX path of (target of front window as alias)', ]); const path = stdout.trim(); if (path && (await fs.stat(path)).isDirectory()) return path; } catch { /* Finder may have no open window or Automation permission */ } try { const items = await getSelectedFinderItems(); if (items.length) { const first = items[0].path; return (await fs.stat(first)).isDirectory() ? first : dirname(first); } } catch { /* Finder selection may be empty or unavailable */ } return undefined; } function textFromBytes(data: Buffer): string { return new TextDecoder("utf-8", { fatal: true }).decode(data); } function contentHits(file: ProjectFile, spans: Array<{ start: number; end: number; type: string; score: number; placeholder: string; original: string; }>, text: string): Hit[] { return toUtf16Offsets(text, spans).map((span, index) => ({ ...span, id: `${file.relPath}:content:${index}`, file: file.relPath, source: file.relPath, kind: "content" as const, context: text.slice(Math.max(0, span.start - 70), Math.min(text.length, span.end + 70)), })); } function documentHits(file: ProjectFile, hits: DocumentHit[]): Hit[] { return hits.map((hit) => ({ ...hit, id: `${file.relPath}:document:${hit.id}`, file: file.relPath, kind: "content" as const })); } function pathSegments(files: ProjectFile[]): PathDraft[] { const map = new Map(); for (const file of files) { const parts = file.relPath.split("/"); for (let index = 0; index < parts.length; index++) { const key = parts.slice(0, index + 1).join("/"); const isFile = index === parts.length - 1; const value = isFile ? basename(parts[index], extname(parts[index])) : parts[index]; if (value && !map.has(key)) map.set(key, { key, value, hits: [] }); } } return [...map.values()]; } async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], whitelist: ProjectWhitelistEntry[], entityTypes: EntityType[], skipped: number, onProgress: (message: string) => void): Promise { await requireProjectCapabilities(); const masks = forceMasks(withoutWhitelistedRules(rules, whitelist)); const latestEntry = (await listRuns(root))[0]; const latest = latestEntry?.run; if (latest?.origin === "imported") { throw new Error("Dieser Quellordner enthält bereits pseudonymisierte Dateien. Für einen neuen Lauf bitte einen Klartext-Quellordner wählen."); } const previousMapping: PlaceholderMapping = latest?.mapping ?? {}; let mapping: PlaceholderMapping = previousMapping; const drafts: Draft[] = []; const retained: Preview["retained"] = []; const draftsByPath = new Map(); const hits: Hit[] = []; const texts = new Map(); const sourceInPlace = Boolean(latest?.inPlace || latest?.sourceInPlace); const previousOutputs = new Set(sourceInPlace ? latest?.files.map((file) => file.output) : []); const previousDirectory = latestEntry?.directory; for (const file of files) { const data = await fs.readFile(file.absPath); const hash = digest(data); const recheckCommentAuthors = entityTypes.includes("PERSON") && file.kind === "document" && extname(file.relPath).toLowerCase() === ".docx" && !latest?.commentAuthorCoverage; if (!recheckCommentAuthors && previousDirectory && previousOutputs.has(file.relPath) && await matchesPreviousOutput(previousDirectory, file.relPath, hash)) { retained.push({ file, hash }); continue; } const draft: Draft = { file, hash, hits: [] }; drafts.push(draft); draftsByPath.set(file.relPath, draft); if (file.kind === "text") texts.set(file.relPath, textFromBytes(data)); } const addHits = (draft: Draft, found: Hit[]) => { draft.hits = withoutWhitelistedHits(found, whitelist); hits.push(...draft.hits); if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); }; const batches = makeTextBatches([...texts].map(([key, text]) => ({ key, text }))); for (const [index, batch] of batches.entries()) { onProgress(`Textdateien analysieren: Paket ${index + 1}/${batches.length}`); const result = await pseudonymize(batch.text, mapping, entityTypes, masks); mapping = result.mapping; const byFile = spansBySegment(batch, result.spans); for (const segment of batch.segments) { const draft = draftsByPath.get(segment.key)!; addHits(draft, contentHits(draft.file, byFile.get(segment.key) ?? [], segment.text)); } } const documents = drafts.filter((draft) => draft.file.kind === "document"); for (const [index, draft] of documents.entries()) { onProgress(`Dokumente analysieren: ${index + 1}/${documents.length}`); const data = await fs.readFile(draft.file.absPath); const result = await pseudonymizeDocument(basename(draft.file.relPath), data, mapping, masks, undefined, entityTypes); mapping = result.mapping; addHits(draft, documentHits(draft.file, result.hits)); } // Recheck names even when an in-place file's contents are unchanged. An // earlier run may have masked the content but left the filename clear. const paths = pathSegments(files); const knownTerms = [...Object.keys(mapping), ...Object.values(mapping).map((entry) => entry.original), ...withoutWhitelistedRules(rules, whitelist).map((rule) => rule.value)]; const pathBatches = makeTextBatches(paths.map((path) => ({ key: path.key, text: filenameScanText(path.value, knownTerms), }))); const byPath = new Map(paths.map((path) => [path.key, path])); for (const [index, batch] of pathBatches.entries()) { onProgress(`Dateinamen analysieren: Paket ${index + 1}/${pathBatches.length}`); const result = await pseudonymize(batch.text, mapping, entityTypes, masks); mapping = result.mapping; const spans = spansBySegment(batch, result.spans); for (const segment of batch.segments) { const path = byPath.get(segment.key)!; path.hits = withoutWhitelistedHits(toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({ ...span, id: `${path.key}:path:${hitIndex}`, file: path.key, source: "Datei- oder Ordnername", kind: "path" as const, context: path.value, })), whitelist); hits.push(...path.hits); if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); } } return { drafts, retained, paths, mapping, previousMapping, matchModes: latest?.matchModes, hits, entityTypes, skipped, sourceInPlace }; } function outputPath(relPath: string, paths: PathDraft[], accepted: Set, document: boolean, merges: Record): string { const lookup = new Map(paths.map((item) => [item.key, item])); const parts = relPath.split("/"); const output = parts.map((part, index) => { const key = parts.slice(0, index + 1).join("/"); const path = lookup.get(key); if (!path) return part; const isFile = index === parts.length - 1; const extension = isFile ? (document && extname(part).toLowerCase() === ".pdf" ? ".docx" : extname(part)) : ""; return replaceSpans(path.value, withPersonMerges(path.hits, merges), accepted) + extension; }); return safeRelativePath(output.join("/")); } function withPersonMerges(hits: Hit[], merges: Record): Hit[] { return hits.map((hit) => hit.type === "PERSON" ? { ...hit, placeholder: canonicalPerson(hit.placeholder, merges) } : hit); } function OccurrenceReview(props: { original: string; placeholder: string; hits: Hit[] }) { return {props.hits.map((hit) => } />)} ; } function HitReview(props: { root: string; rules: ProjectRule[]; whitelist: ProjectWhitelistEntry[]; preview: Preview; onWhitelistChanged: (entries: ProjectWhitelistEntry[]) => void }) { const groups = useMemo(() => groupReviewHits(props.preview.hits), [props.preview.hits]); const [selectedGroups, setSelectedGroups] = useState>(() => new Set(groups.map((group) => group.key))); const [stage, setStage] = useState<"selection" | "merge">("selection"); const [personMerges, setPersonMerges] = useState>({}); const [busy, setBusy] = useState(false); const accepted = useMemo(() => new Set(groups.filter((group) => selectedGroups.has(group.key)) .flatMap((group) => group.hits.map((hit) => hit.id))), [groups, selectedGroups]); const selectedOccurrences = accepted.size; const byType = useMemo(() => { const result = new Map(); for (const group of groups) (result.get(group.type) ?? (result.set(group.type, []), result.get(group.type)!)).push(group); return result; }, [groups]); const people = useMemo(() => { const result = new Map(); for (const group of groups) { if (group.type !== "PERSON" || !selectedGroups.has(group.key)) continue; const originals = result.get(group.placeholder) ?? []; if (!originals.includes(group.original)) originals.push(group.original); result.set(group.placeholder, originals); } return [...result].map(([placeholder, originals]) => ({ placeholder, originals })); }, [groups, selectedGroups]); const earlierPeople = useMemo(() => Object.entries(props.preview.previousMapping) .filter(([placeholder, entry]) => entry.type === "PERSON" && !entry.merged_into && !people.some((person) => person.placeholder === placeholder)) .map(([placeholder, entry]) => ({ placeholder, originals: [entry.original] })), [people, props.preview.previousMapping]); function toggle(key: string) { setSelectedGroups((current) => { const next = new Set(current); if (next.has(key)) next.delete(key); else next.add(key); return next; }); } async function apply(inPlace = false) { if (!props.preview.drafts.length && !accepted.size) { try { const rejected = groups.map((group) => ({ type: group.type, original: group.original })); if (rejected.length) { await saveProjectWhitelist(props.root, [...await loadProjectWhitelist(props.root), ...rejected]); props.onWhitelistChanged(await loadProjectWhitelist(props.root)); } await showToast({ style: Toast.Style.Success, title: "Keine neuen Änderungen", message: rejected.length ? "Der bestehende Projektstand bleibt erhalten. Abgewählte Begriffe wurden als Ausnahmen gespeichert." : "Der bestehende Projektstand bleibt erhalten." }); } catch (error) { await showToast({ style: Toast.Style.Failure, title: "Ausnahmen konnten nicht gespeichert werden", message: (error as Error).message }); } return; } if (inPlace && !(await confirmAlert({ title: "Originaldateien ersetzen?", message: "Die gewählten Dateien und gegebenenfalls ihre Namen werden im Projektordner ersetzt. Die Originale werden vorher außerhalb des Projekts gesichert. PDF-Dateien werden als DOCX ausgegeben.", primaryAction: { title: "Originaldateien ersetzen", style: Alert.ActionStyle.Destructive }, }))) return; setBusy(true); const toast = await showToast({ style: Toast.Style.Animated, title: inPlace ? "Originaldateien werden ersetzt" : "Pseudonymisierte Kopien werden erstellt" }); let directory: string | undefined; let persisted = false; try { const created = await createRun(props.root); directory = created.directory; const { run } = created; const acceptedHits = groups.filter((group) => selectedGroups.has(group.key)) .flatMap((group) => group.hits); const selectedPeople = new Set(people.map((person) => person.placeholder)); const availableTargets = new Set([...selectedPeople, ...earlierPeople.map((person) => person.placeholder)]); const activeMerges = Object.fromEntries(Object.entries(personMerges).filter(([source, target]) => selectedPeople.has(source) && availableTargets.has(canonicalPerson(target, personMerges)))); run.mapping = mappingForSelection(props.preview.mapping, props.preview.previousMapping, acceptedHits, activeMerges); run.commentAuthorCoverage = props.preview.entityTypes.includes("PERSON"); run.matchModes = props.preview.matchModes; run.sourceInPlace = props.preview.sourceInPlace || inPlace; const usedPaths = new Set(); const replacements: RunFile[] = []; for (const retained of props.preview.retained) { const data = await fs.readFile(retained.file.absPath); if (digest(data) !== retained.hash) throw new Error(`Datei wurde seit der Vorschau geändert: ${retained.file.relPath}`); const relPath = outputPath(retained.file.relPath, props.preview.paths, accepted, retained.file.kind === "document", activeMerges); if (usedPaths.has(relPath)) throw new Error(`Zwei Dateien hätten denselben Ausgabenamen: ${relPath}`); usedPaths.add(relPath); const target = resolveWithin(join(directory, "files"), relPath); await fs.mkdir(dirname(target), { recursive: true, mode: 0o700 }); await fs.writeFile(target, data, { mode: 0o600 }); const file: RunFile = { source: retained.file.relPath, output: relPath, kind: retained.file.kind }; run.files.push(file); if (file.source !== file.output) replacements.push(file); } for (const draft of props.preview.drafts) { const data = await fs.readFile(draft.file.absPath); if (digest(data) !== draft.hash) throw new Error(`Datei wurde seit der Vorschau geändert: ${draft.file.relPath}`); const relPath = outputPath(draft.file.relPath, props.preview.paths, accepted, draft.file.kind === "document", activeMerges); if (usedPaths.has(relPath)) throw new Error(`Zwei Dateien hätten denselben Ausgabenamen: ${relPath}`); usedPaths.add(relPath); const target = resolveWithin(join(directory, "files"), relPath); await fs.mkdir(dirname(target), { recursive: true, mode: 0o700 }); let output: Buffer; if (draft.file.kind === "document") { const docIds = draft.hits.filter((hit) => accepted.has(hit.id)).map((hit) => hit.id.split(":document:")[1]); const result = await pseudonymizeDocument( basename(draft.file.relPath), data, run.mapping, forceMasks(withoutWhitelistedRules(props.rules, props.whitelist)), docIds, props.preview.entityTypes, ); output = Buffer.from(result.file_base64, "base64"); } else { output = Buffer.from(replaceSpans(textFromBytes(data), withPersonMerges(draft.hits, activeMerges), accepted), "utf8"); } await fs.writeFile(target, output, { mode: 0o600 }); const file: RunFile = { source: draft.file.relPath, output: relPath, kind: draft.file.kind }; run.files.push(file); replacements.push(file); } await saveRun(directory, run); persisted = true; if (inPlace) { await replaceProjectFilesInPlace(props.root, directory, replacements); run.inPlace = true; await saveRun(directory, run); } const selectedTerms = new Set(groups.filter((group) => selectedGroups.has(group.key)).map(whitelistKey)); const rejected = groups.filter((group) => !selectedGroups.has(group.key) && !selectedTerms.has(whitelistKey(group))) .map((group) => ({ type: group.type, original: group.original })); if (rejected.length) { const nextWhitelist = [...await loadProjectWhitelist(props.root), ...rejected]; await saveProjectWhitelist(props.root, nextWhitelist); props.onWhitelistChanged(await loadProjectWhitelist(props.root)); } toast.style = Toast.Style.Success; toast.title = inPlace ? `${replacements.length} Dateien ersetzt` : `${run.files.length} Kopien erstellt`; await showInFinder(inPlace ? props.root : join(directory, "files")).catch(() => undefined); } catch (error) { if (directory && !persisted) await fs.rm(directory, { recursive: true, force: true }); toast.style = Toast.Style.Failure; toast.title = "Pseudonymisierung fehlgeschlagen"; toast.message = `${(error as Error).message}${persisted && directory ? ` Kopien/Sicherung: ${directory}` : ""}`; } finally { setBusy(false); } } if (stage === "merge") return apply()} /> apply(true)} /> setStage("selection")} /> } /> {people.map((person) => { const canonical = canonicalPerson(person.placeholder, personMerges); const targets = [...people, ...earlierPeople] .filter((other) => canonicalPerson(other.placeholder, personMerges) !== canonical); return {targets.length > 0 && {targets.map((target) => setPersonMerges((current) => mergePeople(current, person.placeholder, target.placeholder))} />)} } {canonical !== person.placeholder && setPersonMerges((current) => { const next = { ...current }; delete next[person.placeholder]; return next; })} />} apply()} /> apply(true)} /> } />; })} ; return setStage("merge")} /> } /> {[...byType].map(([type, entries]) => selectedGroups.has(entry.key)).length}/${entries.length}`}> {entries.map((group) => toggle(group.key)} /> setSelectedGroups((current) => { const next = new Set(current); for (const entry of entries) next.delete(entry.key); return next; })} /> setSelectedGroups((current) => new Set([...current, ...entries.map((entry) => entry.key)]))} /> } /> setStage("merge")} /> } />)} )} ; } function ProjectWhitelist(props: { entries: ProjectWhitelistEntry[]; onRemove: (entry: ProjectWhitelistEntry) => Promise }) { return {props.entries.map((entry) => props.onRemove(entry)} /> } />)} ; } export default function Command() { const { push } = useNavigation(); const [root, setRoot] = useState(""); const [rules, setRules] = useState(""); const [availableTypes, setAvailableTypes] = useState([]); const [selectedTypes, setSelectedTypes] = useState([]); const [whitelist, setWhitelist] = useState([]); const [existingRuns, setExistingRuns] = useState(null); const [busy, setBusy] = useState(false); useEffect(() => { finderFolder().then((path) => { if (path) setRoot((current) => current || path); }); }, []); useEffect(() => { getEntityTypes().then((types) => { setAvailableTypes(types); setSelectedTypes(types.filter((type) => type !== "ORT")); }).catch((error: Error) => showToast({ style: Toast.Style.Failure, title: "Kategorien nicht verfügbar", message: error.message })); }, []); useEffect(() => { if (!root) return; let active = true; loadProjectRules(root).then((found) => { if (active) setRules(found.map((rule) => `${rule.type}: ${rule.value}`).join("\n")); }).catch(() => { if (active) setRules(""); }); return () => { active = false; }; }, [root]); useEffect(() => { if (!root) { setWhitelist([]); return; } let active = true; loadProjectWhitelist(root).then((found) => { if (active) setWhitelist(found); }) .catch((error: Error) => showToast({ style: Toast.Style.Failure, title: "Projekt-Whitelist nicht lesbar", message: error.message })); return () => { active = false; }; }, [root]); useEffect(() => { if (!root) { setExistingRuns(null); return; } setExistingRuns(null); let active = true; listRuns(root).then((found) => { if (active) setExistingRuns(found.length); }) .catch((error: Error) => showToast({ style: Toast.Style.Failure, title: "Projektstatus nicht lesbar", message: error.message })); return () => { active = false; }; }, [root]); async function removeFromWhitelist(entry: ProjectWhitelistEntry) { try { const next = (await loadProjectWhitelist(root)).filter((item) => whitelistKey(item) !== whitelistKey(entry)); await saveProjectWhitelist(root, next); setWhitelist(next); await showToast({ style: Toast.Style.Success, title: "Begriff wird wieder vorgeschlagen" }); } catch (error) { await showToast({ style: Toast.Style.Failure, title: "Entfernen fehlgeschlagen", message: (error as Error).message }); } } async function next() { setBusy(true); const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" }); try { if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen."); if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen."); const parsed = parseProjectRules(rules); await saveProjectRules(root, parsed); toast.title = "Dateien werden erfasst"; const scan = await scanProjectFiles(root); if (!scan.files.length) throw new Error("Keine geeigneten Dateien im Projektordner gefunden."); const currentWhitelist = await loadProjectWhitelist(root); const preview = await prepare(root, scan.files, parsed, currentWhitelist, selectedTypes, scan.skipped.length, (message) => { toast.title = message; }); toast.style = Toast.Style.Success; toast.title = `${groupReviewHits(preview.hits).length} Zuordnungen zur Prüfung`; push(); } catch (error) { toast.style = Toast.Style.Failure; toast.title = "Analyse fehlgeschlagen"; toast.message = (error as Error).message; } finally { setBusy(false); } } async function useFinderFolder() { const path = await finderFolder(); if (path) setRoot(path); else await showToast({ style: Toast.Style.Failure, title: "Kein Finder-Ordner gefunden", message: "Bitte den Ordner im Finder öffnen oder unten manuell auswählen." }); } return
} /> }> setRoot(paths[0] ?? "")} /> {availableTypes.map((type) => )} ; }