Add Velum project workflow and standalone migration tool
This commit is contained in:
muena
308
src/pseudonymize-project-files.tsx
Normal file
308
src/pseudonymize-project-files.tsx
Normal file
@@ -0,0 +1,308 @@
|
||||
import { Action, ActionPanel, Color, Form, getSelectedFinderItems, Icon, List, showInFinder, showToast, Toast, useNavigation } from "@raycast/api";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { basename, dirname, extname, join } from "node:path";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { useEffect, useState } from "react";
|
||||
import type { PlaceholderMapping } from "./types";
|
||||
import { pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
|
||||
import type { DocumentHit } from "./velum";
|
||||
import {
|
||||
createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans,
|
||||
resolveWithin, safeRelativePath, saveProjectRules, saveRun, scanProjectFiles, toUtf16Offsets,
|
||||
} from "./lib/velum-project";
|
||||
import type { ProjectFile, ProjectRule } from "./lib/velum-project";
|
||||
|
||||
const run = promisify(execFile);
|
||||
type Hit = {
|
||||
id: string; file: string; source: string; original: string; placeholder: string;
|
||||
type: string; score: number; context: string; start: number; end: number;
|
||||
kind: "content" | "path";
|
||||
};
|
||||
type Draft = { file: ProjectFile; hash: string; hits: Hit[] };
|
||||
type PathDraft = { key: string; value: string; hits: Hit[] };
|
||||
type Preview = { drafts: Draft[]; paths: PathDraft[]; mapping: PlaceholderMapping;
|
||||
matchModes?: Record<string, "token" | "anywhere">; hits: Hit[] };
|
||||
|
||||
function defaultAccepted(hit: Hit, rules: ProjectRule[]): boolean {
|
||||
if (rules.some((rule) => rule.type === hit.type && rule.value.toLocaleLowerCase() === hit.original.toLocaleLowerCase())) return true;
|
||||
return !["PERSON", "ORT", "ORG"].includes(hit.type);
|
||||
}
|
||||
|
||||
async function finderFolder(): Promise<string | undefined> {
|
||||
try {
|
||||
const items = await getSelectedFinderItems();
|
||||
if (items.length) {
|
||||
const first = items[0].path;
|
||||
return (await fs.stat(first)).isDirectory() ? first : dirname(first);
|
||||
}
|
||||
} catch { /* Finder selection may be empty or unavailable */ }
|
||||
try {
|
||||
const { stdout } = await run("/usr/bin/osascript", ["-e",
|
||||
'tell application "Finder" to get POSIX path of (target of front Finder window as alias)',
|
||||
]);
|
||||
const path = stdout.trim();
|
||||
return path && (await fs.stat(path)).isDirectory() ? path : undefined;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
function textFromBytes(data: Buffer): string {
|
||||
return new TextDecoder("utf-8", { fatal: true }).decode(data);
|
||||
}
|
||||
|
||||
function contentHits(file: ProjectFile, spans: Array<{
|
||||
start: number; end: number; type: string; score: number; placeholder: string; original: string;
|
||||
}>, text: string): Hit[] {
|
||||
return toUtf16Offsets(text, spans).map((span, index) => ({
|
||||
...span, id: `${file.relPath}:content:${index}`, file: file.relPath,
|
||||
source: file.relPath, kind: "content" as const,
|
||||
context: text.slice(Math.max(0, span.start - 70), Math.min(text.length, span.end + 70)),
|
||||
}));
|
||||
}
|
||||
|
||||
function documentHits(file: ProjectFile, hits: DocumentHit[]): Hit[] {
|
||||
return hits.map((hit) => ({ ...hit, id: `${file.relPath}:document:${hit.id}`,
|
||||
file: file.relPath, kind: "content" as const }));
|
||||
}
|
||||
|
||||
function pathSegments(files: ProjectFile[]): PathDraft[] {
|
||||
const map = new Map<string, PathDraft>();
|
||||
for (const file of files) {
|
||||
const parts = file.relPath.split("/");
|
||||
for (let index = 0; index < parts.length; index++) {
|
||||
const key = parts.slice(0, index + 1).join("/");
|
||||
const isFile = index === parts.length - 1;
|
||||
const value = isFile ? basename(parts[index], extname(parts[index])) : parts[index];
|
||||
if (value && !map.has(key)) map.set(key, { key, value, hits: [] });
|
||||
}
|
||||
}
|
||||
return [...map.values()];
|
||||
}
|
||||
|
||||
async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[]): Promise<Preview> {
|
||||
await requireProjectCapabilities();
|
||||
const masks = forceMasks(rules);
|
||||
const latest = (await listRuns(root))[0]?.run;
|
||||
if (latest?.origin === "imported") {
|
||||
throw new Error("Dieser Quellordner enthält bereits pseudonymisierte Dateien. Für einen neuen Lauf bitte einen Klartext-Quellordner wählen.");
|
||||
}
|
||||
let mapping: PlaceholderMapping = latest?.mapping ?? {};
|
||||
const drafts: Draft[] = [];
|
||||
const hits: Hit[] = [];
|
||||
for (const file of files) {
|
||||
const data = await fs.readFile(file.absPath);
|
||||
let found: Hit[];
|
||||
if (file.kind === "document") {
|
||||
const result = await pseudonymizeDocument(basename(file.relPath), data, mapping, masks);
|
||||
mapping = result.mapping;
|
||||
found = documentHits(file, result.hits);
|
||||
} else {
|
||||
const original = textFromBytes(data);
|
||||
const result = await pseudonymize(original, mapping, undefined, masks);
|
||||
mapping = result.mapping;
|
||||
found = contentHits(file, result.spans, original);
|
||||
}
|
||||
drafts.push({ file, hash: digest(data), hits: found });
|
||||
hits.push(...found);
|
||||
if (hits.length > 5000) throw new Error("Mehr als 5.000 Treffer. Bitte weniger Dateien auswählen.");
|
||||
}
|
||||
const paths = pathSegments(files);
|
||||
for (const path of paths) {
|
||||
const result = await pseudonymize(path.value, mapping, undefined, masks);
|
||||
mapping = result.mapping;
|
||||
path.hits = toUtf16Offsets(path.value, result.spans).map((span, index) => ({
|
||||
...span, id: `${path.key}:path:${index}`, file: path.key,
|
||||
source: "Datei- oder Ordnername", kind: "path" as const,
|
||||
context: path.value,
|
||||
}));
|
||||
hits.push(...path.hits);
|
||||
if (hits.length > 5000) throw new Error("Mehr als 5.000 Treffer. Bitte weniger Dateien auswählen.");
|
||||
}
|
||||
return { drafts, paths, mapping, matchModes: latest?.matchModes, hits };
|
||||
}
|
||||
|
||||
function outputPath(relPath: string, paths: PathDraft[], accepted: Set<string>, document: boolean): string {
|
||||
const lookup = new Map(paths.map((item) => [item.key, item]));
|
||||
const parts = relPath.split("/");
|
||||
const output = parts.map((part, index) => {
|
||||
const key = parts.slice(0, index + 1).join("/");
|
||||
const path = lookup.get(key);
|
||||
if (!path) return part;
|
||||
const isFile = index === parts.length - 1;
|
||||
const extension = isFile ? (document && extname(part).toLowerCase() === ".pdf" ? ".docx" : extname(part)) : "";
|
||||
return replaceSpans(path.value, path.hits, accepted) + extension;
|
||||
});
|
||||
return safeRelativePath(output.join("/"));
|
||||
}
|
||||
|
||||
function FileSelection(props: { root: string; rules: ProjectRule[] }) {
|
||||
const { push } = useNavigation();
|
||||
const [files, setFiles] = useState<ProjectFile[]>([]);
|
||||
const [skipped, setSkipped] = useState<string[]>([]);
|
||||
const [selected, setSelected] = useState<Set<string>>(new Set());
|
||||
const [busy, setBusy] = useState(true);
|
||||
useEffect(() => {
|
||||
scanProjectFiles(props.root).then((scan) => {
|
||||
setFiles(scan.files);
|
||||
setSkipped(scan.skipped);
|
||||
setSelected(new Set(scan.files.map((file) => file.relPath)));
|
||||
setBusy(false);
|
||||
}).catch((error: Error) => showToast({ style: Toast.Style.Failure, title: "Projekt nicht lesbar", message: error.message }));
|
||||
}, [props.root]);
|
||||
|
||||
function toggle(path: string) {
|
||||
setSelected((current) => {
|
||||
const next = new Set(current);
|
||||
if (next.has(path)) next.delete(path);
|
||||
else next.add(path);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
async function review() {
|
||||
const chosen = files.filter((file) => selected.has(file.relPath));
|
||||
if (!chosen.length) return;
|
||||
setBusy(true);
|
||||
const toast = await showToast({ style: Toast.Style.Animated, title: "Treffer werden analysiert" });
|
||||
try {
|
||||
const preview = await prepare(props.root, chosen, props.rules);
|
||||
toast.style = Toast.Style.Success;
|
||||
toast.title = `${preview.hits.length} Treffer zur Prüfung`;
|
||||
push(<HitReview root={props.root} rules={props.rules} preview={preview} />);
|
||||
} catch (error) {
|
||||
toast.style = Toast.Style.Failure;
|
||||
toast.title = "Analyse fehlgeschlagen";
|
||||
toast.message = (error as Error).message;
|
||||
} finally { setBusy(false); }
|
||||
}
|
||||
|
||||
return <List isLoading={busy} navigationTitle="Dateien auswählen" searchBarPlaceholder="Dateien filtern">
|
||||
<List.Section title={`${selected.size} von ${files.length} Dateien ausgewählt · ${skipped.length} übersprungen`}>
|
||||
{files.map((file) => <List.Item key={file.relPath}
|
||||
icon={selected.has(file.relPath) ? Icon.CheckCircle : Icon.Circle}
|
||||
title={file.relPath} subtitle={file.kind === "document" ? "Office/PDF" : "Text"}
|
||||
actions={<ActionPanel>
|
||||
<Action title={selected.has(file.relPath) ? "Datei abwählen" : "Datei auswählen"} onAction={() => toggle(file.relPath)} />
|
||||
<Action title="Treffer prüfen" icon={Icon.Eye} onAction={review} />
|
||||
<Action title="Alle auswählen" onAction={() => setSelected(new Set(files.map((item) => item.relPath)))} />
|
||||
<Action title="Keine auswählen" onAction={() => setSelected(new Set())} />
|
||||
</ActionPanel>} />)}
|
||||
</List.Section>
|
||||
{skipped.length > 0 && <List.Section title="Übersprungen (binär, zu groß oder ausgeschlossen)">
|
||||
{skipped.slice(0, 100).map((path) => <List.Item key={path} icon={Icon.XMarkCircle} title={path} />)}
|
||||
</List.Section>}
|
||||
</List>;
|
||||
}
|
||||
|
||||
function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview }) {
|
||||
const [accepted, setAccepted] = useState<Set<string>>(
|
||||
new Set(props.preview.hits.filter((hit) => defaultAccepted(hit, props.rules)).map((hit) => hit.id)),
|
||||
);
|
||||
const [busy, setBusy] = useState(false);
|
||||
function toggle(id: string) {
|
||||
setAccepted((current) => {
|
||||
const next = new Set(current);
|
||||
if (next.has(id)) next.delete(id);
|
||||
else next.add(id);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
async function apply() {
|
||||
setBusy(true);
|
||||
const toast = await showToast({ style: Toast.Style.Animated, title: "Pseudonymisierte Kopien werden erstellt" });
|
||||
let directory: string | undefined;
|
||||
try {
|
||||
const created = await createRun(props.root);
|
||||
directory = created.directory;
|
||||
const { run } = created;
|
||||
run.mapping = props.preview.mapping;
|
||||
run.matchModes = props.preview.matchModes;
|
||||
const usedPaths = new Set<string>();
|
||||
for (const draft of props.preview.drafts) {
|
||||
const data = await fs.readFile(draft.file.absPath);
|
||||
if (digest(data) !== draft.hash) throw new Error(`Datei wurde seit der Vorschau geändert: ${draft.file.relPath}`);
|
||||
const relPath = outputPath(draft.file.relPath, props.preview.paths, accepted, draft.file.kind === "document");
|
||||
if (usedPaths.has(relPath)) throw new Error(`Zwei Dateien hätten denselben Ausgabenamen: ${relPath}`);
|
||||
usedPaths.add(relPath);
|
||||
const target = resolveWithin(join(directory, "files"), relPath);
|
||||
await fs.mkdir(dirname(target), { recursive: true, mode: 0o700 });
|
||||
let output: Buffer;
|
||||
if (draft.file.kind === "document") {
|
||||
const docIds = draft.hits.filter((hit) => accepted.has(hit.id)).map((hit) => hit.id.split(":document:")[1]);
|
||||
const result = await pseudonymizeDocument(
|
||||
basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds,
|
||||
);
|
||||
output = Buffer.from(result.file_base64, "base64");
|
||||
} else {
|
||||
output = Buffer.from(replaceSpans(textFromBytes(data), draft.hits, accepted), "utf8");
|
||||
}
|
||||
await fs.writeFile(target, output, { mode: 0o600 });
|
||||
run.files.push({ source: draft.file.relPath, output: relPath, kind: draft.file.kind });
|
||||
}
|
||||
await saveRun(directory, run);
|
||||
toast.style = Toast.Style.Success;
|
||||
toast.title = `${run.files.length} Kopien erstellt`;
|
||||
await showInFinder(join(directory, "files"));
|
||||
} catch (error) {
|
||||
if (directory) await fs.rm(directory, { recursive: true, force: true });
|
||||
toast.style = Toast.Style.Failure;
|
||||
toast.title = "Pseudonymisierung fehlgeschlagen";
|
||||
toast.message = (error as Error).message;
|
||||
} finally { setBusy(false); }
|
||||
}
|
||||
|
||||
const groups = new Map<string, Hit[]>();
|
||||
for (const hit of props.preview.hits) (groups.get(hit.file) ?? (groups.set(hit.file, []), groups.get(hit.file)!)).push(hit);
|
||||
return <List isLoading={busy} isShowingDetail navigationTitle="Treffer prüfen"
|
||||
searchBarPlaceholder="Treffer, Typ oder Datei suchen">
|
||||
<List.EmptyView title="Keine Treffer" description="Die gewählten Dateien können unverändert kopiert werden."
|
||||
actions={<ActionPanel><Action title="Kopien erstellen" onAction={apply} /></ActionPanel>} />
|
||||
{[...groups].map(([file, hits]) => <List.Section key={file} title={file}>
|
||||
{hits.map((hit) => <List.Item key={hit.id}
|
||||
icon={{ source: accepted.has(hit.id) ? Icon.CheckCircle : Icon.Circle, tintColor: accepted.has(hit.id) ? Color.Green : Color.SecondaryText }}
|
||||
title={`${hit.original} → ${hit.placeholder}`}
|
||||
subtitle={`${hit.type} · ${hit.source}`}
|
||||
detail={<List.Item.Detail markdown={`**${hit.type}** · ${hit.source}\n\n${hit.context.replace(/</g, "<")}`} />}
|
||||
actions={<ActionPanel>
|
||||
<Action title={accepted.has(hit.id) ? "Treffer ausnehmen" : "Treffer übernehmen"} onAction={() => toggle(hit.id)} />
|
||||
<Action title="Gleichen Begriff übernehmen" onAction={() => setAccepted((current) => new Set([
|
||||
...current, ...props.preview.hits.filter((other) => other.type === hit.type && other.original === hit.original).map((other) => other.id),
|
||||
]))} />
|
||||
<Action title="Kopien erstellen" icon={Icon.Document} onAction={apply} />
|
||||
</ActionPanel>} />)}
|
||||
</List.Section>)}
|
||||
</List>;
|
||||
}
|
||||
|
||||
export default function Command() {
|
||||
const { push } = useNavigation();
|
||||
const [root, setRoot] = useState("");
|
||||
const [rules, setRules] = useState("");
|
||||
const [busy, setBusy] = useState(false);
|
||||
useEffect(() => { finderFolder().then((path) => { if (path) setRoot(path); }); }, []);
|
||||
useEffect(() => {
|
||||
if (!root) return;
|
||||
loadProjectRules(root).then((found) => setRules(found.map((rule) => `${rule.type}: ${rule.value}`).join("\n")))
|
||||
.catch(() => setRules(""));
|
||||
}, [root]);
|
||||
async function next() {
|
||||
setBusy(true);
|
||||
try {
|
||||
if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen.");
|
||||
const parsed = parseProjectRules(rules);
|
||||
await saveProjectRules(root, parsed);
|
||||
push(<FileSelection root={root} rules={parsed} />);
|
||||
} catch (error) {
|
||||
await showToast({ style: Toast.Style.Failure, title: "Projekt nicht bereit", message: (error as Error).message });
|
||||
} finally { setBusy(false); }
|
||||
}
|
||||
return <Form isLoading={busy} actions={<ActionPanel><Action title="Dateien auswählen" onAction={next} /></ActionPanel>}>
|
||||
<Form.FilePicker id="projectPath" title="Projektordner" allowMultipleSelection={false}
|
||||
canChooseDirectories canChooseFiles={false} value={root ? [root] : []}
|
||||
onChange={(paths) => setRoot(paths[0] ?? "")} />
|
||||
<Form.TextArea id="projectRules" title="Begriffe nur für dieses Projekt" value={rules}
|
||||
onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"}
|
||||
info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." />
|
||||
<Form.Description title="Ablauf" text="Dateien und Treffer auswählen; erst danach werden Kopien außerhalb des Projektordners erstellt." />
|
||||
</Form>;
|
||||
}
|
||||
Reference in New Issue
Block a user