feat: review project pseudonyms by category
This commit is contained in:
66
src/lib/project-review.ts
Normal file
66
src/lib/project-review.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
const SEPARATOR = "\n\n\n\n";
|
||||
export const MAX_BATCH_CODEPOINTS = 24000;
|
||||
|
||||
export type BatchItem = { key: string; text: string };
|
||||
export type BatchSegment = BatchItem & { start: number; end: number };
|
||||
export type TextBatch = { text: string; segments: BatchSegment[] };
|
||||
|
||||
/** Keep file boundaries while reducing many small text/path API calls to a few batches. */
|
||||
export function makeTextBatches(items: BatchItem[], limit = MAX_BATCH_CODEPOINTS): TextBatch[] {
|
||||
const batches: TextBatch[] = [];
|
||||
let segments: BatchSegment[] = [];
|
||||
let text = "";
|
||||
let length = 0;
|
||||
const separatorLength = [...SEPARATOR].length;
|
||||
const flush = () => {
|
||||
if (segments.length) batches.push({ text, segments });
|
||||
segments = [];
|
||||
text = "";
|
||||
length = 0;
|
||||
};
|
||||
for (const item of items) {
|
||||
if (!item.text) continue;
|
||||
const itemLength = [...item.text].length;
|
||||
if (segments.length && length + separatorLength + itemLength > limit) flush();
|
||||
if (segments.length) { text += SEPARATOR; length += separatorLength; }
|
||||
segments.push({ ...item, start: length, end: length + itemLength });
|
||||
text += item.text;
|
||||
length += itemLength;
|
||||
}
|
||||
flush();
|
||||
return batches;
|
||||
}
|
||||
|
||||
export function spansBySegment<T extends { start: number; end: number }>(
|
||||
batch: TextBatch, spans: T[],
|
||||
): Map<string, T[]> {
|
||||
const result = new Map<string, T[]>(batch.segments.map((segment) => [segment.key, []]));
|
||||
let segmentIndex = 0;
|
||||
for (const span of [...spans].sort((a, b) => a.start - b.start)) {
|
||||
while (segmentIndex < batch.segments.length && span.start >= batch.segments[segmentIndex].end) segmentIndex++;
|
||||
const segment = batch.segments[segmentIndex];
|
||||
if (!segment || span.start < segment.start || span.end > segment.end) continue;
|
||||
result.get(segment.key)!.push({ ...span, start: span.start - segment.start, end: span.end - segment.start });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export type GroupableHit = { id: string; type: string; original: string; placeholder: string };
|
||||
export type ReviewGroup<T extends GroupableHit> = {
|
||||
key: string; type: string; original: string; placeholder: string; hits: T[];
|
||||
};
|
||||
|
||||
export function groupReviewHits<T extends GroupableHit>(hits: T[]): ReviewGroup<T>[] {
|
||||
const groups = new Map<string, ReviewGroup<T>>();
|
||||
for (const hit of hits) {
|
||||
const key = JSON.stringify([hit.type, hit.original, hit.placeholder]);
|
||||
let group = groups.get(key);
|
||||
if (!group) {
|
||||
group = { key, type: hit.type, original: hit.original, placeholder: hit.placeholder,
|
||||
hits: [] };
|
||||
groups.set(key, group);
|
||||
}
|
||||
group.hits.push(hit);
|
||||
}
|
||||
return [...groups.values()].sort((a, b) => a.type.localeCompare(b.type) || a.original.localeCompare(b.original));
|
||||
}
|
||||
@@ -3,17 +3,19 @@ import { promises as fs } from "node:fs";
|
||||
import { basename, dirname, extname, join } from "node:path";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { useEffect, useState } from "react";
|
||||
import type { PlaceholderMapping } from "./types";
|
||||
import { pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import type { EntityType, PlaceholderMapping } from "./types";
|
||||
import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
|
||||
import type { DocumentHit } from "./velum";
|
||||
import {
|
||||
createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans,
|
||||
replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, saveRun, scanProjectFiles, toUtf16Offsets,
|
||||
} from "./lib/velum-project";
|
||||
import type { ProjectFile, ProjectRule } from "./lib/velum-project";
|
||||
import { groupReviewHits, makeTextBatches, spansBySegment } from "./lib/project-review";
|
||||
|
||||
const run = promisify(execFile);
|
||||
const MAX_REVIEW_HITS = 50000;
|
||||
type Hit = {
|
||||
id: string; file: string; source: string; original: string; placeholder: string;
|
||||
type: string; score: number; context: string; start: number; end: number;
|
||||
@@ -22,12 +24,14 @@ type Hit = {
|
||||
type Draft = { file: ProjectFile; hash: string; hits: Hit[] };
|
||||
type PathDraft = { key: string; value: string; hits: Hit[] };
|
||||
type Preview = { drafts: Draft[]; paths: PathDraft[]; mapping: PlaceholderMapping;
|
||||
matchModes?: Record<string, "token" | "anywhere">; hits: Hit[] };
|
||||
matchModes?: Record<string, "token" | "anywhere">; hits: Hit[]; entityTypes: EntityType[]; skipped: number };
|
||||
|
||||
function defaultAccepted(hit: Hit, rules: ProjectRule[]): boolean {
|
||||
if (rules.some((rule) => rule.type === hit.type && rule.value.toLocaleLowerCase() === hit.original.toLocaleLowerCase())) return true;
|
||||
return !["PERSON", "ORT", "ORG"].includes(hit.type);
|
||||
}
|
||||
const typeLabels: Record<string, string> = {
|
||||
PERSON: "Personen", ORT: "Orte", ORG: "Organisationen", FIRMA: "Firmen",
|
||||
KUNDE: "Kunden", ADRESSE: "Adressen", BEGRIFF: "Eigene Begriffe",
|
||||
EMAIL: "E-Mail-Adressen", TELEFON: "Telefonnummern", IBAN: "IBAN",
|
||||
};
|
||||
const labelForType = (type: string) => typeLabels[type] ?? type;
|
||||
|
||||
async function finderFolder(): Promise<string | undefined> {
|
||||
try {
|
||||
@@ -81,7 +85,8 @@ function pathSegments(files: ProjectFile[]): PathDraft[] {
|
||||
return [...map.values()];
|
||||
}
|
||||
|
||||
async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[]): Promise<Preview> {
|
||||
async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], entityTypes: EntityType[],
|
||||
skipped: number, onProgress: (message: string) => void): Promise<Preview> {
|
||||
await requireProjectCapabilities();
|
||||
const masks = forceMasks(rules);
|
||||
const latest = (await listRuns(root))[0]?.run;
|
||||
@@ -90,37 +95,60 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[])
|
||||
}
|
||||
let mapping: PlaceholderMapping = latest?.mapping ?? {};
|
||||
const drafts: Draft[] = [];
|
||||
const draftsByPath = new Map<string, Draft>();
|
||||
const hits: Hit[] = [];
|
||||
const texts = new Map<string, string>();
|
||||
for (const file of files) {
|
||||
const data = await fs.readFile(file.absPath);
|
||||
let found: Hit[];
|
||||
if (file.kind === "document") {
|
||||
const result = await pseudonymizeDocument(basename(file.relPath), data, mapping, masks);
|
||||
mapping = result.mapping;
|
||||
found = documentHits(file, result.hits);
|
||||
} else {
|
||||
const original = textFromBytes(data);
|
||||
const result = await pseudonymize(original, mapping, undefined, masks);
|
||||
mapping = result.mapping;
|
||||
found = contentHits(file, result.spans, original);
|
||||
}
|
||||
drafts.push({ file, hash: digest(data), hits: found });
|
||||
const draft: Draft = { file, hash: digest(data), hits: [] };
|
||||
drafts.push(draft);
|
||||
draftsByPath.set(file.relPath, draft);
|
||||
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
|
||||
}
|
||||
const addHits = (draft: Draft, found: Hit[]) => {
|
||||
draft.hits = found;
|
||||
hits.push(...found);
|
||||
if (hits.length > 5000) throw new Error("Mehr als 5.000 Treffer. Bitte weniger Dateien auswählen.");
|
||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||
};
|
||||
const batches = makeTextBatches([...texts].map(([key, text]) => ({ key, text })));
|
||||
for (const [index, batch] of batches.entries()) {
|
||||
onProgress(`Textdateien analysieren: Paket ${index + 1}/${batches.length}`);
|
||||
const result = await pseudonymize(batch.text, mapping, entityTypes, masks);
|
||||
mapping = result.mapping;
|
||||
const byFile = spansBySegment(batch, result.spans);
|
||||
for (const segment of batch.segments) {
|
||||
const draft = draftsByPath.get(segment.key)!;
|
||||
addHits(draft, contentHits(draft.file, byFile.get(segment.key) ?? [], segment.text));
|
||||
}
|
||||
}
|
||||
const documents = drafts.filter((draft) => draft.file.kind === "document");
|
||||
for (const [index, draft] of documents.entries()) {
|
||||
onProgress(`Dokumente analysieren: ${index + 1}/${documents.length}`);
|
||||
const data = await fs.readFile(draft.file.absPath);
|
||||
const result = await pseudonymizeDocument(basename(draft.file.relPath), data, mapping, masks, undefined, entityTypes);
|
||||
mapping = result.mapping;
|
||||
addHits(draft, documentHits(draft.file, result.hits));
|
||||
}
|
||||
const paths = pathSegments(files);
|
||||
for (const path of paths) {
|
||||
const result = await pseudonymize(path.value, mapping, undefined, masks);
|
||||
const pathBatches = makeTextBatches(paths.map((path) => ({ key: path.key, text: path.value })));
|
||||
const byPath = new Map(paths.map((path) => [path.key, path]));
|
||||
for (const [index, batch] of pathBatches.entries()) {
|
||||
onProgress(`Dateinamen analysieren: Paket ${index + 1}/${pathBatches.length}`);
|
||||
const result = await pseudonymize(batch.text, mapping, entityTypes, masks);
|
||||
mapping = result.mapping;
|
||||
path.hits = toUtf16Offsets(path.value, result.spans).map((span, index) => ({
|
||||
...span, id: `${path.key}:path:${index}`, file: path.key,
|
||||
source: "Datei- oder Ordnername", kind: "path" as const,
|
||||
context: path.value,
|
||||
}));
|
||||
hits.push(...path.hits);
|
||||
if (hits.length > 5000) throw new Error("Mehr als 5.000 Treffer. Bitte weniger Dateien auswählen.");
|
||||
const spans = spansBySegment(batch, result.spans);
|
||||
for (const segment of batch.segments) {
|
||||
const path = byPath.get(segment.key)!;
|
||||
path.hits = toUtf16Offsets(path.value, spans.get(path.key) ?? []).map((span, hitIndex) => ({
|
||||
...span, id: `${path.key}:path:${hitIndex}`, file: path.key,
|
||||
source: "Datei- oder Ordnername", kind: "path" as const,
|
||||
context: path.value,
|
||||
}));
|
||||
hits.push(...path.hits);
|
||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||
}
|
||||
}
|
||||
return { drafts, paths, mapping, matchModes: latest?.matchModes, hits };
|
||||
return { drafts, paths, mapping, matchModes: latest?.matchModes, hits, entityTypes, skipped };
|
||||
}
|
||||
|
||||
function outputPath(relPath: string, paths: PathDraft[], accepted: Set<string>, document: boolean): string {
|
||||
@@ -137,79 +165,31 @@ function outputPath(relPath: string, paths: PathDraft[], accepted: Set<string>,
|
||||
return safeRelativePath(output.join("/"));
|
||||
}
|
||||
|
||||
function FileSelection(props: { root: string; rules: ProjectRule[] }) {
|
||||
const { push } = useNavigation();
|
||||
const [files, setFiles] = useState<ProjectFile[]>([]);
|
||||
const [skipped, setSkipped] = useState<string[]>([]);
|
||||
const [selected, setSelected] = useState<Set<string>>(new Set());
|
||||
const [busy, setBusy] = useState(true);
|
||||
useEffect(() => {
|
||||
scanProjectFiles(props.root).then((scan) => {
|
||||
setFiles(scan.files);
|
||||
setSkipped(scan.skipped);
|
||||
setSelected(new Set(scan.files.map((file) => file.relPath)));
|
||||
}).catch((error: Error) => showToast({ style: Toast.Style.Failure, title: "Projekt nicht lesbar", message: error.message }))
|
||||
.finally(() => setBusy(false));
|
||||
}, [props.root]);
|
||||
|
||||
function toggle(path: string) {
|
||||
setSelected((current) => {
|
||||
const next = new Set(current);
|
||||
if (next.has(path)) next.delete(path);
|
||||
else next.add(path);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
async function review() {
|
||||
const chosen = files.filter((file) => selected.has(file.relPath));
|
||||
if (!chosen.length) {
|
||||
await showToast({ style: Toast.Style.Failure, title: "Keine Dateien ausgewählt" });
|
||||
return;
|
||||
}
|
||||
setBusy(true);
|
||||
const toast = await showToast({ style: Toast.Style.Animated, title: "Treffer werden analysiert" });
|
||||
try {
|
||||
const preview = await prepare(props.root, chosen, props.rules);
|
||||
toast.style = Toast.Style.Success;
|
||||
toast.title = `${preview.hits.length} Treffer zur Prüfung`;
|
||||
push(<HitReview root={props.root} rules={props.rules} preview={preview} />);
|
||||
} catch (error) {
|
||||
toast.style = Toast.Style.Failure;
|
||||
toast.title = "Analyse fehlgeschlagen";
|
||||
toast.message = (error as Error).message;
|
||||
} finally { setBusy(false); }
|
||||
}
|
||||
|
||||
return <List isLoading={busy} navigationTitle="Dateien auswählen" searchBarPlaceholder="Dateien filtern">
|
||||
<List.EmptyView title="Keine geeigneten Dateien" description="Bitte einen anderen Projektordner wählen oder die ausgelassenen Dateien prüfen." />
|
||||
<List.Section title={`${selected.size} von ${files.length} Dateien ausgewählt · ${skipped.length} übersprungen`}>
|
||||
{files.map((file) => <List.Item key={file.relPath}
|
||||
icon={selected.has(file.relPath) ? Icon.CheckCircle : Icon.Circle}
|
||||
title={file.relPath} subtitle={file.kind === "document" ? "Office/PDF" : "Text"}
|
||||
actions={<ActionPanel>
|
||||
<Action title="Treffer prüfen" icon={Icon.Eye} onAction={review} />
|
||||
<Action title={selected.has(file.relPath) ? "Datei abwählen" : "Datei auswählen"} onAction={() => toggle(file.relPath)} />
|
||||
<Action title="Alle auswählen" onAction={() => setSelected(new Set(files.map((item) => item.relPath)))} />
|
||||
<Action title="Keine auswählen" onAction={() => setSelected(new Set())} />
|
||||
</ActionPanel>} />)}
|
||||
</List.Section>
|
||||
{skipped.length > 0 && <List.Section title="Übersprungen (binär, zu groß oder ausgeschlossen)">
|
||||
{skipped.slice(0, 100).map((path) => <List.Item key={path} icon={Icon.XMarkCircle} title={path} />)}
|
||||
</List.Section>}
|
||||
function OccurrenceReview(props: { original: string; placeholder: string; hits: Hit[] }) {
|
||||
return <List isShowingDetail navigationTitle={`${props.original} → ${props.placeholder}`}>
|
||||
{props.hits.map((hit) => <List.Item key={hit.id} title={hit.source}
|
||||
subtitle={hit.kind === "path" ? "Datei- oder Ordnername" : "Dateiinhalt"}
|
||||
detail={<List.Item.Detail markdown={hit.context.replace(/</g, "<")} />} />)}
|
||||
</List>;
|
||||
}
|
||||
|
||||
function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview }) {
|
||||
const [accepted, setAccepted] = useState<Set<string>>(
|
||||
new Set(props.preview.hits.filter((hit) => defaultAccepted(hit, props.rules)).map((hit) => hit.id)),
|
||||
);
|
||||
const groups = useMemo(() => groupReviewHits(props.preview.hits), [props.preview.hits]);
|
||||
const [selectedGroups, setSelectedGroups] = useState<Set<string>>(() => new Set(groups.map((group) => group.key)));
|
||||
const [busy, setBusy] = useState(false);
|
||||
function toggle(id: string) {
|
||||
setAccepted((current) => {
|
||||
const accepted = useMemo(() => new Set(groups.filter((group) => selectedGroups.has(group.key))
|
||||
.flatMap((group) => group.hits.map((hit) => hit.id))), [groups, selectedGroups]);
|
||||
const selectedOccurrences = accepted.size;
|
||||
const byType = useMemo(() => {
|
||||
const result = new Map<string, typeof groups>();
|
||||
for (const group of groups) (result.get(group.type) ?? (result.set(group.type, []), result.get(group.type)!)).push(group);
|
||||
return result;
|
||||
}, [groups]);
|
||||
function toggle(key: string) {
|
||||
setSelectedGroups((current) => {
|
||||
const next = new Set(current);
|
||||
if (next.has(id)) next.delete(id);
|
||||
else next.add(id);
|
||||
if (next.has(key)) next.delete(key);
|
||||
else next.add(key);
|
||||
return next;
|
||||
});
|
||||
}
|
||||
@@ -243,7 +223,7 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
|
||||
if (draft.file.kind === "document") {
|
||||
const docIds = draft.hits.filter((hit) => accepted.has(hit.id)).map((hit) => hit.id.split(":document:")[1]);
|
||||
const result = await pseudonymizeDocument(
|
||||
basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds,
|
||||
basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds, props.preview.entityTypes,
|
||||
);
|
||||
output = Buffer.from(result.file_base64, "base64");
|
||||
} else {
|
||||
@@ -270,28 +250,40 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
|
||||
} finally { setBusy(false); }
|
||||
}
|
||||
|
||||
const groups = new Map<string, Hit[]>();
|
||||
for (const hit of props.preview.hits) (groups.get(hit.file) ?? (groups.set(hit.file, []), groups.get(hit.file)!)).push(hit);
|
||||
return <List isLoading={busy} isShowingDetail navigationTitle="Treffer prüfen"
|
||||
searchBarPlaceholder="Treffer, Typ oder Datei suchen">
|
||||
<List.EmptyView title="Keine Treffer" description="Die gewählten Dateien können unverändert kopiert werden."
|
||||
actions={<ActionPanel>
|
||||
<Action title="Pseudonymisierte Kopien erstellen" onAction={() => apply()} />
|
||||
<Action title="Originaldateien ersetzen" onAction={() => apply(true)} />
|
||||
</ActionPanel>} />
|
||||
{[...groups].map(([file, hits]) => <List.Section key={file} title={file}>
|
||||
{hits.map((hit) => <List.Item key={hit.id}
|
||||
icon={{ source: accepted.has(hit.id) ? Icon.CheckCircle : Icon.Circle, tintColor: accepted.has(hit.id) ? Color.Green : Color.SecondaryText }}
|
||||
title={`${hit.original} → ${hit.placeholder}`}
|
||||
subtitle={`${hit.type} · ${hit.source}`}
|
||||
detail={<List.Item.Detail markdown={`**${hit.type}** · ${hit.source}\n\n${hit.context.replace(/</g, "<")}`} />}
|
||||
return <List isLoading={busy} navigationTitle="Zuordnungen prüfen"
|
||||
searchBarPlaceholder="Begriff oder Platzhalter suchen">
|
||||
<List.Section title="Abschluss">
|
||||
<List.Item key="apply" icon={Icon.Play}
|
||||
title={`${selectedGroups.size} von ${groups.length} Zuordnungen ausgewählt`}
|
||||
subtitle={`${selectedOccurrences} Vorkommen · ${props.preview.drafts.length} Dateien · ${props.preview.skipped} übersprungen`}
|
||||
actions={<ActionPanel>
|
||||
<Action title="Pseudonymisierte Kopien erstellen" icon={Icon.Document} onAction={() => apply()} />
|
||||
<Action title="Originaldateien ersetzen" icon={Icon.Pencil} onAction={() => apply(true)} />
|
||||
<Action title={accepted.has(hit.id) ? "Treffer ausnehmen" : "Treffer übernehmen"} onAction={() => toggle(hit.id)} />
|
||||
<Action title="Gleichen Begriff übernehmen" onAction={() => setAccepted((current) => new Set([
|
||||
...current, ...props.preview.hits.filter((other) => other.type === hit.type && other.original === hit.original).map((other) => other.id),
|
||||
]))} />
|
||||
</ActionPanel>} />
|
||||
</List.Section>
|
||||
{[...byType].map(([type, entries]) => <List.Section key={type}
|
||||
title={`${labelForType(type)} · ${entries.filter((entry) => selectedGroups.has(entry.key)).length}/${entries.length}`}>
|
||||
{entries.map((group) => <List.Item key={group.key}
|
||||
icon={{ source: selectedGroups.has(group.key) ? Icon.CheckCircle : Icon.Circle,
|
||||
tintColor: selectedGroups.has(group.key) ? Color.Green : Color.SecondaryText }}
|
||||
title={group.original} subtitle={`→ ${group.placeholder}`}
|
||||
accessories={[{ text: `${group.hits.length} Vorkommen` }]}
|
||||
keywords={[type, labelForType(type)]}
|
||||
actions={<ActionPanel>
|
||||
<Action title={selectedGroups.has(group.key) ? "Zuordnung abwählen" : "Zuordnung auswählen"}
|
||||
onAction={() => toggle(group.key)} />
|
||||
<Action title={`${labelForType(type)} komplett abwählen`}
|
||||
onAction={() => setSelectedGroups((current) => {
|
||||
const next = new Set(current);
|
||||
for (const entry of entries) next.delete(entry.key);
|
||||
return next;
|
||||
})} />
|
||||
<Action title={`${labelForType(type)} komplett auswählen`}
|
||||
onAction={() => setSelectedGroups((current) => new Set([...current, ...entries.map((entry) => entry.key)]))} />
|
||||
<Action.Push title="Vorkommen ansehen" icon={Icon.Eye}
|
||||
target={<OccurrenceReview original={group.original} placeholder={group.placeholder} hits={group.hits} />} />
|
||||
<Action title="Pseudonymisierte Kopien erstellen" icon={Icon.Document} onAction={() => apply()} />
|
||||
<Action title="Originaldateien ersetzen" icon={Icon.Pencil} onAction={() => apply(true)} />
|
||||
</ActionPanel>} />)}
|
||||
</List.Section>)}
|
||||
</List>;
|
||||
@@ -301,8 +293,17 @@ export default function Command() {
|
||||
const { push } = useNavigation();
|
||||
const [root, setRoot] = useState("");
|
||||
const [rules, setRules] = useState("");
|
||||
const [availableTypes, setAvailableTypes] = useState<EntityType[]>([]);
|
||||
const [selectedTypes, setSelectedTypes] = useState<EntityType[]>([]);
|
||||
const [busy, setBusy] = useState(false);
|
||||
useEffect(() => { finderFolder().then((path) => { if (path) setRoot((current) => current || path); }); }, []);
|
||||
useEffect(() => {
|
||||
getEntityTypes().then((types) => {
|
||||
setAvailableTypes(types);
|
||||
setSelectedTypes(types.filter((type) => type !== "ORT"));
|
||||
}).catch((error: Error) => showToast({ style: Toast.Style.Failure,
|
||||
title: "Kategorien nicht verfügbar", message: error.message }));
|
||||
}, []);
|
||||
useEffect(() => {
|
||||
if (!root) return;
|
||||
let active = true;
|
||||
@@ -313,13 +314,24 @@ export default function Command() {
|
||||
}, [root]);
|
||||
async function next() {
|
||||
setBusy(true);
|
||||
const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" });
|
||||
try {
|
||||
if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen.");
|
||||
if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen.");
|
||||
const parsed = parseProjectRules(rules);
|
||||
await saveProjectRules(root, parsed);
|
||||
push(<FileSelection root={root} rules={parsed} />);
|
||||
toast.title = "Dateien werden erfasst";
|
||||
const scan = await scanProjectFiles(root);
|
||||
if (!scan.files.length) throw new Error("Keine geeigneten Dateien im Projektordner gefunden.");
|
||||
const preview = await prepare(root, scan.files, parsed, selectedTypes, scan.skipped.length,
|
||||
(message) => { toast.title = message; });
|
||||
toast.style = Toast.Style.Success;
|
||||
toast.title = `${groupReviewHits(preview.hits).length} Zuordnungen zur Prüfung`;
|
||||
push(<HitReview root={root} rules={parsed} preview={preview} />);
|
||||
} catch (error) {
|
||||
await showToast({ style: Toast.Style.Failure, title: "Projekt nicht bereit", message: (error as Error).message });
|
||||
toast.style = Toast.Style.Failure;
|
||||
toast.title = "Analyse fehlgeschlagen";
|
||||
toast.message = (error as Error).message;
|
||||
} finally { setBusy(false); }
|
||||
}
|
||||
async function useFinderFolder() {
|
||||
@@ -329,7 +341,7 @@ export default function Command() {
|
||||
message: "Bitte den Ordner im Finder öffnen oder unten manuell auswählen." });
|
||||
}
|
||||
return <Form isLoading={busy} actions={<ActionPanel>
|
||||
<Action title="Dateien auswählen" onAction={next} />
|
||||
<Action title="Zuordnungen prüfen" onAction={next} />
|
||||
<Action title="Geöffneten Finder-Ordner übernehmen" icon={Icon.Folder} onAction={useFinderFolder} />
|
||||
</ActionPanel>}>
|
||||
<Form.FilePicker id="projectPath" title="Projektordner" allowMultipleSelection={false}
|
||||
@@ -338,6 +350,10 @@ export default function Command() {
|
||||
<Form.TextArea id="projectRules" title="Begriffe nur für dieses Projekt" value={rules}
|
||||
onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"}
|
||||
info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." />
|
||||
<Form.Description title="Ablauf" text="Dateien und Treffer auswählen; erst danach werden Kopien außerhalb des Projektordners erstellt." />
|
||||
<Form.TagPicker id="entityTypes" title="Kategorien pseudonymisieren" value={selectedTypes}
|
||||
onChange={setSelectedTypes} info="Alle geeigneten Dateien werden analysiert. Orte sind wegen häufiger Fehlalarme zunächst abgewählt; alle gefundenen Zuordnungen können anschließend einzeln abgewählt werden.">
|
||||
{availableTypes.map((type) => <Form.TagPicker.Item key={type} value={type} title={labelForType(type)} />)}
|
||||
</Form.TagPicker>
|
||||
<Form.Description title="Ablauf" text="Alle geeigneten Dateien erfassen, Zuordnungen je Kategorie prüfen und dann Kopien erstellen oder Originale ersetzen." />
|
||||
</Form>;
|
||||
}
|
||||
|
||||
@@ -118,8 +118,7 @@ export async function pseudonymize(
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
mapping,
|
||||
entity_types:
|
||||
entityTypes && entityTypes.length > 0 ? entityTypes : undefined,
|
||||
entity_types: entityTypes,
|
||||
force_masks: forceMasks,
|
||||
}),
|
||||
});
|
||||
@@ -163,11 +162,13 @@ export async function pseudonymizeDocument(
|
||||
mapping: PlaceholderMapping,
|
||||
forceMasks: Record<string, string[]>,
|
||||
acceptedHitIds?: string[],
|
||||
entityTypes?: EntityType[],
|
||||
): Promise<DocumentResult> {
|
||||
const form = new FormData();
|
||||
form.set("file", new Blob([new Uint8Array(data)]), filename);
|
||||
form.set("mapping", JSON.stringify(mapping));
|
||||
form.set("force_masks", JSON.stringify(forceMasks));
|
||||
if (entityTypes !== undefined) form.set("entity_types", JSON.stringify(entityTypes));
|
||||
if (acceptedHitIds) form.set("accepted_hit_ids", JSON.stringify(acceptedHitIds));
|
||||
const response = await fetchWithAuth(apiUrl("/api/pseudonymize-document"), {
|
||||
method: "POST", body: form,
|
||||
|
||||
Reference in New Issue
Block a user