feat: remember project exclusions and merge selected people
This commit is contained in:
muena
@@ -1,3 +1,7 @@
|
||||
import type { PlaceholderMapping } from "../types";
|
||||
import { whitelistKey } from "./project-whitelist";
|
||||
import type { ProjectWhitelistEntry } from "./project-whitelist";
|
||||
|
||||
const SEPARATOR = "\n\n\n\n";
|
||||
export const MAX_BATCH_CODEPOINTS = 24000;
|
||||
|
||||
@@ -64,3 +68,57 @@ export function groupReviewHits<T extends GroupableHit>(hits: T[]): ReviewGroup<
|
||||
}
|
||||
return [...groups.values()].sort((a, b) => a.type.localeCompare(b.type) || a.original.localeCompare(b.original));
|
||||
}
|
||||
|
||||
export function withoutWhitelistedHits<T extends GroupableHit>(hits: T[], whitelist: ProjectWhitelistEntry[]): T[] {
|
||||
const excluded = new Set(whitelist.map(whitelistKey));
|
||||
return hits.filter((hit) => !excluded.has(whitelistKey(hit)));
|
||||
}
|
||||
|
||||
export function withoutWhitelistedRules<T extends { type: string; value: string }>(
|
||||
rules: T[], whitelist: ProjectWhitelistEntry[],
|
||||
): T[] {
|
||||
const excluded = new Set(whitelist.map(whitelistKey));
|
||||
return rules.filter((rule) => !excluded.has(whitelistKey({ type: rule.type, original: rule.value })));
|
||||
}
|
||||
|
||||
export function canonicalPerson(placeholder: string, merges: Record<string, string>): string {
|
||||
const seen = new Set<string>();
|
||||
let current = placeholder;
|
||||
while (merges[current] && !seen.has(current)) {
|
||||
seen.add(current);
|
||||
current = merges[current];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
export function mergePeople(merges: Record<string, string>, source: string, target: string): Record<string, string> {
|
||||
const from = canonicalPerson(source, merges);
|
||||
const into = canonicalPerson(target, merges);
|
||||
if (from === into) return merges;
|
||||
const next = { ...merges };
|
||||
for (const person of new Set([...Object.keys(next), ...Object.values(next), from])) {
|
||||
if (canonicalPerson(person, merges) === from && person !== into) next[person] = into;
|
||||
}
|
||||
delete next[into];
|
||||
return next;
|
||||
}
|
||||
|
||||
export function mappingForSelection(
|
||||
preview: PlaceholderMapping, previous: PlaceholderMapping, accepted: GroupableHit[],
|
||||
merges: Record<string, string>,
|
||||
): PlaceholderMapping {
|
||||
const keys = new Set([...Object.keys(previous), ...accepted.map((hit) => hit.placeholder)]);
|
||||
const mapping: PlaceholderMapping = {};
|
||||
for (const key of keys) {
|
||||
const entry = preview[key] ?? previous[key];
|
||||
if (entry) mapping[key] = { ...entry };
|
||||
}
|
||||
for (const [source, target] of Object.entries(merges)) {
|
||||
const root = canonicalPerson(target, merges);
|
||||
if (source === root || mapping[source]?.type !== "PERSON" || mapping[root]?.type !== "PERSON") {
|
||||
throw new Error("Ungültige Personen-Zusammenführung.");
|
||||
}
|
||||
mapping[source] = { ...mapping[source], merged_into: root };
|
||||
}
|
||||
return mapping;
|
||||
}
|
||||
|
||||
5
src/lib/project-whitelist.ts
Normal file
5
src/lib/project-whitelist.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export type ProjectWhitelistEntry = { type: string; original: string };
|
||||
|
||||
export function whitelistKey(entry: ProjectWhitelistEntry): string {
|
||||
return JSON.stringify([entry.type.toUpperCase(), entry.original.trim().toLocaleLowerCase("de")]);
|
||||
}
|
||||
@@ -3,6 +3,11 @@ import { constants as fsConstants, promises as fs } from "node:fs";
|
||||
import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
|
||||
import { environment } from "@raycast/api";
|
||||
import type { PlaceholderMapping } from "../types";
|
||||
import { whitelistKey } from "./project-whitelist";
|
||||
import type { ProjectWhitelistEntry } from "./project-whitelist";
|
||||
|
||||
export { whitelistKey } from "./project-whitelist";
|
||||
export type { ProjectWhitelistEntry } from "./project-whitelist";
|
||||
|
||||
export type ProjectRule = { type: "BEGRIFF" | "PERSON" | "ORG" | "ORT" | "FIRMA" | "KUNDE"; value: string };
|
||||
export type ProjectFile = { relPath: string; absPath: string; kind: "text" | "document"; size: number };
|
||||
@@ -19,6 +24,8 @@ export type ProjectRun = {
|
||||
matchModes?: Record<string, "token" | "anywhere">;
|
||||
files: RunFile[];
|
||||
inPlace?: boolean;
|
||||
/** The project source already contains outputs of an earlier in-place run. */
|
||||
sourceInPlace?: boolean;
|
||||
};
|
||||
|
||||
const DOCUMENTS = new Set([".docx", ".xlsx", ".pptx", ".pdf"]);
|
||||
@@ -49,6 +56,38 @@ export function rulesPath(projectPath: string, support = environment.supportPath
|
||||
return join(projectDataRoot(projectPath, support), "rules.json");
|
||||
}
|
||||
|
||||
export function whitelistPath(projectPath: string, support = environment.supportPath): string {
|
||||
return join(projectDataRoot(projectPath, support), "whitelist.json");
|
||||
}
|
||||
|
||||
export async function loadProjectWhitelist(projectPath: string, support?: string): Promise<ProjectWhitelistEntry[]> {
|
||||
try {
|
||||
const value = JSON.parse(await fs.readFile(whitelistPath(projectPath, support), "utf8")) as unknown;
|
||||
if (!Array.isArray(value)) throw new Error("Ungültige Projekt-Whitelist.");
|
||||
return value.filter((entry): entry is ProjectWhitelistEntry => Boolean(entry) && typeof entry === "object"
|
||||
&& typeof entry.type === "string" && typeof entry.original === "string"
|
||||
&& entry.type.length > 0 && entry.original.trim().length > 0);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveProjectWhitelist(projectPath: string, entries: ProjectWhitelistEntry[], support?: string): Promise<void> {
|
||||
const unique = [...new Map(entries.map((entry) => [whitelistKey(entry),
|
||||
{ type: entry.type.toUpperCase(), original: entry.original.trim() }])).values()];
|
||||
const path = whitelistPath(projectPath, support);
|
||||
await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
||||
const temp = `${path}.${randomUUID()}.tmp`;
|
||||
try {
|
||||
await fs.writeFile(temp, JSON.stringify(unique), { encoding: "utf8", mode: 0o600 });
|
||||
await fs.rename(temp, path);
|
||||
} catch (error) {
|
||||
await fs.rm(temp, { force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadProjectRules(projectPath: string, support?: string): Promise<ProjectRule[]> {
|
||||
try {
|
||||
const value = JSON.parse(await fs.readFile(rulesPath(projectPath, support), "utf8")) as unknown;
|
||||
@@ -241,6 +280,17 @@ export async function listRuns(projectPath: string, support = environment.suppor
|
||||
return runs.sort((a, b) => b.run.createdAt.localeCompare(a.run.createdAt));
|
||||
}
|
||||
|
||||
/** An unchanged output from an in-place run can be carried into the next snapshot. */
|
||||
export async function matchesPreviousOutput(directory: string, relPath: string, currentHash: string): Promise<boolean> {
|
||||
try {
|
||||
const previous = await fs.readFile(resolveWithin(join(directory, "files"), relPath));
|
||||
return digest(previous) === currentHash;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function listAllRuns(support = environment.supportPath): Promise<Array<{
|
||||
run: ProjectRun; directory: string;
|
||||
}>> {
|
||||
|
||||
@@ -9,10 +9,14 @@ import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapab
|
||||
import type { DocumentHit } from "./velum";
|
||||
import {
|
||||
createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans,
|
||||
replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, saveRun, scanProjectFiles, toUtf16Offsets,
|
||||
loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules,
|
||||
saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey,
|
||||
} from "./lib/velum-project";
|
||||
import type { ProjectFile, ProjectRule } from "./lib/velum-project";
|
||||
import { groupReviewHits, makeTextBatches, spansBySegment } from "./lib/project-review";
|
||||
import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project";
|
||||
import {
|
||||
canonicalPerson, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment,
|
||||
withoutWhitelistedHits, withoutWhitelistedRules,
|
||||
} from "./lib/project-review";
|
||||
|
||||
const run = promisify(execFile);
|
||||
const MAX_REVIEW_HITS = 50000;
|
||||
@@ -23,8 +27,10 @@ type Hit = {
|
||||
};
|
||||
type Draft = { file: ProjectFile; hash: string; hits: Hit[] };
|
||||
type PathDraft = { key: string; value: string; hits: Hit[] };
|
||||
type Preview = { drafts: Draft[]; paths: PathDraft[]; mapping: PlaceholderMapping;
|
||||
matchModes?: Record<string, "token" | "anywhere">; hits: Hit[]; entityTypes: EntityType[]; skipped: number };
|
||||
type Preview = { drafts: Draft[]; retained: Array<{ file: ProjectFile; hash: string }>;
|
||||
paths: PathDraft[]; mapping: PlaceholderMapping; previousMapping: PlaceholderMapping;
|
||||
matchModes?: Record<string, "token" | "anywhere">; hits: Hit[]; entityTypes: EntityType[];
|
||||
skipped: number; sourceInPlace: boolean };
|
||||
|
||||
const typeLabels: Record<string, string> = {
|
||||
PERSON: "Personen", ORT: "Orte", ORG: "Organisationen", FIRMA: "Firmen",
|
||||
@@ -89,29 +95,41 @@ function pathSegments(files: ProjectFile[]): PathDraft[] {
|
||||
return [...map.values()];
|
||||
}
|
||||
|
||||
async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], entityTypes: EntityType[],
|
||||
async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], whitelist: ProjectWhitelistEntry[], entityTypes: EntityType[],
|
||||
skipped: number, onProgress: (message: string) => void): Promise<Preview> {
|
||||
await requireProjectCapabilities();
|
||||
const masks = forceMasks(rules);
|
||||
const latest = (await listRuns(root))[0]?.run;
|
||||
const masks = forceMasks(withoutWhitelistedRules(rules, whitelist));
|
||||
const latestEntry = (await listRuns(root))[0];
|
||||
const latest = latestEntry?.run;
|
||||
if (latest?.origin === "imported") {
|
||||
throw new Error("Dieser Quellordner enthält bereits pseudonymisierte Dateien. Für einen neuen Lauf bitte einen Klartext-Quellordner wählen.");
|
||||
}
|
||||
let mapping: PlaceholderMapping = latest?.mapping ?? {};
|
||||
const previousMapping: PlaceholderMapping = latest?.mapping ?? {};
|
||||
let mapping: PlaceholderMapping = previousMapping;
|
||||
const drafts: Draft[] = [];
|
||||
const retained: Preview["retained"] = [];
|
||||
const draftsByPath = new Map<string, Draft>();
|
||||
const hits: Hit[] = [];
|
||||
const texts = new Map<string, string>();
|
||||
const sourceInPlace = Boolean(latest?.inPlace || latest?.sourceInPlace);
|
||||
const previousOutputs = new Set(sourceInPlace ? latest?.files.map((file) => file.output) : []);
|
||||
const previousDirectory = latestEntry?.directory;
|
||||
for (const file of files) {
|
||||
const data = await fs.readFile(file.absPath);
|
||||
const draft: Draft = { file, hash: digest(data), hits: [] };
|
||||
const hash = digest(data);
|
||||
if (previousDirectory && previousOutputs.has(file.relPath)
|
||||
&& await matchesPreviousOutput(previousDirectory, file.relPath, hash)) {
|
||||
retained.push({ file, hash });
|
||||
continue;
|
||||
}
|
||||
const draft: Draft = { file, hash, hits: [] };
|
||||
drafts.push(draft);
|
||||
draftsByPath.set(file.relPath, draft);
|
||||
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
|
||||
}
|
||||
const addHits = (draft: Draft, found: Hit[]) => {
|
||||
draft.hits = found;
|
||||
hits.push(...found);
|
||||
draft.hits = withoutWhitelistedHits(found, whitelist);
|
||||
hits.push(...draft.hits);
|
||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||
};
|
||||
const batches = makeTextBatches([...texts].map(([key, text]) => ({ key, text })));
|
||||
@@ -133,7 +151,7 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
||||
mapping = result.mapping;
|
||||
addHits(draft, documentHits(draft.file, result.hits));
|
||||
}
|
||||
const paths = pathSegments(files);
|
||||
const paths = pathSegments(drafts.map((draft) => draft.file));
|
||||
const pathBatches = makeTextBatches(paths.map((path) => ({ key: path.key, text: path.value })));
|
||||
const byPath = new Map(paths.map((path) => [path.key, path]));
|
||||
for (const [index, batch] of pathBatches.entries()) {
|
||||
@@ -143,19 +161,21 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
||||
const spans = spansBySegment(batch, result.spans);
|
||||
for (const segment of batch.segments) {
|
||||
const path = byPath.get(segment.key)!;
|
||||
path.hits = toUtf16Offsets(path.value, spans.get(path.key) ?? []).map((span, hitIndex) => ({
|
||||
path.hits = withoutWhitelistedHits(toUtf16Offsets(path.value, spans.get(path.key) ?? []).map((span, hitIndex) => ({
|
||||
...span, id: `${path.key}:path:${hitIndex}`, file: path.key,
|
||||
source: "Datei- oder Ordnername", kind: "path" as const,
|
||||
context: path.value,
|
||||
}));
|
||||
})), whitelist);
|
||||
hits.push(...path.hits);
|
||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||
}
|
||||
}
|
||||
return { drafts, paths, mapping, matchModes: latest?.matchModes, hits, entityTypes, skipped };
|
||||
return { drafts, retained, paths, mapping, previousMapping,
|
||||
matchModes: latest?.matchModes, hits, entityTypes, skipped, sourceInPlace };
|
||||
}
|
||||
|
||||
function outputPath(relPath: string, paths: PathDraft[], accepted: Set<string>, document: boolean): string {
|
||||
function outputPath(relPath: string, paths: PathDraft[], accepted: Set<string>, document: boolean,
|
||||
merges: Record<string, string>): string {
|
||||
const lookup = new Map(paths.map((item) => [item.key, item]));
|
||||
const parts = relPath.split("/");
|
||||
const output = parts.map((part, index) => {
|
||||
@@ -164,11 +184,16 @@ function outputPath(relPath: string, paths: PathDraft[], accepted: Set<string>,
|
||||
if (!path) return part;
|
||||
const isFile = index === parts.length - 1;
|
||||
const extension = isFile ? (document && extname(part).toLowerCase() === ".pdf" ? ".docx" : extname(part)) : "";
|
||||
return replaceSpans(path.value, path.hits, accepted) + extension;
|
||||
return replaceSpans(path.value, withPersonMerges(path.hits, merges), accepted) + extension;
|
||||
});
|
||||
return safeRelativePath(output.join("/"));
|
||||
}
|
||||
|
||||
function withPersonMerges(hits: Hit[], merges: Record<string, string>): Hit[] {
|
||||
return hits.map((hit) => hit.type === "PERSON"
|
||||
? { ...hit, placeholder: canonicalPerson(hit.placeholder, merges) } : hit);
|
||||
}
|
||||
|
||||
function OccurrenceReview(props: { original: string; placeholder: string; hits: Hit[] }) {
|
||||
return <List isShowingDetail navigationTitle={`${props.original} → ${props.placeholder}`}>
|
||||
{props.hits.map((hit) => <List.Item key={hit.id} title={hit.source}
|
||||
@@ -177,9 +202,12 @@ function OccurrenceReview(props: { original: string; placeholder: string; hits:
|
||||
</List>;
|
||||
}
|
||||
|
||||
function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview }) {
|
||||
function HitReview(props: { root: string; rules: ProjectRule[]; whitelist: ProjectWhitelistEntry[];
|
||||
preview: Preview; onWhitelistChanged: (entries: ProjectWhitelistEntry[]) => void }) {
|
||||
const groups = useMemo(() => groupReviewHits(props.preview.hits), [props.preview.hits]);
|
||||
const [selectedGroups, setSelectedGroups] = useState<Set<string>>(() => new Set(groups.map((group) => group.key)));
|
||||
const [stage, setStage] = useState<"selection" | "merge">("selection");
|
||||
const [personMerges, setPersonMerges] = useState<Record<string, string>>({});
|
||||
const [busy, setBusy] = useState(false);
|
||||
const accepted = useMemo(() => new Set(groups.filter((group) => selectedGroups.has(group.key))
|
||||
.flatMap((group) => group.hits.map((hit) => hit.id))), [groups, selectedGroups]);
|
||||
@@ -189,6 +217,21 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
|
||||
for (const group of groups) (result.get(group.type) ?? (result.set(group.type, []), result.get(group.type)!)).push(group);
|
||||
return result;
|
||||
}, [groups]);
|
||||
const people = useMemo(() => {
|
||||
const result = new Map<string, string[]>();
|
||||
for (const group of groups) {
|
||||
if (group.type !== "PERSON" || !selectedGroups.has(group.key)) continue;
|
||||
const originals = result.get(group.placeholder) ?? [];
|
||||
if (!originals.includes(group.original)) originals.push(group.original);
|
||||
result.set(group.placeholder, originals);
|
||||
}
|
||||
return [...result].map(([placeholder, originals]) => ({ placeholder, originals }));
|
||||
}, [groups, selectedGroups]);
|
||||
const earlierPeople = useMemo(() => Object.entries(props.preview.previousMapping)
|
||||
.filter(([placeholder, entry]) => entry.type === "PERSON" && !entry.merged_into
|
||||
&& !people.some((person) => person.placeholder === placeholder))
|
||||
.map(([placeholder, entry]) => ({ placeholder, originals: [entry.original] })),
|
||||
[people, props.preview.previousMapping]);
|
||||
function toggle(key: string) {
|
||||
setSelectedGroups((current) => {
|
||||
const next = new Set(current);
|
||||
@@ -212,13 +255,34 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
|
||||
const created = await createRun(props.root);
|
||||
directory = created.directory;
|
||||
const { run } = created;
|
||||
run.mapping = props.preview.mapping;
|
||||
const acceptedHits = groups.filter((group) => selectedGroups.has(group.key))
|
||||
.flatMap((group) => group.hits);
|
||||
const selectedPeople = new Set(people.map((person) => person.placeholder));
|
||||
const availableTargets = new Set([...selectedPeople, ...earlierPeople.map((person) => person.placeholder)]);
|
||||
const activeMerges = Object.fromEntries(Object.entries(personMerges).filter(([source, target]) =>
|
||||
selectedPeople.has(source) && availableTargets.has(canonicalPerson(target, personMerges))));
|
||||
run.mapping = mappingForSelection(props.preview.mapping, props.preview.previousMapping,
|
||||
acceptedHits, activeMerges);
|
||||
run.matchModes = props.preview.matchModes;
|
||||
run.sourceInPlace = props.preview.sourceInPlace || inPlace;
|
||||
const usedPaths = new Set<string>();
|
||||
const replacements: RunFile[] = [];
|
||||
for (const retained of props.preview.retained) {
|
||||
const data = await fs.readFile(retained.file.absPath);
|
||||
if (digest(data) !== retained.hash) throw new Error(`Datei wurde seit der Vorschau geändert: ${retained.file.relPath}`);
|
||||
const relPath = safeRelativePath(retained.file.relPath);
|
||||
if (usedPaths.has(relPath)) throw new Error(`Zwei Dateien hätten denselben Ausgabenamen: ${relPath}`);
|
||||
usedPaths.add(relPath);
|
||||
const target = resolveWithin(join(directory, "files"), relPath);
|
||||
await fs.mkdir(dirname(target), { recursive: true, mode: 0o700 });
|
||||
await fs.writeFile(target, data, { mode: 0o600 });
|
||||
run.files.push({ source: relPath, output: relPath, kind: retained.file.kind });
|
||||
}
|
||||
for (const draft of props.preview.drafts) {
|
||||
const data = await fs.readFile(draft.file.absPath);
|
||||
if (digest(data) !== draft.hash) throw new Error(`Datei wurde seit der Vorschau geändert: ${draft.file.relPath}`);
|
||||
const relPath = outputPath(draft.file.relPath, props.preview.paths, accepted, draft.file.kind === "document");
|
||||
const relPath = outputPath(draft.file.relPath, props.preview.paths, accepted,
|
||||
draft.file.kind === "document", activeMerges);
|
||||
if (usedPaths.has(relPath)) throw new Error(`Zwei Dateien hätten denselben Ausgabenamen: ${relPath}`);
|
||||
usedPaths.add(relPath);
|
||||
const target = resolveWithin(join(directory, "files"), relPath);
|
||||
@@ -227,24 +291,36 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
|
||||
if (draft.file.kind === "document") {
|
||||
const docIds = draft.hits.filter((hit) => accepted.has(hit.id)).map((hit) => hit.id.split(":document:")[1]);
|
||||
const result = await pseudonymizeDocument(
|
||||
basename(draft.file.relPath), data, run.mapping, forceMasks(props.rules), docIds, props.preview.entityTypes,
|
||||
basename(draft.file.relPath), data, run.mapping,
|
||||
forceMasks(withoutWhitelistedRules(props.rules, props.whitelist)), docIds, props.preview.entityTypes,
|
||||
);
|
||||
output = Buffer.from(result.file_base64, "base64");
|
||||
} else {
|
||||
output = Buffer.from(replaceSpans(textFromBytes(data), draft.hits, accepted), "utf8");
|
||||
output = Buffer.from(replaceSpans(textFromBytes(data), withPersonMerges(draft.hits, activeMerges), accepted), "utf8");
|
||||
}
|
||||
await fs.writeFile(target, output, { mode: 0o600 });
|
||||
run.files.push({ source: draft.file.relPath, output: relPath, kind: draft.file.kind });
|
||||
const file: RunFile = { source: draft.file.relPath, output: relPath, kind: draft.file.kind };
|
||||
run.files.push(file);
|
||||
replacements.push(file);
|
||||
}
|
||||
await saveRun(directory, run);
|
||||
persisted = true;
|
||||
if (inPlace) {
|
||||
await replaceProjectFilesInPlace(props.root, directory, run.files);
|
||||
await replaceProjectFilesInPlace(props.root, directory, replacements);
|
||||
run.inPlace = true;
|
||||
await saveRun(directory, run);
|
||||
}
|
||||
const selectedTerms = new Set(groups.filter((group) => selectedGroups.has(group.key)).map(whitelistKey));
|
||||
const rejected = groups.filter((group) => !selectedGroups.has(group.key)
|
||||
&& !selectedTerms.has(whitelistKey(group)))
|
||||
.map((group) => ({ type: group.type, original: group.original }));
|
||||
if (rejected.length) {
|
||||
const nextWhitelist = [...await loadProjectWhitelist(props.root), ...rejected];
|
||||
await saveProjectWhitelist(props.root, nextWhitelist);
|
||||
props.onWhitelistChanged(await loadProjectWhitelist(props.root));
|
||||
}
|
||||
toast.style = Toast.Style.Success;
|
||||
toast.title = inPlace ? `${run.files.length} Originaldateien ersetzt` : `${run.files.length} Kopien erstellt`;
|
||||
toast.title = inPlace ? `${replacements.length} Dateien ersetzt` : `${run.files.length} Kopien erstellt`;
|
||||
await showInFinder(inPlace ? props.root : join(directory, "files")).catch(() => undefined);
|
||||
} catch (error) {
|
||||
if (directory && !persisted) await fs.rm(directory, { recursive: true, force: true });
|
||||
@@ -254,15 +330,53 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
|
||||
} finally { setBusy(false); }
|
||||
}
|
||||
|
||||
if (stage === "merge") return <List isLoading={busy} navigationTitle="Personen zusammenfassen"
|
||||
searchBarPlaceholder="Person oder Platzhalter suchen">
|
||||
<List.Section title="Abschluss">
|
||||
<List.Item icon={Icon.Play} title="Pseudonymisierung starten"
|
||||
subtitle={`${selectedGroups.size} Zuordnungen · ${people.length} Personen · ${props.preview.retained.length} unveränderte Dateien übernommen`}
|
||||
actions={<ActionPanel>
|
||||
<Action title="Pseudonymisierte Kopien erstellen" icon={Icon.Document} onAction={() => apply()} />
|
||||
<Action title="Originaldateien ersetzen" icon={Icon.Pencil} onAction={() => apply(true)} />
|
||||
<Action title="Zur Trefferauswahl" icon={Icon.ArrowLeft} onAction={() => setStage("selection")} />
|
||||
</ActionPanel>} />
|
||||
</List.Section>
|
||||
<List.Section title="Ausgewählte Personen · optional zusammenfassen">
|
||||
{people.map((person) => {
|
||||
const canonical = canonicalPerson(person.placeholder, personMerges);
|
||||
const targets = [...people, ...earlierPeople]
|
||||
.filter((other) => canonicalPerson(other.placeholder, personMerges) !== canonical);
|
||||
return <List.Item key={person.placeholder} icon={Icon.Person}
|
||||
title={person.originals.join(" · ")}
|
||||
subtitle={`→ ${canonical}`}
|
||||
accessories={canonical !== person.placeholder ? [{ text: "Zusammengeführt" }] : []}
|
||||
actions={<ActionPanel>
|
||||
{targets.length > 0 && <ActionPanel.Submenu title="Mit Person zusammenfassen" icon={Icon.Link}>
|
||||
{targets.map((target) => <Action key={target.placeholder}
|
||||
title={`${target.originals.join(" · ")} → ${canonicalPerson(target.placeholder, personMerges)}`}
|
||||
onAction={() => setPersonMerges((current) => mergePeople(current,
|
||||
person.placeholder, target.placeholder))} />)}
|
||||
</ActionPanel.Submenu>}
|
||||
{canonical !== person.placeholder && <Action title="Zusammenführung lösen"
|
||||
onAction={() => setPersonMerges((current) => {
|
||||
const next = { ...current }; delete next[person.placeholder]; return next;
|
||||
})} />}
|
||||
<Action title="Pseudonymisierte Kopien erstellen" icon={Icon.Document} onAction={() => apply()} />
|
||||
<Action title="Originaldateien ersetzen" icon={Icon.Pencil} onAction={() => apply(true)} />
|
||||
</ActionPanel>} />;
|
||||
})}
|
||||
</List.Section>
|
||||
</List>;
|
||||
|
||||
return <List isLoading={busy} navigationTitle="Zuordnungen prüfen"
|
||||
searchBarPlaceholder="Begriff oder Platzhalter suchen">
|
||||
<List.Section title="Abschluss">
|
||||
<List.Item key="apply" icon={Icon.Play}
|
||||
title={`${selectedGroups.size} von ${groups.length} Zuordnungen ausgewählt`}
|
||||
subtitle={`${selectedOccurrences} Vorkommen · ${props.preview.drafts.length} Dateien · ${props.preview.skipped} übersprungen`}
|
||||
subtitle={`${selectedOccurrences} Vorkommen · ${props.preview.drafts.length} neue/geänderte Dateien · ${props.preview.retained.length} unverändert · ${props.preview.skipped} übersprungen`}
|
||||
actions={<ActionPanel>
|
||||
<Action title="Pseudonymisierte Kopien erstellen" icon={Icon.Document} onAction={() => apply()} />
|
||||
<Action title="Originaldateien ersetzen" icon={Icon.Pencil} onAction={() => apply(true)} />
|
||||
<Action title="Auswahl abschließen und Personen prüfen" icon={Icon.ArrowRight}
|
||||
onAction={() => setStage("merge")} />
|
||||
</ActionPanel>} />
|
||||
</List.Section>
|
||||
{[...byType].map(([type, entries]) => <List.Section key={type}
|
||||
@@ -286,19 +400,32 @@ function HitReview(props: { root: string; rules: ProjectRule[]; preview: Preview
|
||||
onAction={() => setSelectedGroups((current) => new Set([...current, ...entries.map((entry) => entry.key)]))} />
|
||||
<Action.Push title="Vorkommen ansehen" icon={Icon.Eye}
|
||||
target={<OccurrenceReview original={group.original} placeholder={group.placeholder} hits={group.hits} />} />
|
||||
<Action title="Pseudonymisierte Kopien erstellen" icon={Icon.Document} onAction={() => apply()} />
|
||||
<Action title="Originaldateien ersetzen" icon={Icon.Pencil} onAction={() => apply(true)} />
|
||||
<Action title="Auswahl abschließen und Personen prüfen" icon={Icon.ArrowRight}
|
||||
onAction={() => setStage("merge")} />
|
||||
</ActionPanel>} />)}
|
||||
</List.Section>)}
|
||||
</List>;
|
||||
}
|
||||
|
||||
function ProjectWhitelist(props: { entries: ProjectWhitelistEntry[]; onRemove: (entry: ProjectWhitelistEntry) => Promise<void> }) {
|
||||
return <List navigationTitle="Projekt-Whitelist" searchBarPlaceholder="Ausnahme suchen">
|
||||
<List.Section title={`${props.entries.length} nicht mehr vorgeschlagene Begriffe`}>
|
||||
{props.entries.map((entry) => <List.Item key={whitelistKey(entry)} title={entry.original}
|
||||
subtitle={labelForType(entry.type)} actions={<ActionPanel>
|
||||
<Action title="Aus Projekt-Whitelist entfernen" icon={Icon.Trash}
|
||||
onAction={() => props.onRemove(entry)} />
|
||||
</ActionPanel>} />)}
|
||||
</List.Section>
|
||||
</List>;
|
||||
}
|
||||
|
||||
export default function Command() {
|
||||
const { push } = useNavigation();
|
||||
const [root, setRoot] = useState("");
|
||||
const [rules, setRules] = useState("");
|
||||
const [availableTypes, setAvailableTypes] = useState<EntityType[]>([]);
|
||||
const [selectedTypes, setSelectedTypes] = useState<EntityType[]>([]);
|
||||
const [whitelist, setWhitelist] = useState<ProjectWhitelistEntry[]>([]);
|
||||
const [busy, setBusy] = useState(false);
|
||||
useEffect(() => { finderFolder().then((path) => { if (path) setRoot((current) => current || path); }); }, []);
|
||||
useEffect(() => {
|
||||
@@ -316,6 +443,25 @@ export default function Command() {
|
||||
}).catch(() => { if (active) setRules(""); });
|
||||
return () => { active = false; };
|
||||
}, [root]);
|
||||
useEffect(() => {
|
||||
if (!root) { setWhitelist([]); return; }
|
||||
let active = true;
|
||||
loadProjectWhitelist(root).then((found) => { if (active) setWhitelist(found); })
|
||||
.catch((error: Error) => showToast({ style: Toast.Style.Failure,
|
||||
title: "Projekt-Whitelist nicht lesbar", message: error.message }));
|
||||
return () => { active = false; };
|
||||
}, [root]);
|
||||
async function removeFromWhitelist(entry: ProjectWhitelistEntry) {
|
||||
try {
|
||||
const next = (await loadProjectWhitelist(root)).filter((item) => whitelistKey(item) !== whitelistKey(entry));
|
||||
await saveProjectWhitelist(root, next);
|
||||
setWhitelist(next);
|
||||
await showToast({ style: Toast.Style.Success, title: "Begriff wird wieder vorgeschlagen" });
|
||||
} catch (error) {
|
||||
await showToast({ style: Toast.Style.Failure, title: "Entfernen fehlgeschlagen",
|
||||
message: (error as Error).message });
|
||||
}
|
||||
}
|
||||
async function next() {
|
||||
setBusy(true);
|
||||
const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" });
|
||||
@@ -327,11 +473,13 @@ export default function Command() {
|
||||
toast.title = "Dateien werden erfasst";
|
||||
const scan = await scanProjectFiles(root);
|
||||
if (!scan.files.length) throw new Error("Keine geeigneten Dateien im Projektordner gefunden.");
|
||||
const preview = await prepare(root, scan.files, parsed, selectedTypes, scan.skipped.length,
|
||||
const currentWhitelist = await loadProjectWhitelist(root);
|
||||
const preview = await prepare(root, scan.files, parsed, currentWhitelist, selectedTypes, scan.skipped.length,
|
||||
(message) => { toast.title = message; });
|
||||
toast.style = Toast.Style.Success;
|
||||
toast.title = `${groupReviewHits(preview.hits).length} Zuordnungen zur Prüfung`;
|
||||
push(<HitReview root={root} rules={parsed} preview={preview} />);
|
||||
push(<HitReview root={root} rules={parsed} whitelist={currentWhitelist} preview={preview}
|
||||
onWhitelistChanged={setWhitelist} />);
|
||||
} catch (error) {
|
||||
toast.style = Toast.Style.Failure;
|
||||
toast.title = "Analyse fehlgeschlagen";
|
||||
@@ -347,6 +495,8 @@ export default function Command() {
|
||||
return <Form isLoading={busy} actions={<ActionPanel>
|
||||
<Action title="Zuordnungen prüfen" onAction={next} />
|
||||
<Action title="Geöffneten Finder-Ordner übernehmen" icon={Icon.Folder} onAction={useFinderFolder} />
|
||||
<Action.Push title="Projekt-Whitelist bearbeiten" icon={Icon.List}
|
||||
target={<ProjectWhitelist entries={whitelist} onRemove={removeFromWhitelist} />} />
|
||||
</ActionPanel>}>
|
||||
<Form.FilePicker id="projectPath" title="Projektordner" allowMultipleSelection={false}
|
||||
canChooseDirectories canChooseFiles={false} value={root ? [root] : []}
|
||||
@@ -355,9 +505,10 @@ export default function Command() {
|
||||
onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"}
|
||||
info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." />
|
||||
<Form.TagPicker id="entityTypes" title="Kategorien pseudonymisieren" value={selectedTypes}
|
||||
onChange={setSelectedTypes} info="Alle geeigneten Dateien werden analysiert. Orte sind wegen häufiger Fehlalarme zunächst abgewählt; alle gefundenen Zuordnungen können anschließend einzeln abgewählt werden.">
|
||||
onChange={setSelectedTypes} info="Orte sind wegen häufiger Fehlalarme zunächst abgewählt. Abgewählte Treffer kommen nach dem Erstellen in die Projekt-Whitelist.">
|
||||
{availableTypes.map((type) => <Form.TagPicker.Item key={type} value={type} title={labelForType(type)} />)}
|
||||
</Form.TagPicker>
|
||||
<Form.Description title="Ablauf" text="Alle geeigneten Dateien erfassen, Zuordnungen je Kategorie prüfen und dann Kopien erstellen oder Originale ersetzen." />
|
||||
<Form.Description title="Projekt-Whitelist" text={`${whitelist.length} Begriffe werden bei diesem Ordner nicht mehr vorgeschlagen. Über die Aktionen kannst du sie wieder freigeben.`} />
|
||||
<Form.Description title="Ablauf" text="Kategorien wählen, Zuordnungen auswählen, Personen bei Bedarf zusammenfassen und dann Kopien erstellen oder Originale ersetzen." />
|
||||
</Form>;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user