fix: keep project entity selection and ignore placeholder hits

This commit is contained in:
muena
2026-09-26 15:09:53 +02:00
parent 5877b7cf5f
commit 6cb5b0efc5
5 changed files with 90 additions and 10 deletions

View File

@@ -88,6 +88,12 @@ export function withoutWhitelistedHits<T extends GroupableHit>(hits: T[], whitel
return hits.filter((hit) => !excluded.has(whitelistKey(hit))); return hits.filter((hit) => !excluded.has(whitelistKey(hit)));
} }
/** A later run must not turn an existing Velum placeholder plus adjacent text into a new entity. */
export function withoutPlaceholderHits<T extends { original: string; type: string }>(hits: T[]): T[] {
const placeholder = /(?<![\p{L}\p{N}])[A-Z][A-Z0-9]*_\d{2,}(?![\p{L}\p{N}])/u;
return hits.filter((hit) => hit.type === "PASSWORT" || !placeholder.test(hit.original));
}
export function withoutWhitelistedRules<T extends { type: string; value: string }>( export function withoutWhitelistedRules<T extends { type: string; value: string }>(
rules: T[], whitelist: ProjectWhitelistEntry[], rules: T[], whitelist: ProjectWhitelistEntry[],
): T[] { ): T[] {

View File

@@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto";
import { constants as fsConstants, promises as fs } from "node:fs"; import { constants as fsConstants, promises as fs } from "node:fs";
import { basename, dirname, extname, join, relative, resolve, sep } from "node:path"; import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
import { environment } from "@raycast/api"; import { environment } from "@raycast/api";
import type { PlaceholderMapping } from "../types"; import type { EntityType, PlaceholderMapping } from "../types";
import { whitelistKey } from "./project-whitelist"; import { whitelistKey } from "./project-whitelist";
import type { ProjectWhitelistEntry } from "./project-whitelist"; import type { ProjectWhitelistEntry } from "./project-whitelist";
@@ -68,6 +68,36 @@ export function whitelistPath(projectPath: string, support = environment.support
return join(projectDataRoot(projectPath, support), "whitelist.json"); return join(projectDataRoot(projectPath, support), "whitelist.json");
} }
export function entityTypesPath(projectPath: string, support = environment.supportPath): string {
return join(projectDataRoot(projectPath, support), "entity-types.json");
}
export async function loadProjectEntityTypes(projectPath: string, support?: string): Promise<EntityType[] | null> {
try {
const value = JSON.parse(await fs.readFile(entityTypesPath(projectPath, support), "utf8")) as unknown;
if (!Array.isArray(value) || value.some((type) => typeof type !== "string" || !type)) {
throw new Error("Ungültige Kategorienauswahl für dieses Projekt.");
}
return [...new Set(value)];
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
throw error;
}
}
export async function saveProjectEntityTypes(projectPath: string, types: EntityType[], support?: string): Promise<void> {
const path = entityTypesPath(projectPath, support);
await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 });
const temp = `${path}.${randomUUID()}.tmp`;
try {
await fs.writeFile(temp, JSON.stringify([...new Set(types)]), { encoding: "utf8", mode: 0o600 });
await fs.rename(temp, path);
} catch (error) {
await fs.rm(temp, { force: true });
throw error;
}
}
export async function loadProjectWhitelist(projectPath: string, support?: string): Promise<ProjectWhitelistEntry[]> { export async function loadProjectWhitelist(projectPath: string, support?: string): Promise<ProjectWhitelistEntry[]> {
try { try {
const value = JSON.parse(await fs.readFile(whitelistPath(projectPath, support), "utf8")) as unknown; const value = JSON.parse(await fs.readFile(whitelistPath(projectPath, support), "utf8")) as unknown;

View File

@@ -8,14 +8,14 @@ import type { EntityType, PlaceholderMapping } from "./types";
import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum"; import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
import type { DocumentHit } from "./velum"; import type { DocumentHit } from "./velum";
import { import {
createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans, createRun, digest, forceMasks, listRuns, loadProjectEntityTypes, loadProjectRules, parseProjectRules, replaceSpans,
loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules,
saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey, saveProjectEntityTypes, saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey,
} from "./lib/velum-project"; } from "./lib/velum-project";
import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project"; import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project";
import { import {
canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment, canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment,
withoutWhitelistedHits, withoutWhitelistedRules, withoutPlaceholderHits, withoutWhitelistedHits, withoutWhitelistedRules,
} from "./lib/project-review"; } from "./lib/project-review";
const run = promisify(execFile); const run = promisify(execFile);
@@ -130,7 +130,7 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data)); if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
} }
const addHits = (draft: Draft, found: Hit[]) => { const addHits = (draft: Draft, found: Hit[]) => {
draft.hits = withoutWhitelistedHits(found, whitelist); draft.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist);
hits.push(...draft.hits); hits.push(...draft.hits);
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
}; };
@@ -169,11 +169,12 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
const spans = spansBySegment(batch, result.spans); const spans = spansBySegment(batch, result.spans);
for (const segment of batch.segments) { for (const segment of batch.segments) {
const path = byPath.get(segment.key)!; const path = byPath.get(segment.key)!;
path.hits = withoutWhitelistedHits(toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({ const found = toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({
...span, id: `${path.key}:path:${hitIndex}`, file: path.key, ...span, id: `${path.key}:path:${hitIndex}`, file: path.key,
source: "Datei- oder Ordnername", kind: "path" as const, source: "Datei- oder Ordnername", kind: "path" as const,
context: path.value, context: path.value,
})), whitelist); }));
path.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist);
hits.push(...path.hits); hits.push(...path.hits);
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
} }
@@ -454,6 +455,8 @@ export default function Command() {
const [rules, setRules] = useState(""); const [rules, setRules] = useState("");
const [availableTypes, setAvailableTypes] = useState<EntityType[]>([]); const [availableTypes, setAvailableTypes] = useState<EntityType[]>([]);
const [selectedTypes, setSelectedTypes] = useState<EntityType[]>([]); const [selectedTypes, setSelectedTypes] = useState<EntityType[]>([]);
const [typesLoaded, setTypesLoaded] = useState(false);
const [typesReadyFor, setTypesReadyFor] = useState<string | null>(null);
const [whitelist, setWhitelist] = useState<ProjectWhitelistEntry[]>([]); const [whitelist, setWhitelist] = useState<ProjectWhitelistEntry[]>([]);
const [existingRuns, setExistingRuns] = useState<number | null>(null); const [existingRuns, setExistingRuns] = useState<number | null>(null);
const [busy, setBusy] = useState(false); const [busy, setBusy] = useState(false);
@@ -461,10 +464,24 @@ export default function Command() {
useEffect(() => { useEffect(() => {
getEntityTypes().then((types) => { getEntityTypes().then((types) => {
setAvailableTypes(types); setAvailableTypes(types);
setSelectedTypes(types.filter((type) => type !== "ORT")); setTypesLoaded(true);
}).catch((error: Error) => showToast({ style: Toast.Style.Failure, }).catch((error: Error) => showToast({ style: Toast.Style.Failure,
title: "Kategorien nicht verfügbar", message: error.message })); title: "Kategorien nicht verfügbar", message: error.message }));
}, []); }, []);
useEffect(() => {
if (!typesLoaded) return;
const defaults = availableTypes.filter((type) => type !== "ORT");
if (!root) { setSelectedTypes(defaults); setTypesReadyFor(""); return; }
let active = true;
setTypesReadyFor(null);
loadProjectEntityTypes(root).then((stored) => {
if (!active) return;
setSelectedTypes(stored === null ? defaults : stored.filter((type) => availableTypes.includes(type)));
setTypesReadyFor(root);
}).catch((error: Error) => showToast({ style: Toast.Style.Failure,
title: "Projektkategorien nicht lesbar", message: error.message }));
return () => { active = false; };
}, [root, availableTypes, typesLoaded]);
useEffect(() => { useEffect(() => {
if (!root) return; if (!root) return;
let active = true; let active = true;
@@ -506,8 +523,10 @@ export default function Command() {
const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" }); const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" });
try { try {
if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen."); if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen.");
if (typesReadyFor !== root) throw new Error("Projektkategorien werden noch geladen oder sind nicht lesbar.");
if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen."); if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen.");
const parsed = parseProjectRules(rules); const parsed = parseProjectRules(rules);
await saveProjectEntityTypes(root, selectedTypes);
await saveProjectRules(root, parsed); await saveProjectRules(root, parsed);
toast.title = "Dateien werden erfasst"; toast.title = "Dateien werden erfasst";
const scan = await scanProjectFiles(root); const scan = await scanProjectFiles(root);
@@ -547,7 +566,7 @@ export default function Command() {
onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"} onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"}
info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." /> info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." />
<Form.TagPicker id="entityTypes" title="Kategorien pseudonymisieren" value={selectedTypes} <Form.TagPicker id="entityTypes" title="Kategorien pseudonymisieren" value={selectedTypes}
onChange={setSelectedTypes} info="Orte sind wegen häufiger Fehlalarme zunächst abgewählt. Abgewählte Treffer kommen nach dem Erstellen in die Projekt-Whitelist."> onChange={setSelectedTypes} info="Die Auswahl wird beim Prüfen für diesen Projektordner gespeichert. Bei neuen Projekten sind Orte wegen häufiger Fehlalarme zunächst abgewählt.">
{availableTypes.map((type) => <Form.TagPicker.Item key={type} value={type} title={labelForType(type)} />)} {availableTypes.map((type) => <Form.TagPicker.Item key={type} value={type} title={labelForType(type)} />)}
</Form.TagPicker> </Form.TagPicker>
<Form.Description title="Projekt-Whitelist" text={`${whitelist.length} Begriffe werden bei diesem Ordner nicht mehr vorgeschlagen. Über die Aktionen kannst du sie wieder freigeben.`} /> <Form.Description title="Projekt-Whitelist" text={`${whitelist.length} Begriffe werden bei diesem Ordner nicht mehr vorgeschlagen. Über die Aktionen kannst du sie wieder freigeben.`} />

View File

@@ -1,7 +1,7 @@
import { expect, it } from "bun:test"; import { expect, it } from "bun:test";
import { import {
canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople,
spansBySegment, withoutWhitelistedHits, withoutWhitelistedRules, spansBySegment, withoutPlaceholderHits, withoutWhitelistedHits, withoutWhitelistedRules,
} from "../src/lib/project-review"; } from "../src/lib/project-review";
it("liest Unterstriche in Namen als Leerzeichen und bewahrt bekannte Codebegriffe", () => { it("liest Unterstriche in Namen als Leerzeichen und bewahrt bekannte Codebegriffe", () => {
@@ -58,6 +58,17 @@ it("unterdrückt projektweit nur exakt abgewählte Begriffe samt erzwungener Reg
], whitelist)).toEqual([{ type: "ORG", value: "Max Muster" }]); ], whitelist)).toEqual([{ type: "ORG", value: "Max Muster" }]);
}); });
it("zeigt beim erneuten Lauf keine Treffer, die einen bestehenden Platzhalter einschließen", () => {
const hits = [
{ type: "PERSON", original: "PERSON_39, Roman" },
{ type: "KUNDE", original: "KUNDE_100005" },
{ type: "PERSON", original: "Roman" },
{ type: "PERSON", original: "Anlage44" },
{ type: "PASSWORT", original: "ORT_08!" },
];
expect(withoutPlaceholderHits(hits)).toEqual(hits.slice(2));
});
it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => { it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => {
let merges = mergePeople({}, "PERSON_02", "PERSON_01"); let merges = mergePeople({}, "PERSON_02", "PERSON_01");
merges = mergePeople(merges, "PERSON_03", "PERSON_02"); merges = mergePeople(merges, "PERSON_03", "PERSON_02");

View File

@@ -53,6 +53,20 @@ describe("Velum-Projektstände", () => {
.toEqual([first.run.id, second.run.id].sort()); .toEqual([first.run.id, second.run.id].sort());
}); });
it("behält die Kategorieauswahl pro Projekt für den nächsten Lauf", async () => {
const root = await fs.mkdtemp(join(tmpdir(), "velum-entity-types-test-"));
roots.push(root);
const source = join(root, "Quelle");
const other = join(root, "Anderes Projekt");
const support = join(root, "Raycast");
await fs.mkdir(source);
await fs.mkdir(other);
expect(await project.loadProjectEntityTypes(source, support)).toBeNull();
await project.saveProjectEntityTypes(`${source}/`, ["PERSON", "ORG", "PERSON"], support);
expect(await project.loadProjectEntityTypes(source, support)).toEqual(["PERSON", "ORG"]);
expect(await project.loadProjectEntityTypes(other, support)).toBeNull();
});
it("ersetzt nur bestätigte Vorkommen und stellt Dateinamen wieder her", () => { it("ersetzt nur bestätigte Vorkommen und stellt Dateinamen wieder her", () => {
const hits = [ const hits = [
{ id: "a", start: 0, end: 3, placeholder: "PERSON_01" }, { id: "a", start: 0, end: 3, placeholder: "PERSON_01" },