fix: keep project entity selection and ignore placeholder hits
This commit is contained in:
@@ -88,6 +88,12 @@ export function withoutWhitelistedHits<T extends GroupableHit>(hits: T[], whitel
|
||||
return hits.filter((hit) => !excluded.has(whitelistKey(hit)));
|
||||
}
|
||||
|
||||
/** A later run must not turn an existing Velum placeholder plus adjacent text into a new entity. */
|
||||
export function withoutPlaceholderHits<T extends { original: string; type: string }>(hits: T[]): T[] {
|
||||
const placeholder = /(?<![\p{L}\p{N}])[A-Z][A-Z0-9]*_\d{2,}(?![\p{L}\p{N}])/u;
|
||||
return hits.filter((hit) => hit.type === "PASSWORT" || !placeholder.test(hit.original));
|
||||
}
|
||||
|
||||
export function withoutWhitelistedRules<T extends { type: string; value: string }>(
|
||||
rules: T[], whitelist: ProjectWhitelistEntry[],
|
||||
): T[] {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto";
|
||||
import { constants as fsConstants, promises as fs } from "node:fs";
|
||||
import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
|
||||
import { environment } from "@raycast/api";
|
||||
import type { PlaceholderMapping } from "../types";
|
||||
import type { EntityType, PlaceholderMapping } from "../types";
|
||||
import { whitelistKey } from "./project-whitelist";
|
||||
import type { ProjectWhitelistEntry } from "./project-whitelist";
|
||||
|
||||
@@ -68,6 +68,36 @@ export function whitelistPath(projectPath: string, support = environment.support
|
||||
return join(projectDataRoot(projectPath, support), "whitelist.json");
|
||||
}
|
||||
|
||||
export function entityTypesPath(projectPath: string, support = environment.supportPath): string {
|
||||
return join(projectDataRoot(projectPath, support), "entity-types.json");
|
||||
}
|
||||
|
||||
export async function loadProjectEntityTypes(projectPath: string, support?: string): Promise<EntityType[] | null> {
|
||||
try {
|
||||
const value = JSON.parse(await fs.readFile(entityTypesPath(projectPath, support), "utf8")) as unknown;
|
||||
if (!Array.isArray(value) || value.some((type) => typeof type !== "string" || !type)) {
|
||||
throw new Error("Ungültige Kategorienauswahl für dieses Projekt.");
|
||||
}
|
||||
return [...new Set(value)];
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveProjectEntityTypes(projectPath: string, types: EntityType[], support?: string): Promise<void> {
|
||||
const path = entityTypesPath(projectPath, support);
|
||||
await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
||||
const temp = `${path}.${randomUUID()}.tmp`;
|
||||
try {
|
||||
await fs.writeFile(temp, JSON.stringify([...new Set(types)]), { encoding: "utf8", mode: 0o600 });
|
||||
await fs.rename(temp, path);
|
||||
} catch (error) {
|
||||
await fs.rm(temp, { force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadProjectWhitelist(projectPath: string, support?: string): Promise<ProjectWhitelistEntry[]> {
|
||||
try {
|
||||
const value = JSON.parse(await fs.readFile(whitelistPath(projectPath, support), "utf8")) as unknown;
|
||||
|
||||
@@ -8,14 +8,14 @@ import type { EntityType, PlaceholderMapping } from "./types";
|
||||
import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
|
||||
import type { DocumentHit } from "./velum";
|
||||
import {
|
||||
createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans,
|
||||
createRun, digest, forceMasks, listRuns, loadProjectEntityTypes, loadProjectRules, parseProjectRules, replaceSpans,
|
||||
loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules,
|
||||
saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey,
|
||||
saveProjectEntityTypes, saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey,
|
||||
} from "./lib/velum-project";
|
||||
import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project";
|
||||
import {
|
||||
canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment,
|
||||
withoutWhitelistedHits, withoutWhitelistedRules,
|
||||
withoutPlaceholderHits, withoutWhitelistedHits, withoutWhitelistedRules,
|
||||
} from "./lib/project-review";
|
||||
|
||||
const run = promisify(execFile);
|
||||
@@ -130,7 +130,7 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
||||
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
|
||||
}
|
||||
const addHits = (draft: Draft, found: Hit[]) => {
|
||||
draft.hits = withoutWhitelistedHits(found, whitelist);
|
||||
draft.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist);
|
||||
hits.push(...draft.hits);
|
||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||
};
|
||||
@@ -169,11 +169,12 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
||||
const spans = spansBySegment(batch, result.spans);
|
||||
for (const segment of batch.segments) {
|
||||
const path = byPath.get(segment.key)!;
|
||||
path.hits = withoutWhitelistedHits(toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({
|
||||
const found = toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({
|
||||
...span, id: `${path.key}:path:${hitIndex}`, file: path.key,
|
||||
source: "Datei- oder Ordnername", kind: "path" as const,
|
||||
context: path.value,
|
||||
})), whitelist);
|
||||
}));
|
||||
path.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist);
|
||||
hits.push(...path.hits);
|
||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||
}
|
||||
@@ -454,6 +455,8 @@ export default function Command() {
|
||||
const [rules, setRules] = useState("");
|
||||
const [availableTypes, setAvailableTypes] = useState<EntityType[]>([]);
|
||||
const [selectedTypes, setSelectedTypes] = useState<EntityType[]>([]);
|
||||
const [typesLoaded, setTypesLoaded] = useState(false);
|
||||
const [typesReadyFor, setTypesReadyFor] = useState<string | null>(null);
|
||||
const [whitelist, setWhitelist] = useState<ProjectWhitelistEntry[]>([]);
|
||||
const [existingRuns, setExistingRuns] = useState<number | null>(null);
|
||||
const [busy, setBusy] = useState(false);
|
||||
@@ -461,10 +464,24 @@ export default function Command() {
|
||||
useEffect(() => {
|
||||
getEntityTypes().then((types) => {
|
||||
setAvailableTypes(types);
|
||||
setSelectedTypes(types.filter((type) => type !== "ORT"));
|
||||
setTypesLoaded(true);
|
||||
}).catch((error: Error) => showToast({ style: Toast.Style.Failure,
|
||||
title: "Kategorien nicht verfügbar", message: error.message }));
|
||||
}, []);
|
||||
useEffect(() => {
|
||||
if (!typesLoaded) return;
|
||||
const defaults = availableTypes.filter((type) => type !== "ORT");
|
||||
if (!root) { setSelectedTypes(defaults); setTypesReadyFor(""); return; }
|
||||
let active = true;
|
||||
setTypesReadyFor(null);
|
||||
loadProjectEntityTypes(root).then((stored) => {
|
||||
if (!active) return;
|
||||
setSelectedTypes(stored === null ? defaults : stored.filter((type) => availableTypes.includes(type)));
|
||||
setTypesReadyFor(root);
|
||||
}).catch((error: Error) => showToast({ style: Toast.Style.Failure,
|
||||
title: "Projektkategorien nicht lesbar", message: error.message }));
|
||||
return () => { active = false; };
|
||||
}, [root, availableTypes, typesLoaded]);
|
||||
useEffect(() => {
|
||||
if (!root) return;
|
||||
let active = true;
|
||||
@@ -506,8 +523,10 @@ export default function Command() {
|
||||
const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" });
|
||||
try {
|
||||
if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen.");
|
||||
if (typesReadyFor !== root) throw new Error("Projektkategorien werden noch geladen oder sind nicht lesbar.");
|
||||
if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen.");
|
||||
const parsed = parseProjectRules(rules);
|
||||
await saveProjectEntityTypes(root, selectedTypes);
|
||||
await saveProjectRules(root, parsed);
|
||||
toast.title = "Dateien werden erfasst";
|
||||
const scan = await scanProjectFiles(root);
|
||||
@@ -547,7 +566,7 @@ export default function Command() {
|
||||
onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"}
|
||||
info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." />
|
||||
<Form.TagPicker id="entityTypes" title="Kategorien pseudonymisieren" value={selectedTypes}
|
||||
onChange={setSelectedTypes} info="Orte sind wegen häufiger Fehlalarme zunächst abgewählt. Abgewählte Treffer kommen nach dem Erstellen in die Projekt-Whitelist.">
|
||||
onChange={setSelectedTypes} info="Die Auswahl wird beim Prüfen für diesen Projektordner gespeichert. Bei neuen Projekten sind Orte wegen häufiger Fehlalarme zunächst abgewählt.">
|
||||
{availableTypes.map((type) => <Form.TagPicker.Item key={type} value={type} title={labelForType(type)} />)}
|
||||
</Form.TagPicker>
|
||||
<Form.Description title="Projekt-Whitelist" text={`${whitelist.length} Begriffe werden bei diesem Ordner nicht mehr vorgeschlagen. Über die Aktionen kannst du sie wieder freigeben.`} />
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { expect, it } from "bun:test";
|
||||
import {
|
||||
canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople,
|
||||
spansBySegment, withoutWhitelistedHits, withoutWhitelistedRules,
|
||||
spansBySegment, withoutPlaceholderHits, withoutWhitelistedHits, withoutWhitelistedRules,
|
||||
} from "../src/lib/project-review";
|
||||
|
||||
it("liest Unterstriche in Namen als Leerzeichen und bewahrt bekannte Codebegriffe", () => {
|
||||
@@ -58,6 +58,17 @@ it("unterdrückt projektweit nur exakt abgewählte Begriffe samt erzwungener Reg
|
||||
], whitelist)).toEqual([{ type: "ORG", value: "Max Muster" }]);
|
||||
});
|
||||
|
||||
it("zeigt beim erneuten Lauf keine Treffer, die einen bestehenden Platzhalter einschließen", () => {
|
||||
const hits = [
|
||||
{ type: "PERSON", original: "PERSON_39, Roman" },
|
||||
{ type: "KUNDE", original: "KUNDE_100005" },
|
||||
{ type: "PERSON", original: "Roman" },
|
||||
{ type: "PERSON", original: "Anlage44" },
|
||||
{ type: "PASSWORT", original: "ORT_08!" },
|
||||
];
|
||||
expect(withoutPlaceholderHits(hits)).toEqual(hits.slice(2));
|
||||
});
|
||||
|
||||
it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => {
|
||||
let merges = mergePeople({}, "PERSON_02", "PERSON_01");
|
||||
merges = mergePeople(merges, "PERSON_03", "PERSON_02");
|
||||
|
||||
@@ -53,6 +53,20 @@ describe("Velum-Projektstände", () => {
|
||||
.toEqual([first.run.id, second.run.id].sort());
|
||||
});
|
||||
|
||||
it("behält die Kategorieauswahl pro Projekt für den nächsten Lauf", async () => {
|
||||
const root = await fs.mkdtemp(join(tmpdir(), "velum-entity-types-test-"));
|
||||
roots.push(root);
|
||||
const source = join(root, "Quelle");
|
||||
const other = join(root, "Anderes Projekt");
|
||||
const support = join(root, "Raycast");
|
||||
await fs.mkdir(source);
|
||||
await fs.mkdir(other);
|
||||
expect(await project.loadProjectEntityTypes(source, support)).toBeNull();
|
||||
await project.saveProjectEntityTypes(`${source}/`, ["PERSON", "ORG", "PERSON"], support);
|
||||
expect(await project.loadProjectEntityTypes(source, support)).toEqual(["PERSON", "ORG"]);
|
||||
expect(await project.loadProjectEntityTypes(other, support)).toBeNull();
|
||||
});
|
||||
|
||||
it("ersetzt nur bestätigte Vorkommen und stellt Dateinamen wieder her", () => {
|
||||
const hits = [
|
||||
{ id: "a", start: 0, end: 3, placeholder: "PERSON_01" },
|
||||
|
||||
Reference in New Issue
Block a user