fix: keep project entity selection and ignore placeholder hits
This commit is contained in:
@@ -88,6 +88,12 @@ export function withoutWhitelistedHits<T extends GroupableHit>(hits: T[], whitel
|
|||||||
return hits.filter((hit) => !excluded.has(whitelistKey(hit)));
|
return hits.filter((hit) => !excluded.has(whitelistKey(hit)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A later run must not turn an existing Velum placeholder plus adjacent text into a new entity. */
|
||||||
|
export function withoutPlaceholderHits<T extends { original: string; type: string }>(hits: T[]): T[] {
|
||||||
|
const placeholder = /(?<![\p{L}\p{N}])[A-Z][A-Z0-9]*_\d{2,}(?![\p{L}\p{N}])/u;
|
||||||
|
return hits.filter((hit) => hit.type === "PASSWORT" || !placeholder.test(hit.original));
|
||||||
|
}
|
||||||
|
|
||||||
export function withoutWhitelistedRules<T extends { type: string; value: string }>(
|
export function withoutWhitelistedRules<T extends { type: string; value: string }>(
|
||||||
rules: T[], whitelist: ProjectWhitelistEntry[],
|
rules: T[], whitelist: ProjectWhitelistEntry[],
|
||||||
): T[] {
|
): T[] {
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto";
|
|||||||
import { constants as fsConstants, promises as fs } from "node:fs";
|
import { constants as fsConstants, promises as fs } from "node:fs";
|
||||||
import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
|
import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
|
||||||
import { environment } from "@raycast/api";
|
import { environment } from "@raycast/api";
|
||||||
import type { PlaceholderMapping } from "../types";
|
import type { EntityType, PlaceholderMapping } from "../types";
|
||||||
import { whitelistKey } from "./project-whitelist";
|
import { whitelistKey } from "./project-whitelist";
|
||||||
import type { ProjectWhitelistEntry } from "./project-whitelist";
|
import type { ProjectWhitelistEntry } from "./project-whitelist";
|
||||||
|
|
||||||
@@ -68,6 +68,36 @@ export function whitelistPath(projectPath: string, support = environment.support
|
|||||||
return join(projectDataRoot(projectPath, support), "whitelist.json");
|
return join(projectDataRoot(projectPath, support), "whitelist.json");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function entityTypesPath(projectPath: string, support = environment.supportPath): string {
|
||||||
|
return join(projectDataRoot(projectPath, support), "entity-types.json");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadProjectEntityTypes(projectPath: string, support?: string): Promise<EntityType[] | null> {
|
||||||
|
try {
|
||||||
|
const value = JSON.parse(await fs.readFile(entityTypesPath(projectPath, support), "utf8")) as unknown;
|
||||||
|
if (!Array.isArray(value) || value.some((type) => typeof type !== "string" || !type)) {
|
||||||
|
throw new Error("Ungültige Kategorienauswahl für dieses Projekt.");
|
||||||
|
}
|
||||||
|
return [...new Set(value)];
|
||||||
|
} catch (error) {
|
||||||
|
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function saveProjectEntityTypes(projectPath: string, types: EntityType[], support?: string): Promise<void> {
|
||||||
|
const path = entityTypesPath(projectPath, support);
|
||||||
|
await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
||||||
|
const temp = `${path}.${randomUUID()}.tmp`;
|
||||||
|
try {
|
||||||
|
await fs.writeFile(temp, JSON.stringify([...new Set(types)]), { encoding: "utf8", mode: 0o600 });
|
||||||
|
await fs.rename(temp, path);
|
||||||
|
} catch (error) {
|
||||||
|
await fs.rm(temp, { force: true });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function loadProjectWhitelist(projectPath: string, support?: string): Promise<ProjectWhitelistEntry[]> {
|
export async function loadProjectWhitelist(projectPath: string, support?: string): Promise<ProjectWhitelistEntry[]> {
|
||||||
try {
|
try {
|
||||||
const value = JSON.parse(await fs.readFile(whitelistPath(projectPath, support), "utf8")) as unknown;
|
const value = JSON.parse(await fs.readFile(whitelistPath(projectPath, support), "utf8")) as unknown;
|
||||||
|
|||||||
@@ -8,14 +8,14 @@ import type { EntityType, PlaceholderMapping } from "./types";
|
|||||||
import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
|
import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum";
|
||||||
import type { DocumentHit } from "./velum";
|
import type { DocumentHit } from "./velum";
|
||||||
import {
|
import {
|
||||||
createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans,
|
createRun, digest, forceMasks, listRuns, loadProjectEntityTypes, loadProjectRules, parseProjectRules, replaceSpans,
|
||||||
loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules,
|
loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules,
|
||||||
saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey,
|
saveProjectEntityTypes, saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey,
|
||||||
} from "./lib/velum-project";
|
} from "./lib/velum-project";
|
||||||
import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project";
|
import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project";
|
||||||
import {
|
import {
|
||||||
canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment,
|
canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment,
|
||||||
withoutWhitelistedHits, withoutWhitelistedRules,
|
withoutPlaceholderHits, withoutWhitelistedHits, withoutWhitelistedRules,
|
||||||
} from "./lib/project-review";
|
} from "./lib/project-review";
|
||||||
|
|
||||||
const run = promisify(execFile);
|
const run = promisify(execFile);
|
||||||
@@ -130,7 +130,7 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
|||||||
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
|
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
|
||||||
}
|
}
|
||||||
const addHits = (draft: Draft, found: Hit[]) => {
|
const addHits = (draft: Draft, found: Hit[]) => {
|
||||||
draft.hits = withoutWhitelistedHits(found, whitelist);
|
draft.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist);
|
||||||
hits.push(...draft.hits);
|
hits.push(...draft.hits);
|
||||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||||
};
|
};
|
||||||
@@ -169,11 +169,12 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
|||||||
const spans = spansBySegment(batch, result.spans);
|
const spans = spansBySegment(batch, result.spans);
|
||||||
for (const segment of batch.segments) {
|
for (const segment of batch.segments) {
|
||||||
const path = byPath.get(segment.key)!;
|
const path = byPath.get(segment.key)!;
|
||||||
path.hits = withoutWhitelistedHits(toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({
|
const found = toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({
|
||||||
...span, id: `${path.key}:path:${hitIndex}`, file: path.key,
|
...span, id: `${path.key}:path:${hitIndex}`, file: path.key,
|
||||||
source: "Datei- oder Ordnername", kind: "path" as const,
|
source: "Datei- oder Ordnername", kind: "path" as const,
|
||||||
context: path.value,
|
context: path.value,
|
||||||
})), whitelist);
|
}));
|
||||||
|
path.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist);
|
||||||
hits.push(...path.hits);
|
hits.push(...path.hits);
|
||||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||||
}
|
}
|
||||||
@@ -454,6 +455,8 @@ export default function Command() {
|
|||||||
const [rules, setRules] = useState("");
|
const [rules, setRules] = useState("");
|
||||||
const [availableTypes, setAvailableTypes] = useState<EntityType[]>([]);
|
const [availableTypes, setAvailableTypes] = useState<EntityType[]>([]);
|
||||||
const [selectedTypes, setSelectedTypes] = useState<EntityType[]>([]);
|
const [selectedTypes, setSelectedTypes] = useState<EntityType[]>([]);
|
||||||
|
const [typesLoaded, setTypesLoaded] = useState(false);
|
||||||
|
const [typesReadyFor, setTypesReadyFor] = useState<string | null>(null);
|
||||||
const [whitelist, setWhitelist] = useState<ProjectWhitelistEntry[]>([]);
|
const [whitelist, setWhitelist] = useState<ProjectWhitelistEntry[]>([]);
|
||||||
const [existingRuns, setExistingRuns] = useState<number | null>(null);
|
const [existingRuns, setExistingRuns] = useState<number | null>(null);
|
||||||
const [busy, setBusy] = useState(false);
|
const [busy, setBusy] = useState(false);
|
||||||
@@ -461,10 +464,24 @@ export default function Command() {
|
|||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
getEntityTypes().then((types) => {
|
getEntityTypes().then((types) => {
|
||||||
setAvailableTypes(types);
|
setAvailableTypes(types);
|
||||||
setSelectedTypes(types.filter((type) => type !== "ORT"));
|
setTypesLoaded(true);
|
||||||
}).catch((error: Error) => showToast({ style: Toast.Style.Failure,
|
}).catch((error: Error) => showToast({ style: Toast.Style.Failure,
|
||||||
title: "Kategorien nicht verfügbar", message: error.message }));
|
title: "Kategorien nicht verfügbar", message: error.message }));
|
||||||
}, []);
|
}, []);
|
||||||
|
useEffect(() => {
|
||||||
|
if (!typesLoaded) return;
|
||||||
|
const defaults = availableTypes.filter((type) => type !== "ORT");
|
||||||
|
if (!root) { setSelectedTypes(defaults); setTypesReadyFor(""); return; }
|
||||||
|
let active = true;
|
||||||
|
setTypesReadyFor(null);
|
||||||
|
loadProjectEntityTypes(root).then((stored) => {
|
||||||
|
if (!active) return;
|
||||||
|
setSelectedTypes(stored === null ? defaults : stored.filter((type) => availableTypes.includes(type)));
|
||||||
|
setTypesReadyFor(root);
|
||||||
|
}).catch((error: Error) => showToast({ style: Toast.Style.Failure,
|
||||||
|
title: "Projektkategorien nicht lesbar", message: error.message }));
|
||||||
|
return () => { active = false; };
|
||||||
|
}, [root, availableTypes, typesLoaded]);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!root) return;
|
if (!root) return;
|
||||||
let active = true;
|
let active = true;
|
||||||
@@ -506,8 +523,10 @@ export default function Command() {
|
|||||||
const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" });
|
const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" });
|
||||||
try {
|
try {
|
||||||
if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen.");
|
if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen.");
|
||||||
|
if (typesReadyFor !== root) throw new Error("Projektkategorien werden noch geladen oder sind nicht lesbar.");
|
||||||
if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen.");
|
if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen.");
|
||||||
const parsed = parseProjectRules(rules);
|
const parsed = parseProjectRules(rules);
|
||||||
|
await saveProjectEntityTypes(root, selectedTypes);
|
||||||
await saveProjectRules(root, parsed);
|
await saveProjectRules(root, parsed);
|
||||||
toast.title = "Dateien werden erfasst";
|
toast.title = "Dateien werden erfasst";
|
||||||
const scan = await scanProjectFiles(root);
|
const scan = await scanProjectFiles(root);
|
||||||
@@ -547,7 +566,7 @@ export default function Command() {
|
|||||||
onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"}
|
onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"}
|
||||||
info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." />
|
info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." />
|
||||||
<Form.TagPicker id="entityTypes" title="Kategorien pseudonymisieren" value={selectedTypes}
|
<Form.TagPicker id="entityTypes" title="Kategorien pseudonymisieren" value={selectedTypes}
|
||||||
onChange={setSelectedTypes} info="Orte sind wegen häufiger Fehlalarme zunächst abgewählt. Abgewählte Treffer kommen nach dem Erstellen in die Projekt-Whitelist.">
|
onChange={setSelectedTypes} info="Die Auswahl wird beim Prüfen für diesen Projektordner gespeichert. Bei neuen Projekten sind Orte wegen häufiger Fehlalarme zunächst abgewählt.">
|
||||||
{availableTypes.map((type) => <Form.TagPicker.Item key={type} value={type} title={labelForType(type)} />)}
|
{availableTypes.map((type) => <Form.TagPicker.Item key={type} value={type} title={labelForType(type)} />)}
|
||||||
</Form.TagPicker>
|
</Form.TagPicker>
|
||||||
<Form.Description title="Projekt-Whitelist" text={`${whitelist.length} Begriffe werden bei diesem Ordner nicht mehr vorgeschlagen. Über die Aktionen kannst du sie wieder freigeben.`} />
|
<Form.Description title="Projekt-Whitelist" text={`${whitelist.length} Begriffe werden bei diesem Ordner nicht mehr vorgeschlagen. Über die Aktionen kannst du sie wieder freigeben.`} />
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { expect, it } from "bun:test";
|
import { expect, it } from "bun:test";
|
||||||
import {
|
import {
|
||||||
canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople,
|
canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople,
|
||||||
spansBySegment, withoutWhitelistedHits, withoutWhitelistedRules,
|
spansBySegment, withoutPlaceholderHits, withoutWhitelistedHits, withoutWhitelistedRules,
|
||||||
} from "../src/lib/project-review";
|
} from "../src/lib/project-review";
|
||||||
|
|
||||||
it("liest Unterstriche in Namen als Leerzeichen und bewahrt bekannte Codebegriffe", () => {
|
it("liest Unterstriche in Namen als Leerzeichen und bewahrt bekannte Codebegriffe", () => {
|
||||||
@@ -58,6 +58,17 @@ it("unterdrückt projektweit nur exakt abgewählte Begriffe samt erzwungener Reg
|
|||||||
], whitelist)).toEqual([{ type: "ORG", value: "Max Muster" }]);
|
], whitelist)).toEqual([{ type: "ORG", value: "Max Muster" }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("zeigt beim erneuten Lauf keine Treffer, die einen bestehenden Platzhalter einschließen", () => {
|
||||||
|
const hits = [
|
||||||
|
{ type: "PERSON", original: "PERSON_39, Roman" },
|
||||||
|
{ type: "KUNDE", original: "KUNDE_100005" },
|
||||||
|
{ type: "PERSON", original: "Roman" },
|
||||||
|
{ type: "PERSON", original: "Anlage44" },
|
||||||
|
{ type: "PASSWORT", original: "ORT_08!" },
|
||||||
|
];
|
||||||
|
expect(withoutPlaceholderHits(hits)).toEqual(hits.slice(2));
|
||||||
|
});
|
||||||
|
|
||||||
it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => {
|
it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => {
|
||||||
let merges = mergePeople({}, "PERSON_02", "PERSON_01");
|
let merges = mergePeople({}, "PERSON_02", "PERSON_01");
|
||||||
merges = mergePeople(merges, "PERSON_03", "PERSON_02");
|
merges = mergePeople(merges, "PERSON_03", "PERSON_02");
|
||||||
|
|||||||
@@ -53,6 +53,20 @@ describe("Velum-Projektstände", () => {
|
|||||||
.toEqual([first.run.id, second.run.id].sort());
|
.toEqual([first.run.id, second.run.id].sort());
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("behält die Kategorieauswahl pro Projekt für den nächsten Lauf", async () => {
|
||||||
|
const root = await fs.mkdtemp(join(tmpdir(), "velum-entity-types-test-"));
|
||||||
|
roots.push(root);
|
||||||
|
const source = join(root, "Quelle");
|
||||||
|
const other = join(root, "Anderes Projekt");
|
||||||
|
const support = join(root, "Raycast");
|
||||||
|
await fs.mkdir(source);
|
||||||
|
await fs.mkdir(other);
|
||||||
|
expect(await project.loadProjectEntityTypes(source, support)).toBeNull();
|
||||||
|
await project.saveProjectEntityTypes(`${source}/`, ["PERSON", "ORG", "PERSON"], support);
|
||||||
|
expect(await project.loadProjectEntityTypes(source, support)).toEqual(["PERSON", "ORG"]);
|
||||||
|
expect(await project.loadProjectEntityTypes(other, support)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
it("ersetzt nur bestätigte Vorkommen und stellt Dateinamen wieder her", () => {
|
it("ersetzt nur bestätigte Vorkommen und stellt Dateinamen wieder her", () => {
|
||||||
const hits = [
|
const hits = [
|
||||||
{ id: "a", start: 0, end: 3, placeholder: "PERSON_01" },
|
{ id: "a", start: 0, end: 3, placeholder: "PERSON_01" },
|
||||||
|
|||||||
Reference in New Issue
Block a user