diff --git a/src/lib/project-review.ts b/src/lib/project-review.ts index 8e55730..a3c7cdb 100644 --- a/src/lib/project-review.ts +++ b/src/lib/project-review.ts @@ -88,6 +88,12 @@ export function withoutWhitelistedHits(hits: T[], whitel return hits.filter((hit) => !excluded.has(whitelistKey(hit))); } +/** A later run must not turn an existing Velum placeholder plus adjacent text into a new entity. */ +export function withoutPlaceholderHits(hits: T[]): T[] { + const placeholder = /(? hit.type === "PASSWORT" || !placeholder.test(hit.original)); +} + export function withoutWhitelistedRules( rules: T[], whitelist: ProjectWhitelistEntry[], ): T[] { diff --git a/src/lib/velum-project.ts b/src/lib/velum-project.ts index 58e755f..a1e4202 100644 --- a/src/lib/velum-project.ts +++ b/src/lib/velum-project.ts @@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { constants as fsConstants, promises as fs } from "node:fs"; import { basename, dirname, extname, join, relative, resolve, sep } from "node:path"; import { environment } from "@raycast/api"; -import type { PlaceholderMapping } from "../types"; +import type { EntityType, PlaceholderMapping } from "../types"; import { whitelistKey } from "./project-whitelist"; import type { ProjectWhitelistEntry } from "./project-whitelist"; @@ -68,6 +68,36 @@ export function whitelistPath(projectPath: string, support = environment.support return join(projectDataRoot(projectPath, support), "whitelist.json"); } +export function entityTypesPath(projectPath: string, support = environment.supportPath): string { + return join(projectDataRoot(projectPath, support), "entity-types.json"); +} + +export async function loadProjectEntityTypes(projectPath: string, support?: string): Promise { + try { + const value = JSON.parse(await fs.readFile(entityTypesPath(projectPath, support), "utf8")) as unknown; + if (!Array.isArray(value) || value.some((type) => typeof type !== "string" || !type)) { + throw new Error("Ungültige Kategorienauswahl für dieses Projekt."); + } + return [...new Set(value)]; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} + +export async function saveProjectEntityTypes(projectPath: string, types: EntityType[], support?: string): Promise { + const path = entityTypesPath(projectPath, support); + await fs.mkdir(dirname(path), { recursive: true, mode: 0o700 }); + const temp = `${path}.${randomUUID()}.tmp`; + try { + await fs.writeFile(temp, JSON.stringify([...new Set(types)]), { encoding: "utf8", mode: 0o600 }); + await fs.rename(temp, path); + } catch (error) { + await fs.rm(temp, { force: true }); + throw error; + } +} + export async function loadProjectWhitelist(projectPath: string, support?: string): Promise { try { const value = JSON.parse(await fs.readFile(whitelistPath(projectPath, support), "utf8")) as unknown; diff --git a/src/pseudonymize-project-files.tsx b/src/pseudonymize-project-files.tsx index bcfa178..404eedd 100644 --- a/src/pseudonymize-project-files.tsx +++ b/src/pseudonymize-project-files.tsx @@ -8,14 +8,14 @@ import type { EntityType, PlaceholderMapping } from "./types"; import { getEntityTypes, pseudonymize, pseudonymizeDocument, requireProjectCapabilities } from "./velum"; import type { DocumentHit } from "./velum"; import { - createRun, digest, forceMasks, listRuns, loadProjectRules, parseProjectRules, replaceSpans, + createRun, digest, forceMasks, listRuns, loadProjectEntityTypes, loadProjectRules, parseProjectRules, replaceSpans, loadProjectWhitelist, matchesPreviousOutput, replaceProjectFilesInPlace, resolveWithin, safeRelativePath, saveProjectRules, - saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey, + saveProjectEntityTypes, saveProjectWhitelist, saveRun, scanProjectFiles, toUtf16Offsets, whitelistKey, } from "./lib/velum-project"; import type { ProjectFile, ProjectRule, ProjectWhitelistEntry, RunFile } from "./lib/velum-project"; import { canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, spansBySegment, - withoutWhitelistedHits, withoutWhitelistedRules, + withoutPlaceholderHits, withoutWhitelistedHits, withoutWhitelistedRules, } from "./lib/project-review"; const run = promisify(execFile); @@ -130,7 +130,7 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], if (file.kind === "text") texts.set(file.relPath, textFromBytes(data)); } const addHits = (draft: Draft, found: Hit[]) => { - draft.hits = withoutWhitelistedHits(found, whitelist); + draft.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist); hits.push(...draft.hits); if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); }; @@ -169,11 +169,12 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[], const spans = spansBySegment(batch, result.spans); for (const segment of batch.segments) { const path = byPath.get(segment.key)!; - path.hits = withoutWhitelistedHits(toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({ + const found = toUtf16Offsets(segment.text, spans.get(path.key) ?? []).map((span, hitIndex) => ({ ...span, id: `${path.key}:path:${hitIndex}`, file: path.key, source: "Datei- oder Ordnername", kind: "path" as const, context: path.value, - })), whitelist); + })); + path.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist); hits.push(...path.hits); if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen."); } @@ -454,6 +455,8 @@ export default function Command() { const [rules, setRules] = useState(""); const [availableTypes, setAvailableTypes] = useState([]); const [selectedTypes, setSelectedTypes] = useState([]); + const [typesLoaded, setTypesLoaded] = useState(false); + const [typesReadyFor, setTypesReadyFor] = useState(null); const [whitelist, setWhitelist] = useState([]); const [existingRuns, setExistingRuns] = useState(null); const [busy, setBusy] = useState(false); @@ -461,10 +464,24 @@ export default function Command() { useEffect(() => { getEntityTypes().then((types) => { setAvailableTypes(types); - setSelectedTypes(types.filter((type) => type !== "ORT")); + setTypesLoaded(true); }).catch((error: Error) => showToast({ style: Toast.Style.Failure, title: "Kategorien nicht verfügbar", message: error.message })); }, []); + useEffect(() => { + if (!typesLoaded) return; + const defaults = availableTypes.filter((type) => type !== "ORT"); + if (!root) { setSelectedTypes(defaults); setTypesReadyFor(""); return; } + let active = true; + setTypesReadyFor(null); + loadProjectEntityTypes(root).then((stored) => { + if (!active) return; + setSelectedTypes(stored === null ? defaults : stored.filter((type) => availableTypes.includes(type))); + setTypesReadyFor(root); + }).catch((error: Error) => showToast({ style: Toast.Style.Failure, + title: "Projektkategorien nicht lesbar", message: error.message })); + return () => { active = false; }; + }, [root, availableTypes, typesLoaded]); useEffect(() => { if (!root) return; let active = true; @@ -506,8 +523,10 @@ export default function Command() { const toast = await showToast({ style: Toast.Style.Animated, title: "Projekt wird vorbereitet" }); try { if (!(await fs.stat(root)).isDirectory()) throw new Error("Bitte einen Projektordner wählen."); + if (typesReadyFor !== root) throw new Error("Projektkategorien werden noch geladen oder sind nicht lesbar."); if (!selectedTypes.length) throw new Error("Bitte mindestens eine Kategorie wählen."); const parsed = parseProjectRules(rules); + await saveProjectEntityTypes(root, selectedTypes); await saveProjectRules(root, parsed); toast.title = "Dateien werden erfasst"; const scan = await scanProjectFiles(root); @@ -547,7 +566,7 @@ export default function Command() { onChange={setRules} placeholder={"BEGRIFF: KundenSuite\nPERSON: Max Mustermann"} info="Ein Begriff pro Zeile. Ohne Typ wird BEGRIFF verwendet. Diese Regeln werden nur für dieses Projekt gespeichert und bei der Velum-Anfrage mitgesendet." /> + onChange={setSelectedTypes} info="Die Auswahl wird beim Prüfen für diesen Projektordner gespeichert. Bei neuen Projekten sind Orte wegen häufiger Fehlalarme zunächst abgewählt."> {availableTypes.map((type) => )} diff --git a/tests/project-review.test.ts b/tests/project-review.test.ts index 63eeedd..5a2803e 100644 --- a/tests/project-review.test.ts +++ b/tests/project-review.test.ts @@ -1,7 +1,7 @@ import { expect, it } from "bun:test"; import { canonicalPerson, filenameScanText, groupReviewHits, makeTextBatches, mappingForSelection, mergePeople, - spansBySegment, withoutWhitelistedHits, withoutWhitelistedRules, + spansBySegment, withoutPlaceholderHits, withoutWhitelistedHits, withoutWhitelistedRules, } from "../src/lib/project-review"; it("liest Unterstriche in Namen als Leerzeichen und bewahrt bekannte Codebegriffe", () => { @@ -58,6 +58,17 @@ it("unterdrückt projektweit nur exakt abgewählte Begriffe samt erzwungener Reg ], whitelist)).toEqual([{ type: "ORG", value: "Max Muster" }]); }); +it("zeigt beim erneuten Lauf keine Treffer, die einen bestehenden Platzhalter einschließen", () => { + const hits = [ + { type: "PERSON", original: "PERSON_39, Roman" }, + { type: "KUNDE", original: "KUNDE_100005" }, + { type: "PERSON", original: "Roman" }, + { type: "PERSON", original: "Anlage44" }, + { type: "PASSWORT", original: "ORT_08!" }, + ]; + expect(withoutPlaceholderHits(hits)).toEqual(hits.slice(2)); +}); + it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => { let merges = mergePeople({}, "PERSON_02", "PERSON_01"); merges = mergePeople(merges, "PERSON_03", "PERSON_02"); diff --git a/tests/velum-project.test.ts b/tests/velum-project.test.ts index f0e64d9..aa17b0d 100644 --- a/tests/velum-project.test.ts +++ b/tests/velum-project.test.ts @@ -53,6 +53,20 @@ describe("Velum-Projektstände", () => { .toEqual([first.run.id, second.run.id].sort()); }); + it("behält die Kategorieauswahl pro Projekt für den nächsten Lauf", async () => { + const root = await fs.mkdtemp(join(tmpdir(), "velum-entity-types-test-")); + roots.push(root); + const source = join(root, "Quelle"); + const other = join(root, "Anderes Projekt"); + const support = join(root, "Raycast"); + await fs.mkdir(source); + await fs.mkdir(other); + expect(await project.loadProjectEntityTypes(source, support)).toBeNull(); + await project.saveProjectEntityTypes(`${source}/`, ["PERSON", "ORG", "PERSON"], support); + expect(await project.loadProjectEntityTypes(source, support)).toEqual(["PERSON", "ORG"]); + expect(await project.loadProjectEntityTypes(other, support)).toBeNull(); + }); + it("ersetzt nur bestätigte Vorkommen und stellt Dateinamen wieder her", () => { const hits = [ { id: "a", start: 0, end: 3, placeholder: "PERSON_01" },