fix: allow rerunning imported projects
This commit is contained in:
@@ -89,9 +89,14 @@ export function withoutWhitelistedHits<T extends GroupableHit>(hits: T[], whitel
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** A later run must not turn an existing Velum placeholder plus adjacent text into a new entity. */
|
/** A later run must not turn an existing Velum placeholder plus adjacent text into a new entity. */
|
||||||
export function withoutPlaceholderHits<T extends { original: string; type: string }>(hits: T[]): T[] {
|
export function withoutPlaceholderHits<T extends { original: string; type: string }>(
|
||||||
|
hits: T[], knownPlaceholders: Iterable<string> = [],
|
||||||
|
): T[] {
|
||||||
const placeholder = /(?<![\p{L}\p{N}])[A-Z][A-Z0-9]*_\d{2,}(?![\p{L}\p{N}])/u;
|
const placeholder = /(?<![\p{L}\p{N}])[A-Z][A-Z0-9]*_\d{2,}(?![\p{L}\p{N}])/u;
|
||||||
return hits.filter((hit) => hit.type === "PASSWORT" || !placeholder.test(hit.original));
|
const known = [...knownPlaceholders].filter(Boolean);
|
||||||
|
return hits.filter((hit) => hit.type === "PASSWORT" || (
|
||||||
|
!placeholder.test(hit.original) && !known.some((key) => hit.original.includes(key))
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function withoutWhitelistedRules<T extends { type: string; value: string }>(
|
export function withoutWhitelistedRules<T extends { type: string; value: string }>(
|
||||||
|
|||||||
@@ -101,10 +101,8 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
|||||||
const masks = forceMasks(withoutWhitelistedRules(rules, whitelist));
|
const masks = forceMasks(withoutWhitelistedRules(rules, whitelist));
|
||||||
const latestEntry = (await listRuns(root))[0];
|
const latestEntry = (await listRuns(root))[0];
|
||||||
const latest = latestEntry?.run;
|
const latest = latestEntry?.run;
|
||||||
if (latest?.origin === "imported") {
|
|
||||||
throw new Error("Dieser Quellordner enthält bereits pseudonymisierte Dateien. Für einen neuen Lauf bitte einen Klartext-Quellordner wählen.");
|
|
||||||
}
|
|
||||||
const previousMapping: PlaceholderMapping = latest?.mapping ?? {};
|
const previousMapping: PlaceholderMapping = latest?.mapping ?? {};
|
||||||
|
const previousPlaceholders = Object.keys(previousMapping);
|
||||||
let mapping: PlaceholderMapping = previousMapping;
|
let mapping: PlaceholderMapping = previousMapping;
|
||||||
const drafts: Draft[] = [];
|
const drafts: Draft[] = [];
|
||||||
const retained: Preview["retained"] = [];
|
const retained: Preview["retained"] = [];
|
||||||
@@ -112,6 +110,8 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
|||||||
const hits: Hit[] = [];
|
const hits: Hit[] = [];
|
||||||
const texts = new Map<string, string>();
|
const texts = new Map<string, string>();
|
||||||
const sourceInPlace = Boolean(latest?.inPlace || latest?.sourceInPlace);
|
const sourceInPlace = Boolean(latest?.inPlace || latest?.sourceInPlace);
|
||||||
|
// A migrated snapshot may differ from the source folder. Recheck its files
|
||||||
|
// unless an earlier run actually replaced them in place.
|
||||||
const previousOutputs = new Set(sourceInPlace ? latest?.files.map((file) => file.output) : []);
|
const previousOutputs = new Set(sourceInPlace ? latest?.files.map((file) => file.output) : []);
|
||||||
const previousDirectory = latestEntry?.directory;
|
const previousDirectory = latestEntry?.directory;
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
@@ -131,7 +131,7 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
|||||||
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
|
if (file.kind === "text") texts.set(file.relPath, textFromBytes(data));
|
||||||
}
|
}
|
||||||
const addHits = (draft: Draft, found: Hit[]) => {
|
const addHits = (draft: Draft, found: Hit[]) => {
|
||||||
draft.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist);
|
draft.hits = withoutWhitelistedHits(withoutPlaceholderHits(found, previousPlaceholders), whitelist);
|
||||||
hits.push(...draft.hits);
|
hits.push(...draft.hits);
|
||||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||||
};
|
};
|
||||||
@@ -175,7 +175,7 @@ async function prepare(root: string, files: ProjectFile[], rules: ProjectRule[],
|
|||||||
source: "Datei- oder Ordnername", kind: "path" as const,
|
source: "Datei- oder Ordnername", kind: "path" as const,
|
||||||
context: path.value,
|
context: path.value,
|
||||||
}));
|
}));
|
||||||
path.hits = withoutWhitelistedHits(withoutPlaceholderHits(found), whitelist);
|
path.hits = withoutWhitelistedHits(withoutPlaceholderHits(found, previousPlaceholders), whitelist);
|
||||||
hits.push(...path.hits);
|
hits.push(...path.hits);
|
||||||
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
if (hits.length > MAX_REVIEW_HITS) throw new Error("Zu viele Treffer. Bitte weniger Kategorien auswählen.");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,6 +67,10 @@ it("zeigt beim erneuten Lauf keine Treffer, die einen bestehenden Platzhalter ei
|
|||||||
{ type: "PASSWORT", original: "ORT_08!" },
|
{ type: "PASSWORT", original: "ORT_08!" },
|
||||||
];
|
];
|
||||||
expect(withoutPlaceholderHits(hits)).toEqual(hits.slice(2));
|
expect(withoutPlaceholderHits(hits)).toEqual(hits.slice(2));
|
||||||
|
expect(withoutPlaceholderHits([
|
||||||
|
{ type: "PERSON", original: "KUNDE_100005Service" },
|
||||||
|
{ type: "PERSON", original: "Roman" },
|
||||||
|
], ["KUNDE_100005"])).toEqual([{ type: "PERSON", original: "Roman" }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => {
|
it("fasst ausgewählte Personen zu einem kanonischen Platzhalter zusammen und bewahrt alte Zuordnungen", () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user