🧪 Treat Go table cases with random hex names as the same baseline failure
A table test that names its case with a 16–32 hex id (an org, a token) fails as a different string at the start commit and after the change. The baseline then treats that case as new, so a frontend-only wish cannot pass Go tests that were already red. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
9df1778989
commit
a6b713f5df
@ -227,6 +227,20 @@ export function failedGoTests(output) {
|
||||
return failures.size ? [...failures].sort() : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* A failure's identity without run-to-run IDs: Go table tests often name a case with a 16–32
|
||||
* hex hash, so the same case is a different string at the start commit and after the change.
|
||||
*/
|
||||
export function comparableFailure(name) {
|
||||
return name.replace(/[0-9a-f]{16,}/gi, '*');
|
||||
}
|
||||
|
||||
/** Failures in `now` that were not already failing in `before`, after stripping those IDs. */
|
||||
export function newlyFailing(now, before) {
|
||||
const known = new Set(before.map(comparableFailure));
|
||||
return now.filter(test => !known.has(comparableFailure(test)));
|
||||
}
|
||||
|
||||
async function runCheck(check, cwd, env, quiet = false) {
|
||||
const started = Date.now();
|
||||
if (!quiet) console.log(`[poc-check] START ${check.label}`);
|
||||
@ -302,10 +316,10 @@ async function alreadyFailing(check, workspace, env, output) {
|
||||
env,
|
||||
true,
|
||||
);
|
||||
const before = new Set(parse(atBase.output) ?? []);
|
||||
const added = failed.filter(test => !before.has(test));
|
||||
const before = parse(atBase.output) ?? [];
|
||||
const added = newlyFailing(failed, before);
|
||||
console.log(
|
||||
`[poc-check] BASELINE ${JSON.stringify({ check: check.label, baseCommit: base, failingBefore: before.size, failingNow: failed.length, newFailures: added })}`,
|
||||
`[poc-check] BASELINE ${JSON.stringify({ check: check.label, baseCommit: base, failingBefore: before.length, failingNow: failed.length, newFailures: added })}`,
|
||||
);
|
||||
return added.length ? undefined : failed;
|
||||
} finally {
|
||||
|
||||
@ -15,6 +15,8 @@ import {
|
||||
toolchain,
|
||||
failedTests,
|
||||
failedGoTests,
|
||||
comparableFailure,
|
||||
newlyFailing,
|
||||
runPlan,
|
||||
BASE_REF,
|
||||
} from './poc-project-check.mjs';
|
||||
@ -245,3 +247,17 @@ test('reads the failed Go tests as "package :: Test", and a package that did not
|
||||
]);
|
||||
assert.equal(failedGoTests('ok \tdroboticflow/internal/api\t1s'), undefined);
|
||||
});
|
||||
|
||||
test('a Go table case named with a hex id is the same failure at the start commit', () => {
|
||||
const now = [
|
||||
'droboticflow/internal/api :: TestEmployeeSupplierFixedLifecycle/supplier-default-org-4bc9ed231f699499fa27768fb8ba5304',
|
||||
];
|
||||
const before = [
|
||||
'droboticflow/internal/api :: TestEmployeeSupplierFixedLifecycle/supplier-default-org-d34742d770aec5c061e58d79d19b00f7',
|
||||
];
|
||||
assert.equal(comparableFailure(now[0]), comparableFailure(before[0]));
|
||||
assert.deepEqual(newlyFailing(now, before), []);
|
||||
assert.deepEqual(newlyFailing([...now, 'droboticflow/internal/api :: TestNew'], before), [
|
||||
'droboticflow/internal/api :: TestNew',
|
||||
]);
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user