"""Mandatory synthetic work/recovery gate; loopback Mongo only, no customer writes.""" import json import os import signal from pathlib import Path import subprocess import sys import time import uuid from lib.release_checks import check_environment, write_log, source_fingerprint ROOT = Path(__file__).resolve().parents[1] CORE = {'TestSourceAdvisoryCompleteWorkAndReviewFlow', 'TestSourceAdvisoryPublishedReviewDoesNotRequireFullAudit', 'TestSourceAdvisoryPreparesIndexDespiteCadenceAuditFailure', 'TestSourceAdvisoryTaskScopeCannotBeBypassed', 'TestSourceAdvisoryIsolatedReworkDoesNotTrapRandomClaim', 'TestSourceAdvisoryOwnedFailureCanRetryTimeline', 'TestSubmissionLossGuardArchiveFailureBlocksReplacement', 'TestSubmissionLossGuardArchiveSurvivesRestartAndIsIdempotent', 'TestSubmissionLossGuardArchivesFailedInputBeforeReplacement', 'TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter', 'TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker', 'TestCombinedSubmissionFinalCASFailureDoesNotPublishQualityTask', 'TestDeletedAnnotatorSubmissionWinsCASRace', 'TestDeletedAnnotatorRestartFinishesResetOnce', 'TestReviewCannotMutateAnnotation', 'TestSourceAdvisoryDoesNotGateOnFullAudit', 'TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate', 'TestReadObservationIsPrivateConcurrentAndRequestScoped'} MONGO = {'TestSourceReadinessMongoParityReadOnly', 'TestSourceAdvisoryMongoEquivalenceAndBuffer', 'TestSourceAdmissionMongoClaimFence', 'TestSourceAdmissionMongoBatchMatchesLegacyPipeline', 'TestSourceAdmissionCurrentOutboxProofMongo', 'TestSourceAdmissionExistingTaskUpdateDoesNotEnqueueMongo', 'TestSourceValidationLegacyDetailsGETDoesNotEnqueueMongo', 'TestSubmissionRecoveryEventMongo'} def verdict(stdout, code, required): events = [] for line in stdout.splitlines(): try: events.append(json.loads(line)) except ValueError: continue passed = {e.get('Test') for e in events if e.get('Action') == 'pass'} skipped = sorted((e['Test'] for e in events if e.get('Action') == 'skip' and e.get('Test'))) missing = sorted(required - passed) return {'passed': code == 0 and (not skipped) and (not missing), 'missingOrFailed': missing, 'skipped': skipped} def phases(): return [('core', CORE, {}), ('mongo', MONGO, {'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'SUBMISSION_RECOVERY_LOCAL_MONGO': '1', 'POOL_LOCAL_MONGO_URI': 'mongodb://127.0.0.1:27018'}), ('advisory-workbench', {'TestSourceAdvisoryWorkbenchBrowser'}, {'SOURCE_ADVISORY_BROWSER_TEST': '1'}), ('advisory-recovery', {'TestReworkBoundaryBrowserReplica'}, {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'REWORK_BOUNDARY_BROWSER_TEST': '1'})] def run_phase(argv, env): # Terminate only this synthetic phase's process group on timeout, including # Go-launched Node/browser children. Never terminate a shared local service. child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True) try: stdout, stderr = child.communicate(timeout=360) except subprocess.TimeoutExpired: try: os.killpg(child.pid, signal.SIGTERM) except ProcessLookupError: pass try: stdout, stderr = child.communicate(timeout=5) except subprocess.TimeoutExpired: try: os.killpg(child.pid, signal.SIGKILL) except ProcessLookupError: pass stdout, stderr = child.communicate() return subprocess.CompletedProcess(argv, 124, stdout, stderr + '\nphase timed out') return subprocess.CompletedProcess(argv, child.returncode, stdout, stderr) def main(): folder = ROOT / 'local-data/test-results' / ('work-availability-' + uuid.uuid4().hex) folder.mkdir(parents=True, mode=448) before = source_fingerprint(ROOT) report = {'inputDigest': before, 'scope': 'synthetic local fixtures; no production acceptance/capacity claim', 'checks': [], 'passed': False} for name, required, switches in phases(): argv = ['go', 'test', '-json', './internal/api', './internal/store', '-run', '^(' + '|'.join(sorted(required)) + ')$', '-count=1', '-timeout=5m'] env = {**check_environment(), **switches, 'VERIFICATION_RUN_ID': uuid.uuid4().hex} try: r = run_phase(argv, env) row = verdict(r.stdout, r.returncode, required) output = r.stdout + '\n' + r.stderr except (OSError, subprocess.TimeoutExpired) as e: row = {'passed': False, 'error': 'runner failed or timed out'} output = str(e) report['inputsChangedDuringRun'] = source_fingerprint(ROOT) != before if report['inputsChangedDuringRun'] or not before: row.update(passed=False, error='Source identity changed or unavailable') row.update(phase=name, argv=argv, artifactRunId=env['VERIFICATION_RUN_ID']) row.update(write_log(folder, argv, output, 'passed' if row['passed'] else 'failed')) report['checks'].append(row) print(json.dumps(row), flush=True) (folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n') if report['inputsChangedDuringRun'] or not before: report['remainingUnverified'] = [p[0] for p in phases()[len(report['checks']):]] (folder/'report.json').write_text(json.dumps(report,indent=2)+'\n') return 2 report['passed'] = all(row['passed'] for row in report['checks']) (folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n') print('Work availability gate ' + ('passed' if report['passed'] else 'failed') + '; production writes and capacity remain separate. Report: ' + str(folder / 'report.json')) return 0 if report['passed'] else 1 if __name__ == '__main__': sys.exit(main())