droboticflow-new/scripts/test_work_availability.py

86 lines
5.8 KiB
Python

"""Mandatory synthetic work/recovery gate; loopback Mongo only, no customer writes."""
import json
import os
import signal
from pathlib import Path
import subprocess
import sys
import time
import uuid
from lib.release_checks import check_environment, write_log, source_fingerprint
ROOT = Path(__file__).resolve().parents[1]
CORE = {'TestSourceAdvisoryCompleteWorkAndReviewFlow', 'TestSourceAdvisoryPublishedReviewDoesNotRequireFullAudit', 'TestSourceAdvisoryPreparesIndexDespiteCadenceAuditFailure', 'TestSourceAdvisoryTaskScopeCannotBeBypassed', 'TestSourceAdvisoryIsolatedReworkDoesNotTrapRandomClaim', 'TestSourceAdvisoryOwnedFailureCanRetryTimeline', 'TestSubmissionLossGuardArchiveFailureBlocksReplacement', 'TestSubmissionLossGuardArchiveSurvivesRestartAndIsIdempotent', 'TestSubmissionLossGuardArchivesFailedInputBeforeReplacement', 'TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter', 'TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker', 'TestCombinedSubmissionFinalCASFailureDoesNotPublishQualityTask', 'TestDeletedAnnotatorSubmissionWinsCASRace', 'TestDeletedAnnotatorRestartFinishesResetOnce', 'TestReviewCannotMutateAnnotation', 'TestSourceAdvisoryDoesNotGateOnFullAudit', 'TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate', 'TestReadObservationIsPrivateConcurrentAndRequestScoped'}
MONGO = {'TestSourceReadinessMongoParityReadOnly', 'TestSourceAdvisoryMongoEquivalenceAndBuffer', 'TestSourceAdmissionMongoClaimFence', 'TestSourceAdmissionMongoBatchMatchesLegacyPipeline', 'TestSourceAdmissionCurrentOutboxProofMongo', 'TestSourceAdmissionExistingTaskUpdateDoesNotEnqueueMongo', 'TestSourceValidationLegacyDetailsGETDoesNotEnqueueMongo', 'TestSubmissionRecoveryEventMongo'}
def verdict(stdout, code, required):
events = []
for line in stdout.splitlines():
try:
events.append(json.loads(line))
except ValueError:
continue
passed = {e.get('Test') for e in events if e.get('Action') == 'pass'}
skipped = sorted((e['Test'] for e in events if e.get('Action') == 'skip' and e.get('Test')))
missing = sorted(required - passed)
return {'passed': code == 0 and (not skipped) and (not missing), 'missingOrFailed': missing, 'skipped': skipped}
def phases():
return [('core', CORE, {}), ('mongo', MONGO, {'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'SUBMISSION_RECOVERY_LOCAL_MONGO': '1', 'POOL_LOCAL_MONGO_URI': 'mongodb://127.0.0.1:27018'}), ('advisory-workbench', {'TestSourceAdvisoryWorkbenchBrowser'}, {'SOURCE_ADVISORY_BROWSER_TEST': '1'}), ('advisory-recovery', {'TestReworkBoundaryBrowserReplica'}, {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'REWORK_BOUNDARY_BROWSER_TEST': '1'})]
def run_phase(argv, env):
# Terminate only this synthetic phase's process group on timeout, including
# Go-launched Node/browser children. Never terminate a shared local service.
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, start_new_session=True)
try:
stdout, stderr = child.communicate(timeout=360)
except subprocess.TimeoutExpired:
try:
os.killpg(child.pid, signal.SIGTERM)
except ProcessLookupError:
pass
try:
stdout, stderr = child.communicate(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(child.pid, signal.SIGKILL)
except ProcessLookupError:
pass
stdout, stderr = child.communicate()
return subprocess.CompletedProcess(argv, 124, stdout, stderr + '\nphase timed out')
return subprocess.CompletedProcess(argv, child.returncode, stdout, stderr)
def main():
folder = ROOT / 'local-data/test-results' / ('work-availability-' + uuid.uuid4().hex)
folder.mkdir(parents=True, mode=448)
before = source_fingerprint(ROOT)
report = {'inputDigest': before, 'scope': 'synthetic local fixtures; no production acceptance/capacity claim', 'checks': [], 'passed': False}
for name, required, switches in phases():
argv = ['go', 'test', '-json', './internal/api', './internal/store', '-run', '^(' + '|'.join(sorted(required)) + ')$', '-count=1', '-timeout=5m']
env = {**check_environment(), **switches, 'VERIFICATION_RUN_ID': uuid.uuid4().hex}
try:
r = run_phase(argv, env)
row = verdict(r.stdout, r.returncode, required)
output = r.stdout + '\n' + r.stderr
except (OSError, subprocess.TimeoutExpired) as e:
row = {'passed': False, 'error': 'runner failed or timed out'}
output = str(e)
report['inputsChangedDuringRun'] = source_fingerprint(ROOT) != before
if report['inputsChangedDuringRun'] or not before:
row.update(passed=False, error='Source identity changed or unavailable')
row.update(phase=name, argv=argv, artifactRunId=env['VERIFICATION_RUN_ID'])
row.update(write_log(folder, argv, output, 'passed' if row['passed'] else 'failed'))
report['checks'].append(row)
print(json.dumps(row), flush=True)
(folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
if report['inputsChangedDuringRun'] or not before:
report['remainingUnverified'] = [p[0] for p in phases()[len(report['checks']):]]
(folder/'report.json').write_text(json.dumps(report,indent=2)+'\n')
return 2
report['passed'] = all(row['passed'] for row in report['checks'])
(folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
print('Work availability gate ' + ('passed' if report['passed'] else 'failed') + '; production writes and capacity remain separate. Report: ' + str(folder / 'report.json'))
return 0 if report['passed'] else 1
if __name__ == '__main__':
sys.exit(main())