86 lines
5.8 KiB
Python
86 lines
5.8 KiB
Python
"""Mandatory synthetic work/recovery gate; loopback Mongo only, no customer writes."""
|
|
import json
|
|
import os
|
|
import signal
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from lib.release_checks import check_environment, write_log, source_fingerprint
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CORE = {'TestSourceAdvisoryCompleteWorkAndReviewFlow', 'TestSourceAdvisoryPublishedReviewDoesNotRequireFullAudit', 'TestSourceAdvisoryPreparesIndexDespiteCadenceAuditFailure', 'TestSourceAdvisoryTaskScopeCannotBeBypassed', 'TestSourceAdvisoryIsolatedReworkDoesNotTrapRandomClaim', 'TestSourceAdvisoryOwnedFailureCanRetryTimeline', 'TestSubmissionLossGuardArchiveFailureBlocksReplacement', 'TestSubmissionLossGuardArchiveSurvivesRestartAndIsIdempotent', 'TestSubmissionLossGuardArchivesFailedInputBeforeReplacement', 'TestFailedSliceRetryPreservesBothSnapshotsAgainstLateWriter', 'TestDurableAnnotationOutboxSurvivesRestartAndFencesOldWorker', 'TestCombinedSubmissionFinalCASFailureDoesNotPublishQualityTask', 'TestDeletedAnnotatorSubmissionWinsCASRace', 'TestDeletedAnnotatorRestartFinishesResetOnce', 'TestReviewCannotMutateAnnotation', 'TestSourceAdvisoryDoesNotGateOnFullAudit', 'TestSourceReadinessIsScopedReadOnlyAndNotAValidityCertificate', 'TestReadObservationIsPrivateConcurrentAndRequestScoped'}
|
|
MONGO = {'TestSourceReadinessMongoParityReadOnly', 'TestSourceAdvisoryMongoEquivalenceAndBuffer', 'TestSourceAdmissionMongoClaimFence', 'TestSourceAdmissionMongoBatchMatchesLegacyPipeline', 'TestSourceAdmissionCurrentOutboxProofMongo', 'TestSourceAdmissionExistingTaskUpdateDoesNotEnqueueMongo', 'TestSourceValidationLegacyDetailsGETDoesNotEnqueueMongo', 'TestSubmissionRecoveryEventMongo'}
|
|
|
|
def verdict(stdout, code, required):
|
|
events = []
|
|
for line in stdout.splitlines():
|
|
try:
|
|
events.append(json.loads(line))
|
|
except ValueError:
|
|
continue
|
|
passed = {e.get('Test') for e in events if e.get('Action') == 'pass'}
|
|
skipped = sorted((e['Test'] for e in events if e.get('Action') == 'skip' and e.get('Test')))
|
|
missing = sorted(required - passed)
|
|
return {'passed': code == 0 and (not skipped) and (not missing), 'missingOrFailed': missing, 'skipped': skipped}
|
|
|
|
def phases():
|
|
return [('core', CORE, {}), ('mongo', MONGO, {'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'SUBMISSION_RECOVERY_LOCAL_MONGO': '1', 'POOL_LOCAL_MONGO_URI': 'mongodb://127.0.0.1:27018'}), ('advisory-workbench', {'TestSourceAdvisoryWorkbenchBrowser'}, {'SOURCE_ADVISORY_BROWSER_TEST': '1'}), ('advisory-recovery', {'TestReworkBoundaryBrowserReplica'}, {'ANNOTATION_SOURCE_VALIDATION_MODE': 'advisory', 'REWORK_BOUNDARY_BROWSER_TEST': '1'})]
|
|
|
|
def run_phase(argv, env):
|
|
# Terminate only this synthetic phase's process group on timeout, including
|
|
# Go-launched Node/browser children. Never terminate a shared local service.
|
|
child = subprocess.Popen(argv, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
text=True, start_new_session=True)
|
|
try:
|
|
stdout, stderr = child.communicate(timeout=360)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(child.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
pass
|
|
try:
|
|
stdout, stderr = child.communicate(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(child.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
stdout, stderr = child.communicate()
|
|
return subprocess.CompletedProcess(argv, 124, stdout, stderr + '\nphase timed out')
|
|
return subprocess.CompletedProcess(argv, child.returncode, stdout, stderr)
|
|
|
|
def main():
|
|
folder = ROOT / 'local-data/test-results' / ('work-availability-' + uuid.uuid4().hex)
|
|
folder.mkdir(parents=True, mode=448)
|
|
before = source_fingerprint(ROOT)
|
|
report = {'inputDigest': before, 'scope': 'synthetic local fixtures; no production acceptance/capacity claim', 'checks': [], 'passed': False}
|
|
for name, required, switches in phases():
|
|
argv = ['go', 'test', '-json', './internal/api', './internal/store', '-run', '^(' + '|'.join(sorted(required)) + ')$', '-count=1', '-timeout=5m']
|
|
env = {**check_environment(), **switches, 'VERIFICATION_RUN_ID': uuid.uuid4().hex}
|
|
try:
|
|
r = run_phase(argv, env)
|
|
row = verdict(r.stdout, r.returncode, required)
|
|
output = r.stdout + '\n' + r.stderr
|
|
except (OSError, subprocess.TimeoutExpired) as e:
|
|
row = {'passed': False, 'error': 'runner failed or timed out'}
|
|
output = str(e)
|
|
report['inputsChangedDuringRun'] = source_fingerprint(ROOT) != before
|
|
if report['inputsChangedDuringRun'] or not before:
|
|
row.update(passed=False, error='Source identity changed or unavailable')
|
|
row.update(phase=name, argv=argv, artifactRunId=env['VERIFICATION_RUN_ID'])
|
|
row.update(write_log(folder, argv, output, 'passed' if row['passed'] else 'failed'))
|
|
report['checks'].append(row)
|
|
print(json.dumps(row), flush=True)
|
|
(folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
if report['inputsChangedDuringRun'] or not before:
|
|
report['remainingUnverified'] = [p[0] for p in phases()[len(report['checks']):]]
|
|
(folder/'report.json').write_text(json.dumps(report,indent=2)+'\n')
|
|
return 2
|
|
report['passed'] = all(row['passed'] for row in report['checks'])
|
|
(folder / 'report.json').write_text(json.dumps(report, indent=2) + '\n')
|
|
print('Work availability gate ' + ('passed' if report['passed'] else 'failed') + '; production writes and capacity remain separate. Report: ' + str(folder / 'report.json'))
|
|
return 0 if report['passed'] else 1
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|