droboticflow/tests/release/test_reliability_checks.py

125 lines
8.2 KiB
Python

import datetime as dt
import copy
import importlib.util
import json
import os
from pathlib import Path
import tempfile
import time
import sys
import subprocess
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
def module(name, path):
s = importlib.util.spec_from_file_location(name, ROOT / path)
m = importlib.util.module_from_spec(s)
s.loader.exec_module(m)
return m
m = module('release_checks', 'scripts/lib/release_checks.py')
g = module('work_availability', 'scripts/test_work_availability.py')
ready = module('source_readiness', 'scripts/lib/source_readiness.py')
class CheckEvidenceTests(unittest.TestCase):
def test_private_bounded_redaction_including_multiline_secrets(self):
with tempfile.TemporaryDirectory() as d:
result = m.write_log(d, ['fake-check'], 'password:\n"canary-pass"\nAuthorization: Bearer canary-token\nmongodb://user:canary-db@host/db\nhttps://host/x?signature=canary-url\nAWS_ACCESS_KEY_ID=canary-aws\nBOS_SK=canary-bos\nTraceback: visible failure', 'failed')
p = Path(result['log'])
text = p.read_text()
for value in ['canary-pass', 'canary-token', 'canary-db', 'canary-url','canary-aws','canary-bos']:
self.assertNotIn(value, text)
self.assertIn('visible failure', text)
self.assertEqual(p.stat().st_mode & 511, 384)
def test_large_unstructured_output_is_redacted_with_bounded_runtime(self):
code = "import sys;sys.path.insert(0," + repr(str(ROOT/'scripts')) + ");from lib.release_checks import redact;assert len(redact('a'*3000000))<1000"
subprocess.run([sys.executable,'-c',code],check=True,capture_output=True,timeout=5)
def test_tampering_expiry_identity_and_external_checks_never_reuse(self):
with tempfile.TemporaryDirectory() as d:
e = {'result': 'passed', 'identity': {'commit': 'x', 'complete': True}, 'finishedUnix': time.time(), **m.write_log(d, ['test'], 'ok', 'passed')}
self.assertTrue(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
self.assertFalse(m.reusable(e, {'commit': 'y', 'complete': True}, ['npm', 'run', 'lint:web']))
for args in [['npm', 'run', 'test:browser:annotation-persistence'], ['python3', 'scripts/test_work_availability.py']]:
self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, args))
self.assertFalse(m.reusable({**e, 'finishedUnix': 0}, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
Path(e['log']).write_text('tampered')
self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
def test_production_credentials_and_inherited_switches_removed(self):
with patch.dict(os.environ, {'POOL_LOCAL_MONGO_URI': 'production', 'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'MONGO_URI': 'production', 'BOS_SECRET_KEY': 'secret', 'SOURCE_ADVISORY_BROWSER_TEST': '1', 'ANNOTATION_SOURCE_VALIDATION_MODE': 'strict'}):
e = m.check_environment()
for key in ['POOL_LOCAL_MONGO_URI', 'SOURCE_ADMISSION_LOCAL_MONGO', 'MONGO_URI', 'BOS_SECRET_KEY', 'SOURCE_ADVISORY_BROWSER_TEST', 'ANNOTATION_SOURCE_VALIDATION_MODE']:
self.assertNotIn(key, e)
def test_installed_dependency_and_symlink_changes_invalidate_identity(self):
with tempfile.TemporaryDirectory() as d, patch.object(m.subprocess, 'check_output', return_value=b'version'):
root = Path(d)
pkg = root / 'node_modules/p'
pkg.mkdir(parents=True)
(pkg / 'index.js').write_text('a')
a = m.identity(root, 'x', 'p')
(pkg / 'index.js').write_text('b')
self.assertNotEqual(a, m.identity(root, 'x', 'p'))
linked = root / 'linked'
linked.mkdir()
(linked / 'index.js').write_text('c')
(root / 'node_modules/link').symlink_to(linked, target_is_directory=True)
a = m.identity(root, 'x', 'p')
(linked / 'index.js').write_text('d')
self.assertNotEqual(a, m.identity(root, 'x', 'p'))
def test_required_case_skip_missing_and_failure_are_not_passes(self):
out = '\n'.join((json.dumps(e) for e in [{'Action': 'pass', 'Test': 'Required'}, {'Action': 'skip', 'Test': 'Required/child'}]))
self.assertFalse(g.verdict(out, 0, {'Required'})['passed'])
self.assertFalse(g.verdict('', 0, {'Required'})['passed'])
self.assertFalse(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 1, {'Required'})['passed'])
self.assertTrue(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 0, {'Required'})['passed'])
def test_existing_checks_are_wired_not_reimplemented(self):
policy = json.loads((ROOT / 'docs/release/fanyu-policy.json').read_text())
self.assertIn(['python3', 'scripts/test_work_availability.py'], policy['localChecks'])
self.assertIn('npm run test:work-availability', (ROOT / '.gitea/workflows/web-ci.yml').read_text())
self.assertIn('TestSourceAdvisoryCompleteWorkAndReviewFlow', g.CORE)
self.assertIn('TestSubmissionRecoveryEventMongo', g.MONGO)
def test_independent_checks_continue_but_mixed_sources_stop(self):
checks = [('first', {'Required'}, {}), ('second', {'Required'}, {})]
output = json.dumps({'Action': 'pass', 'Test': 'Required'})
with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', return_value='stable'), patch.object(g, 'run_phase', side_effect=[subprocess.CompletedProcess([], 1, '', 'first failed'), subprocess.CompletedProcess([], 0, output, '')]) as run:
self.assertEqual(g.main(), 1)
self.assertEqual(run.call_count, 2)
report = json.loads(next(Path(d).rglob('report.json')).read_text())
self.assertFalse(report['passed'])
self.assertTrue(report['checks'][1]['passed'])
with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', side_effect=['before', 'after']), patch.object(g, 'run_phase', return_value=subprocess.CompletedProcess([], 0, output, '')) as run:
self.assertEqual(g.main(), 2)
self.assertEqual(run.call_count, 1)
report = json.loads(next(Path(d).rglob('report.json')).read_text())
self.assertEqual(report['remainingUnverified'], ['second'])
class StrictReadinessTests(unittest.TestCase):
def report(self):
return {'schema': 'droboticflow.source-readiness/v1', 'namespace': 'n', 'database': 'db', 'scope': 'deployment', 'organizationId': '', 'projectId': '', 'targetMode': 'strict', 'complete': True, 'legacySourceCutoff': '', 'buildCommit': 'c' * 40, 'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'rows': [{'organizationId': 'o', 'stage': 'quality_review', 'status': 'ready', 'total': 2, 'allowed': 2, 'unverified': 0, 'pending': 0, 'failed': 0}]}
def validate(self, r):
with tempfile.TemporaryDirectory() as d:
p = Path(d) / 'report.json'
p.write_text(json.dumps(r))
return ready.validate_report(p, namespace='n', database='db', cutoff=None, live_commit='c' * 40)
def test_only_complete_current_deployment_inventory_can_activate(self):
self.assertEqual(self.validate(self.report())['totals']['allowed'], 2)
for patcher in [lambda r: r.update(scope='organization', organizationId='o'), lambda r: r.update(projectId='p'), lambda r: r.update(complete=False), lambda r: r.update(database='local-synthetic'), lambda r: r.update(namespace='other'), lambda r: r.update(buildCommit='a' * 40), lambda r: r.update(generatedAt='2000-01-01T00:00:00Z'), lambda r: r.update(legacySourceCutoff='2026-11-12T00:00:00Z'), lambda r: r['rows'][0].update(total=3), lambda r: r['rows'][0].update(allowed=True), lambda r: r['rows'][0].update(allowed=1, pending=1), lambda r: r['rows'].append(copy.deepcopy(r['rows'][0]))]:
r = self.report()
patcher(r)
with self.subTest(r=r), self.assertRaises(ValueError):
self.validate(r)
if __name__ == '__main__':
unittest.main()