forked from longtao.wu/droboticflow
125 lines
8.2 KiB
Python
125 lines
8.2 KiB
Python
import datetime as dt
|
|
import copy
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import time
|
|
import sys
|
|
import subprocess
|
|
import unittest
|
|
from unittest.mock import patch
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
|
|
def module(name, path):
|
|
s = importlib.util.spec_from_file_location(name, ROOT / path)
|
|
m = importlib.util.module_from_spec(s)
|
|
s.loader.exec_module(m)
|
|
return m
|
|
m = module('release_checks', 'scripts/lib/release_checks.py')
|
|
g = module('work_availability', 'scripts/test_work_availability.py')
|
|
ready = module('source_readiness', 'scripts/lib/source_readiness.py')
|
|
|
|
class CheckEvidenceTests(unittest.TestCase):
|
|
|
|
def test_private_bounded_redaction_including_multiline_secrets(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
result = m.write_log(d, ['fake-check'], 'password:\n"canary-pass"\nAuthorization: Bearer canary-token\nmongodb://user:canary-db@host/db\nhttps://host/x?signature=canary-url\nAWS_ACCESS_KEY_ID=canary-aws\nBOS_SK=canary-bos\nTraceback: visible failure', 'failed')
|
|
p = Path(result['log'])
|
|
text = p.read_text()
|
|
for value in ['canary-pass', 'canary-token', 'canary-db', 'canary-url','canary-aws','canary-bos']:
|
|
self.assertNotIn(value, text)
|
|
self.assertIn('visible failure', text)
|
|
self.assertEqual(p.stat().st_mode & 511, 384)
|
|
|
|
|
|
def test_large_unstructured_output_is_redacted_with_bounded_runtime(self):
|
|
code = "import sys;sys.path.insert(0," + repr(str(ROOT/'scripts')) + ");from lib.release_checks import redact;assert len(redact('a'*3000000))<1000"
|
|
subprocess.run([sys.executable,'-c',code],check=True,capture_output=True,timeout=5)
|
|
|
|
def test_tampering_expiry_identity_and_external_checks_never_reuse(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
e = {'result': 'passed', 'identity': {'commit': 'x', 'complete': True}, 'finishedUnix': time.time(), **m.write_log(d, ['test'], 'ok', 'passed')}
|
|
self.assertTrue(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
|
|
self.assertFalse(m.reusable(e, {'commit': 'y', 'complete': True}, ['npm', 'run', 'lint:web']))
|
|
for args in [['npm', 'run', 'test:browser:annotation-persistence'], ['python3', 'scripts/test_work_availability.py']]:
|
|
self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, args))
|
|
self.assertFalse(m.reusable({**e, 'finishedUnix': 0}, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
|
|
Path(e['log']).write_text('tampered')
|
|
self.assertFalse(m.reusable(e, {'commit': 'x', 'complete': True}, ['npm', 'run', 'lint:web']))
|
|
|
|
def test_production_credentials_and_inherited_switches_removed(self):
|
|
with patch.dict(os.environ, {'POOL_LOCAL_MONGO_URI': 'production', 'SOURCE_ADMISSION_LOCAL_MONGO': '1', 'MONGO_URI': 'production', 'BOS_SECRET_KEY': 'secret', 'SOURCE_ADVISORY_BROWSER_TEST': '1', 'ANNOTATION_SOURCE_VALIDATION_MODE': 'strict'}):
|
|
e = m.check_environment()
|
|
for key in ['POOL_LOCAL_MONGO_URI', 'SOURCE_ADMISSION_LOCAL_MONGO', 'MONGO_URI', 'BOS_SECRET_KEY', 'SOURCE_ADVISORY_BROWSER_TEST', 'ANNOTATION_SOURCE_VALIDATION_MODE']:
|
|
self.assertNotIn(key, e)
|
|
|
|
def test_installed_dependency_and_symlink_changes_invalidate_identity(self):
|
|
with tempfile.TemporaryDirectory() as d, patch.object(m.subprocess, 'check_output', return_value=b'version'):
|
|
root = Path(d)
|
|
pkg = root / 'node_modules/p'
|
|
pkg.mkdir(parents=True)
|
|
(pkg / 'index.js').write_text('a')
|
|
a = m.identity(root, 'x', 'p')
|
|
(pkg / 'index.js').write_text('b')
|
|
self.assertNotEqual(a, m.identity(root, 'x', 'p'))
|
|
linked = root / 'linked'
|
|
linked.mkdir()
|
|
(linked / 'index.js').write_text('c')
|
|
(root / 'node_modules/link').symlink_to(linked, target_is_directory=True)
|
|
a = m.identity(root, 'x', 'p')
|
|
(linked / 'index.js').write_text('d')
|
|
self.assertNotEqual(a, m.identity(root, 'x', 'p'))
|
|
|
|
def test_required_case_skip_missing_and_failure_are_not_passes(self):
|
|
out = '\n'.join((json.dumps(e) for e in [{'Action': 'pass', 'Test': 'Required'}, {'Action': 'skip', 'Test': 'Required/child'}]))
|
|
self.assertFalse(g.verdict(out, 0, {'Required'})['passed'])
|
|
self.assertFalse(g.verdict('', 0, {'Required'})['passed'])
|
|
self.assertFalse(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 1, {'Required'})['passed'])
|
|
self.assertTrue(g.verdict(json.dumps({'Action': 'pass', 'Test': 'Required'}), 0, {'Required'})['passed'])
|
|
|
|
def test_existing_checks_are_wired_not_reimplemented(self):
|
|
policy = json.loads((ROOT / 'docs/release/fanyu-policy.json').read_text())
|
|
self.assertIn(['python3', 'scripts/test_work_availability.py'], policy['localChecks'])
|
|
self.assertIn('npm run test:work-availability', (ROOT / '.gitea/workflows/web-ci.yml').read_text())
|
|
self.assertIn('TestSourceAdvisoryCompleteWorkAndReviewFlow', g.CORE)
|
|
self.assertIn('TestSubmissionRecoveryEventMongo', g.MONGO)
|
|
|
|
def test_independent_checks_continue_but_mixed_sources_stop(self):
|
|
checks = [('first', {'Required'}, {}), ('second', {'Required'}, {})]
|
|
output = json.dumps({'Action': 'pass', 'Test': 'Required'})
|
|
with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', return_value='stable'), patch.object(g, 'run_phase', side_effect=[subprocess.CompletedProcess([], 1, '', 'first failed'), subprocess.CompletedProcess([], 0, output, '')]) as run:
|
|
self.assertEqual(g.main(), 1)
|
|
self.assertEqual(run.call_count, 2)
|
|
report = json.loads(next(Path(d).rglob('report.json')).read_text())
|
|
self.assertFalse(report['passed'])
|
|
self.assertTrue(report['checks'][1]['passed'])
|
|
with tempfile.TemporaryDirectory() as d, patch.object(g, 'ROOT', Path(d)), patch.object(g, 'phases', return_value=checks), patch.object(g, 'source_fingerprint', side_effect=['before', 'after']), patch.object(g, 'run_phase', return_value=subprocess.CompletedProcess([], 0, output, '')) as run:
|
|
self.assertEqual(g.main(), 2)
|
|
self.assertEqual(run.call_count, 1)
|
|
report = json.loads(next(Path(d).rglob('report.json')).read_text())
|
|
self.assertEqual(report['remainingUnverified'], ['second'])
|
|
|
|
class StrictReadinessTests(unittest.TestCase):
|
|
|
|
def report(self):
|
|
return {'schema': 'droboticflow.source-readiness/v1', 'namespace': 'n', 'database': 'db', 'scope': 'deployment', 'organizationId': '', 'projectId': '', 'targetMode': 'strict', 'complete': True, 'legacySourceCutoff': '', 'buildCommit': 'c' * 40, 'generatedAt': dt.datetime.now(dt.timezone.utc).isoformat(), 'rows': [{'organizationId': 'o', 'stage': 'quality_review', 'status': 'ready', 'total': 2, 'allowed': 2, 'unverified': 0, 'pending': 0, 'failed': 0}]}
|
|
|
|
def validate(self, r):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
p = Path(d) / 'report.json'
|
|
p.write_text(json.dumps(r))
|
|
return ready.validate_report(p, namespace='n', database='db', cutoff=None, live_commit='c' * 40)
|
|
|
|
def test_only_complete_current_deployment_inventory_can_activate(self):
|
|
self.assertEqual(self.validate(self.report())['totals']['allowed'], 2)
|
|
for patcher in [lambda r: r.update(scope='organization', organizationId='o'), lambda r: r.update(projectId='p'), lambda r: r.update(complete=False), lambda r: r.update(database='local-synthetic'), lambda r: r.update(namespace='other'), lambda r: r.update(buildCommit='a' * 40), lambda r: r.update(generatedAt='2000-01-01T00:00:00Z'), lambda r: r.update(legacySourceCutoff='2026-11-12T00:00:00Z'), lambda r: r['rows'][0].update(total=3), lambda r: r['rows'][0].update(allowed=True), lambda r: r['rows'][0].update(allowed=1, pending=1), lambda r: r['rows'].append(copy.deepcopy(r['rows'][0]))]:
|
|
r = self.report()
|
|
patcher(r)
|
|
with self.subTest(r=r), self.assertRaises(ValueError):
|
|
self.validate(r)
|
|
if __name__ == '__main__':
|
|
unittest.main()
|