droboticflow/tests/tooling/test_verification.py

185 lines
10 KiB
Python

import importlib.util
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
spec = importlib.util.spec_from_file_location('verification', ROOT/'scripts/verify-change.py')
v = importlib.util.module_from_spec(spec)
spec.loader.exec_module(v)
POLICY = json.loads((ROOT/'scripts/verification-scopes.json').read_text())
class VerificationTest(unittest.TestCase):
def plan(self, files, mode='quick'):
return v.plan_checks(ROOT, files, POLICY, mode, 'HEAD')
def test_documentation_is_small(self):
plan = self.plan(['docs/example.md', 'AGENTS.md'])
self.assertEqual(len(plan['commands']), 2)
self.assertEqual(plan['deferred'], [])
self.assertEqual(plan['unmapped'], [])
def test_common_page_expands_and_deduplicates(self):
plan = self.plan(['src/pages/AnnotationOperations.tsx'])
self.assertGreaterEqual(len(plan['scopes']), 4)
self.assertTrue(plan['deferred'])
self.assertFalse(any('go' == c['argv'][0] and c['env'] for c in plan['commands']))
node_tests = [c for c in plan['commands'] if c['argv'][:2] == ['node', '--test']]
self.assertEqual(len(node_tests), 1)
self.assertEqual(len(node_tests[0]['argv'][2:]), len(set(node_tests[0]['argv'][2:])))
def test_quick_scope_excludes_opt_in_and_interactive_previews(self):
plan = self.plan(['src/pages/AnnotationOperations.tsx'])
patterns = [c['argv'][c['argv'].index('-run') + 1] for c in plan['commands'] if c['argv'][:2] == ['go', 'test']]
import re
for name in ['TestSourceValidationBrowserReplica', 'TestRandomClaimLocalPreview']:
self.assertFalse(any(re.search(p, name) for p in patterns), name)
self.assertTrue(any(re.search(p, 'TestHumanPoolSearchBeyondFirst500') for p in patterns))
def test_pool_task_explicitly_runs_mongo_parity_and_large_payload(self):
plan = self.plan(['internal/store/human_pool_search.go'], 'task')
self.assertTrue(any(c['env'].get('POOL_LOCAL_MONGO_URI', '').startswith('mongodb://127.0.0.1:') for c in plan['commands']))
def test_failed_check_does_not_hide_independent_checks_or_allow_stale_boot(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
commands = [v.command([sys.executable, '-c', 'raise SystemExit(1)']),
v.command([sys.executable, '-c', 'print("independent")']),
{**v.command([sys.executable, '-c', 'raise RuntimeError("must not run")']), 'requires': [[sys.executable, '-c', 'raise SystemExit(1)']]}]
results = v.run_checks(commands, root, root, 5)
self.assertEqual([x['status'] for x in results], ['failed', 'passed', 'not-run'])
self.assertIn('dependency', results[-1]['reason'])
def test_source_task_includes_real_mongo_and_browser(self):
plan = self.plan(['internal/store/annotation_source_admission.go'], 'task')
self.assertTrue(any(c['env'].get('SOURCE_ADMISSION_LOCAL_MONGO') for c in plan['commands']))
self.assertTrue(any(c['env'].get('SOURCE_VALIDATION_BROWSER_TEST') for c in plan['commands']))
self.assertIn('source-and-buffer', plan['scopes'])
self.assertEqual(plan['unmapped'], [])
def test_dependency_task_boot_requires_the_current_build(self):
plan = self.plan(['package-lock.json'], 'task')
boot = next(c for c in plan['commands'] if c['argv'] == ['npm', 'run', 'test:production:boot'])
self.assertEqual(boot['requires'], [['npm', 'run', 'build:web']])
def test_script_alias_only_does_not_trigger_dependency_suite(self):
plan = v.plan_checks(ROOT, ['package.json'], POLICY, 'quick', 'HEAD', tooling_package=True)
self.assertEqual(plan['deferred'], [])
self.assertFalse(any('lint:web' in c['argv'] for c in plan['commands']))
self.assertTrue(any('unittest' in c['argv'] for c in plan['commands']))
def test_css_does_not_run_business_suites(self):
plan = self.plan(['src/example.css'])
self.assertEqual(plan['deferred'], [])
self.assertFalse(any('lint:web' in c['argv'] for c in plan['commands']))
def test_integration_does_not_repeat_targeted_frontend_tests(self):
plan = self.plan(['src/pages/AnnotationOperations.tsx'], 'integration')
self.assertFalse(any(c['argv'][:2] == ['node', '--test'] for c in plan['commands']))
self.assertEqual(sum('test:frontend:gate' in c['argv'] for c in plan['commands']), 1)
def test_integration_keeps_tooling_regressions(self):
plan = self.plan(['scripts/verify-change.py'], 'integration')
self.assertTrue(any('unittest' in c['argv'] for c in plan['commands']))
def test_new_frontend_falls_back_at_task_level(self):
plan = self.plan(['src/components/FutureShared.tsx'])
self.assertTrue(any('test:frontend:gate' in c['argv'] for c in plan['deferred']))
self.assertFalse(any('test:frontend:gate' in c['argv'] for c in plan['commands']))
def test_unknown_script_is_not_silently_covered(self):
self.assertTrue(self.plan(['scripts/new-data-migration.py'])['unmapped'])
self.assertTrue(self.plan(['tests/browser/new-flow.mjs'])['unmapped'])
def test_known_failure_file_uses_official_gate_without_expanding_baseline(self):
plan = self.plan(['tests/frontend/fanyu-comment-followup.test.mjs'])
self.assertTrue(any(c['argv'] == ['npm', 'run', 'test:frontend:gate'] for c in plan['commands']))
self.assertFalse(any(c['argv'][:2] == ['node', '--test'] for c in plan['commands']))
def test_known_go_failure_uses_official_gate_and_does_not_repeat_raw_suite(self):
plan = self.plan(['internal/worker/runtime.go'], 'task')
self.assertEqual(sum(c['argv'] == ['npm', 'run', 'test:go:gate'] for c in plan['commands']), 1)
self.assertFalse(any(c['argv'][:2] == ['go', 'test'] and './internal/worker' in c['argv'] for c in plan['commands']))
def test_changed_frontend_test_is_always_included(self):
f = 'tests/frontend/annotation-buffer-summary.test.mjs'
plan = self.plan([f])
self.assertTrue(any(f in c['argv'] for c in plan['commands']))
def test_deleted_go_file_gets_package_fallback(self):
plan = self.plan(['internal/store/removed.go'])
self.assertTrue(any('./internal/store' in c['argv'] for c in plan['deferred']))
def test_scope_tests_and_browser_names_exist(self):
for scope in POLICY['scopes']:
for pattern in scope.get('frontend', []):
self.assertTrue(list(ROOT.glob(pattern)), pattern)
for c in scope.get('task', []):
if c['argv'][0] == 'go':
package = ROOT/c['argv'][2]
self.assertTrue(package.is_dir())
if c['argv'][:2] == ['npm', 'run']:
self.assertIn(c['argv'][2], json.loads((ROOT/'package.json').read_text())['scripts'])
def test_changes_include_committed_staged_unstaged_untracked_and_renamed(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
def git(*args):
return subprocess.check_output(['git', *args], cwd=root, stderr=subprocess.DEVNULL).decode().strip()
git('init')
git('config', 'user.email', 'test@example.invalid')
git('config', 'user.name', 'Test')
(root/'a').write_text('initial')
git('add', 'a'); git('commit', '-m', 'initial')
base = git('rev-parse', 'HEAD')
git('mv', 'a', 'b'); git('commit', '-m', 'rename')
(root/'staged').write_text('staged'); git('add', 'staged')
(root/'b').write_text('modified')
(root/'new').write_text('untracked')
_, files = v.changed_files(root, base)
self.assertEqual(files, ['a', 'b', 'new', 'staged'])
before = v.fingerprint(root)
(root/'new').write_text('changed during check')
self.assertNotEqual(before, v.fingerprint(root))
with self.assertRaises(subprocess.CalledProcessError):
v.changed_files(root, 'does-not-exist')
def test_runner_failure_skip_empty_and_timeout_never_pass(self):
with tempfile.TemporaryDirectory() as d:
for code, expected in [
('print("ok")', 'passed'),
('raise SystemExit(3)', 'failed'),
('print("--- SKIP: TestOptIn")', 'incomplete'),
('print("testing: warning: no tests to run")', 'incomplete'),
('import time; time.sleep(3)', 'incomplete'),
]:
result = v.run_check(v.command([sys.executable, '-c', code], tests=True), Path(d), Path(d)/'run.log', .1)
self.assertEqual(result['status'], expected, code)
def test_runner_only_enables_declared_opt_ins(self):
from unittest.mock import patch
with tempfile.TemporaryDirectory() as d, patch.dict(os.environ, {'RUN_OUTPUT_SNAPSHOT_BENCHMARK': '1', 'TEST_WORKFORCE_MONGO_URI': 'remote', 'MONGO_URI': 'remote'}):
code = 'import os; assert "RUN_OUTPUT_SNAPSHOT_BENCHMARK" not in os.environ; assert "TEST_WORKFORCE_MONGO_URI" not in os.environ; assert "MONGO_URI" not in os.environ; assert os.environ["SOURCE_ADMISSION_LOCAL_MONGO"] == "1"'
c = v.command([sys.executable, '-c', code], env={'SOURCE_ADMISSION_LOCAL_MONGO': '1'})
self.assertEqual(v.run_check(c, Path(d), Path(d)/'run.log', 2)['status'], 'passed')
def test_timeout_terminates_descendants(self):
with tempfile.TemporaryDirectory() as d:
marker = Path(d)/'should-not-exist'
nested = f'import time; from pathlib import Path; time.sleep(1); Path({str(marker)!r}).touch()'
code = f'import subprocess,sys,time; subprocess.Popen([sys.executable,"-c",{nested!r}]); time.sleep(5)'
result = v.run_check(v.command([sys.executable, '-c', code]), Path(d), Path(d)/'run.log', .1)
self.assertEqual(result['status'], 'incomplete')
import time
time.sleep(1.1)
self.assertFalse(marker.exists())
if __name__ == '__main__':
unittest.main()