import importlib.util import json import os from pathlib import Path import subprocess import sys import tempfile import unittest ROOT = Path(__file__).resolve().parents[2] spec = importlib.util.spec_from_file_location('verification', ROOT/'scripts/verify-change.py') v = importlib.util.module_from_spec(spec) spec.loader.exec_module(v) POLICY = json.loads((ROOT/'scripts/verification-scopes.json').read_text()) class VerificationTest(unittest.TestCase): def plan(self, files, mode='quick'): return v.plan_checks(ROOT, files, POLICY, mode, 'HEAD') def test_documentation_is_small(self): plan = self.plan(['docs/example.md', 'AGENTS.md']) self.assertEqual(len(plan['commands']), 2) self.assertEqual(plan['deferred'], []) self.assertEqual(plan['unmapped'], []) def test_common_page_expands_and_deduplicates(self): plan = self.plan(['src/pages/AnnotationOperations.tsx']) self.assertGreaterEqual(len(plan['scopes']), 4) self.assertTrue(plan['deferred']) self.assertFalse(any('go' == c['argv'][0] and c['env'] for c in plan['commands'])) node_tests = [c for c in plan['commands'] if c['argv'][:2] == ['node', '--test']] self.assertEqual(len(node_tests), 1) self.assertEqual(len(node_tests[0]['argv'][2:]), len(set(node_tests[0]['argv'][2:]))) def test_quick_scope_excludes_opt_in_and_interactive_previews(self): plan = self.plan(['src/pages/AnnotationOperations.tsx']) patterns = [c['argv'][c['argv'].index('-run') + 1] for c in plan['commands'] if c['argv'][:2] == ['go', 'test']] import re for name in ['TestSourceValidationBrowserReplica', 'TestRandomClaimLocalPreview']: self.assertFalse(any(re.search(p, name) for p in patterns), name) self.assertTrue(any(re.search(p, 'TestHumanPoolSearchBeyondFirst500') for p in patterns)) def test_pool_task_explicitly_runs_mongo_parity_and_large_payload(self): plan = self.plan(['internal/store/human_pool_search.go'], 'task') self.assertTrue(any(c['env'].get('POOL_LOCAL_MONGO_URI', '').startswith('mongodb://127.0.0.1:') for c in plan['commands'])) def test_failed_check_does_not_hide_independent_checks_or_allow_stale_boot(self): with tempfile.TemporaryDirectory() as d: root = Path(d) commands = [v.command([sys.executable, '-c', 'raise SystemExit(1)']), v.command([sys.executable, '-c', 'print("independent")']), {**v.command([sys.executable, '-c', 'raise RuntimeError("must not run")']), 'requires': [[sys.executable, '-c', 'raise SystemExit(1)']]}] results = v.run_checks(commands, root, root, 5) self.assertEqual([x['status'] for x in results], ['failed', 'passed', 'not-run']) self.assertIn('dependency', results[-1]['reason']) def test_source_task_includes_real_mongo_and_browser(self): plan = self.plan(['internal/store/annotation_source_admission.go'], 'task') self.assertTrue(any(c['env'].get('SOURCE_ADMISSION_LOCAL_MONGO') for c in plan['commands'])) self.assertTrue(any(c['env'].get('SOURCE_VALIDATION_BROWSER_TEST') for c in plan['commands'])) self.assertIn('source-and-buffer', plan['scopes']) self.assertEqual(plan['unmapped'], []) def test_dependency_task_boot_requires_the_current_build(self): plan = self.plan(['package-lock.json'], 'task') boot = next(c for c in plan['commands'] if c['argv'] == ['npm', 'run', 'test:production:boot']) self.assertEqual(boot['requires'], [['npm', 'run', 'build:web']]) def test_script_alias_only_does_not_trigger_dependency_suite(self): plan = v.plan_checks(ROOT, ['package.json'], POLICY, 'quick', 'HEAD', tooling_package=True) self.assertEqual(plan['deferred'], []) self.assertFalse(any('lint:web' in c['argv'] for c in plan['commands'])) self.assertTrue(any('unittest' in c['argv'] for c in plan['commands'])) def test_css_does_not_run_business_suites(self): plan = self.plan(['src/example.css']) self.assertEqual(plan['deferred'], []) self.assertFalse(any('lint:web' in c['argv'] for c in plan['commands'])) def test_integration_does_not_repeat_targeted_frontend_tests(self): plan = self.plan(['src/pages/AnnotationOperations.tsx'], 'integration') self.assertFalse(any(c['argv'][:2] == ['node', '--test'] for c in plan['commands'])) self.assertEqual(sum('test:frontend:gate' in c['argv'] for c in plan['commands']), 1) def test_integration_keeps_tooling_regressions(self): plan = self.plan(['scripts/verify-change.py'], 'integration') self.assertTrue(any('unittest' in c['argv'] for c in plan['commands'])) def test_new_frontend_falls_back_at_task_level(self): plan = self.plan(['src/components/FutureShared.tsx']) self.assertTrue(any('test:frontend:gate' in c['argv'] for c in plan['deferred'])) self.assertFalse(any('test:frontend:gate' in c['argv'] for c in plan['commands'])) def test_unknown_script_is_not_silently_covered(self): self.assertTrue(self.plan(['scripts/new-data-migration.py'])['unmapped']) self.assertTrue(self.plan(['tests/browser/new-flow.mjs'])['unmapped']) def test_known_failure_file_uses_official_gate_without_expanding_baseline(self): plan = self.plan(['tests/frontend/fanyu-comment-followup.test.mjs']) self.assertTrue(any(c['argv'] == ['npm', 'run', 'test:frontend:gate'] for c in plan['commands'])) self.assertFalse(any(c['argv'][:2] == ['node', '--test'] for c in plan['commands'])) def test_known_go_failure_uses_official_gate_and_does_not_repeat_raw_suite(self): plan = self.plan(['internal/worker/runtime.go'], 'task') self.assertEqual(sum(c['argv'] == ['npm', 'run', 'test:go:gate'] for c in plan['commands']), 1) self.assertFalse(any(c['argv'][:2] == ['go', 'test'] and './internal/worker' in c['argv'] for c in plan['commands'])) def test_changed_frontend_test_is_always_included(self): f = 'tests/frontend/annotation-buffer-summary.test.mjs' plan = self.plan([f]) self.assertTrue(any(f in c['argv'] for c in plan['commands'])) def test_deleted_go_file_gets_package_fallback(self): plan = self.plan(['internal/store/removed.go']) self.assertTrue(any('./internal/store' in c['argv'] for c in plan['deferred'])) def test_scope_tests_and_browser_names_exist(self): for scope in POLICY['scopes']: for pattern in scope.get('frontend', []): self.assertTrue(list(ROOT.glob(pattern)), pattern) for c in scope.get('task', []): if c['argv'][0] == 'go': package = ROOT/c['argv'][2] self.assertTrue(package.is_dir()) if c['argv'][:2] == ['npm', 'run']: self.assertIn(c['argv'][2], json.loads((ROOT/'package.json').read_text())['scripts']) def test_changes_include_committed_staged_unstaged_untracked_and_renamed(self): with tempfile.TemporaryDirectory() as d: root = Path(d) def git(*args): return subprocess.check_output(['git', *args], cwd=root, stderr=subprocess.DEVNULL).decode().strip() git('init') git('config', 'user.email', 'test@example.invalid') git('config', 'user.name', 'Test') (root/'a').write_text('initial') git('add', 'a'); git('commit', '-m', 'initial') base = git('rev-parse', 'HEAD') git('mv', 'a', 'b'); git('commit', '-m', 'rename') (root/'staged').write_text('staged'); git('add', 'staged') (root/'b').write_text('modified') (root/'new').write_text('untracked') _, files = v.changed_files(root, base) self.assertEqual(files, ['a', 'b', 'new', 'staged']) before = v.fingerprint(root) (root/'new').write_text('changed during check') self.assertNotEqual(before, v.fingerprint(root)) with self.assertRaises(subprocess.CalledProcessError): v.changed_files(root, 'does-not-exist') def test_runner_failure_skip_empty_and_timeout_never_pass(self): with tempfile.TemporaryDirectory() as d: for code, expected in [ ('print("ok")', 'passed'), ('raise SystemExit(3)', 'failed'), ('print("--- SKIP: TestOptIn")', 'incomplete'), ('print("testing: warning: no tests to run")', 'incomplete'), ('import time; time.sleep(3)', 'incomplete'), ]: result = v.run_check(v.command([sys.executable, '-c', code], tests=True), Path(d), Path(d)/'run.log', .1) self.assertEqual(result['status'], expected, code) def test_runner_only_enables_declared_opt_ins(self): from unittest.mock import patch with tempfile.TemporaryDirectory() as d, patch.dict(os.environ, {'RUN_OUTPUT_SNAPSHOT_BENCHMARK': '1', 'TEST_WORKFORCE_MONGO_URI': 'remote', 'MONGO_URI': 'remote'}): code = 'import os; assert "RUN_OUTPUT_SNAPSHOT_BENCHMARK" not in os.environ; assert "TEST_WORKFORCE_MONGO_URI" not in os.environ; assert "MONGO_URI" not in os.environ; assert os.environ["SOURCE_ADMISSION_LOCAL_MONGO"] == "1"' c = v.command([sys.executable, '-c', code], env={'SOURCE_ADMISSION_LOCAL_MONGO': '1'}) self.assertEqual(v.run_check(c, Path(d), Path(d)/'run.log', 2)['status'], 'passed') def test_timeout_terminates_descendants(self): with tempfile.TemporaryDirectory() as d: marker = Path(d)/'should-not-exist' nested = f'import time; from pathlib import Path; time.sleep(1); Path({str(marker)!r}).touch()' code = f'import subprocess,sys,time; subprocess.Popen([sys.executable,"-c",{nested!r}]); time.sleep(5)' result = v.run_check(v.command([sys.executable, '-c', code]), Path(d), Path(d)/'run.log', .1) self.assertEqual(result['status'], 'incomplete') import time time.sleep(1.1) self.assertFalse(marker.exists()) if __name__ == '__main__': unittest.main()