forked from longtao.wu/droboticflow
185 lines
10 KiB
Python
185 lines
10 KiB
Python
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
spec = importlib.util.spec_from_file_location('verification', ROOT/'scripts/verify-change.py')
|
|
v = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(v)
|
|
POLICY = json.loads((ROOT/'scripts/verification-scopes.json').read_text())
|
|
|
|
|
|
class VerificationTest(unittest.TestCase):
|
|
def plan(self, files, mode='quick'):
|
|
return v.plan_checks(ROOT, files, POLICY, mode, 'HEAD')
|
|
|
|
def test_documentation_is_small(self):
|
|
plan = self.plan(['docs/example.md', 'AGENTS.md'])
|
|
self.assertEqual(len(plan['commands']), 2)
|
|
self.assertEqual(plan['deferred'], [])
|
|
self.assertEqual(plan['unmapped'], [])
|
|
|
|
def test_common_page_expands_and_deduplicates(self):
|
|
plan = self.plan(['src/pages/AnnotationOperations.tsx'])
|
|
self.assertGreaterEqual(len(plan['scopes']), 4)
|
|
self.assertTrue(plan['deferred'])
|
|
self.assertFalse(any('go' == c['argv'][0] and c['env'] for c in plan['commands']))
|
|
node_tests = [c for c in plan['commands'] if c['argv'][:2] == ['node', '--test']]
|
|
self.assertEqual(len(node_tests), 1)
|
|
self.assertEqual(len(node_tests[0]['argv'][2:]), len(set(node_tests[0]['argv'][2:])))
|
|
|
|
def test_quick_scope_excludes_opt_in_and_interactive_previews(self):
|
|
plan = self.plan(['src/pages/AnnotationOperations.tsx'])
|
|
patterns = [c['argv'][c['argv'].index('-run') + 1] for c in plan['commands'] if c['argv'][:2] == ['go', 'test']]
|
|
import re
|
|
for name in ['TestSourceValidationBrowserReplica', 'TestRandomClaimLocalPreview']:
|
|
self.assertFalse(any(re.search(p, name) for p in patterns), name)
|
|
self.assertTrue(any(re.search(p, 'TestHumanPoolSearchBeyondFirst500') for p in patterns))
|
|
|
|
def test_pool_task_explicitly_runs_mongo_parity_and_large_payload(self):
|
|
plan = self.plan(['internal/store/human_pool_search.go'], 'task')
|
|
self.assertTrue(any(c['env'].get('POOL_LOCAL_MONGO_URI', '').startswith('mongodb://127.0.0.1:') for c in plan['commands']))
|
|
|
|
def test_failed_check_does_not_hide_independent_checks_or_allow_stale_boot(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root = Path(d)
|
|
commands = [v.command([sys.executable, '-c', 'raise SystemExit(1)']),
|
|
v.command([sys.executable, '-c', 'print("independent")']),
|
|
{**v.command([sys.executable, '-c', 'raise RuntimeError("must not run")']), 'requires': [[sys.executable, '-c', 'raise SystemExit(1)']]}]
|
|
results = v.run_checks(commands, root, root, 5)
|
|
self.assertEqual([x['status'] for x in results], ['failed', 'passed', 'not-run'])
|
|
self.assertIn('dependency', results[-1]['reason'])
|
|
|
|
def test_source_task_includes_real_mongo_and_browser(self):
|
|
plan = self.plan(['internal/store/annotation_source_admission.go'], 'task')
|
|
self.assertTrue(any(c['env'].get('SOURCE_ADMISSION_LOCAL_MONGO') for c in plan['commands']))
|
|
self.assertTrue(any(c['env'].get('SOURCE_VALIDATION_BROWSER_TEST') for c in plan['commands']))
|
|
self.assertIn('source-and-buffer', plan['scopes'])
|
|
self.assertEqual(plan['unmapped'], [])
|
|
|
|
def test_dependency_task_boot_requires_the_current_build(self):
|
|
plan = self.plan(['package-lock.json'], 'task')
|
|
boot = next(c for c in plan['commands'] if c['argv'] == ['npm', 'run', 'test:production:boot'])
|
|
self.assertEqual(boot['requires'], [['npm', 'run', 'build:web']])
|
|
|
|
def test_script_alias_only_does_not_trigger_dependency_suite(self):
|
|
plan = v.plan_checks(ROOT, ['package.json'], POLICY, 'quick', 'HEAD', tooling_package=True)
|
|
self.assertEqual(plan['deferred'], [])
|
|
self.assertFalse(any('lint:web' in c['argv'] for c in plan['commands']))
|
|
self.assertTrue(any('unittest' in c['argv'] for c in plan['commands']))
|
|
|
|
def test_css_does_not_run_business_suites(self):
|
|
plan = self.plan(['src/example.css'])
|
|
self.assertEqual(plan['deferred'], [])
|
|
self.assertFalse(any('lint:web' in c['argv'] for c in plan['commands']))
|
|
|
|
def test_integration_does_not_repeat_targeted_frontend_tests(self):
|
|
plan = self.plan(['src/pages/AnnotationOperations.tsx'], 'integration')
|
|
self.assertFalse(any(c['argv'][:2] == ['node', '--test'] for c in plan['commands']))
|
|
self.assertEqual(sum('test:frontend:gate' in c['argv'] for c in plan['commands']), 1)
|
|
|
|
def test_integration_keeps_tooling_regressions(self):
|
|
plan = self.plan(['scripts/verify-change.py'], 'integration')
|
|
self.assertTrue(any('unittest' in c['argv'] for c in plan['commands']))
|
|
|
|
def test_new_frontend_falls_back_at_task_level(self):
|
|
plan = self.plan(['src/components/FutureShared.tsx'])
|
|
self.assertTrue(any('test:frontend:gate' in c['argv'] for c in plan['deferred']))
|
|
self.assertFalse(any('test:frontend:gate' in c['argv'] for c in plan['commands']))
|
|
|
|
def test_unknown_script_is_not_silently_covered(self):
|
|
self.assertTrue(self.plan(['scripts/new-data-migration.py'])['unmapped'])
|
|
self.assertTrue(self.plan(['tests/browser/new-flow.mjs'])['unmapped'])
|
|
|
|
def test_known_failure_file_uses_official_gate_without_expanding_baseline(self):
|
|
plan = self.plan(['tests/frontend/fanyu-comment-followup.test.mjs'])
|
|
self.assertTrue(any(c['argv'] == ['npm', 'run', 'test:frontend:gate'] for c in plan['commands']))
|
|
self.assertFalse(any(c['argv'][:2] == ['node', '--test'] for c in plan['commands']))
|
|
|
|
def test_known_go_failure_uses_official_gate_and_does_not_repeat_raw_suite(self):
|
|
plan = self.plan(['internal/worker/runtime.go'], 'task')
|
|
self.assertEqual(sum(c['argv'] == ['npm', 'run', 'test:go:gate'] for c in plan['commands']), 1)
|
|
self.assertFalse(any(c['argv'][:2] == ['go', 'test'] and './internal/worker' in c['argv'] for c in plan['commands']))
|
|
|
|
def test_changed_frontend_test_is_always_included(self):
|
|
f = 'tests/frontend/annotation-buffer-summary.test.mjs'
|
|
plan = self.plan([f])
|
|
self.assertTrue(any(f in c['argv'] for c in plan['commands']))
|
|
|
|
def test_deleted_go_file_gets_package_fallback(self):
|
|
plan = self.plan(['internal/store/removed.go'])
|
|
self.assertTrue(any('./internal/store' in c['argv'] for c in plan['deferred']))
|
|
|
|
def test_scope_tests_and_browser_names_exist(self):
|
|
for scope in POLICY['scopes']:
|
|
for pattern in scope.get('frontend', []):
|
|
self.assertTrue(list(ROOT.glob(pattern)), pattern)
|
|
for c in scope.get('task', []):
|
|
if c['argv'][0] == 'go':
|
|
package = ROOT/c['argv'][2]
|
|
self.assertTrue(package.is_dir())
|
|
if c['argv'][:2] == ['npm', 'run']:
|
|
self.assertIn(c['argv'][2], json.loads((ROOT/'package.json').read_text())['scripts'])
|
|
|
|
def test_changes_include_committed_staged_unstaged_untracked_and_renamed(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
root = Path(d)
|
|
def git(*args):
|
|
return subprocess.check_output(['git', *args], cwd=root, stderr=subprocess.DEVNULL).decode().strip()
|
|
git('init')
|
|
git('config', 'user.email', 'test@example.invalid')
|
|
git('config', 'user.name', 'Test')
|
|
(root/'a').write_text('initial')
|
|
git('add', 'a'); git('commit', '-m', 'initial')
|
|
base = git('rev-parse', 'HEAD')
|
|
git('mv', 'a', 'b'); git('commit', '-m', 'rename')
|
|
(root/'staged').write_text('staged'); git('add', 'staged')
|
|
(root/'b').write_text('modified')
|
|
(root/'new').write_text('untracked')
|
|
_, files = v.changed_files(root, base)
|
|
self.assertEqual(files, ['a', 'b', 'new', 'staged'])
|
|
before = v.fingerprint(root)
|
|
(root/'new').write_text('changed during check')
|
|
self.assertNotEqual(before, v.fingerprint(root))
|
|
with self.assertRaises(subprocess.CalledProcessError):
|
|
v.changed_files(root, 'does-not-exist')
|
|
|
|
def test_runner_failure_skip_empty_and_timeout_never_pass(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
for code, expected in [
|
|
('print("ok")', 'passed'),
|
|
('raise SystemExit(3)', 'failed'),
|
|
('print("--- SKIP: TestOptIn")', 'incomplete'),
|
|
('print("testing: warning: no tests to run")', 'incomplete'),
|
|
('import time; time.sleep(3)', 'incomplete'),
|
|
]:
|
|
result = v.run_check(v.command([sys.executable, '-c', code], tests=True), Path(d), Path(d)/'run.log', .1)
|
|
self.assertEqual(result['status'], expected, code)
|
|
|
|
def test_runner_only_enables_declared_opt_ins(self):
|
|
from unittest.mock import patch
|
|
with tempfile.TemporaryDirectory() as d, patch.dict(os.environ, {'RUN_OUTPUT_SNAPSHOT_BENCHMARK': '1', 'TEST_WORKFORCE_MONGO_URI': 'remote', 'MONGO_URI': 'remote'}):
|
|
code = 'import os; assert "RUN_OUTPUT_SNAPSHOT_BENCHMARK" not in os.environ; assert "TEST_WORKFORCE_MONGO_URI" not in os.environ; assert "MONGO_URI" not in os.environ; assert os.environ["SOURCE_ADMISSION_LOCAL_MONGO"] == "1"'
|
|
c = v.command([sys.executable, '-c', code], env={'SOURCE_ADMISSION_LOCAL_MONGO': '1'})
|
|
self.assertEqual(v.run_check(c, Path(d), Path(d)/'run.log', 2)['status'], 'passed')
|
|
|
|
def test_timeout_terminates_descendants(self):
|
|
with tempfile.TemporaryDirectory() as d:
|
|
marker = Path(d)/'should-not-exist'
|
|
nested = f'import time; from pathlib import Path; time.sleep(1); Path({str(marker)!r}).touch()'
|
|
code = f'import subprocess,sys,time; subprocess.Popen([sys.executable,"-c",{nested!r}]); time.sleep(5)'
|
|
result = v.run_check(v.command([sys.executable, '-c', code]), Path(d), Path(d)/'run.log', .1)
|
|
self.assertEqual(result['status'], 'incomplete')
|
|
import time
|
|
time.sleep(1.1)
|
|
self.assertFalse(marker.exists())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|