#!/usr/bin/env python3 """Select local development checks; never substitutes for release.py evidence.""" import argparse import ast import hashlib import json import os from pathlib import Path import re import signal import subprocess import sys import time ROOT = Path(__file__).resolve().parents[1] POLICY = ROOT / 'scripts/verification-scopes.json' def git(root, *args): return subprocess.check_output(['git', *args], cwd=root, stderr=subprocess.PIPE).decode().strip('\n') def changed_files(root, base): revision = git(root, 'rev-parse', '--verify', base + '^{commit}') changed = git(root, 'diff', '--name-only', '--no-renames', '-z', revision, '--').split('\0') untracked = git(root, 'ls-files', '--others', '--exclude-standard', '-z').split('\0') return revision, sorted(set(filter(None, changed + untracked))) def command(argv, env=None, tests=False): result = {'argv': argv, 'env': env or {}, 'tests': tests} if argv == ['npm', 'run', 'test:production:boot']: result['requires'] = [['npm', 'run', 'build:web']] return result def tooling_package_change(root, base): try: old = json.loads(git(root, 'show', base + ':package.json')) new = json.loads((root/'package.json').read_text()) except (OSError, ValueError, subprocess.CalledProcessError): return False old_scripts, new_scripts = old.pop('scripts', {}), new.pop('scripts', {}) names = {name for name in set(old_scripts) | set(new_scripts) if old_scripts.get(name) != new_scripts.get(name)} allowed = {'verify:plan', 'verify:quick', 'verify:task', 'verify:integration', 'test:verification', 'predev', 'predev:full', 'predev:api', 'predev:web', 'predev:worker', 'predev:api:replica'} return old == new and bool(names) and names <= allowed def plan_checks(root, files, policy, mode, base, tooling_package=False): quick = [command(['git', 'diff', '--check', base, '--']), command(['node', 'scripts/check-git-hooks.mjs', '--install'])] task = [] warnings = [] unresolved = [] scopes = [] frontend = set() go_patterns = {} matched = set() for scope in policy['scopes']: hits = [f for f in files if any(re.search(p, f) for p in scope['paths'])] if not hits: continue scopes.append(scope['name']) matched.update(hits) for pattern in scope.get('frontend', []): found = list(root.glob(pattern)) if not found: unresolved.append('测试映射不存在: ' + pattern) frontend.update(str(p.relative_to(root)) for p in found) for pkg, patterns in scope.get('go', {}).items(): go_patterns.setdefault(pkg, set()).update(patterns) task.extend(scope.get('task', [])) warnings.extend(scope.get('warnings', [])) web = any((f.startswith('src/') and not f.endswith('.css') or re.match(r'(package(-lock)?\.json|tsconfig.*|vite\.config.*)$', f)) and not (f == 'package.json' and tooling_package) for f in files) if web: quick.append(command(['npm', 'run', 'lint:web'])) formatting = [f for f in files if (root/f).is_file() and (f.startswith('src/') and re.search(r'\.(ts|tsx|css|js)$', f) or f.startswith('tests/frontend/') and f.endswith('.mjs'))] if formatting: quick.append(command(['node', 'node_modules/prettier/bin/prettier.cjs', '--check', *formatting])) go_files = [f for f in files if f.endswith('.go') and (root/f).is_file()] if go_files: quick.append(command([sys.executable, 'scripts/verify-change.py', '--gofmt', *go_files])) for f in files: path = root/f if f.startswith('tests/frontend/') and f.endswith('.test.mjs') and path.exists(): frontend.add(f) continue if f.endswith('.go'): pkg = './' + str(Path(f).parent) test_file = path if f.endswith('_test.go') else path.with_name(path.stem + '_test.go') fast = [] if test_file.exists(): names = re.findall(r'^func (Test\w+)\(', test_file.read_text(), re.M) fast = [name for name in names if not re.search(r'Browser|Mongo|Capacity|Soak|Load|Local|Benchmark|Preview', name)] if fast: go_patterns.setdefault(pkg, set()).update('^' + name + '$' for name in fast) if len(fast) != len(names) and f not in matched: unresolved.append(f + ': 外部环境测试需要显式任务级映射,不能将跳过当通过。') if len(fast) != len(names): warnings.append(f + ': 外部环境/容量测试不计入快速检查;核对任务级覆盖。') if f not in matched and (not f.endswith('_test.go') or not fast): # Package-wide fallback is explicit at task level, not every edit. task.append(command(['go', 'test', pkg, '-timeout=5m', '-v'], tests=True)) continue if f in ('scripts/verify-change.py', 'scripts/verification-scopes.json') or f.startswith('tests/tooling/'): quick.append(command([sys.executable, '-m', 'unittest', 'discover', '-s', 'tests/tooling', '-p', 'test_verification.py'], tests=True)) continue if f in matched or f.endswith('.md') or f.startswith('docs/') and f.endswith(('.png', '.jpg', '.svg')): continue if f.startswith('src/') and f.endswith('.css'): warnings.append(f + ': 样式改动需人工查看受影响页面及异常状态。') continue if f.startswith('src/'): task.append(command(['npm', 'run', 'test:frontend:gate'])) warnings.append(f + ': 未建立业务映射,任务级扩大到前端基线门禁;仍需人工确认浏览器流程。') continue if f.startswith('tests/browser/'): unresolved.append(f + ': 为该浏览器夹具补充可运行的任务级映射。') continue if f == 'package.json' and tooling_package: quick.append(command([sys.executable, '-m', 'unittest', 'discover', '-s', 'tests/tooling', '-p', 'test_verification.py'], tests=True)) continue if f in ('package.json', 'package-lock.json', 'go.mod', 'go.sum') or re.match(r'(tsconfig|vite\.config)', f): task.extend([command(['npm', 'run', 'test:frontend:gate']), command(['npm', 'run', 'test:go:gate']), command(['npm', 'run', 'build:web']), command(['npm', 'run', 'test:production:boot'])]) warnings.append('依赖/公共配置变化:任务级扩大到集成检查,核对安装与锁文件。') continue if path.exists() and f.endswith('.py'): quick.append(command([sys.executable, 'scripts/verify-change.py', '--python-syntax', f])) elif path.exists() and f.endswith('.mjs'): quick.append(command(['node', '--check', f])) elif path.exists() and f.endswith('.sh'): quick.append(command(['bash', '-n', f])) unresolved.append(f + ': 尚无行为验证映射;补充映射/专项证据,语法通过不代表功能通过。') if frontend: known = json.loads((root/'tests/known-failures.json').read_text()).get('frontend', []) known_files = {item.split(' :: ', 1)[0] for item in known} if any(Path(name).name in known_files for name in frontend): quick.append(command(['npm', 'run', 'test:frontend:gate'])) else: quick.append(command(['node', '--test', *sorted(frontend)], tests=True)) for pkg, patterns in sorted(go_patterns.items()): names = set() for test_file in (root/pkg).glob('*_test.go'): names.update(re.findall(r'^func (Test\w+)\(', test_file.read_text(), re.M)) selected_names = sorted(name for name in names if any(re.search(p, name) for p in patterns) and not re.search(r'Browser|Mongo|Capacity|Soak|Load|Local|Benchmark|Preview', name)) if selected_names: known_go = json.loads((root/'tests/known-failures.json').read_text()).get('go', []) if any(pkg.removeprefix('./') + ' :: ' + name in known_go for name in selected_names): quick.append(command(['npm', 'run', 'test:go:gate'], tests=True)) else: quick.append(command(['go', 'test', pkg, '-run', '|'.join('^' + name + '$' for name in selected_names), '-timeout=2m', '-v'], tests=True)) integration = [command(['npm', 'run', x]) for x in ['lint:web', 'format:check', 'test:frontend:gate', 'test:go:gate', 'build:web', 'test:production:boot']] def unique(commands): out, seen = [], set() for c in commands: c = {**command(c['argv']), **c} key = json.dumps(c, sort_keys=True) if key not in seen: out.append(c) seen.add(key) return out quick, task = unique(quick), unique(task) # Full suite gates subsume targeted frontend/Go tests, not tooling checks. integration_extra = [c for c in quick if c['argv'][:2] not in (['node', '--test'], ['go', 'test']) and c['argv'] != ['npm', 'run', 'lint:web'] and 'node_modules/prettier/bin/prettier.cjs' not in c['argv']] selected = quick if mode == 'quick' else unique(integration_extra + task + integration) if mode == 'integration' else unique(quick + task) return {'mode': mode, 'files': files, 'scopes': scopes, 'commands': selected, 'deferred': task if mode == 'quick' else [], 'unmapped': sorted(set(unresolved)), 'notes': sorted(set(warnings)), 'notCovered': [ '生产业务写验收、生产数据处置及正式发布门禁', '120 人完整作业、8 小时运行、节点故障及真实媒体容量', '业务影响需人工核对;文件映射不能证明所有间接依赖均已覆盖']} def fingerprint(root): digest = hashlib.sha256() paths = git(root, 'ls-files', '-z', '--cached', '--others', '--exclude-standard').split('\0') for name in sorted(set(filter(None, paths))): p = root/name digest.update(name.encode()) if p.is_symlink(): digest.update(os.readlink(p).encode()) elif p.is_file(): digest.update(p.read_bytes()) else: digest.update(b'') return digest.hexdigest() def run_check(check, root, logfile, timeout): start = time.monotonic() env = dict(os.environ) # Avoid accidentally enabling unrelated opt-in suites in an inherited shell. for key in list(env): if key.startswith(('TEST_', 'RUN_')) or key in ('MONGO_URI', 'DATABASE_URL') or re.search(r'(_BROWSER_TEST|_LOCAL_MONGO|_MONGO_HTTP|_MONGO_STORE|_LOAD_TEST|_CAPACITY_TEST)$', key): env.pop(key) env.update(check['env']) status, code = 'failed', None with logfile.open('w') as stream: try: child = subprocess.Popen(check['argv'], cwd=root, env=env, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) try: code = child.wait(timeout=timeout) status = 'passed' if code == 0 else 'failed' except (subprocess.TimeoutExpired, KeyboardInterrupt): status = 'incomplete' try: os.killpg(child.pid, signal.SIGTERM) except ProcessLookupError: pass try: child.wait(timeout=2) except subprocess.TimeoutExpired: pass # Kill lingering grandchildren even if the parent exited on TERM. try: os.killpg(child.pid, signal.SIGKILL) except ProcessLookupError: pass child.wait() except OSError as error: stream.write(str(error)) output = logfile.read_text() if status == 'passed' and check.get('tests'): if re.search(r'--- SKIP:|\[no test files\]|no tests to run|# SKIP|# skipped [1-9]|ℹ skipped [1-9]|skipped=[1-9]', output): status = 'incomplete' if re.search(r'Ran 0 tests|# tests 0|ℹ tests 0', output): status = 'incomplete' return {**check, 'status': status, 'exitCode': code, 'seconds': round(time.monotonic()-start, 3), 'log': str(logfile)} def run_checks(checks, root, folder, budget): results = [] start = time.monotonic() for index, check in enumerate(checks): remaining = budget - (time.monotonic() - start) if remaining <= 0: results.append({**check, 'status': 'not-run', 'reason': 'budget exhausted'}) continue if any(not any(r['argv'] == required and r['status'] == 'passed' for r in results) for required in check.get('requires', [])): results.append({**check, 'status': 'not-run', 'reason': 'dependency did not pass'}) continue result = run_check(check, root, folder/f'{index:02}.log', remaining) results.append(result) print(f"{result['status']}: {' '.join(check['argv'])} ({result['seconds']}s)", flush=True) return results def runtime_identity(root): versions = {'python': sys.version.split()[0]} for name, argv in [('node', ['node', '--version']), ('go', ['go', 'version'])]: try: versions[name] = subprocess.check_output(argv, cwd=root, stderr=subprocess.PIPE, timeout=5).decode().strip() except (OSError, subprocess.SubprocessError): versions[name] = 'unavailable' lock = root/'node_modules/.package-lock.json' versions['installedNpmLockDigest'] = hashlib.sha256(lock.read_bytes()).hexdigest() if lock.is_file() else 'unavailable' return versions def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--mode', choices=['quick', 'task', 'integration'], default='quick') parser.add_argument('--base', default='HEAD', help='包含该提交以来的提交、暂存、未暂存及未跟踪改动;默认仅当前未提交改动') parser.add_argument('--plan', action='store_true', help='只打印计划,不运行检查') parser.add_argument('--budget-seconds', type=int, help='总时间预算,超时报告待验证') parser.add_argument('--gofmt', nargs='+', help=argparse.SUPPRESS) parser.add_argument('--python-syntax', help=argparse.SUPPRESS) args = parser.parse_args(argv) if args.gofmt: result = subprocess.check_output(['gofmt', '-l', *args.gofmt], cwd=ROOT).decode() print(result, end='') return 1 if result.strip() else 0 if args.python_syntax: ast.parse((ROOT/args.python_syntax).read_text()) return 0 budget = args.budget_seconds if args.budget_seconds is not None else {'quick': 180, 'task': 600, 'integration': 3600}[args.mode] if budget <= 0: parser.error('--budget-seconds must be positive') base, files = changed_files(ROOT, args.base) plan = plan_checks(ROOT, files, json.loads(POLICY.read_text()), args.mode, base, tooling_package_change(ROOT, base)) print(json.dumps(plan, ensure_ascii=False, indent=2), flush=True) if args.plan: return 0 folder = ROOT/'local-data/test-results'/f'verify-{time.time_ns()}' folder.mkdir(parents=True, mode=0o700) before = fingerprint(ROOT) report = {**plan, 'base': base, 'head': git(ROOT, 'rev-parse', 'HEAD'), 'inputDigest': before, 'runtime': runtime_identity(ROOT), 'environmentDigest': hashlib.sha256(json.dumps(dict(os.environ), sort_keys=True).encode()).hexdigest(), 'startedAt': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'results': []} start = time.monotonic() report['results'] = run_checks(plan['commands'], ROOT, folder, budget) report['inputsChangedDuringRun'] = fingerprint(ROOT) != before statuses = {r['status'] for r in report['results']} code = 1 if 'failed' in statuses else 2 if (statuses - {'passed'} or plan['unmapped'] or report['inputsChangedDuringRun']) else 0 report['status'] = 'failed' if code == 1 else 'incomplete' if code == 2 else 'selected-checks-passed' report['seconds'] = round(time.monotonic()-start, 3) (folder/'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n') print(f"{report['status']}; 报告: {folder/'report.json'}") if plan['deferred']: print('任务级检查尚未执行;本结果不代表发布/容量验收通过。') return code if __name__ == '__main__': try: sys.exit(main()) except (OSError, ValueError, subprocess.CalledProcessError) as error: print(f'检查入口失败: {type(error).__name__}', file=sys.stderr) sys.exit(2)