data-gen-bpu/scripts/migration_inventory.py

123 lines
6.7 KiB
Python

#!/usr/bin/env python3
"""Read-only migration inventory. Reads source/config, never keys, DBs, or data trees."""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
import re
import shutil
import sys
import yaml
# Direct CLI and package imports use the same runtime path resolver.
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness.paths import resolve_installation
from scripts.release_files import source_files
ABSOLUTE = re.compile(r'''(?:/home/|/mnt/|/opt/|/workspace/|/userdata/)[^\s'"`<>;,\)\]\}]+''')
STORAGE = ('state', 'runs', 'data', 'cache', 'datasets', 'reports')
def metadata(raw, root, executable=False, directory=False):
expanded = os.path.expandvars(os.path.expanduser(str(raw)))
if '$' in expanded:
return {'configured': str(raw), 'resolved': expanded, 'status': 'unresolved_environment'}
p = Path(expanded)
if executable and '/' not in expanded:
found = shutil.which(expanded)
return {'configured': str(raw), 'resolved': found, 'status': 'present' if found else 'missing'}
if not p.is_absolute():
p = root / p
# stat only: no execution, content reads or directory traversal.
try:
exists = p.exists()
result = {'configured': str(raw), 'resolved': str(p.absolute()),
'status': 'present' if exists else 'missing', 'symlink': p.is_symlink()}
if p.is_symlink():
result['link_target'] = os.readlink(p)
if exists:
result['kind'] = 'directory' if p.is_dir() else 'file' if p.is_file() else 'other'
if (directory and not p.is_dir()) or (executable and not p.is_file()):
result['status'] = 'wrong_type'
if exists and executable:
result['executable'] = os.access(p, os.X_OK)
if not result['executable']:
result['status'] = 'not_executable'
return result
except OSError as exc:
return {'configured': str(raw), 'resolved': str(p), 'status': 'unreadable_metadata', 'error': type(exc).__name__}
def inventory(root, config, scopes):
root, config, state = resolve_installation(root=root, config_path=config)
cfg = yaml.safe_load(config.read_text()) or {}
paths = cfg.get('paths') or {}
checks = []
def check(name, value, scope, executable=False, directory=False):
item = {'name': name, 'scope': scope, 'required': scope in scopes}
item.update(metadata(value, root, executable, directory) if value else {'status': 'unconfigured'})
checks.append(item)
check('harness_python', paths.get('harness_python', 'python3'), 'control', True)
for kind, value in (paths.get('venv') or {}).items():
check(f'paths.venv.{kind}.python', str(value) + '/bin/python' if value else None, 'compile', True)
for name, spec in (cfg.get('toolchains') or {}).items():
check(f'toolchains.{name}.python', str(spec['venv']) + '/bin/python' if spec.get('venv') else None, 'compile', True)
for name, value in (cfg.get('model_runtimes') or {}).items():
check('model_runtimes.' + name, value, 'compile', True)
for name in ('pi_bin', 'node_bin', 'horizon'):
check('paths.' + name, paths.get(name), 'pi', name.endswith('_bin'), directory=name == 'horizon')
check('runners.opencode.bin', (cfg.get('runners') or {}).get('opencode', {}).get('bin'), 'opencode', True)
check('agent.opencode.plugin', 'agent/opencode/node_modules/@opencode-ai/plugin/package.json', 'opencode')
for name in STORAGE:
check('storage.' + name, state if name == 'state' else root / name, 'storage', directory=True)
check('paths.work_root', paths.get('work_root'), 'storage', directory=True)
env_names = set()
for ep in (cfg.get('endpoints') or {}).values():
if ep.get('api_key_env'):
env_names.add((str(ep['api_key_env']), 'agent'))
boards_path = root / 'registry/boards.yaml'
if boards_path.is_file():
boards = yaml.safe_load(boards_path.read_text()) or {}
for board in (boards.get('boards') or {}).values():
if board.get('enabled', True):
for key, name in board.items():
if key.endswith('_env'):
env_names.add((str(name), 'board'))
for name, scope in sorted(env_names):
# Presence only; never print env values or open .env.local / SSH keys.
checks.append({'name': name, 'scope': scope, 'required': scope in scopes,
'status': 'set' if os.environ.get(name) else 'unset', 'kind': 'environment'})
references = []
for file in source_files(root):
if file.suffix not in {'.py', '.sh', '.ts', '.yaml', '.yml', '.csv'}:
continue
rel = file.relative_to(root).as_posix()
category = ('test' if file.name.startswith('test_') else
'migration-helper' if file.name in {'migration_inventory.py', 'package_release.py'} else
'operational-script' if rel.startswith('scripts/') else 'runtime-or-registry')
for no, line in enumerate(file.read_text(errors='replace').splitlines(), 1):
if line.lstrip().startswith('#'):
continue
for raw in sorted(set(ABSOLUTE.findall(line))):
references.append({'file': rel, 'line': no, 'path': raw.rstrip(':/.'), 'category': category})
failures = [x['name'] for x in checks if x['required'] and x['status'] not in ('present', 'set')]
return {'schema': 'bench-migration-inventory.v1', 'source_root': str(root), 'config': str(config), 'state': str(state),
'scopes_checked': sorted(scopes), 'metadata_only': True, 'checks': checks,
'required_missing': failures, 'absolute_source_references': references,
'limits': ['Presence is not runtime, ABI, license, connectivity or board acceptance.',
'Absolute references are static candidates; selected recipes determine actual requirements.',
'No secret files, runtime DB, data tree or model files are opened.',
'Existing venvs may embed original Python paths; rebuild or preserve compatible paths.']}
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--root', type=Path)
parser.add_argument('--config', type=Path)
parser.add_argument('--require', action='append', default=[], choices=['control','compile','storage','pi','opencode','agent','board'])
args = parser.parse_args(argv)
root, config, _ = resolve_installation(root=args.root, config_path=args.config)
result = inventory(root, config, set(args.require))
print(json.dumps(result, indent=2, ensure_ascii=False))
return 2 if result['required_missing'] else 0
if __name__ == '__main__':
sys.exit(main())