data-gen-bpu/scripts/run_natural_terminal_probe.py

217 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""One real GLM autonomous CPU classification pilot; prepare and execute separately.
No prewritten candidate implementation. Existing ONNX is reused, labels hidden.
"""
from __future__ import annotations
import argparse
import copy
import hashlib
import json
import os
from pathlib import Path
import shutil
import sqlite3
import subprocess
import sys
import tarfile
import time
import yaml
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from harness import paths
from harness.execution import Execution
from harness import collection_runtime as runtime, rollout, watchdog
from harness.stageworker import Worker
from harness.quota import Broker, QuotaPolicy, Upstream
def save(path, value):
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')
path.chmod(0o600)
def prepare(args):
out = args.output.resolve()
if not out.is_relative_to(ROOT): raise ValueError('OUTPUT_MUST_BE_STAGING')
out.mkdir(parents=True, exist_ok=False)
cfg = yaml.safe_load((args.production_root / 'config.yaml').read_text())
cfg = copy.deepcopy(cfg)
cfg.setdefault('terminal', {}).update(enabled=True, board_enabled=False) # This pilot config only; never production config.
cfg['paths']['work_root'] = str(out / 'work')
cfg['paths']['harness_python'] = sys.executable
cfg['rollouts'].update(max_turns=12, max_tokens=150000, hard_ceiling_s=1800,
tools=['read', 'write', 'edit', 'terminal_exec'], private_home=True, jail=True)
life = cfg['collection_v1']['lifecycle']
life.update(execution_deadline_s=1800, queue_deadline_s=120, ordinary_tool_deadline_s=30,
managed_tool_deadline_s=900, suspect_after_s=600)
cfg['pools'] = {'cpu': {'workers': 1}, 'agent': {'workers': 1}}
config = out / 'config.yaml'; config.write_text(yaml.safe_dump(cfg, sort_keys=False))
run_id = args.run_id
if not run_id or any(c not in 'abcdefghijklmnopqrstuvwxyz0123456789-' for c in run_id):
raise ValueError('INVALID_PROBE_RUN_ID')
rd = ROOT / 'runs' / run_id / 'mobilenetv2-cpu' / 'rollouts/00'
for name in ('agent', 'workspace/.pi', 'workspace/.agent-home/.pi/agent', 'handles', 'stages', 'materials'):
(rd / name).mkdir(parents=True, exist_ok=False if name == 'agent' else True)
workspace = rd / 'workspace'
model = args.production_root / 'cache/onnx/8c730a98be77249c5424ffc1ff2e1deb3efd8e05ee0fbe742ba4d7ec719fabd0/cache-v1/15b2d67a9c13d0089af631bcd8084dacf5e601a0d04e635d6218bbacb3f80e17/mobilenet.v2.onnx'
data = args.production_root / 'data/t1/imagenet1k/n100-s20260914/20260914.tar'
if not model.is_file() or not data.is_file(): raise ValueError('FIXED_MATERIALS_UNAVAILABLE')
shutil.copyfile(model, workspace / 'model.onnx')
with tarfile.open(data) as archive:
manifest = json.loads(archive.extractfile('manifest.json').read())
first = manifest['items'][0]
(workspace / 'diagnostic.jpg').write_bytes(archive.extractfile(first['file']).read())
job = {'run_id': run_id, 'job_id': 'mobilenetv2-cpu', 'task': 'cls', 'model': 'mobilenet.v2',
'march': 'nash-p', 'backend': 'ptq', 'qcfg': 'fp16int8',
'model_spec': {'input_spec': '1x3x224x224'},
'data': {'t1': {'path': str(data), 'n': len(manifest['items']),
'manifest_sha': hashlib.sha256(data.read_bytes()).hexdigest(),
'content_sha': manifest['sha256']}},
'terminal_targets': ['development'],
'task_scope': 'CPU classification only; board unavailable, board/HBM/BPU provenance unverified',
'gates': {'G3': {'metric': 'top1', 'tol': 5, 'kind': 'relative'}}}
save(rd.parent.parent / 'job.json', job)
save(rd / 'rollout.json', {'schema': 'bench.rollout/1', 'run_id': run_id, 'job_id': job['job_id'],
'kk': 0, 'mode': 'e2e', 'policy': 'teacher', 'runner': 'pi', 'runner_version': 'pi-sdk-v1',
'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home', 'state': 'PREPARED',
'max_turns': 12, 'max_tokens': 150000, 'model': cfg['endpoints']['teacher']['model']})
settings = {'defaultProvider': 'dr-gateway', 'defaultModel': cfg['endpoints']['teacher']['model'],
'compaction': {'enabled': False}, 'skills': [], 'retry': {'enabled': False, 'provider': {'maxRetries': 0}},
'extensions': [str(ROOT / 'agent/extensions' / name) for name in ('path-guard.ts', 'terminal.ts')]}
save(workspace / '.pi/settings.json', settings)
endpoint = cfg['endpoints']['teacher']
save(workspace / '.agent-home/.pi/agent/models.json', {'providers': {'dr-gateway': {
'baseUrl': endpoint['base_url'], 'api': 'openai-completions', 'apiKey': 'BENCH_RUNNER_API_KEY',
'models': [{'id': endpoint['model'], 'name': endpoint['model'], 'reasoning': False,
'input': ['text'], 'contextWindow': 128000, 'maxTokens': 4096,
'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}})
prompt = '''我有一个已导出的 MobileNetV2 ImageNet 分类模型 model.onnx,另有 diagnostic.jpg 供你先做单图诊断。
请在这个任务目录中实现自己的图像预处理和 ONNX CPU 推理程序,先确认能运行,再在正式给定的 100 张图上执行,保存每张图的有序 Top-5 类别 ID,并报告实际评测结果和产物位置。
模型已经导出,不需要为这次任务重新导出。请把新增依赖安装到任务私有目录,留下实际依赖版本和安装记录,不修改共享环境。预处理按 ImageNet RGB、短边 256 / 中心裁剪 224、mean=[0.485,0.456,0.406]、std=[0.229,0.224,0.225]、NCHW float32;遇到依赖或运行错误可以自行诊断修改。
目前没有可用的隔离板端环境,因此不要启动板卡或编译任务,也不要把 CPU 分数说成板端/HBM/BPU 验证通过;最后明确已完成和未验证的范围。'''
system = '''你是处理真实工程任务的 Agent。自由实现候选代码,不要求固定脚本名、claim文件或固定stage流程。
本任务所有 shell 命令使用 terminal_exec(target="development");不要请求 board。
普通工具 read/write/edit 的路径按当前工作目录解释。受管终端内同一任务目录挂载为 /workspace;命令的 cwd 参数是任务相对路径。平台已有 Python 运行时,但不要修改它,所需依赖安装到任务目录(例如 private_deps),保存安装日志和版本记录。
正式评测通过同一个 terminal_exec 的 evaluate=true 提交你的实际命令;source="." 会冻结整个任务目录并仍挂载到 /workspace。正式输入是 /input/samples.json,其 items 包含 sample_id 和相对图片文件名 file;图片在 /input。正式标签不向候选开放。运行时输出写 /out(冻结的 /workspace 只读),predictions参数填写相对 /out 的文件名。每行JSON需要 sample_id 和 top5(5个不同的0~999整数,按置信度降序)。平台会评分并封存这一输出,不要自己猜标签。正式进程无网络,依赖须提前就绪。
如果前面已诊断好,正式评分直接使用你的实际推理实现,不另跑平台推理脚本。terminal_exec等待真实终态期间不需要模型轮询。最终依据真实结果说明,来源未核实就保留未验证。'''
(rd / 'agent/prompt.md').write_text(prompt)
(rd / 'agent/append-system.md').write_text(system)
save(out / 'prepared.json', {'rd': str(rd), 'config': str(config), 'state': str(out / 'state'),
'model_sha256': hashlib.sha256(model.read_bytes()).hexdigest(), 'model_source': str(model),
't1_sha256': job['data']['t1']['manifest_sha'], 'n': len(manifest['items']),
'candidate_code_prepared': False, 'terminal_ready_required': True})
print(json.dumps({'prepared': str(out / 'prepared.json'), 'rd': str(rd), 'provider_calls': 0}))
def configure(out):
meta = json.loads((out / 'prepared.json').read_text())
paths.CONFIG_PATH = Path(meta['config']); paths.STATE = Path(meta['state'])
paths.DB_PATH = paths.STATE / 'jobs.sqlite'; paths.config.cache_clear()
os.environ.update(BENCH_CONFIG=meta['config'], BENCH_STATE_DIR=meta['state'], BENCH_ROOT=str(ROOT))
return Path(meta['rd'])
def jail_probe(out):
rd = configure(out)
env = rollout.agent_env(rd, 'e2e')
# No credential or provider invocation in this preflight.
env.pop('BENCH_RUNNER_API_KEY', None)
cmd = rollout.jail_prefix(rd, 'e2e') + ['/usr/bin/python3', '-c',
'from pathlib import Path; import json; p=Path(' + repr(str(ROOT)) + '); '
'assert not (p/"data").exists(); assert not (p/"cache").exists(); '
'assert Path("model.onnx").is_file(); print(json.dumps({"natural_jail":True,"old_data_hidden":True,"old_cache_hidden":True,"task_model_visible":True}))']
result = subprocess.run(cmd, cwd=rd / 'workspace', env=env, capture_output=True, text=True, timeout=30)
(out / 'jail.stdout.log').write_text(result.stdout); (out / 'jail.stderr.log').write_text(result.stderr)
if result.returncode: raise RuntimeError('NATURAL_AGENT_JAIL_FAILED')
save(out / 'jail-accepted.json', {'exit_code': result.returncode, 'checked_at': time.time()})
print(result.stdout.strip())
def execute(args):
out = args.output.resolve(); rd = configure(out)
if not (out / 'jail-accepted.json').is_file(): raise ValueError('JAIL_ACCEPTANCE_REQUIRED')
marker = out / 'real-provider-started.json'
with marker.open('x') as f: json.dump({'started_at': time.time(), 'max_sessions': 1}, f)
save(out / 'executed-source-hashes.json', {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
for name in ['scripts/run_natural_terminal_probe.py', 'agent/pi_session.js', 'agent/runner-budget.js',
'agent/extensions/terminal.ts', 'scripts/jail_setup.py', 'scripts/terminal_jail.py', 'harness/terminal.py']})
cfg = paths.config(); endpoint = cfg['endpoints']['teacher']; shared = cfg['collection_v1']
if endpoint['model'] != 'glm-5.3': raise ValueError('UNEXPECTED_TEACHER_MODEL')
prior = paths.BENCH_ROOT
try:
paths.BENCH_ROOT = args.production_root; paths.load_env_local.cache_clear()
key = paths.secret(endpoint['api_key_env'])
finally:
paths.BENCH_ROOT = prior; paths.load_env_local.cache_clear()
if not key: raise ValueError('TEACHER_CREDENTIAL_UNAVAILABLE')
groups = {name: QuotaPolicy(**p) for name, p in shared['quota_groups'].items()}
broker = Broker(Path(shared['quota_ledger']), groups, {'teacher': Upstream(
shared['credential_groups']['teacher'], endpoint['base_url'], key, (endpoint['model'],))})
broker.start(); runtime._broker = broker; runtime._configuration = json.dumps(shared, sort_keys=True)
ex = Execution(rd); ex.start(shared['lifecycle'])
scope = runtime.configure_pi(rd, ex, 'teacher')
env = rollout.agent_env(rd, 'e2e'); env['BENCH_RUNNER_API_KEY'] = scope.api_key
deadlines = runtime.ToolDeadlines(rd, ex)
cmd = rollout.jail_prefix(rd, 'e2e') + [cfg['paths']['node_bin'], str(ROOT / 'agent/pi_session.js'), str(rd / 'agent/pi-session.json')]
try:
with ex.scope(), Worker(rd, execution=ex):
result = watchdog.run(cmd, cwd=rd / 'workspace', env=env,
stdout=rd / 'agent/events.jsonl', stderr=rd / 'agent/pi.stderr.log',
execution=ex, role='agent', hard_ceiling_s=1800, poll_s=1,
stop_check=deadlines.poll, on_finish=lambda _: deadlines.poll())
finalization_error = None
checkpoint = {}
try:
runtime.capture_context(rd)
checkpoint = runtime.freeze_checkpoint(rd, ex)
except (OSError, ValueError, KeyError) as error:
finalization_error = str(error)
# Preserve failure bytes separately; never revise an old checkpoint
# hash, fabricate agent_end, or label a last partial generation done.
save(out / 'finalization-error.json', {'type': type(error).__name__, 'reason': str(error)})
raw_checkpoint = rd / 'workspace/.bench-session.json'
if raw_checkpoint.is_file():
shutil.copyfile(raw_checkpoint, out / 'checkpoint-partial-original.json')
old = json.loads(raw_checkpoint.read_text())
native = Path(old['session_path'])
if native.is_relative_to(rd / 'workspace/.pi-sessions') and native.is_file():
shutil.copyfile(native, out / 'native-session-partial.jsonl')
state = ex.finish(result.exit_code, protocol_complete=not finalization_error and checkpoint.get('complete') is True,
cleanup_state=result.cleanup_state,
stop_reason=result.stalled_reason or ('INFRA_FINALIZATION_FAILED' if finalization_error else None))
status = broker.execution_status(scope.execution_id)
save(out / 'quota-status.json', status)
with sqlite3.connect(broker.path) as con:
con.row_factory = sqlite3.Row
attempts = [dict(row) for row in con.execute('''SELECT attempt_id,generation_id,state,sent,http_status,
charged_tokens,usage_certainty,reason FROM attempts WHERE generation_id IN
(SELECT generation_id FROM generations WHERE execution_id=?)''', (scope.execution_id,))]
save(out / 'quota-attempts.json', attempts)
terminal = json.loads((rd / 'terminal_result.json').read_text()) if (rd / 'terminal_result.json').exists() else None
summary = {'rd': str(rd), 'execution_id': scope.execution_id, 'provider': 'real glm-5.3',
'returncode': result.exit_code, 'execution_outcome': state['execution_outcome'],
'cleanup_state': state['cleanup_state'], 'checkpoint': checkpoint, 'finalization_error': finalization_error,
'terminal_result': terminal, 'quota_charged_tokens_including_estimates': sum(a['charged_tokens'] or 0 for a in attempts),
'provider_reported_tokens': sum(a['charged_tokens'] or 0 for a in attempts if a['usage_certainty'] == 'REPORTED'),
'estimated_tokens': sum(a['charged_tokens'] or 0 for a in attempts if a['usage_certainty'] == 'ESTIMATED'),
'monetary_cost': 'UNKNOWN', 'board_verified': False, 'sft_eligible': False}
save(out / 'summary.json', summary)
print(json.dumps(summary, ensure_ascii=False))
finally:
broker.cancel(scope.execution_id); broker.close()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--production-root', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
parser.add_argument('--run-id', default='natural-terminal-live-20260922')
parser.add_argument('action', choices=['prepare', 'jail-probe', 'execute'])
args = parser.parse_args()
if args.action == 'prepare': prepare(args)
elif args.action == 'jail-probe': jail_probe(args.output.resolve())
else: execute(args)
if __name__ == '__main__': main()