217 lines
15 KiB
Python
217 lines
15 KiB
Python
"""One real GLM autonomous CPU classification pilot; prepare and execute separately.
|
||
No prewritten candidate implementation. Existing ONNX is reused, labels hidden.
|
||
"""
|
||
from __future__ import annotations
|
||
import argparse
|
||
import copy
|
||
import hashlib
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import shutil
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
import tarfile
|
||
import time
|
||
import yaml
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT))
|
||
from harness import paths
|
||
from harness.execution import Execution
|
||
from harness import collection_runtime as runtime, rollout, watchdog
|
||
from harness.stageworker import Worker
|
||
from harness.quota import Broker, QuotaPolicy, Upstream
|
||
|
||
|
||
def save(path, value):
|
||
path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')
|
||
path.chmod(0o600)
|
||
|
||
|
||
def prepare(args):
|
||
out = args.output.resolve()
|
||
if not out.is_relative_to(ROOT): raise ValueError('OUTPUT_MUST_BE_STAGING')
|
||
out.mkdir(parents=True, exist_ok=False)
|
||
cfg = yaml.safe_load((args.production_root / 'config.yaml').read_text())
|
||
cfg = copy.deepcopy(cfg)
|
||
cfg.setdefault('terminal', {}).update(enabled=True, board_enabled=False) # This pilot config only; never production config.
|
||
cfg['paths']['work_root'] = str(out / 'work')
|
||
cfg['paths']['harness_python'] = sys.executable
|
||
cfg['rollouts'].update(max_turns=12, max_tokens=150000, hard_ceiling_s=1800,
|
||
tools=['read', 'write', 'edit', 'terminal_exec'], private_home=True, jail=True)
|
||
life = cfg['collection_v1']['lifecycle']
|
||
life.update(execution_deadline_s=1800, queue_deadline_s=120, ordinary_tool_deadline_s=30,
|
||
managed_tool_deadline_s=900, suspect_after_s=600)
|
||
cfg['pools'] = {'cpu': {'workers': 1}, 'agent': {'workers': 1}}
|
||
config = out / 'config.yaml'; config.write_text(yaml.safe_dump(cfg, sort_keys=False))
|
||
run_id = args.run_id
|
||
if not run_id or any(c not in 'abcdefghijklmnopqrstuvwxyz0123456789-' for c in run_id):
|
||
raise ValueError('INVALID_PROBE_RUN_ID')
|
||
rd = ROOT / 'runs' / run_id / 'mobilenetv2-cpu' / 'rollouts/00'
|
||
for name in ('agent', 'workspace/.pi', 'workspace/.agent-home/.pi/agent', 'handles', 'stages', 'materials'):
|
||
(rd / name).mkdir(parents=True, exist_ok=False if name == 'agent' else True)
|
||
workspace = rd / 'workspace'
|
||
model = args.production_root / 'cache/onnx/8c730a98be77249c5424ffc1ff2e1deb3efd8e05ee0fbe742ba4d7ec719fabd0/cache-v1/15b2d67a9c13d0089af631bcd8084dacf5e601a0d04e635d6218bbacb3f80e17/mobilenet.v2.onnx'
|
||
data = args.production_root / 'data/t1/imagenet1k/n100-s20260914/20260914.tar'
|
||
if not model.is_file() or not data.is_file(): raise ValueError('FIXED_MATERIALS_UNAVAILABLE')
|
||
shutil.copyfile(model, workspace / 'model.onnx')
|
||
with tarfile.open(data) as archive:
|
||
manifest = json.loads(archive.extractfile('manifest.json').read())
|
||
first = manifest['items'][0]
|
||
(workspace / 'diagnostic.jpg').write_bytes(archive.extractfile(first['file']).read())
|
||
job = {'run_id': run_id, 'job_id': 'mobilenetv2-cpu', 'task': 'cls', 'model': 'mobilenet.v2',
|
||
'march': 'nash-p', 'backend': 'ptq', 'qcfg': 'fp16int8',
|
||
'model_spec': {'input_spec': '1x3x224x224'},
|
||
'data': {'t1': {'path': str(data), 'n': len(manifest['items']),
|
||
'manifest_sha': hashlib.sha256(data.read_bytes()).hexdigest(),
|
||
'content_sha': manifest['sha256']}},
|
||
'terminal_targets': ['development'],
|
||
'task_scope': 'CPU classification only; board unavailable, board/HBM/BPU provenance unverified',
|
||
'gates': {'G3': {'metric': 'top1', 'tol': 5, 'kind': 'relative'}}}
|
||
save(rd.parent.parent / 'job.json', job)
|
||
save(rd / 'rollout.json', {'schema': 'bench.rollout/1', 'run_id': run_id, 'job_id': job['job_id'],
|
||
'kk': 0, 'mode': 'e2e', 'policy': 'teacher', 'runner': 'pi', 'runner_version': 'pi-sdk-v1',
|
||
'natural_agent_version': 'natural-agent-v1', 'jail_policy': 'private-home', 'state': 'PREPARED',
|
||
'max_turns': 12, 'max_tokens': 150000, 'model': cfg['endpoints']['teacher']['model']})
|
||
settings = {'defaultProvider': 'dr-gateway', 'defaultModel': cfg['endpoints']['teacher']['model'],
|
||
'compaction': {'enabled': False}, 'skills': [], 'retry': {'enabled': False, 'provider': {'maxRetries': 0}},
|
||
'extensions': [str(ROOT / 'agent/extensions' / name) for name in ('path-guard.ts', 'terminal.ts')]}
|
||
save(workspace / '.pi/settings.json', settings)
|
||
endpoint = cfg['endpoints']['teacher']
|
||
save(workspace / '.agent-home/.pi/agent/models.json', {'providers': {'dr-gateway': {
|
||
'baseUrl': endpoint['base_url'], 'api': 'openai-completions', 'apiKey': 'BENCH_RUNNER_API_KEY',
|
||
'models': [{'id': endpoint['model'], 'name': endpoint['model'], 'reasoning': False,
|
||
'input': ['text'], 'contextWindow': 128000, 'maxTokens': 4096,
|
||
'cost': {'input': 0, 'output': 0, 'cacheRead': 0, 'cacheWrite': 0}}]}}})
|
||
prompt = '''我有一个已导出的 MobileNetV2 ImageNet 分类模型 model.onnx,另有 diagnostic.jpg 供你先做单图诊断。
|
||
请在这个任务目录中实现自己的图像预处理和 ONNX CPU 推理程序,先确认能运行,再在正式给定的 100 张图上执行,保存每张图的有序 Top-5 类别 ID,并报告实际评测结果和产物位置。
|
||
模型已经导出,不需要为这次任务重新导出。请把新增依赖安装到任务私有目录,留下实际依赖版本和安装记录,不修改共享环境。预处理按 ImageNet RGB、短边 256 / 中心裁剪 224、mean=[0.485,0.456,0.406]、std=[0.229,0.224,0.225]、NCHW float32;遇到依赖或运行错误可以自行诊断修改。
|
||
目前没有可用的隔离板端环境,因此不要启动板卡或编译任务,也不要把 CPU 分数说成板端/HBM/BPU 验证通过;最后明确已完成和未验证的范围。'''
|
||
system = '''你是处理真实工程任务的 Agent。自由实现候选代码,不要求固定脚本名、claim文件或固定stage流程。
|
||
本任务所有 shell 命令使用 terminal_exec(target="development");不要请求 board。
|
||
普通工具 read/write/edit 的路径按当前工作目录解释。受管终端内同一任务目录挂载为 /workspace;命令的 cwd 参数是任务相对路径。平台已有 Python 运行时,但不要修改它,所需依赖安装到任务目录(例如 private_deps),保存安装日志和版本记录。
|
||
正式评测通过同一个 terminal_exec 的 evaluate=true 提交你的实际命令;source="." 会冻结整个任务目录并仍挂载到 /workspace。正式输入是 /input/samples.json,其 items 包含 sample_id 和相对图片文件名 file;图片在 /input。正式标签不向候选开放。运行时输出写 /out(冻结的 /workspace 只读),predictions参数填写相对 /out 的文件名。每行JSON需要 sample_id 和 top5(5个不同的0~999整数,按置信度降序)。平台会评分并封存这一输出,不要自己猜标签。正式进程无网络,依赖须提前就绪。
|
||
如果前面已诊断好,正式评分直接使用你的实际推理实现,不另跑平台推理脚本。terminal_exec等待真实终态期间不需要模型轮询。最终依据真实结果说明,来源未核实就保留未验证。'''
|
||
(rd / 'agent/prompt.md').write_text(prompt)
|
||
(rd / 'agent/append-system.md').write_text(system)
|
||
save(out / 'prepared.json', {'rd': str(rd), 'config': str(config), 'state': str(out / 'state'),
|
||
'model_sha256': hashlib.sha256(model.read_bytes()).hexdigest(), 'model_source': str(model),
|
||
't1_sha256': job['data']['t1']['manifest_sha'], 'n': len(manifest['items']),
|
||
'candidate_code_prepared': False, 'terminal_ready_required': True})
|
||
print(json.dumps({'prepared': str(out / 'prepared.json'), 'rd': str(rd), 'provider_calls': 0}))
|
||
|
||
|
||
def configure(out):
|
||
meta = json.loads((out / 'prepared.json').read_text())
|
||
paths.CONFIG_PATH = Path(meta['config']); paths.STATE = Path(meta['state'])
|
||
paths.DB_PATH = paths.STATE / 'jobs.sqlite'; paths.config.cache_clear()
|
||
os.environ.update(BENCH_CONFIG=meta['config'], BENCH_STATE_DIR=meta['state'], BENCH_ROOT=str(ROOT))
|
||
return Path(meta['rd'])
|
||
|
||
|
||
def jail_probe(out):
|
||
rd = configure(out)
|
||
env = rollout.agent_env(rd, 'e2e')
|
||
# No credential or provider invocation in this preflight.
|
||
env.pop('BENCH_RUNNER_API_KEY', None)
|
||
cmd = rollout.jail_prefix(rd, 'e2e') + ['/usr/bin/python3', '-c',
|
||
'from pathlib import Path; import json; p=Path(' + repr(str(ROOT)) + '); '
|
||
'assert not (p/"data").exists(); assert not (p/"cache").exists(); '
|
||
'assert Path("model.onnx").is_file(); print(json.dumps({"natural_jail":True,"old_data_hidden":True,"old_cache_hidden":True,"task_model_visible":True}))']
|
||
result = subprocess.run(cmd, cwd=rd / 'workspace', env=env, capture_output=True, text=True, timeout=30)
|
||
(out / 'jail.stdout.log').write_text(result.stdout); (out / 'jail.stderr.log').write_text(result.stderr)
|
||
if result.returncode: raise RuntimeError('NATURAL_AGENT_JAIL_FAILED')
|
||
save(out / 'jail-accepted.json', {'exit_code': result.returncode, 'checked_at': time.time()})
|
||
print(result.stdout.strip())
|
||
|
||
|
||
def execute(args):
|
||
out = args.output.resolve(); rd = configure(out)
|
||
if not (out / 'jail-accepted.json').is_file(): raise ValueError('JAIL_ACCEPTANCE_REQUIRED')
|
||
marker = out / 'real-provider-started.json'
|
||
with marker.open('x') as f: json.dump({'started_at': time.time(), 'max_sessions': 1}, f)
|
||
save(out / 'executed-source-hashes.json', {name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
||
for name in ['scripts/run_natural_terminal_probe.py', 'agent/pi_session.js', 'agent/runner-budget.js',
|
||
'agent/extensions/terminal.ts', 'scripts/jail_setup.py', 'scripts/terminal_jail.py', 'harness/terminal.py']})
|
||
cfg = paths.config(); endpoint = cfg['endpoints']['teacher']; shared = cfg['collection_v1']
|
||
if endpoint['model'] != 'glm-5.3': raise ValueError('UNEXPECTED_TEACHER_MODEL')
|
||
prior = paths.BENCH_ROOT
|
||
try:
|
||
paths.BENCH_ROOT = args.production_root; paths.load_env_local.cache_clear()
|
||
key = paths.secret(endpoint['api_key_env'])
|
||
finally:
|
||
paths.BENCH_ROOT = prior; paths.load_env_local.cache_clear()
|
||
if not key: raise ValueError('TEACHER_CREDENTIAL_UNAVAILABLE')
|
||
groups = {name: QuotaPolicy(**p) for name, p in shared['quota_groups'].items()}
|
||
broker = Broker(Path(shared['quota_ledger']), groups, {'teacher': Upstream(
|
||
shared['credential_groups']['teacher'], endpoint['base_url'], key, (endpoint['model'],))})
|
||
broker.start(); runtime._broker = broker; runtime._configuration = json.dumps(shared, sort_keys=True)
|
||
ex = Execution(rd); ex.start(shared['lifecycle'])
|
||
scope = runtime.configure_pi(rd, ex, 'teacher')
|
||
env = rollout.agent_env(rd, 'e2e'); env['BENCH_RUNNER_API_KEY'] = scope.api_key
|
||
deadlines = runtime.ToolDeadlines(rd, ex)
|
||
cmd = rollout.jail_prefix(rd, 'e2e') + [cfg['paths']['node_bin'], str(ROOT / 'agent/pi_session.js'), str(rd / 'agent/pi-session.json')]
|
||
try:
|
||
with ex.scope(), Worker(rd, execution=ex):
|
||
result = watchdog.run(cmd, cwd=rd / 'workspace', env=env,
|
||
stdout=rd / 'agent/events.jsonl', stderr=rd / 'agent/pi.stderr.log',
|
||
execution=ex, role='agent', hard_ceiling_s=1800, poll_s=1,
|
||
stop_check=deadlines.poll, on_finish=lambda _: deadlines.poll())
|
||
finalization_error = None
|
||
checkpoint = {}
|
||
try:
|
||
runtime.capture_context(rd)
|
||
checkpoint = runtime.freeze_checkpoint(rd, ex)
|
||
except (OSError, ValueError, KeyError) as error:
|
||
finalization_error = str(error)
|
||
# Preserve failure bytes separately; never revise an old checkpoint
|
||
# hash, fabricate agent_end, or label a last partial generation done.
|
||
save(out / 'finalization-error.json', {'type': type(error).__name__, 'reason': str(error)})
|
||
raw_checkpoint = rd / 'workspace/.bench-session.json'
|
||
if raw_checkpoint.is_file():
|
||
shutil.copyfile(raw_checkpoint, out / 'checkpoint-partial-original.json')
|
||
old = json.loads(raw_checkpoint.read_text())
|
||
native = Path(old['session_path'])
|
||
if native.is_relative_to(rd / 'workspace/.pi-sessions') and native.is_file():
|
||
shutil.copyfile(native, out / 'native-session-partial.jsonl')
|
||
state = ex.finish(result.exit_code, protocol_complete=not finalization_error and checkpoint.get('complete') is True,
|
||
cleanup_state=result.cleanup_state,
|
||
stop_reason=result.stalled_reason or ('INFRA_FINALIZATION_FAILED' if finalization_error else None))
|
||
status = broker.execution_status(scope.execution_id)
|
||
save(out / 'quota-status.json', status)
|
||
with sqlite3.connect(broker.path) as con:
|
||
con.row_factory = sqlite3.Row
|
||
attempts = [dict(row) for row in con.execute('''SELECT attempt_id,generation_id,state,sent,http_status,
|
||
charged_tokens,usage_certainty,reason FROM attempts WHERE generation_id IN
|
||
(SELECT generation_id FROM generations WHERE execution_id=?)''', (scope.execution_id,))]
|
||
save(out / 'quota-attempts.json', attempts)
|
||
terminal = json.loads((rd / 'terminal_result.json').read_text()) if (rd / 'terminal_result.json').exists() else None
|
||
summary = {'rd': str(rd), 'execution_id': scope.execution_id, 'provider': 'real glm-5.3',
|
||
'returncode': result.exit_code, 'execution_outcome': state['execution_outcome'],
|
||
'cleanup_state': state['cleanup_state'], 'checkpoint': checkpoint, 'finalization_error': finalization_error,
|
||
'terminal_result': terminal, 'quota_charged_tokens_including_estimates': sum(a['charged_tokens'] or 0 for a in attempts),
|
||
'provider_reported_tokens': sum(a['charged_tokens'] or 0 for a in attempts if a['usage_certainty'] == 'REPORTED'),
|
||
'estimated_tokens': sum(a['charged_tokens'] or 0 for a in attempts if a['usage_certainty'] == 'ESTIMATED'),
|
||
'monetary_cost': 'UNKNOWN', 'board_verified': False, 'sft_eligible': False}
|
||
save(out / 'summary.json', summary)
|
||
print(json.dumps(summary, ensure_ascii=False))
|
||
finally:
|
||
broker.cancel(scope.execution_id); broker.close()
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument('--production-root', type=Path, required=True)
|
||
parser.add_argument('--output', type=Path, required=True)
|
||
parser.add_argument('--run-id', default='natural-terminal-live-20260922')
|
||
parser.add_argument('action', choices=['prepare', 'jail-probe', 'execute'])
|
||
args = parser.parse_args()
|
||
if args.action == 'prepare': prepare(args)
|
||
elif args.action == 'jail-probe': jail_probe(args.output.resolve())
|
||
else: execute(args)
|
||
|
||
|
||
if __name__ == '__main__': main()
|