data-gen-bpu/harness/salty_dev.py
bench 8fb9349907 Agent no longer gets the platform's model code; each task states its I/O contract
The agent could read and copy harness/adapters (model loading, preprocessing, export, decoding), and every
bench dev container carried the whole harness, registry, scripts and export trees, evaluator and gates
included. That did the model-specific half of the deployment for it.

- guard and private-home jail no longer expose harness/adapters; bench dev packs only container_site
- the evaluator still decodes outputs with the platform's own code, so agent/io-contracts states what the
  graph's inputs and outputs mean (9 families, 71 models so far); the prompt appends the task's contract
- SYSTEM.md: says where official sources, weights and environments are; the compile.yaml template is
  replaced by the facts that differ from the docs
- e2e prompt: whole-class int16/float fallback and >1.5x board latency do not pass
- agent/VERSION 24

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-22 21:51:17 +08:00

273 lines
15 KiB
Python

"""`bench dev`: one agent command in a Salty container that has the toolchain (hb_compile, hmct,
onnxruntime, torch). The agent's bash runs on the controller VM, which has none of it.
The workspace lives twice: the controller copy the agent edits, and the development copy on personal
storage (the same absolute path inside containers, so 04/05 read the graph and calibration made here).
Before a command, files the agent changed locally since the last sync go up; afterwards, files the
command changed come back if small, and big ones (graphs, calibration sets) stay remote and are listed.
The worker sandboxes the command (salty_remote_worker.dev): only toolchains, caches and data read-only
plus this workspace are visible, no capabilities, no board credentials.
"""
from __future__ import annotations
import fcntl
import hashlib
import io
import json
import os
import re
import shutil
import tarfile
import time
import uuid
from contextlib import contextmanager
from pathlib import Path, PurePosixPath
from types import SimpleNamespace
from . import execution, paths, salty_remote
from .salty_client import TransportError
from .watchdog import RunResult
# .bench-session.json.tmp: pi_session's atomic write; listed, then renamed away before tar read it
# (10-20 20:17 and twice on 10-16: FileNotFoundError, the dev command died as STAGE_EXECUTOR_EXIT).
EXCLUDED = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.bench-session.json.tmp', '.env', '.env.local', 'guard.log'}
UPLOAD_LIMIT = 256 << 20 # the Mac mini uplink is ~1 MB/s; big inputs are made inside bench dev
GRACE_S = 300
FETCH_MARGIN_S = 120 # the command ends this long before the request deadline so its results still come back
def selected(cfg=None) -> bool:
policy = ((cfg or paths.config()).get('execution') or {}).get('salty_remote') or {}
return policy.get('enabled') is True and 'terminal' in policy.get('stages', [])
def local_files(ws: Path) -> dict:
files = {}
for root, dirs, names in os.walk(ws):
top = Path(root) == ws
dirs[:] = [d for d in dirs if not (top and d in EXCLUDED) and not os.path.islink(os.path.join(root, d))]
for name in names:
p = Path(root) / name
if (top and name in EXCLUDED) or p.is_symlink() or not p.is_file():
continue
st = p.stat()
files[p.relative_to(ws).as_posix()] = [st.st_size, st.st_mtime_ns]
return files
@contextmanager
def synced(rd: Path):
"""{relative path: [size, mtime_ns]} of the controller copy as last synced; changes since are uploads.
Held only around reads/updates, so dev commands of one rollout still run in parallel."""
path = rd / 'stages' / '_dev-sync.json'
path.parent.mkdir(parents=True, exist_ok=True)
with (rd / 'stages' / '_dev-sync.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
state = json.loads(path.read_text()) if path.exists() else {}
yield state
execution.atomic_json(path, state) # fsynced: an empty manifest after a crash would fail every dev command
def memory_gi(policy, ctx, rd: Path) -> int:
"""The heaviest dev use is compiling this model: at least what its formal compiles needed (salty_remote._memory
doubles after an OOM; 10-14: vit.h.14 died at 16Gi in bench dev). A dev job of this rollout that failed (an OOM
gives no reason) doubles what it had, cap 128Gi: the agent cannot raise the limit, and retrying at the same size
failed again every time (10-20: efficientnet.b6 3x at 16Gi, unidepth.v2.vit.l 2x at 32Gi)."""
gi = max(int(str(m).removesuffix('Gi')) for m in (
(policy.get('memory_by_stage') or {}).get('terminal', policy.get('memory', '16Gi')),
salty_remote._memory(policy, SimpleNamespace(name='05-compile'), ctx)))
# ponytail: this rollout only; a new rollout of the same model starts from the formal compiles' floor again.
for f in (rd / 'stages' / '_queue').glob('terminal-*/salty/remote.json'):
state = json.loads(f.read_text())
if state.get('job_state') == 'failed' and state.get('memory'):
gi = max(gi, 2 * int(state['memory'].removesuffix('Gi')))
return min(gi, 128)
def bundle(path: Path, ws: Path, uploads: list, request: dict) -> None:
with tarfile.open(path, 'x') as tar:
data = json.dumps(request).encode()
info = tarfile.TarInfo('request.json'); info.size = len(data)
tar.addfile(info, io.BytesIO(data))
for rel in uploads:
tar.add(ws / rel, 'ws/' + rel, recursive=False)
# Only what the sandbox itself needs (onnxruntime thread cap): platform code (adapters, export tools, the
# evaluator and gates) would do the model-specific half of the task for the agent.
tar.add(paths.BENCH_ROOT / 'harness/container_site/sitecustomize.py', 'code/harness/container_site/sitecustomize.py')
def install(ws: Path, archive: Path, result: dict) -> list:
"""Changed small files back into the controller copy; only regular files under the workspace."""
data = archive.read_bytes()
if hashlib.sha256(data).hexdigest() != result['archive']['sha256']:
raise ValueError('dev output archive bytes differ from its receipt')
installed = []
with tarfile.open(fileobj=io.BytesIO(data)) as tar:
for member in tar.getmembers():
rel = PurePosixPath(member.name)
if not member.isfile() or rel.is_absolute() or '..' in rel.parts or rel.parts[0] in EXCLUDED:
raise ValueError('unsafe dev output member: ' + member.name)
body = tar.extractfile(member).read()
if hashlib.sha256(body).hexdigest() != result['outputs'][member.name]['sha256']:
raise ValueError('dev output bytes fail verification: ' + member.name)
target = ws / rel
if any(p.is_symlink() for p in [target, *target.parents] if p.is_relative_to(ws) and p != ws):
raise ValueError('dev output would follow a workspace symlink: ' + member.name)
target.parent.mkdir(parents=True, exist_ok=True)
tmp = target.with_name(target.name + '.dev-tmp')
tmp.write_bytes(body)
salty_remote._fsync(tmp)
os.replace(tmp, target)
installed.append(member.name)
return installed
def _storage(rd: Path, rel: str):
policy = salty_remote._policy(SimpleNamespace(cfg=paths.config()))
cli = salty_remote._Cli(policy.get('salty', 'salty'), policy.get('outage_seconds', 1800), time.sleep, time.time, lambda m: None)
remote = 'personal:/bench-root/runs/' + (rd / 'workspace').relative_to(paths.BENCH_ROOT / 'runs').as_posix()
if rel not in ('', '.'):
remote += '/' + PurePosixPath(rel).as_posix()
return cli, remote
def remote_size(rd: Path, rel: str, cap: int, max_dirs: int = 40) -> int | None:
"""Bytes under the development workspace's <rel>, or None once past cap bytes or max_dirs directories
(one `salty storage ls` per directory, ~3 s each)."""
cli, root = _storage(rd, rel)
todo, total = [root], 0
while todo:
if max_dirs <= 0:
return None
max_dirs -= 1
here = todo.pop()
_, out, _ = cli.call('storage', 'ls', here, ok_404=True)
for line in (out or '').splitlines()[1:]: # header: 名称 类型 大小 修改时间
name, kind, size = (line.split('\t') + ['', '', ''])[:3]
if kind == '目录':
todo.append(here + '/' + name)
elif size.isdigit():
total += int(size)
if total > cap:
return None
return total
def download_dir(rd: Path, rel: str, destination: Path) -> None:
"""The development workspace's copy of <rel> (files over the copy-back limit only exist there); a directory
never synced up gives an empty destination."""
cli, remote = _storage(rd, rel)
part = destination.with_name(destination.name + '.part')
def fresh():
shutil.rmtree(part, ignore_errors=True); part.mkdir(parents=True)
rc, _, err = cli.call('storage', 'cp', remote, str(part), timeout=1800, ok_404=True, before=fresh)
got = part / PurePosixPath(remote).name # `salty storage cp <dir> <existing dir>` lands in <existing dir>/<name>
if rc == 0 and got.is_dir():
os.replace(got, destination)
else:
destination.mkdir(parents=True)
shutil.rmtree(part)
def calib_images(rd: Path, job: dict) -> str:
"""The job's calibration images for agent v18+ (calib-raw mirrors calib; 10-15 night: told only to
"replace /data/calib/", agents searched /data/calib-raw and spent 5-8 commands finding it). From v21 the
original training images (images/, from 36's JuiceFS); v18-v20 rollouts keep the resized png/ they started with."""
version = int(json.loads((rd / 'rollout.json').read_text()).get('agent_config_version', 0))
return job['data']['calib']['path'].replace('/data/calib/', '/data/calib-raw/', 1) + ('/images' if version >= 21 else '/png')
def execute(ctx, hd: Path, req: dict, deadline: float, stdout: Path, stderr: Path) -> RunResult:
"""Run req['command'] remotely; returns a watchdog-shaped result for terminal.run."""
started = time.time()
rd = ctx.stage_root.parent
ws = rd / 'workspace'
adapter = hd / 'salty'
adapter.mkdir(exist_ok=True)
state_file = adapter / 'remote.json'
policy = salty_remote._policy(SimpleNamespace(cfg=paths.config()))
def log(message):
with (adapter / 'transport.log').open('a') as f:
f.write(f"{time.strftime('%F %T')} {message}\n")
cli = salty_remote._Cli(policy.get('salty', 'salty'), policy.get('outage_seconds', 1800), time.sleep, time.time, log)
state = {'name': 'bench-dev-' + str(uuid.uuid4()), 'job_id': None}
state['remote'] = 'personal:/' + state['name']
remaining = lambda: deadline - time.monotonic()
def done(st, code, reason=None, cleanup='VERIFIED'):
stdout.touch(); stderr.touch()
return RunResult(st, code, time.time() - started, stdout, stderr, stalled_reason=reason, cleanup_state=cleanup)
with synced(rd) as manifest:
local = local_files(ws)
uploads = sorted(rel for rel, st in local.items() if manifest.get(rel) != st)
total = sum(local[rel][0] for rel in uploads)
if total > UPLOAD_LIMIT:
stderr.write_text(f'bench dev: {total >> 20} MiB of local changes to upload (limit {UPLOAD_LIMIT >> 20} MiB); '
'make big files inside bench dev instead\n')
return done('FAIL', 2)
request = {'identity': {'stage': 'terminal', 'operation_id': hd.name}, 'bench_root': str(paths.BENCH_ROOT),
'dev': {'workspace': str(ws), 'command': (paths.AGENT / 'pipe-note.sh').read_text().strip() + ' ' + req['command'], 'cwd': req.get('cwd', '.'),
'deadline_wall': time.time() + remaining() - FETCH_MARGIN_S,
'timeout': req.get('timeout'), # the command's own time, counted from container start
'own_inputs': bool(ctx.extra.get('own_inputs_required')),
'calib_images': calib_images(rd, ctx.job) if ctx.extra.get('own_inputs_required') else None,
'cpu': str((policy.get('cpu_by_stage') or {}).get('terminal', policy.get('cpu', 4)))}}
bundle(adapter / 'bundle.tar', ws, uploads, request)
shutil.copyfile(salty_remote.WORKER, adapter / 'worker.py')
try:
memory = state['memory'] = f'{memory_gi(policy, ctx, rd)}Gi'
state['job_id'] = salty_remote._submit(cli, state, policy, SimpleNamespace(name='terminal'), adapter, memory)
execution.atomic_json(state_file, state)
with synced(rd) as manifest: # the remote copy has them now, whatever the command does
manifest.update({rel: local[rel] for rel in uploads})
job_state = None
while job_state not in ('succeeded', 'failed', 'stopped'):
if remaining() < -GRACE_S: # the worker's own timeout should have ended it already
cli.call('stop', state['job_id'], ok_404=True)
time.sleep(5 if time.time() - started < 120 else 30) # short commands answer fast; long ones poll gently
out = cli.call('describe', state['job_id'])[1]
job_state = (re.search(r'^状态:\s*(\w+)', out, re.MULTILINE) or [None, None])[1]
if job_state != 'succeeded':
state['job_state'] = job_state
execution.atomic_json(state_file, state)
salty_remote._remote_logs(policy, state['job_id'], stderr)
# A container OOM gives no reason; its memory log shows it (the agent cannot raise the limit itself).
if cli.once(['storage', 'cp', state['remote'] + '/usage.log', str(adapter / 'usage.log')])[0] == 0:
more = f"; the next bench dev command gets {min(2 * int(memory.removesuffix('Gi')), 128)}Gi" if job_state == 'failed' else ''
with stderr.open('a') as f:
f.write(f'\n[bench dev container: {memory} memory; usage every 15s — a peak near the limit means it ran out of memory{more}]\n'
+ (adapter / 'usage.log').read_text()[-4000:])
return done('FAIL', None, 'TERMINAL_DEADLINE_EXCEEDED' if remaining() < 0 else 'REMOTE_WORKER_FAILED')
for f in ('result.json', 'outputs.tar', 'stdout.log', 'stderr.log'):
salty_remote._fetch(cli, f"{state['remote']}/result/{f}", adapter / f)
result = json.loads((adapter / 'result.json').read_text())
if result.get('identity') != request['identity']:
raise ValueError('remote result belongs to another dev command')
installed = install(ws, adapter / 'outputs.tar', result)
with synced(rd) as manifest:
for rel in installed:
st = (ws / rel).stat()
manifest[rel] = [st.st_size, st.st_mtime_ns]
shutil.copyfile(adapter / 'stdout.log', stdout)
shutil.copyfile(adapter / 'stderr.log', stderr)
with stdout.open('a') as f:
if installed:
f.write(f"\n[bench dev] copied back to the workspace: {', '.join(installed[:50])}\n")
if result.get('remote_only'):
f.write('[bench dev] kept only in the development workspace (too big to copy back; usable from bench dev '
'and compile.yaml): ' + ', '.join(f'{r} ({s >> 20} MiB)' for r, s in result['remote_only'][:50]) + '\n')
code = result.get('exit_code')
return done('OK' if code == 0 else 'FAIL', code, 'TERMINAL_DEADLINE_EXCEEDED' if code == 124 else None)
except (TransportError, ValueError, OSError) as exc:
with stderr.open('a') as f:
f.write(f'\n[bench dev] {type(exc).__name__}: {exc}\n')
# The container ends by its own timeout and writes only to this workspace.
return done('FAIL', None, 'REMOTE_UNAVAILABLE', 'REMOTE_BOUNDED')
finally:
if state['job_id']:
salty_remote._cleanup(cli, state, state_file)
for f in ('bundle.tar', 'outputs.tar', 'worker.py'):
(adapter / f).unlink(missing_ok=True)