The agent could read and copy harness/adapters (model loading, preprocessing, export, decoding), and every bench dev container carried the whole harness, registry, scripts and export trees, evaluator and gates included. That did the model-specific half of the deployment for it. - guard and private-home jail no longer expose harness/adapters; bench dev packs only container_site - the evaluator still decodes outputs with the platform's own code, so agent/io-contracts states what the graph's inputs and outputs mean (9 families, 71 models so far); the prompt appends the task's contract - SYSTEM.md: says where official sources, weights and environments are; the compile.yaml template is replaced by the facts that differ from the docs - e2e prompt: whole-class int16/float fallback and >1.5x board latency do not pass - agent/VERSION 24 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
273 lines
15 KiB
Python
273 lines
15 KiB
Python
"""`bench dev`: one agent command in a Salty container that has the toolchain (hb_compile, hmct,
|
|
onnxruntime, torch). The agent's bash runs on the controller VM, which has none of it.
|
|
|
|
The workspace lives twice: the controller copy the agent edits, and the development copy on personal
|
|
storage (the same absolute path inside containers, so 04/05 read the graph and calibration made here).
|
|
Before a command, files the agent changed locally since the last sync go up; afterwards, files the
|
|
command changed come back if small, and big ones (graphs, calibration sets) stay remote and are listed.
|
|
The worker sandboxes the command (salty_remote_worker.dev): only toolchains, caches and data read-only
|
|
plus this workspace are visible, no capabilities, no board credentials.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import tarfile
|
|
import time
|
|
import uuid
|
|
from contextlib import contextmanager
|
|
from pathlib import Path, PurePosixPath
|
|
from types import SimpleNamespace
|
|
|
|
from . import execution, paths, salty_remote
|
|
from .salty_client import TransportError
|
|
from .watchdog import RunResult
|
|
|
|
# .bench-session.json.tmp: pi_session's atomic write; listed, then renamed away before tar read it
|
|
# (10-20 20:17 and twice on 10-16: FileNotFoundError, the dev command died as STAGE_EXECUTOR_EXIT).
|
|
EXCLUDED = {'.pi', '.pi-sessions', '.agent-home', '.bench-session.json', '.bench-session.json.tmp', '.env', '.env.local', 'guard.log'}
|
|
UPLOAD_LIMIT = 256 << 20 # the Mac mini uplink is ~1 MB/s; big inputs are made inside bench dev
|
|
GRACE_S = 300
|
|
FETCH_MARGIN_S = 120 # the command ends this long before the request deadline so its results still come back
|
|
|
|
|
|
def selected(cfg=None) -> bool:
|
|
policy = ((cfg or paths.config()).get('execution') or {}).get('salty_remote') or {}
|
|
return policy.get('enabled') is True and 'terminal' in policy.get('stages', [])
|
|
|
|
|
|
def local_files(ws: Path) -> dict:
|
|
files = {}
|
|
for root, dirs, names in os.walk(ws):
|
|
top = Path(root) == ws
|
|
dirs[:] = [d for d in dirs if not (top and d in EXCLUDED) and not os.path.islink(os.path.join(root, d))]
|
|
for name in names:
|
|
p = Path(root) / name
|
|
if (top and name in EXCLUDED) or p.is_symlink() or not p.is_file():
|
|
continue
|
|
st = p.stat()
|
|
files[p.relative_to(ws).as_posix()] = [st.st_size, st.st_mtime_ns]
|
|
return files
|
|
|
|
|
|
@contextmanager
|
|
def synced(rd: Path):
|
|
"""{relative path: [size, mtime_ns]} of the controller copy as last synced; changes since are uploads.
|
|
Held only around reads/updates, so dev commands of one rollout still run in parallel."""
|
|
path = rd / 'stages' / '_dev-sync.json'
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with (rd / 'stages' / '_dev-sync.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
state = json.loads(path.read_text()) if path.exists() else {}
|
|
yield state
|
|
execution.atomic_json(path, state) # fsynced: an empty manifest after a crash would fail every dev command
|
|
|
|
|
|
def memory_gi(policy, ctx, rd: Path) -> int:
|
|
"""The heaviest dev use is compiling this model: at least what its formal compiles needed (salty_remote._memory
|
|
doubles after an OOM; 10-14: vit.h.14 died at 16Gi in bench dev). A dev job of this rollout that failed (an OOM
|
|
gives no reason) doubles what it had, cap 128Gi: the agent cannot raise the limit, and retrying at the same size
|
|
failed again every time (10-20: efficientnet.b6 3x at 16Gi, unidepth.v2.vit.l 2x at 32Gi)."""
|
|
gi = max(int(str(m).removesuffix('Gi')) for m in (
|
|
(policy.get('memory_by_stage') or {}).get('terminal', policy.get('memory', '16Gi')),
|
|
salty_remote._memory(policy, SimpleNamespace(name='05-compile'), ctx)))
|
|
# ponytail: this rollout only; a new rollout of the same model starts from the formal compiles' floor again.
|
|
for f in (rd / 'stages' / '_queue').glob('terminal-*/salty/remote.json'):
|
|
state = json.loads(f.read_text())
|
|
if state.get('job_state') == 'failed' and state.get('memory'):
|
|
gi = max(gi, 2 * int(state['memory'].removesuffix('Gi')))
|
|
return min(gi, 128)
|
|
|
|
|
|
def bundle(path: Path, ws: Path, uploads: list, request: dict) -> None:
|
|
with tarfile.open(path, 'x') as tar:
|
|
data = json.dumps(request).encode()
|
|
info = tarfile.TarInfo('request.json'); info.size = len(data)
|
|
tar.addfile(info, io.BytesIO(data))
|
|
for rel in uploads:
|
|
tar.add(ws / rel, 'ws/' + rel, recursive=False)
|
|
# Only what the sandbox itself needs (onnxruntime thread cap): platform code (adapters, export tools, the
|
|
# evaluator and gates) would do the model-specific half of the task for the agent.
|
|
tar.add(paths.BENCH_ROOT / 'harness/container_site/sitecustomize.py', 'code/harness/container_site/sitecustomize.py')
|
|
|
|
|
|
def install(ws: Path, archive: Path, result: dict) -> list:
|
|
"""Changed small files back into the controller copy; only regular files under the workspace."""
|
|
data = archive.read_bytes()
|
|
if hashlib.sha256(data).hexdigest() != result['archive']['sha256']:
|
|
raise ValueError('dev output archive bytes differ from its receipt')
|
|
installed = []
|
|
with tarfile.open(fileobj=io.BytesIO(data)) as tar:
|
|
for member in tar.getmembers():
|
|
rel = PurePosixPath(member.name)
|
|
if not member.isfile() or rel.is_absolute() or '..' in rel.parts or rel.parts[0] in EXCLUDED:
|
|
raise ValueError('unsafe dev output member: ' + member.name)
|
|
body = tar.extractfile(member).read()
|
|
if hashlib.sha256(body).hexdigest() != result['outputs'][member.name]['sha256']:
|
|
raise ValueError('dev output bytes fail verification: ' + member.name)
|
|
target = ws / rel
|
|
if any(p.is_symlink() for p in [target, *target.parents] if p.is_relative_to(ws) and p != ws):
|
|
raise ValueError('dev output would follow a workspace symlink: ' + member.name)
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = target.with_name(target.name + '.dev-tmp')
|
|
tmp.write_bytes(body)
|
|
salty_remote._fsync(tmp)
|
|
os.replace(tmp, target)
|
|
installed.append(member.name)
|
|
return installed
|
|
|
|
|
|
def _storage(rd: Path, rel: str):
|
|
policy = salty_remote._policy(SimpleNamespace(cfg=paths.config()))
|
|
cli = salty_remote._Cli(policy.get('salty', 'salty'), policy.get('outage_seconds', 1800), time.sleep, time.time, lambda m: None)
|
|
remote = 'personal:/bench-root/runs/' + (rd / 'workspace').relative_to(paths.BENCH_ROOT / 'runs').as_posix()
|
|
if rel not in ('', '.'):
|
|
remote += '/' + PurePosixPath(rel).as_posix()
|
|
return cli, remote
|
|
|
|
|
|
def remote_size(rd: Path, rel: str, cap: int, max_dirs: int = 40) -> int | None:
|
|
"""Bytes under the development workspace's <rel>, or None once past cap bytes or max_dirs directories
|
|
(one `salty storage ls` per directory, ~3 s each)."""
|
|
cli, root = _storage(rd, rel)
|
|
todo, total = [root], 0
|
|
while todo:
|
|
if max_dirs <= 0:
|
|
return None
|
|
max_dirs -= 1
|
|
here = todo.pop()
|
|
_, out, _ = cli.call('storage', 'ls', here, ok_404=True)
|
|
for line in (out or '').splitlines()[1:]: # header: 名称 类型 大小 修改时间
|
|
name, kind, size = (line.split('\t') + ['', '', ''])[:3]
|
|
if kind == '目录':
|
|
todo.append(here + '/' + name)
|
|
elif size.isdigit():
|
|
total += int(size)
|
|
if total > cap:
|
|
return None
|
|
return total
|
|
|
|
|
|
def download_dir(rd: Path, rel: str, destination: Path) -> None:
|
|
"""The development workspace's copy of <rel> (files over the copy-back limit only exist there); a directory
|
|
never synced up gives an empty destination."""
|
|
cli, remote = _storage(rd, rel)
|
|
part = destination.with_name(destination.name + '.part')
|
|
def fresh():
|
|
shutil.rmtree(part, ignore_errors=True); part.mkdir(parents=True)
|
|
rc, _, err = cli.call('storage', 'cp', remote, str(part), timeout=1800, ok_404=True, before=fresh)
|
|
got = part / PurePosixPath(remote).name # `salty storage cp <dir> <existing dir>` lands in <existing dir>/<name>
|
|
if rc == 0 and got.is_dir():
|
|
os.replace(got, destination)
|
|
else:
|
|
destination.mkdir(parents=True)
|
|
shutil.rmtree(part)
|
|
|
|
|
|
def calib_images(rd: Path, job: dict) -> str:
|
|
"""The job's calibration images for agent v18+ (calib-raw mirrors calib; 10-15 night: told only to
|
|
"replace /data/calib/", agents searched /data/calib-raw and spent 5-8 commands finding it). From v21 the
|
|
original training images (images/, from 36's JuiceFS); v18-v20 rollouts keep the resized png/ they started with."""
|
|
version = int(json.loads((rd / 'rollout.json').read_text()).get('agent_config_version', 0))
|
|
return job['data']['calib']['path'].replace('/data/calib/', '/data/calib-raw/', 1) + ('/images' if version >= 21 else '/png')
|
|
|
|
|
|
def execute(ctx, hd: Path, req: dict, deadline: float, stdout: Path, stderr: Path) -> RunResult:
|
|
"""Run req['command'] remotely; returns a watchdog-shaped result for terminal.run."""
|
|
started = time.time()
|
|
rd = ctx.stage_root.parent
|
|
ws = rd / 'workspace'
|
|
adapter = hd / 'salty'
|
|
adapter.mkdir(exist_ok=True)
|
|
state_file = adapter / 'remote.json'
|
|
policy = salty_remote._policy(SimpleNamespace(cfg=paths.config()))
|
|
|
|
def log(message):
|
|
with (adapter / 'transport.log').open('a') as f:
|
|
f.write(f"{time.strftime('%F %T')} {message}\n")
|
|
cli = salty_remote._Cli(policy.get('salty', 'salty'), policy.get('outage_seconds', 1800), time.sleep, time.time, log)
|
|
state = {'name': 'bench-dev-' + str(uuid.uuid4()), 'job_id': None}
|
|
state['remote'] = 'personal:/' + state['name']
|
|
remaining = lambda: deadline - time.monotonic()
|
|
|
|
def done(st, code, reason=None, cleanup='VERIFIED'):
|
|
stdout.touch(); stderr.touch()
|
|
return RunResult(st, code, time.time() - started, stdout, stderr, stalled_reason=reason, cleanup_state=cleanup)
|
|
|
|
with synced(rd) as manifest:
|
|
local = local_files(ws)
|
|
uploads = sorted(rel for rel, st in local.items() if manifest.get(rel) != st)
|
|
total = sum(local[rel][0] for rel in uploads)
|
|
if total > UPLOAD_LIMIT:
|
|
stderr.write_text(f'bench dev: {total >> 20} MiB of local changes to upload (limit {UPLOAD_LIMIT >> 20} MiB); '
|
|
'make big files inside bench dev instead\n')
|
|
return done('FAIL', 2)
|
|
request = {'identity': {'stage': 'terminal', 'operation_id': hd.name}, 'bench_root': str(paths.BENCH_ROOT),
|
|
'dev': {'workspace': str(ws), 'command': (paths.AGENT / 'pipe-note.sh').read_text().strip() + ' ' + req['command'], 'cwd': req.get('cwd', '.'),
|
|
'deadline_wall': time.time() + remaining() - FETCH_MARGIN_S,
|
|
'timeout': req.get('timeout'), # the command's own time, counted from container start
|
|
'own_inputs': bool(ctx.extra.get('own_inputs_required')),
|
|
'calib_images': calib_images(rd, ctx.job) if ctx.extra.get('own_inputs_required') else None,
|
|
'cpu': str((policy.get('cpu_by_stage') or {}).get('terminal', policy.get('cpu', 4)))}}
|
|
bundle(adapter / 'bundle.tar', ws, uploads, request)
|
|
shutil.copyfile(salty_remote.WORKER, adapter / 'worker.py')
|
|
try:
|
|
memory = state['memory'] = f'{memory_gi(policy, ctx, rd)}Gi'
|
|
state['job_id'] = salty_remote._submit(cli, state, policy, SimpleNamespace(name='terminal'), adapter, memory)
|
|
execution.atomic_json(state_file, state)
|
|
with synced(rd) as manifest: # the remote copy has them now, whatever the command does
|
|
manifest.update({rel: local[rel] for rel in uploads})
|
|
job_state = None
|
|
while job_state not in ('succeeded', 'failed', 'stopped'):
|
|
if remaining() < -GRACE_S: # the worker's own timeout should have ended it already
|
|
cli.call('stop', state['job_id'], ok_404=True)
|
|
time.sleep(5 if time.time() - started < 120 else 30) # short commands answer fast; long ones poll gently
|
|
out = cli.call('describe', state['job_id'])[1]
|
|
job_state = (re.search(r'^状态:\s*(\w+)', out, re.MULTILINE) or [None, None])[1]
|
|
if job_state != 'succeeded':
|
|
state['job_state'] = job_state
|
|
execution.atomic_json(state_file, state)
|
|
salty_remote._remote_logs(policy, state['job_id'], stderr)
|
|
# A container OOM gives no reason; its memory log shows it (the agent cannot raise the limit itself).
|
|
if cli.once(['storage', 'cp', state['remote'] + '/usage.log', str(adapter / 'usage.log')])[0] == 0:
|
|
more = f"; the next bench dev command gets {min(2 * int(memory.removesuffix('Gi')), 128)}Gi" if job_state == 'failed' else ''
|
|
with stderr.open('a') as f:
|
|
f.write(f'\n[bench dev container: {memory} memory; usage every 15s — a peak near the limit means it ran out of memory{more}]\n'
|
|
+ (adapter / 'usage.log').read_text()[-4000:])
|
|
return done('FAIL', None, 'TERMINAL_DEADLINE_EXCEEDED' if remaining() < 0 else 'REMOTE_WORKER_FAILED')
|
|
for f in ('result.json', 'outputs.tar', 'stdout.log', 'stderr.log'):
|
|
salty_remote._fetch(cli, f"{state['remote']}/result/{f}", adapter / f)
|
|
result = json.loads((adapter / 'result.json').read_text())
|
|
if result.get('identity') != request['identity']:
|
|
raise ValueError('remote result belongs to another dev command')
|
|
installed = install(ws, adapter / 'outputs.tar', result)
|
|
with synced(rd) as manifest:
|
|
for rel in installed:
|
|
st = (ws / rel).stat()
|
|
manifest[rel] = [st.st_size, st.st_mtime_ns]
|
|
shutil.copyfile(adapter / 'stdout.log', stdout)
|
|
shutil.copyfile(adapter / 'stderr.log', stderr)
|
|
with stdout.open('a') as f:
|
|
if installed:
|
|
f.write(f"\n[bench dev] copied back to the workspace: {', '.join(installed[:50])}\n")
|
|
if result.get('remote_only'):
|
|
f.write('[bench dev] kept only in the development workspace (too big to copy back; usable from bench dev '
|
|
'and compile.yaml): ' + ', '.join(f'{r} ({s >> 20} MiB)' for r, s in result['remote_only'][:50]) + '\n')
|
|
code = result.get('exit_code')
|
|
return done('OK' if code == 0 else 'FAIL', code, 'TERMINAL_DEADLINE_EXCEEDED' if code == 124 else None)
|
|
except (TransportError, ValueError, OSError) as exc:
|
|
with stderr.open('a') as f:
|
|
f.write(f'\n[bench dev] {type(exc).__name__}: {exc}\n')
|
|
# The container ends by its own timeout and writes only to this workspace.
|
|
return done('FAIL', None, 'REMOTE_UNAVAILABLE', 'REMOTE_BOUNDED')
|
|
finally:
|
|
if state['job_id']:
|
|
salty_remote._cleanup(cli, state, state_file)
|
|
for f in ('bundle.tar', 'outputs.tar', 'worker.py'):
|
|
(adapter / f).unlink(missing_ok=True)
|