amos c64c79b31d Run the Pi agent in Salty jobs over a one-time SSH tunnel and export SFT rows
rollout.salty_run keeps the gateway, grading and evidence on 36 and moves only the
agent step into a Salty job. The gateway listens on 127.0.0.1 with a per-attempt
bearer token; the job reaches it over port 22 with a restricted one-time key that is
removed after each attempt. rollout.batch gains --backend salty and repeatable
--credential-file, and writes pass/fail SFT exports through rollout.sft at the end.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 17:45:28 +08:00

624 lines
29 KiB
Python

"""Small local supervisor. Agent execution always requires Docker."""
from contextlib import contextmanager
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import signal
import stat
import subprocess
import sys
import time
import uuid
from urllib.parse import urlsplit
MAX_FILE_BYTES = 1024 * 1024
MAX_CHECKER_BYTES = 64 * 1024
MAX_TRACE_BYTES = 64 * 1024 * 1024 # Must match agent/pi.mjs.
FIXTURES = {"gold": "PASS", "template": "FAIL", "noop": "FAIL", "cheat": "FAIL"}
# shortcut: fixed floor for 36's shared /home (user lowered it from 50 GiB on 2026-10-17); make it a flag when another host needs a different one.
MIN_FREE_BYTES = 30 * 1024 ** 3
MAX_ATTEMPT_BYTES = 2 * 1024 ** 3
def _path(path):
path = Path(os.path.abspath(path))
for part in (path, *path.parents):
if part.is_symlink():
raise ValueError("symlink is not allowed: " + str(part))
return path
def _relative(value):
if not isinstance(value, str) or not value or "\\" in value:
raise ValueError("expected a nonempty relative file path")
path = Path(value)
if path.is_absolute() or any(p in (".", "..", "") for p in value.split("/")):
raise ValueError("unsafe relative path: " + value)
return path
def _read(root, relative, limit=MAX_FILE_BYTES):
path = _path(root / _relative(relative))
if root not in path.parents:
raise ValueError("unsafe file path: " + str(path))
try:
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
except OSError as error:
raise ValueError("missing or unsafe file: " + str(path)) from error
with os.fdopen(descriptor, "rb") as stream:
if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
raise ValueError("missing or unsafe file: " + str(path))
data = stream.read(limit + 1)
if len(data) > limit:
raise ValueError("file exceeds its size limit: " + str(path))
return data
def _create(path, data):
"""Write a new file exclusively; remove only our own inode if writing fails."""
created = None
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
created = os.fstat(descriptor)
with os.fdopen(descriptor, "wb") as stream:
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
except BaseException:
if created is not None:
try:
current = path.lstat()
except FileNotFoundError:
current = None
if current and (current.st_dev, current.st_ino) == (created.st_dev, created.st_ino):
path.unlink()
raise
return path
def _write_tree(root, files, file_mode, dir_mode):
root.mkdir(exist_ok=True)
root.chmod(dir_mode)
for name, data in files.items():
path = root
for part in _relative(name).parts[:-1]:
path = path / part
path.mkdir(exist_ok=True)
path.chmod(dir_mode)
path = root / name
path.write_bytes(data)
path.chmod(file_mode)
def _digest(files):
return hashlib.sha256(json.dumps(
{name: hashlib.sha256(data).hexdigest() for name, data in files.items()},
sort_keys=True, separators=(",", ":"),
).encode()).hexdigest()
def load_task(task_path):
"""Validate and capture the fixed task package; private keys hold its snapshot."""
root = _path(task_path)
if not root.is_dir():
raise ValueError("task directory does not exist")
for entry in root.rglob("*"):
if entry.is_symlink():
raise ValueError("task contains a symlink: " + str(entry))
raw = _read(root, "task.json")
try:
task = json.loads(raw)
except (ValueError, UnicodeDecodeError) as error:
raise ValueError("invalid task.json") from error
if not isinstance(task, dict):
raise ValueError("task.json must be an object")
for key in ("id", "version", "family", "source"):
if not isinstance(task.get(key), str) or not task[key].strip():
raise ValueError("missing or invalid field: " + key)
if task.get("template") != "template" or task.get("verifier") != "verify.py":
raise ValueError("template/verifier must be template and verify.py")
candidates = task.get("candidate_files")
if (not isinstance(candidates, list) or not 1 <= len(candidates) <= 16 or
len(set(map(str, candidates))) != len(candidates)):
raise ValueError("candidate_files must list 1..16 unique relative files")
for name in candidates:
_relative(name)
fixtures = task.setdefault("fixtures", dict(FIXTURES))
if (not isinstance(fixtures, dict) or len(fixtures) > 12 or
any(fixtures.get(name) != wanted for name, wanted in FIXTURES.items()) or
any(not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", name) or wanted not in ("PASS", "FAIL")
for name, wanted in fixtures.items())):
raise ValueError("fixtures must keep gold/template/noop/cheat and map at most 12 names to PASS/FAIL")
graders = task.setdefault("grader_files", [])
if not isinstance(graders, list) or len(graders) > 16 or len(set(map(str, graders))) != len(graders):
raise ValueError("grader_files must list at most 16 unique files")
for name in graders:
if (_relative(name).parts[0] in set(fixtures) | {"template"} or
name in ("task.json", "prompt.md", "verify.py", "dataset.json")):
raise ValueError("grader file collides with task material: " + name)
timeout = task.get("timeout_seconds")
if type(timeout) is not int or not 1 <= timeout <= 300:
raise ValueError("timeout_seconds must be an integer in 1..300")
template = sorted(entry.relative_to(root / "template").as_posix()
for entry in (root / "template").rglob("*") if not entry.is_dir())
if not 1 <= len(template) <= 64 or any(name not in template for name in candidates):
raise ValueError("template must hold 1..64 files including every candidate file")
names = ["task.json", "prompt.md", "verify.py", *graders, *("template/" + name for name in template)]
names += [fixture + "/" + name for fixture in fixtures if fixture != "template" for name in candidates]
files = {name: _read(root, name) for name in names}
if not files["prompt.md"].strip() or not files["verify.py"].strip():
raise ValueError("prompt and verifier must not be empty")
try:
files["prompt.md"].decode("utf-8")
except UnicodeDecodeError as error:
raise ValueError("prompt must be UTF-8") from error
task.update(_root=root, _files=files, _digest=_digest(files))
return task
def _fixture(task, fixture):
return {name: task["_files"][fixture + "/" + name] for name in task["candidate_files"]}
def _template(task):
return {name[len("template/"):]: data for name, data in task["_files"].items()
if name.startswith("template/")}
def _event(attempt, kind, **fields):
with (attempt / "events.jsonl").open("a", encoding="utf-8") as stream:
stream.write(json.dumps(dict(time=time.time(), event=kind, **fields), sort_keys=True) + "\n")
stream.flush()
def _attempt(task, output_root, mode):
root = _path(output_root)
if root == task["_root"] or task["_root"] in root.parents:
raise ValueError("output must be outside the task directory")
root.mkdir(parents=True, exist_ok=True)
attempt = root / (mode + "-" + uuid.uuid4().hex)
attempt.mkdir(mode=0o700)
_write_tree(attempt / "grader", {name: task["_files"][name] for name in
("verify.py", *task["grader_files"])}, 0o600, 0o700)
_event(attempt, "created", mode=mode, task_id=task["id"], task_digest=task["_digest"])
return attempt
def _receipt(task, mode, image_id=None):
return dict(mode=mode, task_id=task["id"], task_version=task["version"],
task_digest=task["_digest"],
manifest_digest=hashlib.sha256(task["_files"]["task.json"]).hexdigest(),
verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(),
image_id=image_id, status="UNVERIFIED", task_verdict="UNVERIFIED",
cleanup="confirmed", supervisor_only=True,
training_disposition="NOT_TRAIN_READY")
def _finish(attempt, receipt):
_event(attempt, "finished", status=receipt["status"], cleanup=receipt["cleanup"])
temporary = attempt / "receipt.tmp"
with temporary.open("x", encoding="utf-8") as stream:
json.dump(receipt, stream, indent=2, sort_keys=True)
stream.write("\n")
stream.flush()
os.fsync(stream.fileno())
temporary.replace(attempt / "receipt.json")
return attempt
@contextmanager
def _sigterm():
"""Deliver SIGTERM as KeyboardInterrupt so cleanup still runs."""
previous = signal.getsignal(signal.SIGTERM)
installed = False
def interrupt(signum, frame):
raise KeyboardInterrupt
try:
signal.signal(signal.SIGTERM, interrupt)
installed = True
except ValueError: # Only the main thread can own signal handlers.
pass
try:
yield
finally:
if installed:
signal.signal(signal.SIGTERM, previous)
@contextmanager
def _execution(attempt, receipt):
"""Turn cancellation into a durable terminal receipt after resource cleanup."""
try:
with _sigterm():
yield
except KeyboardInterrupt:
receipt.update(status="UNVERIFIED", task_verdict="UNVERIFIED", termination="interrupted")
except ValueError as error:
receipt.update(status="UNVERIFIED", task_verdict="UNVERIFIED", error=str(error))
except Exception as error:
receipt.update(status="INFRA_ERROR", task_verdict="INFRA_ERROR", error=str(error))
finally:
_finish(attempt, receipt)
def _disk_problem(root, max_bytes=None):
"""Name the disk limit an attempt has hit, if any."""
if shutil.disk_usage(root).free < MIN_FREE_BYTES:
return "disk_low"
if max_bytes is None:
return None
used = 0
# ponytail: directories the Agent made unreadable are skipped; the free-space floor still bounds them.
for directory, _, names in os.walk(root):
for name in names:
try:
used += os.lstat(os.path.join(directory, name)).st_blocks * 512
except OSError:
pass
return "attempt_size" if used > max_bytes else None
def _process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False, watch=None):
facts = dict(returncode=None, termination="launch_error", cleanup="confirmed")
# Host execution is only for explicitly trusted calibration fixtures.
env = dict(os.environ) if controller else {"PATH": os.environ.get("PATH", os.defpath), "LANG": "C.UTF-8"}
with stdout.open("wb") as out, stderr.open("wb") as err:
try:
proc = subprocess.Popen(argv, cwd=cwd, stdin=subprocess.PIPE if stdin else subprocess.DEVNULL,
stdout=out, stderr=err, env=env, start_new_session=True)
except OSError as error:
facts["detail"] = str(error)
return facts
try:
deadline = time.monotonic() + timeout
while True:
try:
proc.communicate(stdin, timeout=max(0, min(2, deadline - time.monotonic())))
facts["termination"] = "exit"
break
except subprocess.TimeoutExpired:
stdin = None # Input was already sent; communicate() refuses it twice.
problem = _disk_problem(watch, MAX_ATTEMPT_BYTES) if watch else None
if problem or time.monotonic() >= deadline:
facts["termination"] = problem or "timeout"
break
except KeyboardInterrupt:
facts["termination"] = "interrupted"
finally:
try:
os.killpg(proc.pid, signal.SIGKILL)
except ProcessLookupError:
pass
except OSError as error:
facts["cleanup"] = "unknown"
facts["cleanup_detail"] = str(error)
proc.wait()
try:
os.killpg(proc.pid, 0)
facts["cleanup"] = "unknown"
facts["cleanup_detail"] = "process group still exists after termination"
except ProcessLookupError:
pass
except OSError as error:
facts["cleanup"] = "unknown"
facts["cleanup_detail"] = str(error)
facts["returncode"] = proc.returncode
return facts
def _cli(argv):
try:
return subprocess.run(argv, capture_output=True, text=True, timeout=10)
except (OSError, subprocess.TimeoutExpired) as error:
raise RuntimeError("Docker unavailable: " + str(error)) from error
def doctor():
"""Only inspect local dependencies; never install or start services."""
result = dict(status="UNAVAILABLE", python=sys.version.split()[0], docker={})
if not shutil.which("docker"):
result["docker"] = dict(status="UNAVAILABLE", reason="docker command not found")
return result
try:
check = _cli(["docker", "info", "--format", "{{.ServerVersion}}"])
if check.returncode:
raise RuntimeError(check.stderr.strip() or "daemon not ready")
result.update(status="AVAILABLE", docker=dict(status="AVAILABLE", version=check.stdout.strip()))
except RuntimeError as error:
result["docker"] = dict(status="UNAVAILABLE", reason=str(error))
return result
def _image(image):
if not isinstance(image, str) or not image.strip() or image.startswith("-"):
raise ValueError("a Docker image reference is required")
result = _cli(["docker", "image", "inspect", image, "--format", "{{.Id}}"])
identity = result.stdout.strip()
if result.returncode or not re.fullmatch(r"sha256:[a-f0-9]{64}", identity):
raise RuntimeError("image is unavailable locally: " + image)
return identity
def _docker(attempt, image_id, argv, mounts, prefix, timeout, stdin=None, network=False,
model_base_url=None):
name = "rollout-" + uuid.uuid4().hex
memory = "2g" if prefix == "agent" else "512m"
command = ["docker", "run", "--name", name, "--read-only", "--cap-drop", "ALL",
"--security-opt", "no-new-privileges", "--pids-limit", "128", "--memory", memory, "--cpus", "1",
"--tmpfs", "/tmp:rw,exec,nosuid,nodev,size=128m", "--entrypoint", ""]
if stdin is not None:
command.append("-i")
if not network:
command += ["--network", "none"]
command += ["--user", "0:0", "--cap-add", "SETUID", "--cap-add", "SETGID", "--cap-add", "CHOWN",
"--cap-add", "DAC_OVERRIDE", "--env", "ROLLOUT_GRADE_ISOLATION=uid65534"]
elif prefix == "agent":
command += ["--network", network]
if model_base_url:
command += ["--env", "ROLLOUT_MODEL_BASE_URL=" + model_base_url]
for source, target, writable in mounts:
# --mount rejects ambiguous comma-delimited host paths.
if "," in str(source):
raise ValueError("Docker bind paths must not contain commas")
command += ["--mount", "type=bind,src=" + str(source) + ",dst=" + target +
("" if writable else ",readonly")]
command += ["-w", "/workspace" if prefix == "agent" else "/tmp", image_id, *argv]
if _disk_problem(attempt):
facts = dict(returncode=None, termination="disk_low", cleanup="confirmed")
_event(attempt, "container_skipped", name=name, role=prefix, **facts)
return facts
_event(attempt, "container_requested", name=name, role=prefix, image_id=image_id)
facts = dict(returncode=None, termination="launch_error", cleanup="unknown")
try:
facts = _process(command, attempt, timeout, attempt / (prefix + ".stdout"),
attempt / (prefix + ".stderr"), stdin, controller=True,
watch=attempt if prefix == "agent" else None)
except KeyboardInterrupt:
facts["termination"] = "interrupted"
finally:
try:
removal = _cli(["docker", "rm", "-f", name])
check = _cli(["docker", "container", "inspect", name])
absent = check.returncode != 0 and any(
phrase in check.stderr.lower() for phrase in ("no such object", "no such container"))
# Killing the CLI does not cancel a daemon create request still in flight.
finished = (facts["termination"] == "exit" and
type(facts["returncode"]) is int and facts["returncode"] >= 0)
confirmed = absent and finished
facts["container_cleanup"] = "confirmed" if confirmed else "unknown"
facts["cleanup"] = "confirmed" if confirmed and facts["cleanup"] == "confirmed" else "unknown"
if not absent:
facts["cleanup_detail"] = removal.stderr.strip() or check.stderr.strip()
elif not finished:
facts["cleanup_detail"] = "Docker CLI did not finish; daemon container creation may still be pending"
except RuntimeError as error:
facts["cleanup"] = "unknown"
facts["cleanup_detail"] = str(error)
except KeyboardInterrupt:
facts.update(cleanup="unknown", termination="interrupted", cleanup_detail="cleanup interrupted")
_event(attempt, "container_cleanup", name=name, **facts)
return facts
def _checker(facts, stdout):
if facts["cleanup"] != "confirmed":
return dict(status="INFRA_ERROR", checks=0, detail="cleanup was not confirmed")
if facts["termination"] == "interrupted":
return dict(status="UNVERIFIED", checks=0, detail="verification interrupted")
if facts["termination"] != "exit" or facts["returncode"] in (125, 126, 127):
return dict(status="INFRA_ERROR", checks=0, detail=facts["termination"])
try:
if stdout.stat().st_size > MAX_CHECKER_BYTES:
raise ValueError("grader output exceeds 64 KiB")
lines = stdout.read_text(encoding="utf-8").splitlines()
if len(lines) != 1:
raise ValueError("grader must emit exactly one JSON line")
verdict = json.loads(lines[0])
status, checks = verdict["status"], verdict["checks"]
if type(checks) is not int or checks < 0:
raise ValueError("invalid checks count")
expected = {"PASS": 0, "FAIL": 1, "INFRA_ERROR": 2, "UNVERIFIED": 2}
if status not in expected or expected[status] != facts["returncode"]:
raise ValueError("grader exit/status conflict")
if status in ("PASS", "FAIL") and checks == 0:
raise ValueError("no behavior checks ran")
return verdict
except (ValueError, KeyError, TypeError, UnicodeDecodeError) as error:
return dict(status="UNVERIFIED", checks=0, detail=str(error))
def _grade(task, attempt, candidate, prefix, image_id):
argv = ["python3", "-I", "/grader/verify.py", "/candidate"]
if image_id:
facts = _docker(attempt, image_id, argv,
[(candidate, "/candidate", False), (attempt / "grader", "/grader", False)],
prefix, task["timeout_seconds"])
else:
facts = _process([sys.executable, str(attempt / "grader/verify.py"), str(candidate)],
attempt, task["timeout_seconds"], attempt / (prefix + ".stdout"),
attempt / (prefix + ".stderr"))
verdict = _checker(facts, attempt / (prefix + ".stdout"))
_event(attempt, "graded", role=prefix, verdict=verdict, process=facts)
return dict(verdict=verdict, process=facts)
def calibrate(task_path, output_root, trusted_local=False, image=None):
"""Check every declared fixture, each in five fresh candidates."""
if type(trusted_local) is not bool or bool(trusted_local) == bool(image):
raise ValueError("choose exactly one of trusted_local=True or image")
task = load_task(task_path)
image_id = _image(image) if image else None
attempt = _attempt(task, output_root, "calibration")
receipt = _receipt(task, "calibration", image_id)
receipt.update(trusted_local=bool(trusted_local), trials=[])
with _execution(attempt, receipt):
for fixture, expected in task["fixtures"].items():
files = _fixture(task, fixture)
for index in range(5):
prefix = fixture + "-" + str(index + 1)
candidate = attempt / prefix
_write_tree(candidate, files, 0o644, 0o755)
receipt["cleanup"] = "unknown"
trial = _grade(task, attempt, candidate, prefix, image_id)
trial.update(fixture=fixture, repeat=index + 1, expected=expected, candidate_digest=_digest(files))
receipt["trials"].append(trial)
receipt["cleanup"] = ("confirmed" if all(t["process"]["cleanup"] == "confirmed"
for t in receipt["trials"]) else "unknown")
if trial["process"]["termination"] == "interrupted":
raise KeyboardInterrupt
statuses = [t["verdict"]["status"] for t in receipt["trials"]]
receipt["status"] = ("INFRA_ERROR" if "INFRA_ERROR" in statuses else
"UNVERIFIED" if "UNVERIFIED" in statuses else
"PASS" if all(t["verdict"]["status"] == t["expected"]
for t in receipt["trials"]) else "FAIL")
receipt["task_verdict"] = receipt["status"]
return attempt
def _calibration(task, image_id, path):
path = _path(path)
if path.is_dir():
path = _path(path / "receipt.json")
try:
raw = _read(path.parent, path.name)
receipt = json.loads(raw)
except (ValueError, UnicodeDecodeError) as error:
raise ValueError("invalid calibration receipt") from error
expected = dict(mode="calibration", status="PASS", task_verdict="PASS", cleanup="confirmed",
task_digest=task["_digest"], image_id=image_id,
verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(),
trusted_local=False)
if not isinstance(receipt, dict) or any(receipt.get(k) != v for k, v in expected.items()):
raise ValueError("calibration must pass in the same task, verifier and Docker image")
if receipt.get("trusted_local") is not False:
raise ValueError("host calibration cannot authorize Docker execution")
trials = receipt.get("trials")
fixtures = task["fixtures"]
if not isinstance(trials, list) or len(trials) != 5 * len(fixtures):
raise ValueError("calibration must include five trials per fixture")
seen = set()
try:
for trial in trials:
fixture, repeat = trial["fixture"], trial["repeat"]
if fixture not in fixtures or type(repeat) is not int or not 1 <= repeat <= 5:
raise ValueError("invalid calibration trial identity")
if (fixture, repeat) in seen:
raise ValueError("duplicate calibration trial")
seen.add((fixture, repeat))
wanted = fixtures[fixture]
verdict, process = trial["verdict"], trial["process"]
if (trial["expected"] != wanted or verdict["status"] != wanted or
type(verdict["checks"]) is not int or verdict["checks"] <= 0 or
process["cleanup"] != "confirmed" or process["termination"] != "exit" or
process["returncode"] != (0 if wanted == "PASS" else 1) or
trial["candidate_digest"] != _digest(_fixture(task, fixture))):
raise ValueError("calibration trial did not meet its expected result")
except (KeyError, TypeError) as error:
raise ValueError("malformed calibration trial") from error
return dict(path=str(path), receipt_digest=hashlib.sha256(raw).hexdigest())
def _native_trace(attempt):
"""Capture the one adapter artifact after the Agent container is gone."""
result = dict(path="trace/native.jsonl", status="UNVERIFIED")
try:
if not os.path.lexists(attempt / result["path"]):
return dict(result, status="MISSING")
data = _read(attempt, result["path"], MAX_TRACE_BYTES)
return dict(result, status="AVAILABLE", bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
except (ValueError, OSError) as error:
return dict(result, detail=str(error))
def _model_url(value):
if value is None:
return None
if not isinstance(value, str):
raise ValueError("model_base_url must be a URL without credentials")
parsed = urlsplit(value)
if (parsed.scheme not in ("http", "https") or not parsed.hostname or
parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment):
raise ValueError("model_base_url must be an HTTP(S) URL without credentials, query or fragment")
parsed.port # Validate the optional port before recording or starting an attempt.
return value
def run(task_path, output_root, image, agent_argv, agent_identity, calibration,
agent_timeout=900, model_base_url=None, agent_network=None, agent_runner=None):
"""Run a configured Agent in Docker (or agent_runner), freeze its output, then grade independently.
agent_runner(attempt, image_id, argv, timeout, model_base_url) must leave the Agent's
workspace/, trace/ and agent.stdout/.stderr in attempt and return _docker-style facts.
"""
task = load_task(task_path)
if not isinstance(agent_identity, str) or not agent_identity.strip():
raise ValueError("agent_identity is required")
if not isinstance(agent_argv, (list, tuple)) or not agent_argv or any(
not isinstance(arg, str) or "\0" in arg for arg in agent_argv) or not agent_argv[0]:
raise ValueError("agent_argv must be a nonempty argv list")
if type(agent_timeout) is not int or not 1 <= agent_timeout <= 86400:
raise ValueError("agent_timeout must be an integer in 1..86400 seconds")
model_base_url = _model_url(model_base_url)
if agent_network is not None and (not isinstance(agent_network, str) or not re.fullmatch(
r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", agent_network)):
raise ValueError("agent_network must be a Docker network name")
image_id = _image(image)
if agent_network is not None and agent_runner is None:
# Only the in-network model proxy may be reachable; an open bridge reaches the internet.
inspected = _cli(["docker", "network", "inspect", agent_network, "--format", "{{.Internal}}"])
if inspected.returncode or inspected.stdout.strip() != "true":
raise ValueError("agent_network must be an existing internal Docker network")
calibration_identity = _calibration(task, image_id, calibration)
attempt = _attempt(task, output_root, "run")
receipt = _receipt(task, "run", image_id)
receipt.update(agent_identity=agent_identity, agent_argv=list(agent_argv), calibration=calibration_identity,
agent_exit=None, termination="not_started", agent_timeout_seconds=agent_timeout,
agent_memory="2g", model_base_url=model_base_url, agent_network=agent_network,
native_trace=dict(path="trace/native.jsonl", status="MISSING"))
workspace = attempt / "workspace"
_write_tree(workspace, _template(task), 0o666, 0o777)
trace = attempt / "trace"
trace.mkdir(mode=0o777)
trace.chmod(0o777)
prompt = attempt / "prompt.md"
prompt.write_bytes(task["_files"]["prompt.md"])
prompt.chmod(0o644)
with _execution(attempt, receipt):
receipt["cleanup"] = "unknown"
if agent_runner is None:
agent = _docker(attempt, image_id, list(agent_argv),
[(workspace, "/workspace", True), (prompt, "/task/prompt.md", False),
(trace, "/trace", True)],
"agent", agent_timeout, task["_files"]["prompt.md"], network=agent_network or "none",
model_base_url=model_base_url)
else:
agent = agent_runner(attempt, image_id, list(agent_argv), agent_timeout, model_base_url)
receipt.update(agent_exit=agent["returncode"], termination=agent["termination"],
cleanup=agent["cleanup"], agent_process=agent)
if agent["cleanup"] == "confirmed":
receipt["native_trace"] = _native_trace(attempt)
if agent["termination"] == "interrupted":
raise KeyboardInterrupt
if agent["cleanup"] != "confirmed":
raise RuntimeError("agent cleanup was not confirmed; candidate cannot be frozen")
files = {name: _read(workspace, name) for name in task["candidate_files"]}
candidate = attempt / "candidate"
_write_tree(candidate, files, 0o644, 0o755)
receipt["candidate_digest"] = _digest(files)
_event(attempt, "candidate_frozen", candidate_digest=receipt["candidate_digest"])
receipt["cleanup"] = "unknown"
grade = _grade(task, attempt, candidate, "grade", image_id)
receipt["cleanup"] = grade["process"]["cleanup"]
receipt.update(status=grade["verdict"]["status"], task_verdict=grade["verdict"]["status"], grade=grade)
if grade["process"]["cleanup"] != "confirmed":
receipt["cleanup"] = "unknown"
if grade["process"]["termination"] == "interrupted":
raise KeyboardInterrupt
return attempt