The other users of /home left 57 GiB free; the user chose to keep running and stop only at 30 GiB. Batch start and runtime disk_low now share that floor. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
617 lines
29 KiB
Python
617 lines
29 KiB
Python
"""Small local supervisor. Agent execution always requires Docker."""
|
|
|
|
from contextlib import contextmanager
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from urllib.parse import urlsplit
|
|
|
|
MAX_FILE_BYTES = 1024 * 1024
|
|
MAX_CHECKER_BYTES = 64 * 1024
|
|
MAX_TRACE_BYTES = 64 * 1024 * 1024 # Must match agent/pi.mjs.
|
|
FIXTURES = {"gold": "PASS", "template": "FAIL", "noop": "FAIL", "cheat": "FAIL"}
|
|
# shortcut: fixed floor for 36's shared /home (user lowered it from 50 GiB on 2026-10-17); make it a flag when another host needs a different one.
|
|
MIN_FREE_BYTES = 30 * 1024 ** 3
|
|
MAX_ATTEMPT_BYTES = 2 * 1024 ** 3
|
|
|
|
|
|
def _path(path):
|
|
path = Path(os.path.abspath(path))
|
|
for part in (path, *path.parents):
|
|
if part.is_symlink():
|
|
raise ValueError("symlink is not allowed: " + str(part))
|
|
return path
|
|
|
|
|
|
def _relative(value):
|
|
if not isinstance(value, str) or not value or "\\" in value:
|
|
raise ValueError("expected a nonempty relative file path")
|
|
path = Path(value)
|
|
if path.is_absolute() or any(p in (".", "..", "") for p in value.split("/")):
|
|
raise ValueError("unsafe relative path: " + value)
|
|
return path
|
|
|
|
|
|
def _read(root, relative, limit=MAX_FILE_BYTES):
|
|
path = _path(root / _relative(relative))
|
|
if root not in path.parents:
|
|
raise ValueError("unsafe file path: " + str(path))
|
|
try:
|
|
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
except OSError as error:
|
|
raise ValueError("missing or unsafe file: " + str(path)) from error
|
|
with os.fdopen(descriptor, "rb") as stream:
|
|
if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
|
|
raise ValueError("missing or unsafe file: " + str(path))
|
|
data = stream.read(limit + 1)
|
|
if len(data) > limit:
|
|
raise ValueError("file exceeds its size limit: " + str(path))
|
|
return data
|
|
|
|
|
|
def _create(path, data):
|
|
"""Write a new file exclusively; remove only our own inode if writing fails."""
|
|
created = None
|
|
try:
|
|
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
created = os.fstat(descriptor)
|
|
with os.fdopen(descriptor, "wb") as stream:
|
|
stream.write(data)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
except BaseException:
|
|
if created is not None:
|
|
try:
|
|
current = path.lstat()
|
|
except FileNotFoundError:
|
|
current = None
|
|
if current and (current.st_dev, current.st_ino) == (created.st_dev, created.st_ino):
|
|
path.unlink()
|
|
raise
|
|
return path
|
|
|
|
|
|
def _write_tree(root, files, file_mode, dir_mode):
|
|
root.mkdir(exist_ok=True)
|
|
root.chmod(dir_mode)
|
|
for name, data in files.items():
|
|
path = root
|
|
for part in _relative(name).parts[:-1]:
|
|
path = path / part
|
|
path.mkdir(exist_ok=True)
|
|
path.chmod(dir_mode)
|
|
path = root / name
|
|
path.write_bytes(data)
|
|
path.chmod(file_mode)
|
|
|
|
|
|
def _digest(files):
|
|
return hashlib.sha256(json.dumps(
|
|
{name: hashlib.sha256(data).hexdigest() for name, data in files.items()},
|
|
sort_keys=True, separators=(",", ":"),
|
|
).encode()).hexdigest()
|
|
|
|
|
|
def load_task(task_path):
|
|
"""Validate and capture the fixed task package; private keys hold its snapshot."""
|
|
root = _path(task_path)
|
|
if not root.is_dir():
|
|
raise ValueError("task directory does not exist")
|
|
for entry in root.rglob("*"):
|
|
if entry.is_symlink():
|
|
raise ValueError("task contains a symlink: " + str(entry))
|
|
raw = _read(root, "task.json")
|
|
try:
|
|
task = json.loads(raw)
|
|
except (ValueError, UnicodeDecodeError) as error:
|
|
raise ValueError("invalid task.json") from error
|
|
if not isinstance(task, dict):
|
|
raise ValueError("task.json must be an object")
|
|
for key in ("id", "version", "family", "source"):
|
|
if not isinstance(task.get(key), str) or not task[key].strip():
|
|
raise ValueError("missing or invalid field: " + key)
|
|
if task.get("template") != "template" or task.get("verifier") != "verify.py":
|
|
raise ValueError("template/verifier must be template and verify.py")
|
|
candidates = task.get("candidate_files")
|
|
if (not isinstance(candidates, list) or not 1 <= len(candidates) <= 16 or
|
|
len(set(map(str, candidates))) != len(candidates)):
|
|
raise ValueError("candidate_files must list 1..16 unique relative files")
|
|
for name in candidates:
|
|
_relative(name)
|
|
fixtures = task.setdefault("fixtures", dict(FIXTURES))
|
|
if (not isinstance(fixtures, dict) or len(fixtures) > 12 or
|
|
any(fixtures.get(name) != wanted for name, wanted in FIXTURES.items()) or
|
|
any(not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", name) or wanted not in ("PASS", "FAIL")
|
|
for name, wanted in fixtures.items())):
|
|
raise ValueError("fixtures must keep gold/template/noop/cheat and map at most 12 names to PASS/FAIL")
|
|
graders = task.setdefault("grader_files", [])
|
|
if not isinstance(graders, list) or len(graders) > 16 or len(set(map(str, graders))) != len(graders):
|
|
raise ValueError("grader_files must list at most 16 unique files")
|
|
for name in graders:
|
|
if (_relative(name).parts[0] in set(fixtures) | {"template"} or
|
|
name in ("task.json", "prompt.md", "verify.py", "dataset.json")):
|
|
raise ValueError("grader file collides with task material: " + name)
|
|
timeout = task.get("timeout_seconds")
|
|
if type(timeout) is not int or not 1 <= timeout <= 300:
|
|
raise ValueError("timeout_seconds must be an integer in 1..300")
|
|
template = sorted(entry.relative_to(root / "template").as_posix()
|
|
for entry in (root / "template").rglob("*") if not entry.is_dir())
|
|
if not 1 <= len(template) <= 64 or any(name not in template for name in candidates):
|
|
raise ValueError("template must hold 1..64 files including every candidate file")
|
|
names = ["task.json", "prompt.md", "verify.py", *graders, *("template/" + name for name in template)]
|
|
names += [fixture + "/" + name for fixture in fixtures if fixture != "template" for name in candidates]
|
|
files = {name: _read(root, name) for name in names}
|
|
if not files["prompt.md"].strip() or not files["verify.py"].strip():
|
|
raise ValueError("prompt and verifier must not be empty")
|
|
try:
|
|
files["prompt.md"].decode("utf-8")
|
|
except UnicodeDecodeError as error:
|
|
raise ValueError("prompt must be UTF-8") from error
|
|
task.update(_root=root, _files=files, _digest=_digest(files))
|
|
return task
|
|
|
|
|
|
def _fixture(task, fixture):
|
|
return {name: task["_files"][fixture + "/" + name] for name in task["candidate_files"]}
|
|
|
|
|
|
def _template(task):
|
|
return {name[len("template/"):]: data for name, data in task["_files"].items()
|
|
if name.startswith("template/")}
|
|
|
|
|
|
def _event(attempt, kind, **fields):
|
|
with (attempt / "events.jsonl").open("a", encoding="utf-8") as stream:
|
|
stream.write(json.dumps(dict(time=time.time(), event=kind, **fields), sort_keys=True) + "\n")
|
|
stream.flush()
|
|
|
|
|
|
def _attempt(task, output_root, mode):
|
|
root = _path(output_root)
|
|
if root == task["_root"] or task["_root"] in root.parents:
|
|
raise ValueError("output must be outside the task directory")
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
attempt = root / (mode + "-" + uuid.uuid4().hex)
|
|
attempt.mkdir(mode=0o700)
|
|
_write_tree(attempt / "grader", {name: task["_files"][name] for name in
|
|
("verify.py", *task["grader_files"])}, 0o600, 0o700)
|
|
_event(attempt, "created", mode=mode, task_id=task["id"], task_digest=task["_digest"])
|
|
return attempt
|
|
|
|
|
|
def _receipt(task, mode, image_id=None):
|
|
return dict(mode=mode, task_id=task["id"], task_version=task["version"],
|
|
task_digest=task["_digest"],
|
|
manifest_digest=hashlib.sha256(task["_files"]["task.json"]).hexdigest(),
|
|
verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(),
|
|
image_id=image_id, status="UNVERIFIED", task_verdict="UNVERIFIED",
|
|
cleanup="confirmed", supervisor_only=True,
|
|
training_disposition="NOT_TRAIN_READY")
|
|
|
|
|
|
def _finish(attempt, receipt):
|
|
_event(attempt, "finished", status=receipt["status"], cleanup=receipt["cleanup"])
|
|
temporary = attempt / "receipt.tmp"
|
|
with temporary.open("x", encoding="utf-8") as stream:
|
|
json.dump(receipt, stream, indent=2, sort_keys=True)
|
|
stream.write("\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
temporary.replace(attempt / "receipt.json")
|
|
return attempt
|
|
|
|
|
|
@contextmanager
|
|
def _sigterm():
|
|
"""Deliver SIGTERM as KeyboardInterrupt so cleanup still runs."""
|
|
previous = signal.getsignal(signal.SIGTERM)
|
|
installed = False
|
|
|
|
def interrupt(signum, frame):
|
|
raise KeyboardInterrupt
|
|
|
|
try:
|
|
signal.signal(signal.SIGTERM, interrupt)
|
|
installed = True
|
|
except ValueError: # Only the main thread can own signal handlers.
|
|
pass
|
|
try:
|
|
yield
|
|
finally:
|
|
if installed:
|
|
signal.signal(signal.SIGTERM, previous)
|
|
|
|
|
|
@contextmanager
|
|
def _execution(attempt, receipt):
|
|
"""Turn cancellation into a durable terminal receipt after resource cleanup."""
|
|
try:
|
|
with _sigterm():
|
|
yield
|
|
except KeyboardInterrupt:
|
|
receipt.update(status="UNVERIFIED", task_verdict="UNVERIFIED", termination="interrupted")
|
|
except ValueError as error:
|
|
receipt.update(status="UNVERIFIED", task_verdict="UNVERIFIED", error=str(error))
|
|
except Exception as error:
|
|
receipt.update(status="INFRA_ERROR", task_verdict="INFRA_ERROR", error=str(error))
|
|
finally:
|
|
_finish(attempt, receipt)
|
|
|
|
|
|
def _disk_problem(root, max_bytes=None):
|
|
"""Name the disk limit an attempt has hit, if any."""
|
|
if shutil.disk_usage(root).free < MIN_FREE_BYTES:
|
|
return "disk_low"
|
|
if max_bytes is None:
|
|
return None
|
|
used = 0
|
|
# ponytail: directories the Agent made unreadable are skipped; the free-space floor still bounds them.
|
|
for directory, _, names in os.walk(root):
|
|
for name in names:
|
|
try:
|
|
used += os.lstat(os.path.join(directory, name)).st_blocks * 512
|
|
except OSError:
|
|
pass
|
|
return "attempt_size" if used > max_bytes else None
|
|
|
|
|
|
def _process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False, watch=None):
|
|
facts = dict(returncode=None, termination="launch_error", cleanup="confirmed")
|
|
# Host execution is only for explicitly trusted calibration fixtures.
|
|
env = dict(os.environ) if controller else {"PATH": os.environ.get("PATH", os.defpath), "LANG": "C.UTF-8"}
|
|
with stdout.open("wb") as out, stderr.open("wb") as err:
|
|
try:
|
|
proc = subprocess.Popen(argv, cwd=cwd, stdin=subprocess.PIPE if stdin else subprocess.DEVNULL,
|
|
stdout=out, stderr=err, env=env, start_new_session=True)
|
|
except OSError as error:
|
|
facts["detail"] = str(error)
|
|
return facts
|
|
try:
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
try:
|
|
proc.communicate(stdin, timeout=max(0, min(2, deadline - time.monotonic())))
|
|
facts["termination"] = "exit"
|
|
break
|
|
except subprocess.TimeoutExpired:
|
|
stdin = None # Input was already sent; communicate() refuses it twice.
|
|
problem = _disk_problem(watch, MAX_ATTEMPT_BYTES) if watch else None
|
|
if problem or time.monotonic() >= deadline:
|
|
facts["termination"] = problem or "timeout"
|
|
break
|
|
except KeyboardInterrupt:
|
|
facts["termination"] = "interrupted"
|
|
finally:
|
|
try:
|
|
os.killpg(proc.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
except OSError as error:
|
|
facts["cleanup"] = "unknown"
|
|
facts["cleanup_detail"] = str(error)
|
|
proc.wait()
|
|
try:
|
|
os.killpg(proc.pid, 0)
|
|
facts["cleanup"] = "unknown"
|
|
facts["cleanup_detail"] = "process group still exists after termination"
|
|
except ProcessLookupError:
|
|
pass
|
|
except OSError as error:
|
|
facts["cleanup"] = "unknown"
|
|
facts["cleanup_detail"] = str(error)
|
|
facts["returncode"] = proc.returncode
|
|
return facts
|
|
|
|
|
|
def _cli(argv):
|
|
try:
|
|
return subprocess.run(argv, capture_output=True, text=True, timeout=10)
|
|
except (OSError, subprocess.TimeoutExpired) as error:
|
|
raise RuntimeError("Docker unavailable: " + str(error)) from error
|
|
|
|
|
|
def doctor():
|
|
"""Only inspect local dependencies; never install or start services."""
|
|
result = dict(status="UNAVAILABLE", python=sys.version.split()[0], docker={})
|
|
if not shutil.which("docker"):
|
|
result["docker"] = dict(status="UNAVAILABLE", reason="docker command not found")
|
|
return result
|
|
try:
|
|
check = _cli(["docker", "info", "--format", "{{.ServerVersion}}"])
|
|
if check.returncode:
|
|
raise RuntimeError(check.stderr.strip() or "daemon not ready")
|
|
result.update(status="AVAILABLE", docker=dict(status="AVAILABLE", version=check.stdout.strip()))
|
|
except RuntimeError as error:
|
|
result["docker"] = dict(status="UNAVAILABLE", reason=str(error))
|
|
return result
|
|
|
|
|
|
def _image(image):
|
|
if not isinstance(image, str) or not image.strip() or image.startswith("-"):
|
|
raise ValueError("a Docker image reference is required")
|
|
result = _cli(["docker", "image", "inspect", image, "--format", "{{.Id}}"])
|
|
identity = result.stdout.strip()
|
|
if result.returncode or not re.fullmatch(r"sha256:[a-f0-9]{64}", identity):
|
|
raise RuntimeError("image is unavailable locally: " + image)
|
|
return identity
|
|
|
|
|
|
def _docker(attempt, image_id, argv, mounts, prefix, timeout, stdin=None, network=False,
|
|
model_base_url=None):
|
|
name = "rollout-" + uuid.uuid4().hex
|
|
memory = "2g" if prefix == "agent" else "512m"
|
|
command = ["docker", "run", "--name", name, "--read-only", "--cap-drop", "ALL",
|
|
"--security-opt", "no-new-privileges", "--pids-limit", "128", "--memory", memory, "--cpus", "1",
|
|
"--tmpfs", "/tmp:rw,exec,nosuid,nodev,size=128m", "--entrypoint", ""]
|
|
if stdin is not None:
|
|
command.append("-i")
|
|
if not network:
|
|
command += ["--network", "none"]
|
|
command += ["--user", "0:0", "--cap-add", "SETUID", "--cap-add", "SETGID", "--cap-add", "CHOWN",
|
|
"--cap-add", "DAC_OVERRIDE", "--env", "ROLLOUT_GRADE_ISOLATION=uid65534"]
|
|
elif prefix == "agent":
|
|
command += ["--network", network]
|
|
if model_base_url:
|
|
command += ["--env", "ROLLOUT_MODEL_BASE_URL=" + model_base_url]
|
|
for source, target, writable in mounts:
|
|
# --mount rejects ambiguous comma-delimited host paths.
|
|
if "," in str(source):
|
|
raise ValueError("Docker bind paths must not contain commas")
|
|
command += ["--mount", "type=bind,src=" + str(source) + ",dst=" + target +
|
|
("" if writable else ",readonly")]
|
|
command += ["-w", "/workspace" if prefix == "agent" else "/tmp", image_id, *argv]
|
|
if _disk_problem(attempt):
|
|
facts = dict(returncode=None, termination="disk_low", cleanup="confirmed")
|
|
_event(attempt, "container_skipped", name=name, role=prefix, **facts)
|
|
return facts
|
|
_event(attempt, "container_requested", name=name, role=prefix, image_id=image_id)
|
|
facts = dict(returncode=None, termination="launch_error", cleanup="unknown")
|
|
try:
|
|
facts = _process(command, attempt, timeout, attempt / (prefix + ".stdout"),
|
|
attempt / (prefix + ".stderr"), stdin, controller=True,
|
|
watch=attempt if prefix == "agent" else None)
|
|
except KeyboardInterrupt:
|
|
facts["termination"] = "interrupted"
|
|
finally:
|
|
try:
|
|
removal = _cli(["docker", "rm", "-f", name])
|
|
check = _cli(["docker", "container", "inspect", name])
|
|
absent = check.returncode != 0 and any(
|
|
phrase in check.stderr.lower() for phrase in ("no such object", "no such container"))
|
|
# Killing the CLI does not cancel a daemon create request still in flight.
|
|
finished = (facts["termination"] == "exit" and
|
|
type(facts["returncode"]) is int and facts["returncode"] >= 0)
|
|
confirmed = absent and finished
|
|
facts["container_cleanup"] = "confirmed" if confirmed else "unknown"
|
|
facts["cleanup"] = "confirmed" if confirmed and facts["cleanup"] == "confirmed" else "unknown"
|
|
if not absent:
|
|
facts["cleanup_detail"] = removal.stderr.strip() or check.stderr.strip()
|
|
elif not finished:
|
|
facts["cleanup_detail"] = "Docker CLI did not finish; daemon container creation may still be pending"
|
|
except RuntimeError as error:
|
|
facts["cleanup"] = "unknown"
|
|
facts["cleanup_detail"] = str(error)
|
|
except KeyboardInterrupt:
|
|
facts.update(cleanup="unknown", termination="interrupted", cleanup_detail="cleanup interrupted")
|
|
_event(attempt, "container_cleanup", name=name, **facts)
|
|
return facts
|
|
|
|
|
|
def _checker(facts, stdout):
|
|
if facts["cleanup"] != "confirmed":
|
|
return dict(status="INFRA_ERROR", checks=0, detail="cleanup was not confirmed")
|
|
if facts["termination"] == "interrupted":
|
|
return dict(status="UNVERIFIED", checks=0, detail="verification interrupted")
|
|
if facts["termination"] != "exit" or facts["returncode"] in (125, 126, 127):
|
|
return dict(status="INFRA_ERROR", checks=0, detail=facts["termination"])
|
|
try:
|
|
if stdout.stat().st_size > MAX_CHECKER_BYTES:
|
|
raise ValueError("grader output exceeds 64 KiB")
|
|
lines = stdout.read_text(encoding="utf-8").splitlines()
|
|
if len(lines) != 1:
|
|
raise ValueError("grader must emit exactly one JSON line")
|
|
verdict = json.loads(lines[0])
|
|
status, checks = verdict["status"], verdict["checks"]
|
|
if type(checks) is not int or checks < 0:
|
|
raise ValueError("invalid checks count")
|
|
expected = {"PASS": 0, "FAIL": 1, "INFRA_ERROR": 2, "UNVERIFIED": 2}
|
|
if status not in expected or expected[status] != facts["returncode"]:
|
|
raise ValueError("grader exit/status conflict")
|
|
if status in ("PASS", "FAIL") and checks == 0:
|
|
raise ValueError("no behavior checks ran")
|
|
return verdict
|
|
except (ValueError, KeyError, TypeError, UnicodeDecodeError) as error:
|
|
return dict(status="UNVERIFIED", checks=0, detail=str(error))
|
|
|
|
|
|
def _grade(task, attempt, candidate, prefix, image_id):
|
|
argv = ["python3", "-I", "/grader/verify.py", "/candidate"]
|
|
if image_id:
|
|
facts = _docker(attempt, image_id, argv,
|
|
[(candidate, "/candidate", False), (attempt / "grader", "/grader", False)],
|
|
prefix, task["timeout_seconds"])
|
|
else:
|
|
facts = _process([sys.executable, str(attempt / "grader/verify.py"), str(candidate)],
|
|
attempt, task["timeout_seconds"], attempt / (prefix + ".stdout"),
|
|
attempt / (prefix + ".stderr"))
|
|
verdict = _checker(facts, attempt / (prefix + ".stdout"))
|
|
_event(attempt, "graded", role=prefix, verdict=verdict, process=facts)
|
|
return dict(verdict=verdict, process=facts)
|
|
|
|
|
|
def calibrate(task_path, output_root, trusted_local=False, image=None):
|
|
"""Check every declared fixture, each in five fresh candidates."""
|
|
if type(trusted_local) is not bool or bool(trusted_local) == bool(image):
|
|
raise ValueError("choose exactly one of trusted_local=True or image")
|
|
task = load_task(task_path)
|
|
image_id = _image(image) if image else None
|
|
attempt = _attempt(task, output_root, "calibration")
|
|
receipt = _receipt(task, "calibration", image_id)
|
|
receipt.update(trusted_local=bool(trusted_local), trials=[])
|
|
with _execution(attempt, receipt):
|
|
for fixture, expected in task["fixtures"].items():
|
|
files = _fixture(task, fixture)
|
|
for index in range(5):
|
|
prefix = fixture + "-" + str(index + 1)
|
|
candidate = attempt / prefix
|
|
_write_tree(candidate, files, 0o644, 0o755)
|
|
receipt["cleanup"] = "unknown"
|
|
trial = _grade(task, attempt, candidate, prefix, image_id)
|
|
trial.update(fixture=fixture, repeat=index + 1, expected=expected, candidate_digest=_digest(files))
|
|
receipt["trials"].append(trial)
|
|
receipt["cleanup"] = ("confirmed" if all(t["process"]["cleanup"] == "confirmed"
|
|
for t in receipt["trials"]) else "unknown")
|
|
if trial["process"]["termination"] == "interrupted":
|
|
raise KeyboardInterrupt
|
|
statuses = [t["verdict"]["status"] for t in receipt["trials"]]
|
|
receipt["status"] = ("INFRA_ERROR" if "INFRA_ERROR" in statuses else
|
|
"UNVERIFIED" if "UNVERIFIED" in statuses else
|
|
"PASS" if all(t["verdict"]["status"] == t["expected"]
|
|
for t in receipt["trials"]) else "FAIL")
|
|
receipt["task_verdict"] = receipt["status"]
|
|
return attempt
|
|
|
|
|
|
def _calibration(task, image_id, path):
|
|
path = _path(path)
|
|
if path.is_dir():
|
|
path = _path(path / "receipt.json")
|
|
try:
|
|
raw = _read(path.parent, path.name)
|
|
receipt = json.loads(raw)
|
|
except (ValueError, UnicodeDecodeError) as error:
|
|
raise ValueError("invalid calibration receipt") from error
|
|
expected = dict(mode="calibration", status="PASS", task_verdict="PASS", cleanup="confirmed",
|
|
task_digest=task["_digest"], image_id=image_id,
|
|
verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(),
|
|
trusted_local=False)
|
|
if not isinstance(receipt, dict) or any(receipt.get(k) != v for k, v in expected.items()):
|
|
raise ValueError("calibration must pass in the same task, verifier and Docker image")
|
|
if receipt.get("trusted_local") is not False:
|
|
raise ValueError("host calibration cannot authorize Docker execution")
|
|
trials = receipt.get("trials")
|
|
fixtures = task["fixtures"]
|
|
if not isinstance(trials, list) or len(trials) != 5 * len(fixtures):
|
|
raise ValueError("calibration must include five trials per fixture")
|
|
seen = set()
|
|
try:
|
|
for trial in trials:
|
|
fixture, repeat = trial["fixture"], trial["repeat"]
|
|
if fixture not in fixtures or type(repeat) is not int or not 1 <= repeat <= 5:
|
|
raise ValueError("invalid calibration trial identity")
|
|
if (fixture, repeat) in seen:
|
|
raise ValueError("duplicate calibration trial")
|
|
seen.add((fixture, repeat))
|
|
wanted = fixtures[fixture]
|
|
verdict, process = trial["verdict"], trial["process"]
|
|
if (trial["expected"] != wanted or verdict["status"] != wanted or
|
|
type(verdict["checks"]) is not int or verdict["checks"] <= 0 or
|
|
process["cleanup"] != "confirmed" or process["termination"] != "exit" or
|
|
process["returncode"] != (0 if wanted == "PASS" else 1) or
|
|
trial["candidate_digest"] != _digest(_fixture(task, fixture))):
|
|
raise ValueError("calibration trial did not meet its expected result")
|
|
except (KeyError, TypeError) as error:
|
|
raise ValueError("malformed calibration trial") from error
|
|
return dict(path=str(path), receipt_digest=hashlib.sha256(raw).hexdigest())
|
|
|
|
|
|
def _native_trace(attempt):
|
|
"""Capture the one adapter artifact after the Agent container is gone."""
|
|
result = dict(path="trace/native.jsonl", status="UNVERIFIED")
|
|
try:
|
|
if not os.path.lexists(attempt / result["path"]):
|
|
return dict(result, status="MISSING")
|
|
data = _read(attempt, result["path"], MAX_TRACE_BYTES)
|
|
return dict(result, status="AVAILABLE", bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
|
|
except (ValueError, OSError) as error:
|
|
return dict(result, detail=str(error))
|
|
|
|
|
|
def _model_url(value):
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, str):
|
|
raise ValueError("model_base_url must be a URL without credentials")
|
|
parsed = urlsplit(value)
|
|
if (parsed.scheme not in ("http", "https") or not parsed.hostname or
|
|
parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment):
|
|
raise ValueError("model_base_url must be an HTTP(S) URL without credentials, query or fragment")
|
|
parsed.port # Validate the optional port before recording or starting an attempt.
|
|
return value
|
|
|
|
|
|
def run(task_path, output_root, image, agent_argv, agent_identity, calibration,
|
|
agent_timeout=900, model_base_url=None, agent_network=None):
|
|
"""Run a configured Agent in Docker, freeze its output, then grade independently."""
|
|
task = load_task(task_path)
|
|
if not isinstance(agent_identity, str) or not agent_identity.strip():
|
|
raise ValueError("agent_identity is required")
|
|
if not isinstance(agent_argv, (list, tuple)) or not agent_argv or any(
|
|
not isinstance(arg, str) or "\0" in arg for arg in agent_argv) or not agent_argv[0]:
|
|
raise ValueError("agent_argv must be a nonempty argv list")
|
|
if type(agent_timeout) is not int or not 1 <= agent_timeout <= 86400:
|
|
raise ValueError("agent_timeout must be an integer in 1..86400 seconds")
|
|
model_base_url = _model_url(model_base_url)
|
|
if agent_network is not None and (not isinstance(agent_network, str) or not re.fullmatch(
|
|
r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", agent_network)):
|
|
raise ValueError("agent_network must be a Docker network name")
|
|
image_id = _image(image)
|
|
if agent_network is not None:
|
|
# Only the in-network model proxy may be reachable; an open bridge reaches the internet.
|
|
inspected = _cli(["docker", "network", "inspect", agent_network, "--format", "{{.Internal}}"])
|
|
if inspected.returncode or inspected.stdout.strip() != "true":
|
|
raise ValueError("agent_network must be an existing internal Docker network")
|
|
calibration_identity = _calibration(task, image_id, calibration)
|
|
attempt = _attempt(task, output_root, "run")
|
|
receipt = _receipt(task, "run", image_id)
|
|
receipt.update(agent_identity=agent_identity, agent_argv=list(agent_argv), calibration=calibration_identity,
|
|
agent_exit=None, termination="not_started", agent_timeout_seconds=agent_timeout,
|
|
agent_memory="2g", model_base_url=model_base_url, agent_network=agent_network,
|
|
native_trace=dict(path="trace/native.jsonl", status="MISSING"))
|
|
workspace = attempt / "workspace"
|
|
_write_tree(workspace, _template(task), 0o666, 0o777)
|
|
trace = attempt / "trace"
|
|
trace.mkdir(mode=0o777)
|
|
trace.chmod(0o777)
|
|
prompt = attempt / "prompt.md"
|
|
prompt.write_bytes(task["_files"]["prompt.md"])
|
|
prompt.chmod(0o644)
|
|
with _execution(attempt, receipt):
|
|
receipt["cleanup"] = "unknown"
|
|
agent = _docker(attempt, image_id, list(agent_argv),
|
|
[(workspace, "/workspace", True), (prompt, "/task/prompt.md", False),
|
|
(trace, "/trace", True)],
|
|
"agent", agent_timeout, task["_files"]["prompt.md"], network=agent_network or "none",
|
|
model_base_url=model_base_url)
|
|
receipt.update(agent_exit=agent["returncode"], termination=agent["termination"],
|
|
cleanup=agent["cleanup"], agent_process=agent)
|
|
if agent["cleanup"] == "confirmed":
|
|
receipt["native_trace"] = _native_trace(attempt)
|
|
if agent["termination"] == "interrupted":
|
|
raise KeyboardInterrupt
|
|
if agent["cleanup"] != "confirmed":
|
|
raise RuntimeError("agent cleanup was not confirmed; candidate cannot be frozen")
|
|
files = {name: _read(workspace, name) for name in task["candidate_files"]}
|
|
candidate = attempt / "candidate"
|
|
_write_tree(candidate, files, 0o644, 0o755)
|
|
receipt["candidate_digest"] = _digest(files)
|
|
_event(attempt, "candidate_frozen", candidate_digest=receipt["candidate_digest"])
|
|
receipt["cleanup"] = "unknown"
|
|
grade = _grade(task, attempt, candidate, "grade", image_id)
|
|
receipt["cleanup"] = grade["process"]["cleanup"]
|
|
receipt.update(status=grade["verdict"]["status"], task_verdict=grade["verdict"]["status"], grade=grade)
|
|
if grade["process"]["cleanup"] != "confirmed":
|
|
receipt["cleanup"] = "unknown"
|
|
if grade["process"]["termination"] == "interrupted":
|
|
raise KeyboardInterrupt
|
|
return attempt
|