"""Small local supervisor. Agent execution always requires Docker.""" from contextlib import contextmanager import hashlib import json import os from pathlib import Path import re import shutil import signal import stat import subprocess import sys import time import uuid from urllib.parse import urlsplit MAX_FILE_BYTES = 1024 * 1024 MAX_CHECKER_BYTES = 64 * 1024 MAX_TRACE_BYTES = 64 * 1024 * 1024 # Must match agent/pi.mjs. FIXTURES = {"gold": "PASS", "template": "FAIL", "noop": "FAIL", "cheat": "FAIL"} # shortcut: fixed floor for 36's shared /home (user lowered it from 50 GiB on 2026-10-17); make it a flag when another host needs a different one. MIN_FREE_BYTES = 30 * 1024 ** 3 MAX_ATTEMPT_BYTES = 2 * 1024 ** 3 def _path(path): path = Path(os.path.abspath(path)) for part in (path, *path.parents): if part.is_symlink(): raise ValueError("symlink is not allowed: " + str(part)) return path def _relative(value): if not isinstance(value, str) or not value or "\\" in value: raise ValueError("expected a nonempty relative file path") path = Path(value) if path.is_absolute() or any(p in (".", "..", "") for p in value.split("/")): raise ValueError("unsafe relative path: " + value) return path def _read(root, relative, limit=MAX_FILE_BYTES): path = _path(root / _relative(relative)) if root not in path.parents: raise ValueError("unsafe file path: " + str(path)) try: descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) except OSError as error: raise ValueError("missing or unsafe file: " + str(path)) from error with os.fdopen(descriptor, "rb") as stream: if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): raise ValueError("missing or unsafe file: " + str(path)) data = stream.read(limit + 1) if len(data) > limit: raise ValueError("file exceeds its size limit: " + str(path)) return data def _create(path, data): """Write a new file exclusively; remove only our own inode if writing fails.""" created = None try: descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) created = os.fstat(descriptor) with os.fdopen(descriptor, "wb") as stream: stream.write(data) stream.flush() os.fsync(stream.fileno()) except BaseException: if created is not None: try: current = path.lstat() except FileNotFoundError: current = None if current and (current.st_dev, current.st_ino) == (created.st_dev, created.st_ino): path.unlink() raise return path def _write_tree(root, files, file_mode, dir_mode): root.mkdir(exist_ok=True) root.chmod(dir_mode) for name, data in files.items(): path = root for part in _relative(name).parts[:-1]: path = path / part path.mkdir(exist_ok=True) path.chmod(dir_mode) path = root / name path.write_bytes(data) path.chmod(file_mode) def _digest(files): return hashlib.sha256(json.dumps( {name: hashlib.sha256(data).hexdigest() for name, data in files.items()}, sort_keys=True, separators=(",", ":"), ).encode()).hexdigest() def load_task(task_path): """Validate and capture the fixed task package; private keys hold its snapshot.""" root = _path(task_path) if not root.is_dir(): raise ValueError("task directory does not exist") for entry in root.rglob("*"): if entry.is_symlink(): raise ValueError("task contains a symlink: " + str(entry)) raw = _read(root, "task.json") try: task = json.loads(raw) except (ValueError, UnicodeDecodeError) as error: raise ValueError("invalid task.json") from error if not isinstance(task, dict): raise ValueError("task.json must be an object") for key in ("id", "version", "family", "source"): if not isinstance(task.get(key), str) or not task[key].strip(): raise ValueError("missing or invalid field: " + key) if task.get("template") != "template" or task.get("verifier") != "verify.py": raise ValueError("template/verifier must be template and verify.py") candidates = task.get("candidate_files") if (not isinstance(candidates, list) or not 1 <= len(candidates) <= 16 or len(set(map(str, candidates))) != len(candidates)): raise ValueError("candidate_files must list 1..16 unique relative files") for name in candidates: _relative(name) fixtures = task.setdefault("fixtures", dict(FIXTURES)) if (not isinstance(fixtures, dict) or len(fixtures) > 12 or any(fixtures.get(name) != wanted for name, wanted in FIXTURES.items()) or any(not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", name) or wanted not in ("PASS", "FAIL") for name, wanted in fixtures.items())): raise ValueError("fixtures must keep gold/template/noop/cheat and map at most 12 names to PASS/FAIL") graders = task.setdefault("grader_files", []) if not isinstance(graders, list) or len(graders) > 16 or len(set(map(str, graders))) != len(graders): raise ValueError("grader_files must list at most 16 unique files") for name in graders: if (_relative(name).parts[0] in set(fixtures) | {"template"} or name in ("task.json", "prompt.md", "verify.py", "dataset.json")): raise ValueError("grader file collides with task material: " + name) timeout = task.get("timeout_seconds") if type(timeout) is not int or not 1 <= timeout <= 300: raise ValueError("timeout_seconds must be an integer in 1..300") template = sorted(entry.relative_to(root / "template").as_posix() for entry in (root / "template").rglob("*") if not entry.is_dir()) if not 1 <= len(template) <= 64 or any(name not in template for name in candidates): raise ValueError("template must hold 1..64 files including every candidate file") names = ["task.json", "prompt.md", "verify.py", *graders, *("template/" + name for name in template)] names += [fixture + "/" + name for fixture in fixtures if fixture != "template" for name in candidates] files = {name: _read(root, name) for name in names} if not files["prompt.md"].strip() or not files["verify.py"].strip(): raise ValueError("prompt and verifier must not be empty") try: files["prompt.md"].decode("utf-8") except UnicodeDecodeError as error: raise ValueError("prompt must be UTF-8") from error task.update(_root=root, _files=files, _digest=_digest(files)) return task def _fixture(task, fixture): return {name: task["_files"][fixture + "/" + name] for name in task["candidate_files"]} def _template(task): return {name[len("template/"):]: data for name, data in task["_files"].items() if name.startswith("template/")} def _event(attempt, kind, **fields): with (attempt / "events.jsonl").open("a", encoding="utf-8") as stream: stream.write(json.dumps(dict(time=time.time(), event=kind, **fields), sort_keys=True) + "\n") stream.flush() def _attempt(task, output_root, mode): root = _path(output_root) if root == task["_root"] or task["_root"] in root.parents: raise ValueError("output must be outside the task directory") root.mkdir(parents=True, exist_ok=True) attempt = root / (mode + "-" + uuid.uuid4().hex) attempt.mkdir(mode=0o700) _write_tree(attempt / "grader", {name: task["_files"][name] for name in ("verify.py", *task["grader_files"])}, 0o600, 0o700) _event(attempt, "created", mode=mode, task_id=task["id"], task_digest=task["_digest"]) return attempt def _receipt(task, mode, image_id=None): return dict(mode=mode, task_id=task["id"], task_version=task["version"], task_digest=task["_digest"], manifest_digest=hashlib.sha256(task["_files"]["task.json"]).hexdigest(), verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(), image_id=image_id, status="UNVERIFIED", task_verdict="UNVERIFIED", cleanup="confirmed", supervisor_only=True, training_disposition="NOT_TRAIN_READY") def _finish(attempt, receipt): _event(attempt, "finished", status=receipt["status"], cleanup=receipt["cleanup"]) temporary = attempt / "receipt.tmp" with temporary.open("x", encoding="utf-8") as stream: json.dump(receipt, stream, indent=2, sort_keys=True) stream.write("\n") stream.flush() os.fsync(stream.fileno()) temporary.replace(attempt / "receipt.json") return attempt @contextmanager def _sigterm(): """Deliver SIGTERM as KeyboardInterrupt so cleanup still runs.""" previous = signal.getsignal(signal.SIGTERM) installed = False def interrupt(signum, frame): raise KeyboardInterrupt try: signal.signal(signal.SIGTERM, interrupt) installed = True except ValueError: # Only the main thread can own signal handlers. pass try: yield finally: if installed: signal.signal(signal.SIGTERM, previous) @contextmanager def _execution(attempt, receipt): """Turn cancellation into a durable terminal receipt after resource cleanup.""" try: with _sigterm(): yield except KeyboardInterrupt: receipt.update(status="UNVERIFIED", task_verdict="UNVERIFIED", termination="interrupted") except ValueError as error: receipt.update(status="UNVERIFIED", task_verdict="UNVERIFIED", error=str(error)) except Exception as error: receipt.update(status="INFRA_ERROR", task_verdict="INFRA_ERROR", error=str(error)) finally: _finish(attempt, receipt) def _disk_problem(root, max_bytes=None): """Name the disk limit an attempt has hit, if any.""" if shutil.disk_usage(root).free < MIN_FREE_BYTES: return "disk_low" if max_bytes is None: return None used = 0 # ponytail: directories the Agent made unreadable are skipped; the free-space floor still bounds them. for directory, _, names in os.walk(root): for name in names: try: used += os.lstat(os.path.join(directory, name)).st_blocks * 512 except OSError: pass return "attempt_size" if used > max_bytes else None def _process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False, watch=None): facts = dict(returncode=None, termination="launch_error", cleanup="confirmed") # Host execution is only for explicitly trusted calibration fixtures. env = dict(os.environ) if controller else {"PATH": os.environ.get("PATH", os.defpath), "LANG": "C.UTF-8"} with stdout.open("wb") as out, stderr.open("wb") as err: try: proc = subprocess.Popen(argv, cwd=cwd, stdin=subprocess.PIPE if stdin else subprocess.DEVNULL, stdout=out, stderr=err, env=env, start_new_session=True) except OSError as error: facts["detail"] = str(error) return facts try: deadline = time.monotonic() + timeout while True: try: proc.communicate(stdin, timeout=max(0, min(2, deadline - time.monotonic()))) facts["termination"] = "exit" break except subprocess.TimeoutExpired: stdin = None # Input was already sent; communicate() refuses it twice. problem = _disk_problem(watch, MAX_ATTEMPT_BYTES) if watch else None if problem or time.monotonic() >= deadline: facts["termination"] = problem or "timeout" break except KeyboardInterrupt: facts["termination"] = "interrupted" finally: try: os.killpg(proc.pid, signal.SIGKILL) except ProcessLookupError: pass except OSError as error: facts["cleanup"] = "unknown" facts["cleanup_detail"] = str(error) proc.wait() try: os.killpg(proc.pid, 0) facts["cleanup"] = "unknown" facts["cleanup_detail"] = "process group still exists after termination" except ProcessLookupError: pass except OSError as error: facts["cleanup"] = "unknown" facts["cleanup_detail"] = str(error) facts["returncode"] = proc.returncode return facts def _cli(argv): try: return subprocess.run(argv, capture_output=True, text=True, timeout=10) except (OSError, subprocess.TimeoutExpired) as error: raise RuntimeError("Docker unavailable: " + str(error)) from error def doctor(): """Only inspect local dependencies; never install or start services.""" result = dict(status="UNAVAILABLE", python=sys.version.split()[0], docker={}) if not shutil.which("docker"): result["docker"] = dict(status="UNAVAILABLE", reason="docker command not found") return result try: check = _cli(["docker", "info", "--format", "{{.ServerVersion}}"]) if check.returncode: raise RuntimeError(check.stderr.strip() or "daemon not ready") result.update(status="AVAILABLE", docker=dict(status="AVAILABLE", version=check.stdout.strip())) except RuntimeError as error: result["docker"] = dict(status="UNAVAILABLE", reason=str(error)) return result def _image(image): if not isinstance(image, str) or not image.strip() or image.startswith("-"): raise ValueError("a Docker image reference is required") result = _cli(["docker", "image", "inspect", image, "--format", "{{.Id}}"]) identity = result.stdout.strip() if result.returncode or not re.fullmatch(r"sha256:[a-f0-9]{64}", identity): raise RuntimeError("image is unavailable locally: " + image) return identity def _docker(attempt, image_id, argv, mounts, prefix, timeout, stdin=None, network=False, model_base_url=None): name = "rollout-" + uuid.uuid4().hex memory = "2g" if prefix == "agent" else "512m" command = ["docker", "run", "--name", name, "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "128", "--memory", memory, "--cpus", "1", "--tmpfs", "/tmp:rw,exec,nosuid,nodev,size=128m", "--entrypoint", ""] if stdin is not None: command.append("-i") if not network: command += ["--network", "none"] command += ["--user", "0:0", "--cap-add", "SETUID", "--cap-add", "SETGID", "--cap-add", "CHOWN", "--cap-add", "DAC_OVERRIDE", "--env", "ROLLOUT_GRADE_ISOLATION=uid65534"] elif prefix == "agent": command += ["--network", network] if model_base_url: command += ["--env", "ROLLOUT_MODEL_BASE_URL=" + model_base_url] for source, target, writable in mounts: # --mount rejects ambiguous comma-delimited host paths. if "," in str(source): raise ValueError("Docker bind paths must not contain commas") command += ["--mount", "type=bind,src=" + str(source) + ",dst=" + target + ("" if writable else ",readonly")] command += ["-w", "/workspace" if prefix == "agent" else "/tmp", image_id, *argv] if _disk_problem(attempt): facts = dict(returncode=None, termination="disk_low", cleanup="confirmed") _event(attempt, "container_skipped", name=name, role=prefix, **facts) return facts _event(attempt, "container_requested", name=name, role=prefix, image_id=image_id) facts = dict(returncode=None, termination="launch_error", cleanup="unknown") try: facts = _process(command, attempt, timeout, attempt / (prefix + ".stdout"), attempt / (prefix + ".stderr"), stdin, controller=True, watch=attempt if prefix == "agent" else None) except KeyboardInterrupt: facts["termination"] = "interrupted" finally: try: removal = _cli(["docker", "rm", "-f", name]) check = _cli(["docker", "container", "inspect", name]) absent = check.returncode != 0 and any( phrase in check.stderr.lower() for phrase in ("no such object", "no such container")) # Killing the CLI does not cancel a daemon create request still in flight. finished = (facts["termination"] == "exit" and type(facts["returncode"]) is int and facts["returncode"] >= 0) confirmed = absent and finished facts["container_cleanup"] = "confirmed" if confirmed else "unknown" facts["cleanup"] = "confirmed" if confirmed and facts["cleanup"] == "confirmed" else "unknown" if not absent: facts["cleanup_detail"] = removal.stderr.strip() or check.stderr.strip() elif not finished: facts["cleanup_detail"] = "Docker CLI did not finish; daemon container creation may still be pending" except RuntimeError as error: facts["cleanup"] = "unknown" facts["cleanup_detail"] = str(error) except KeyboardInterrupt: facts.update(cleanup="unknown", termination="interrupted", cleanup_detail="cleanup interrupted") _event(attempt, "container_cleanup", name=name, **facts) return facts def _checker(facts, stdout): if facts["cleanup"] != "confirmed": return dict(status="INFRA_ERROR", checks=0, detail="cleanup was not confirmed") if facts["termination"] == "interrupted": return dict(status="UNVERIFIED", checks=0, detail="verification interrupted") if facts["termination"] != "exit" or facts["returncode"] in (125, 126, 127): return dict(status="INFRA_ERROR", checks=0, detail=facts["termination"]) try: if stdout.stat().st_size > MAX_CHECKER_BYTES: raise ValueError("grader output exceeds 64 KiB") lines = stdout.read_text(encoding="utf-8").splitlines() if len(lines) != 1: raise ValueError("grader must emit exactly one JSON line") verdict = json.loads(lines[0]) status, checks = verdict["status"], verdict["checks"] if type(checks) is not int or checks < 0: raise ValueError("invalid checks count") expected = {"PASS": 0, "FAIL": 1, "INFRA_ERROR": 2, "UNVERIFIED": 2} if status not in expected or expected[status] != facts["returncode"]: raise ValueError("grader exit/status conflict") if status in ("PASS", "FAIL") and checks == 0: raise ValueError("no behavior checks ran") return verdict except (ValueError, KeyError, TypeError, UnicodeDecodeError) as error: return dict(status="UNVERIFIED", checks=0, detail=str(error)) def _grade(task, attempt, candidate, prefix, image_id): argv = ["python3", "-I", "/grader/verify.py", "/candidate"] if image_id: facts = _docker(attempt, image_id, argv, [(candidate, "/candidate", False), (attempt / "grader", "/grader", False)], prefix, task["timeout_seconds"]) else: facts = _process([sys.executable, str(attempt / "grader/verify.py"), str(candidate)], attempt, task["timeout_seconds"], attempt / (prefix + ".stdout"), attempt / (prefix + ".stderr")) verdict = _checker(facts, attempt / (prefix + ".stdout")) _event(attempt, "graded", role=prefix, verdict=verdict, process=facts) return dict(verdict=verdict, process=facts) def calibrate(task_path, output_root, trusted_local=False, image=None): """Check every declared fixture, each in five fresh candidates.""" if type(trusted_local) is not bool or bool(trusted_local) == bool(image): raise ValueError("choose exactly one of trusted_local=True or image") task = load_task(task_path) image_id = _image(image) if image else None attempt = _attempt(task, output_root, "calibration") receipt = _receipt(task, "calibration", image_id) receipt.update(trusted_local=bool(trusted_local), trials=[]) with _execution(attempt, receipt): for fixture, expected in task["fixtures"].items(): files = _fixture(task, fixture) for index in range(5): prefix = fixture + "-" + str(index + 1) candidate = attempt / prefix _write_tree(candidate, files, 0o644, 0o755) receipt["cleanup"] = "unknown" trial = _grade(task, attempt, candidate, prefix, image_id) trial.update(fixture=fixture, repeat=index + 1, expected=expected, candidate_digest=_digest(files)) receipt["trials"].append(trial) receipt["cleanup"] = ("confirmed" if all(t["process"]["cleanup"] == "confirmed" for t in receipt["trials"]) else "unknown") if trial["process"]["termination"] == "interrupted": raise KeyboardInterrupt statuses = [t["verdict"]["status"] for t in receipt["trials"]] receipt["status"] = ("INFRA_ERROR" if "INFRA_ERROR" in statuses else "UNVERIFIED" if "UNVERIFIED" in statuses else "PASS" if all(t["verdict"]["status"] == t["expected"] for t in receipt["trials"]) else "FAIL") receipt["task_verdict"] = receipt["status"] return attempt def _calibration(task, image_id, path): path = _path(path) if path.is_dir(): path = _path(path / "receipt.json") try: raw = _read(path.parent, path.name) receipt = json.loads(raw) except (ValueError, UnicodeDecodeError) as error: raise ValueError("invalid calibration receipt") from error expected = dict(mode="calibration", status="PASS", task_verdict="PASS", cleanup="confirmed", task_digest=task["_digest"], image_id=image_id, verifier_digest=hashlib.sha256(task["_files"]["verify.py"]).hexdigest(), trusted_local=False) if not isinstance(receipt, dict) or any(receipt.get(k) != v for k, v in expected.items()): raise ValueError("calibration must pass in the same task, verifier and Docker image") if receipt.get("trusted_local") is not False: raise ValueError("host calibration cannot authorize Docker execution") trials = receipt.get("trials") fixtures = task["fixtures"] if not isinstance(trials, list) or len(trials) != 5 * len(fixtures): raise ValueError("calibration must include five trials per fixture") seen = set() try: for trial in trials: fixture, repeat = trial["fixture"], trial["repeat"] if fixture not in fixtures or type(repeat) is not int or not 1 <= repeat <= 5: raise ValueError("invalid calibration trial identity") if (fixture, repeat) in seen: raise ValueError("duplicate calibration trial") seen.add((fixture, repeat)) wanted = fixtures[fixture] verdict, process = trial["verdict"], trial["process"] if (trial["expected"] != wanted or verdict["status"] != wanted or type(verdict["checks"]) is not int or verdict["checks"] <= 0 or process["cleanup"] != "confirmed" or process["termination"] != "exit" or process["returncode"] != (0 if wanted == "PASS" else 1) or trial["candidate_digest"] != _digest(_fixture(task, fixture))): raise ValueError("calibration trial did not meet its expected result") except (KeyError, TypeError) as error: raise ValueError("malformed calibration trial") from error return dict(path=str(path), receipt_digest=hashlib.sha256(raw).hexdigest()) def _native_trace(attempt): """Capture the one adapter artifact after the Agent container is gone.""" result = dict(path="trace/native.jsonl", status="UNVERIFIED") try: if not os.path.lexists(attempt / result["path"]): return dict(result, status="MISSING") data = _read(attempt, result["path"], MAX_TRACE_BYTES) return dict(result, status="AVAILABLE", bytes=len(data), sha256=hashlib.sha256(data).hexdigest()) except (ValueError, OSError) as error: return dict(result, detail=str(error)) def _model_url(value): if value is None: return None if not isinstance(value, str): raise ValueError("model_base_url must be a URL without credentials") parsed = urlsplit(value) if (parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username is not None or parsed.password is not None or parsed.query or parsed.fragment): raise ValueError("model_base_url must be an HTTP(S) URL without credentials, query or fragment") parsed.port # Validate the optional port before recording or starting an attempt. return value def run(task_path, output_root, image, agent_argv, agent_identity, calibration, agent_timeout=900, model_base_url=None, agent_network=None): """Run a configured Agent in Docker, freeze its output, then grade independently.""" task = load_task(task_path) if not isinstance(agent_identity, str) or not agent_identity.strip(): raise ValueError("agent_identity is required") if not isinstance(agent_argv, (list, tuple)) or not agent_argv or any( not isinstance(arg, str) or "\0" in arg for arg in agent_argv) or not agent_argv[0]: raise ValueError("agent_argv must be a nonempty argv list") if type(agent_timeout) is not int or not 1 <= agent_timeout <= 86400: raise ValueError("agent_timeout must be an integer in 1..86400 seconds") model_base_url = _model_url(model_base_url) if agent_network is not None and (not isinstance(agent_network, str) or not re.fullmatch( r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", agent_network)): raise ValueError("agent_network must be a Docker network name") image_id = _image(image) if agent_network is not None: # Only the in-network model proxy may be reachable; an open bridge reaches the internet. inspected = _cli(["docker", "network", "inspect", agent_network, "--format", "{{.Internal}}"]) if inspected.returncode or inspected.stdout.strip() != "true": raise ValueError("agent_network must be an existing internal Docker network") calibration_identity = _calibration(task, image_id, calibration) attempt = _attempt(task, output_root, "run") receipt = _receipt(task, "run", image_id) receipt.update(agent_identity=agent_identity, agent_argv=list(agent_argv), calibration=calibration_identity, agent_exit=None, termination="not_started", agent_timeout_seconds=agent_timeout, agent_memory="2g", model_base_url=model_base_url, agent_network=agent_network, native_trace=dict(path="trace/native.jsonl", status="MISSING")) workspace = attempt / "workspace" _write_tree(workspace, _template(task), 0o666, 0o777) trace = attempt / "trace" trace.mkdir(mode=0o777) trace.chmod(0o777) prompt = attempt / "prompt.md" prompt.write_bytes(task["_files"]["prompt.md"]) prompt.chmod(0o644) with _execution(attempt, receipt): receipt["cleanup"] = "unknown" agent = _docker(attempt, image_id, list(agent_argv), [(workspace, "/workspace", True), (prompt, "/task/prompt.md", False), (trace, "/trace", True)], "agent", agent_timeout, task["_files"]["prompt.md"], network=agent_network or "none", model_base_url=model_base_url) receipt.update(agent_exit=agent["returncode"], termination=agent["termination"], cleanup=agent["cleanup"], agent_process=agent) if agent["cleanup"] == "confirmed": receipt["native_trace"] = _native_trace(attempt) if agent["termination"] == "interrupted": raise KeyboardInterrupt if agent["cleanup"] != "confirmed": raise RuntimeError("agent cleanup was not confirmed; candidate cannot be frozen") files = {name: _read(workspace, name) for name in task["candidate_files"]} candidate = attempt / "candidate" _write_tree(candidate, files, 0o644, 0o755) receipt["candidate_digest"] = _digest(files) _event(attempt, "candidate_frozen", candidate_digest=receipt["candidate_digest"]) receipt["cleanup"] = "unknown" grade = _grade(task, attempt, candidate, "grade", image_id) receipt["cleanup"] = grade["process"]["cleanup"] receipt.update(status=grade["verdict"]["status"], task_verdict=grade["verdict"]["status"], grade=grade) if grade["process"]["cleanup"] != "confirmed": receipt["cleanup"] = "unknown" if grade["process"]["termination"] == "interrupted": raise KeyboardInterrupt return attempt