- Tasks may declare up to 16 candidate files; the whole template becomes the Agent workspace while only candidate files are frozen and graded. - Optional fixtures add more cheats or legal alternatives (five trials each); optional grader_files ship private tests next to verify.py. - Tasks without an argv oracle export call_type=verifier ground truth. - Upstream model routes live in one MODELS table. - One bounded no-follow reader, one exclusive writer and one SIGTERM helper replace per-module copies; drop unused validate_task. frame-decoder keeps its task digest, calibration receipts and exported query. Verified on host 36: internal network probe, 20-trial calibration and a real GLM rollout (PASS, evidence VERIFIED, cleanup confirmed). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
401 lines
23 KiB
Python
401 lines
23 KiB
Python
import json
|
|
import hashlib
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from rollout import core
|
|
|
|
|
|
class CoreTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temp.cleanup)
|
|
self.root = Path(self.temp.name).resolve()
|
|
self.task = self.root / "task"
|
|
self.task.mkdir()
|
|
self.output = self.root / "out"
|
|
self.manifest = dict(id="parser", version="1", family="protocol", source="synthetic",
|
|
template="template", verifier="verify.py", candidate_files=["decoder.c"],
|
|
timeout_seconds=15)
|
|
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
|
|
self.task.joinpath("prompt.md").write_text("Repair decoder.c")
|
|
self.task.joinpath("verify.py").write_text(
|
|
"import json,sys\nfrom pathlib import Path\n"
|
|
"ok=Path(sys.argv[1], 'decoder.c').read_text() == 'gold'\n"
|
|
"print(json.dumps({'status':'PASS' if ok else 'FAIL','checks':1}))\n"
|
|
"sys.exit(0 if ok else 1)\n")
|
|
for relative, value in (("template", "buggy"), ("gold", "gold"),
|
|
("noop", "noop"), ("cheat", "cheat")):
|
|
directory = self.task / relative
|
|
directory.mkdir(parents=True)
|
|
directory.joinpath("decoder.c").write_text(value)
|
|
|
|
def receipt(self, attempt):
|
|
return json.loads(attempt.joinpath("receipt.json").read_text())
|
|
|
|
def mock_calibration(self):
|
|
def grade(task, attempt, candidate, prefix, image_id):
|
|
ok = candidate.joinpath("decoder.c").read_text() == "gold"
|
|
return dict(verdict=dict(status="PASS" if ok else "FAIL", checks=1),
|
|
process=dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed"))
|
|
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(
|
|
core, "_grade", side_effect=grade):
|
|
return core.calibrate(self.task, self.output, image="tag")
|
|
|
|
def test_task_rejects_unsafe_and_missing_material(self):
|
|
self.assertEqual(core.load_task(self.task)["id"], "parser")
|
|
for bad in (None, [], ["../decoder.c"], ["/tmp/decoder.c"]):
|
|
self.manifest["candidate_files"] = bad
|
|
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
|
|
with self.assertRaises(ValueError):
|
|
core.load_task(self.task)
|
|
self.manifest["candidate_files"] = ["decoder.c"]
|
|
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
|
|
fixture = self.task / "gold/decoder.c"
|
|
fixture.unlink()
|
|
fixture.symlink_to(self.task / "template/decoder.c")
|
|
with self.assertRaises(ValueError):
|
|
core.load_task(self.task)
|
|
fixture.unlink()
|
|
with self.assertRaises(ValueError):
|
|
core.load_task(self.task)
|
|
|
|
def test_size_and_output_boundaries(self):
|
|
self.task.joinpath("template/decoder.c").write_bytes(b"x" * (core.MAX_FILE_BYTES + 1))
|
|
with self.assertRaises(ValueError):
|
|
core.load_task(self.task)
|
|
self.task.joinpath("template/decoder.c").write_text("buggy")
|
|
with self.assertRaises(ValueError):
|
|
core.calibrate(self.task, self.task / "out", trusted_local=True)
|
|
with self.assertRaises(ValueError):
|
|
core.calibrate(self.task, self.output)
|
|
with self.assertRaises(ValueError):
|
|
core.calibrate(self.task, self.output, trusted_local=True, image="image")
|
|
|
|
def test_trusted_calibration_is_five_fresh_trials_per_fixture(self):
|
|
attempt = core.calibrate(self.task, self.output, trusted_local=True)
|
|
result = self.receipt(attempt)
|
|
self.assertEqual(result["status"], "PASS")
|
|
self.assertEqual(len(result["trials"]), 20)
|
|
self.assertEqual(result["cleanup"], "confirmed")
|
|
self.assertEqual(result["training_disposition"], "NOT_TRAIN_READY")
|
|
second = core.calibrate(self.task, self.output, trusted_local=True)
|
|
self.assertNotEqual(attempt, second)
|
|
self.assertTrue(attempt.joinpath("events.jsonl").exists())
|
|
|
|
def test_grader_cannot_pass_empty_conflicting_or_malformed_output(self):
|
|
stdout = self.root / "stdout"
|
|
facts = dict(returncode=0, termination="exit", cleanup="confirmed")
|
|
for output in ('{"status":"PASS","checks":0}', '{"status":"FAIL","checks":1}',
|
|
'noise\n{"status":"PASS","checks":1}', '[]', 'not json'):
|
|
stdout.write_text(output)
|
|
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
|
|
stdout.write_text('{"status":"PASS","checks":1}')
|
|
self.assertEqual(core._checker(facts, stdout)["status"], "PASS")
|
|
facts["cleanup"] = "unknown"
|
|
self.assertEqual(core._checker(facts, stdout)["status"], "INFRA_ERROR")
|
|
facts["cleanup"] = "confirmed"
|
|
facts["returncode"] = 2
|
|
stdout.write_text('{"status":"UNVERIFIED","checks":0}')
|
|
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
|
|
stdout.write_text("x" * (core.MAX_CHECKER_BYTES + 1))
|
|
self.assertEqual(core._checker(facts, stdout)["status"], "UNVERIFIED")
|
|
|
|
def test_interrupt_stops_calibration_and_preserves_terminal_receipt(self):
|
|
with patch.object(core, "_grade", side_effect=KeyboardInterrupt) as grade:
|
|
attempt = core.calibrate(self.task, self.output, trusted_local=True)
|
|
result = self.receipt(attempt)
|
|
self.assertEqual(result["status"], "UNVERIFIED")
|
|
self.assertEqual(result["termination"], "interrupted")
|
|
self.assertEqual(grade.call_count, 1)
|
|
self.assertEqual(result["cleanup"], "unknown")
|
|
|
|
def test_sigterm_is_recorded_after_active_grader_cleanup(self):
|
|
script = ("import os,signal,time\n"
|
|
"os.kill(os.getppid(),signal.SIGTERM)\ntime.sleep(10)\n")
|
|
self.task.joinpath("verify.py").write_text(script)
|
|
attempt = core.calibrate(self.task, self.output, trusted_local=True)
|
|
result = self.receipt(attempt)
|
|
self.assertEqual(result["status"], "UNVERIFIED")
|
|
self.assertEqual(result["termination"], "interrupted")
|
|
self.assertEqual(result["cleanup"], "confirmed")
|
|
self.assertEqual(len(result["trials"]), 1)
|
|
|
|
def test_timeout_is_not_a_pass_and_has_process_cleanup(self):
|
|
self.task.joinpath("verify.py").write_text("import time\ntime.sleep(10)\n")
|
|
self.manifest["timeout_seconds"] = 1
|
|
self.task.joinpath("task.json").write_text(json.dumps(self.manifest))
|
|
task = core.load_task(self.task)
|
|
attempt = core._attempt(task, self.output, "test")
|
|
candidate = attempt / "candidate"
|
|
candidate.mkdir()
|
|
candidate.joinpath("decoder.c").write_text("gold")
|
|
result = core._grade(task, attempt, candidate, "grade", None)
|
|
self.assertEqual(result["verdict"]["status"], "INFRA_ERROR")
|
|
self.assertEqual(result["process"]["termination"], "timeout")
|
|
self.assertEqual(result["process"]["cleanup"], "confirmed")
|
|
|
|
def test_docker_run_freezes_and_grades_independent_mounts(self):
|
|
commands = []
|
|
deadlines = []
|
|
image_id = "sha256:" + "a" * 64
|
|
calibration = self.mock_calibration()
|
|
|
|
def cli(argv):
|
|
commands.append(argv)
|
|
if argv[1:3] == ["image", "inspect"]:
|
|
return subprocess.CompletedProcess(argv, 0, image_id + "\n", "")
|
|
if argv[1:3] == ["network", "inspect"]:
|
|
return subprocess.CompletedProcess(argv, 0, "true\n", "")
|
|
if argv[1:3] == ["container", "inspect"]:
|
|
return subprocess.CompletedProcess(argv, 1, "", "No such container")
|
|
return subprocess.CompletedProcess(argv, 0, "", "")
|
|
|
|
def process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False):
|
|
commands.append(argv)
|
|
deadlines.append(timeout)
|
|
stdout.write_text("")
|
|
stderr.write_text("")
|
|
if stdout.name == "agent.stdout":
|
|
cwd.joinpath("workspace/decoder.c").write_text("gold")
|
|
cwd.joinpath("trace/native.jsonl").write_text('{"event":"test"}\n')
|
|
return dict(returncode=1, termination="exit", cleanup="confirmed")
|
|
stdout.write_text('{"status":"PASS","checks":1}\n')
|
|
return dict(returncode=0, termination="exit", cleanup="confirmed")
|
|
|
|
with patch.object(core, "_cli", side_effect=cli), patch.object(core, "_process", side_effect=process):
|
|
attempt = core.run(self.task, self.output, "tag", ["agent", "--arg"], "fake-agent@1", calibration,
|
|
agent_timeout=120, model_base_url="http://172.22.0.1:8123/v1",
|
|
agent_network="rollout-test-network")
|
|
result = self.receipt(attempt)
|
|
self.assertEqual(result["status"], "PASS")
|
|
self.assertEqual(result["agent_exit"], 1)
|
|
self.assertEqual(result["image_id"], image_id)
|
|
self.assertEqual(result["agent_timeout_seconds"], 120)
|
|
self.assertEqual(deadlines, [120, 15])
|
|
self.assertEqual(result["native_trace"]["status"], "AVAILABLE")
|
|
self.assertEqual(result["native_trace"]["sha256"], hashlib.sha256(b'{"event":"test"}\n').hexdigest())
|
|
self.assertEqual(result["training_disposition"], "NOT_TRAIN_READY")
|
|
self.assertEqual(attempt.joinpath("candidate/decoder.c").read_text(), "gold")
|
|
self.assertEqual(sum(c[1:3] == ["image", "inspect"] for c in commands), 1)
|
|
runs = [c for c in commands if c[1] == "run"]
|
|
for command in runs:
|
|
self.assertIn("exec", command[command.index("--tmpfs") + 1].split(":", 1)[1].split(","))
|
|
self.assertNotIn("/grader", " ".join(runs[0]))
|
|
self.assertNotIn("/gold", " ".join(runs[0]))
|
|
self.assertNotIn("docker.sock", " ".join(runs[0]))
|
|
self.assertIn("dst=/trace", " ".join(runs[0]))
|
|
self.assertNotIn("dst=/trace", " ".join(runs[1]))
|
|
self.assertEqual(runs[0][runs[0].index("--memory") + 1], "2g")
|
|
self.assertEqual(runs[1][runs[1].index("--memory") + 1], "512m")
|
|
self.assertIn("rollout-test-network", runs[0])
|
|
self.assertNotIn("--add-host", runs[0])
|
|
self.assertIn("ROLLOUT_MODEL_BASE_URL=http://172.22.0.1:8123/v1", runs[0])
|
|
self.assertNotIn("ROLLOUT_MODEL_BASE_URL", " ".join(runs[1]))
|
|
self.assertIn("none", runs[1])
|
|
self.assertIn("ROLLOUT_GRADE_ISOLATION=uid65534", runs[1])
|
|
self.assertIn("SETUID", runs[1])
|
|
self.assertIn("SETGID", runs[1])
|
|
self.assertIn("CHOWN", runs[1])
|
|
self.assertEqual(attempt.stat().st_mode & 0o777, 0o700)
|
|
self.assertEqual(attempt.joinpath("workspace").stat().st_mode & 0o777, 0o777)
|
|
self.assertEqual(attempt.joinpath("workspace/decoder.c").stat().st_mode & 0o777, 0o666)
|
|
self.assertEqual(attempt.joinpath("grader").stat().st_mode & 0o777, 0o700)
|
|
self.assertEqual(attempt.joinpath("grader/verify.py").stat().st_mode & 0o777, 0o600)
|
|
self.assertEqual(runs[0][-2:], ["agent", "--arg"])
|
|
|
|
def test_early_absence_cannot_confirm_an_unfinished_docker_run(self):
|
|
for termination, code, expected in (("timeout", -9, "unknown"),
|
|
("interrupted", -9, "unknown"),
|
|
("exit", -9, "unknown"),
|
|
("exit", 0, "confirmed"),
|
|
("exit", 1, "confirmed")):
|
|
with self.subTest(termination=termination, returncode=code):
|
|
task = core.load_task(self.task)
|
|
attempt = core._attempt(task, self.output, "test")
|
|
missing = subprocess.CompletedProcess([], 1, "", "Error: No such container")
|
|
with patch.object(core, "_process", return_value=dict(
|
|
returncode=code, termination=termination, cleanup="confirmed")), \
|
|
patch.object(core, "_cli", return_value=missing) as cli:
|
|
facts = core._docker(attempt, "sha256:" + "a" * 64, ["true"], [], "grade", 15)
|
|
self.assertEqual(facts["cleanup"], expected)
|
|
self.assertEqual(facts["container_cleanup"], expected)
|
|
self.assertEqual(cli.call_args_list[0].args[0][1:3], ["rm", "-f"])
|
|
self.assertEqual(cli.call_args_list[1].args[0][1:3], ["container", "inspect"])
|
|
recorded = json.loads(attempt.joinpath("events.jsonl").read_text().splitlines()[-1])
|
|
self.assertEqual(recorded["cleanup"], expected)
|
|
if expected == "unknown":
|
|
self.assertIn("may still be pending", facts["cleanup_detail"])
|
|
|
|
def test_agent_configuration_is_validated_before_docker(self):
|
|
with patch.object(core, "_image") as image:
|
|
for timeout in (0, True, 86401, "900"):
|
|
with self.assertRaises(ValueError):
|
|
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", agent_timeout=timeout)
|
|
for url in ("file:///tmp/key", "http://user:secret@localhost/v1", "http://localhost?token=secret",
|
|
"http://localhost#secret", "http://localhost:bad", "http:///v1", 12):
|
|
with self.assertRaises(ValueError):
|
|
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", model_base_url=url)
|
|
for network in ("", "--host", "a,b", 12):
|
|
with self.assertRaises(ValueError):
|
|
core.run(self.task, self.output, "tag", ["agent"], "fake@1", "unused", agent_network=network)
|
|
image.assert_not_called()
|
|
|
|
def test_agent_gets_no_network_or_only_an_internal_one(self):
|
|
calibration = self.mock_calibration()
|
|
|
|
def cli(internal):
|
|
def call(argv):
|
|
if argv[1:3] == ["image", "inspect"]:
|
|
return subprocess.CompletedProcess(argv, 0, "sha256:" + "a" * 64 + "\n", "")
|
|
if argv[1:3] == ["network", "inspect"]:
|
|
return subprocess.CompletedProcess(argv, 0, internal, "")
|
|
return subprocess.CompletedProcess(argv, 1, "", "No such container")
|
|
return call
|
|
|
|
for internal in ("false\n", ""):
|
|
with patch.object(core, "_cli", side_effect=cli(internal)), \
|
|
patch.object(core, "_process") as process:
|
|
with self.assertRaises(ValueError):
|
|
core.run(self.task, self.output, "tag", ["agent"], "fake@1", calibration,
|
|
agent_network="open-bridge")
|
|
process.assert_not_called()
|
|
launch = dict(returncode=None, termination="launch_error", cleanup="confirmed")
|
|
with patch.object(core, "_cli", side_effect=cli("")), \
|
|
patch.object(core, "_process", return_value=launch) as process:
|
|
core.run(self.task, self.output, "tag", ["agent"], "fake@1", calibration)
|
|
agent = process.call_args_list[0].args[0]
|
|
self.assertEqual(agent[agent.index("--network") + 1], "none")
|
|
|
|
def test_native_trace_rejects_missing_symlink_and_oversized_file(self):
|
|
attempt = self.root / "attempt"
|
|
trace = attempt / "trace"
|
|
trace.mkdir(parents=True)
|
|
path = trace / "native.jsonl"
|
|
self.assertEqual(core._native_trace(attempt)["status"], "MISSING")
|
|
path.symlink_to(self.task / "gold/decoder.c")
|
|
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
|
|
path.unlink()
|
|
path.mkdir()
|
|
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
|
|
path.rmdir()
|
|
with path.open("wb") as stream:
|
|
stream.truncate(core.MAX_TRACE_BYTES + 1)
|
|
self.assertEqual(core._native_trace(attempt)["status"], "UNVERIFIED")
|
|
|
|
def test_identity_docker_dependency_and_cleanup_failure(self):
|
|
with self.assertRaises(ValueError):
|
|
core.run(self.task, self.output, "image", ["agent"], "", "unused")
|
|
with patch.object(core.shutil, "which", return_value=None):
|
|
self.assertEqual(core.doctor()["status"], "UNAVAILABLE")
|
|
calibration = self.mock_calibration()
|
|
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(
|
|
core, "_docker", return_value=dict(returncode=0, termination="exit", cleanup="unknown")):
|
|
attempt = core.run(self.task, self.output, "image", ["agent"], "fake@1", calibration)
|
|
result = self.receipt(attempt)
|
|
self.assertEqual(result["status"], "INFRA_ERROR")
|
|
self.assertEqual(result["cleanup"], "unknown")
|
|
self.assertFalse(attempt.joinpath("candidate").exists())
|
|
|
|
def multi_file_task(self):
|
|
root = self.root / "multi"
|
|
manifest = dict(id="multi", version="1", family="driver", source="synthetic", template="template",
|
|
verifier="verify.py", candidate_files=["src/a.c", "inc/a.h"], timeout_seconds=15,
|
|
fixtures=dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", alternative="PASS"),
|
|
grader_files=["tests.py"])
|
|
files = {"task.json": json.dumps(manifest), "prompt.md": "Implement src/a.c", "tests.py": "secret",
|
|
"verify.py": "import json,sys\nfrom pathlib import Path\nc=Path(sys.argv[1])\n"
|
|
"ok=(c/'src/a.c').read_text() in ('ok','alt') and (c/'inc/a.h').read_text()=='h' and "
|
|
"Path(__file__).with_name('tests.py').read_text()=='secret' and not (c/'README.md').exists()\n"
|
|
"print(json.dumps({'status':'PASS' if ok else 'FAIL','checks':1}))\nsys.exit(0 if ok else 1)\n",
|
|
"template/README.md": "spec", "template/src/a.c": "todo", "template/inc/a.h": "h"}
|
|
for fixture, value in (("gold", "ok"), ("noop", ""), ("cheat", "puts"), ("alternative", "alt")):
|
|
files[fixture + "/src/a.c"] = value
|
|
files[fixture + "/inc/a.h"] = "h"
|
|
for name, value in files.items():
|
|
root.joinpath(name).parent.mkdir(parents=True, exist_ok=True)
|
|
root.joinpath(name).write_text(value)
|
|
return root, manifest
|
|
|
|
def test_multi_file_task_calibrates_every_fixture_and_freezes_only_candidates(self):
|
|
root, manifest = self.multi_file_task()
|
|
task = core.load_task(root)
|
|
self.assertEqual(sorted(core._template(task)), ["README.md", "inc/a.h", "src/a.c"])
|
|
receipt = self.receipt(core.calibrate(root, self.output, trusted_local=True))
|
|
self.assertEqual((receipt["status"], len(receipt["trials"])), ("PASS", 25))
|
|
self.assertEqual({t["fixture"]: t["verdict"]["status"] for t in receipt["trials"]}["alternative"], "PASS")
|
|
|
|
def process(argv, cwd, timeout, stdout, stderr, stdin=None, controller=False):
|
|
stdout.write_text("")
|
|
stderr.write_text("")
|
|
if stdout.name == "agent.stdout":
|
|
cwd.joinpath("workspace/src/a.c").write_text("ok")
|
|
cwd.joinpath("workspace/extra.c").write_text("ignored")
|
|
return dict(returncode=0, termination="exit", cleanup="confirmed")
|
|
ok = cwd.joinpath("candidate/src/a.c").read_text() == "ok"
|
|
stdout.write_text(json.dumps(dict(status="PASS" if ok else "FAIL", checks=1)) + "\n")
|
|
return dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed")
|
|
|
|
receipt.update(trusted_local=False, image_id="sha256:" + "a" * 64)
|
|
calibration = self.root / "docker-calibration.json"
|
|
calibration.write_text(json.dumps(receipt))
|
|
missing = subprocess.CompletedProcess([], 1, "", "No such container")
|
|
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), \
|
|
patch.object(core, "_cli", return_value=missing), patch.object(core, "_process", side_effect=process):
|
|
attempt = core.run(root, self.output, "tag", ["agent"], "fake@1", calibration)
|
|
result = self.receipt(attempt)
|
|
self.assertEqual(result["status"], "PASS")
|
|
self.assertEqual(attempt.joinpath("workspace/README.md").read_text(), "spec")
|
|
self.assertEqual(attempt.joinpath("workspace/src").stat().st_mode & 0o777, 0o777)
|
|
self.assertEqual(sorted(p.relative_to(attempt / "candidate").as_posix()
|
|
for p in attempt.joinpath("candidate").rglob("*") if p.is_file()), ["inc/a.h", "src/a.c"])
|
|
self.assertEqual(result["candidate_digest"], core._digest({"src/a.c": b"ok", "inc/a.h": b"h"}))
|
|
self.assertEqual(attempt.joinpath("grader/tests.py").stat().st_mode & 0o777, 0o600)
|
|
altered = dict(receipt, trials=receipt["trials"][:20])
|
|
calibration.write_text(json.dumps(altered))
|
|
with self.assertRaises(ValueError):
|
|
core._calibration(task, "sha256:" + "a" * 64, calibration)
|
|
|
|
def test_multi_file_contract_rejects_unsafe_or_incomplete_declarations(self):
|
|
root, manifest = self.multi_file_task()
|
|
for key, value in (("candidate_files", ["src/missing.c"]), ("candidate_files", ["src/a.c", "src/a.c"]),
|
|
("fixtures", dict(gold="FAIL", template="FAIL", noop="FAIL", cheat="FAIL")),
|
|
("fixtures", dict(gold="PASS", template="FAIL", noop="FAIL")),
|
|
("fixtures", dict(gold="PASS", template="FAIL", noop="FAIL", cheat="FAIL", Bad="FAIL")),
|
|
("grader_files", ["template/src/a.c"]), ("grader_files", ["../x"]),
|
|
("grader_files", ["gold/x"])):
|
|
with self.subTest(key=key, value=value):
|
|
root.joinpath("task.json").write_text(json.dumps(dict(manifest, **{key: value})))
|
|
with self.assertRaises(ValueError):
|
|
core.load_task(root)
|
|
root.joinpath("task.json").write_text(json.dumps(manifest))
|
|
root.joinpath("alternative/inc/a.h").unlink()
|
|
with self.assertRaises(ValueError):
|
|
core.load_task(root)
|
|
|
|
def test_run_requires_full_matching_docker_calibration(self):
|
|
calibration = self.mock_calibration()
|
|
receipt = calibration / "receipt.json"
|
|
original = receipt.read_text()
|
|
for key, value in (("trusted_local", True), ("status", "FAIL"), ("image_id", "other"),
|
|
("task_digest", "other"), ("verifier_digest", "other"), ("trials", [])):
|
|
altered = json.loads(original)
|
|
altered[key] = value
|
|
receipt.write_text(json.dumps(altered))
|
|
with patch.object(core, "_image", return_value="sha256:" + "a" * 64), patch.object(core, "_docker") as docker:
|
|
with self.assertRaises(ValueError):
|
|
core.run(self.task, self.output, "image", ["agent"], "fake@1", calibration)
|
|
docker.assert_not_called()
|
|
altered = json.loads(original)
|
|
altered["trials"][0]["verdict"]["checks"] = 0
|
|
receipt.write_text(json.dumps(altered))
|
|
with self.assertRaises(ValueError):
|
|
core._calibration(core.load_task(self.task), "sha256:" + "a" * 64, calibration)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|