rollout.salty_run keeps the gateway, grading and evidence on 36 and moves only the agent step into a Salty job. The gateway listens on 127.0.0.1 with a per-attempt bearer token; the job reaches it over port 22 with a restricted one-time key that is removed after each attempt. rollout.batch gains --backend salty and repeatable --credential-file, and writes pass/fail SFT exports through rollout.sft at the end. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
104 lines
5.9 KiB
Python
104 lines
5.9 KiB
Python
import contextlib
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
from rollout import sft
|
|
|
|
SYSTEM = dict(role="system", content="You are pi.")
|
|
USER = dict(role="user", content=[dict(type="text", text="fix rtc_client.c")])
|
|
CALL = dict(role="assistant", content="", reasoning_content="read first",
|
|
tool_calls=[dict(id="call-1", type="function", function=dict(name="read", arguments='{"path":"a.c"}'))])
|
|
RESULT = dict(role="tool", content="int x;", tool_call_id="call-1")
|
|
TOOLS = [dict(type="function", function=dict(name="read", description="Read", parameters=dict(type="object")))]
|
|
FINAL = dict(role="assistant", content=[dict(type="thinking", thinking="done"), dict(type="text", text="完成")])
|
|
|
|
|
|
def attempt(root, final=FINAL, **receipt_changes):
|
|
"""A two-request GLM rollout recorded as pi/2 prefix deltas, plus its top-level receipt."""
|
|
run = root / "run-1"
|
|
(run / "trace").mkdir(parents=True)
|
|
events = [dict(type="trace_header", schema="embedded-rollout.pi/2"),
|
|
dict(type="model_payload", generation_id=0, delta=dict(field="messages", **{"from": 0}),
|
|
payload=dict(model="glm-5.3", messages=[SYSTEM, USER], tools=TOOLS)),
|
|
dict(type="turn_end", message=dict(role="assistant", content=[dict(type="toolCall", id="call-1")])),
|
|
dict(type="model_payload", generation_id=1, delta=dict(field="messages", **{"from": 2}),
|
|
payload=dict(model="glm-5.3", messages=[CALL, RESULT], tools=TOOLS)),
|
|
dict(type="turn_end", message=final),
|
|
dict(type="session_end", complete=True, closed=True)]
|
|
raw = "".join(json.dumps(event, ensure_ascii=False) + "\n" for event in events).encode()
|
|
(run / "trace/native.jsonl").write_bytes(raw)
|
|
receipt = dict(attempt="/elsewhere/run-1", status="PASS", cleanup="confirmed", contamination=[],
|
|
model_api="openai-completions", model_id="glm-5.3", image_id="sha256:" + "a" * 64,
|
|
task_id="rtc", task_digest="d" * 64,
|
|
pi_evidence=dict(status="VERIFIED", trace_sha256=hashlib.sha256(raw).hexdigest()))
|
|
receipt.update(receipt_changes)
|
|
(root / "receipt.json").write_text(json.dumps(receipt))
|
|
return root / "receipt.json"
|
|
|
|
|
|
class SftTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name).resolve()
|
|
|
|
def test_verified_rollout_becomes_one_ultradata_style_conversation(self):
|
|
row = sft.record(attempt(self.root))
|
|
self.assertEqual(set(row), {"uuid", "messages", "tools", "source", "domain", "rollout"})
|
|
self.assertEqual(row["messages"], [
|
|
dict(role="system", content="You are pi."), dict(role="user", content="fix rtc_client.c"),
|
|
dict(role="assistant", content="", reasoning_content="read first",
|
|
tool_calls=[dict(type="function", function=dict(name="read", arguments={"path": "a.c"}))]),
|
|
dict(role="tool", content="int x;"),
|
|
dict(role="assistant", content="完成", reasoning_content="done")])
|
|
self.assertEqual(row["tools"], TOOLS)
|
|
self.assertEqual((row["source"], row["domain"]), ("embedded-rollout:rtc", "Code_Agent"))
|
|
self.assertEqual(row["rollout"]["verdict"], "PASS")
|
|
self.assertTrue(row["uuid"].startswith("Code_Agent_"))
|
|
|
|
def test_batch_export_keeps_pass_and_fail_apart_and_lists_skips(self):
|
|
batch = self.root / "batch"
|
|
for name, changes in (("salty-a", {}), ("salty-b", dict(status="FAIL")),
|
|
("pi-c", dict(status="INFRA_ERROR")), ("salty-d", dict(contamination=["/data"]))):
|
|
(batch / name).mkdir(parents=True)
|
|
attempt(batch / name, **changes)
|
|
(batch / "report-x.json").write_text("{}")
|
|
output, summary = sft.export([batch], self.root / "export")
|
|
rows = {name: [json.loads(line) for line in (output / name).read_text().splitlines()]
|
|
for name in ("pass.jsonl", "fail.jsonl")}
|
|
self.assertEqual([row["rollout"]["verdict"] for row in rows["pass.jsonl"]], ["PASS"])
|
|
self.assertEqual([row["rollout"]["verdict"] for row in rows["fail.jsonl"]], ["FAIL"])
|
|
skipped = json.loads((output / "skipped.json").read_text())["skipped"]
|
|
self.assertEqual(sorted(Path(item["controller"]).name for item in skipped), ["pi-c", "salty-d"])
|
|
self.assertEqual((summary["passed"], summary["failed"]), (1, 1))
|
|
with self.assertRaises(FileExistsError):
|
|
sft.export([batch], output)
|
|
|
|
def test_unverified_contaminated_or_infra_rollouts_are_rejected(self):
|
|
for changes in (dict(pi_evidence=dict(status="UNVERIFIED")), dict(contamination=["/data/personal"]),
|
|
dict(status="INFRA_ERROR"), dict(cleanup="unknown"), dict(model_api="openai-responses")):
|
|
with self.subTest(changes=changes), tempfile.TemporaryDirectory() as temporary:
|
|
with self.assertRaises(ValueError):
|
|
sft.record(attempt(Path(temporary).resolve(), **changes))
|
|
|
|
def test_trace_must_match_receipt_and_end_with_a_reply(self):
|
|
receipt = attempt(self.root, pi_evidence=dict(status="VERIFIED", trace_sha256="0" * 64))
|
|
with self.assertRaisesRegex(ValueError, "differs"):
|
|
sft.record(receipt)
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
pending = dict(role="assistant", content=[dict(type="toolCall", id="call-2")])
|
|
with self.assertRaisesRegex(ValueError, "final assistant"):
|
|
sft.record(attempt(Path(temporary).resolve(), final=pending))
|
|
|
|
def test_cli_rejects_a_missing_batch_without_writing(self):
|
|
with contextlib.redirect_stderr(io.StringIO()):
|
|
self.assertEqual(sft.main([str(self.root / "missing"), "--output", str(self.root / "out")]), 2)
|
|
self.assertFalse((self.root / "out").exists())
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|