data-gen-ebd/tests/test_sft.py
amos c64c79b31d Run the Pi agent in Salty jobs over a one-time SSH tunnel and export SFT rows
rollout.salty_run keeps the gateway, grading and evidence on 36 and moves only the
agent step into a Salty job. The gateway listens on 127.0.0.1 with a per-attempt
bearer token; the job reaches it over port 22 with a restricted one-time key that is
removed after each attempt. rollout.batch gains --backend salty and repeatable
--credential-file, and writes pass/fail SFT exports through rollout.sft at the end.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 17:45:28 +08:00

104 lines
5.9 KiB
Python

import contextlib
import hashlib
import io
import json
from pathlib import Path
import tempfile
import unittest
from rollout import sft
SYSTEM = dict(role="system", content="You are pi.")
USER = dict(role="user", content=[dict(type="text", text="fix rtc_client.c")])
CALL = dict(role="assistant", content="", reasoning_content="read first",
tool_calls=[dict(id="call-1", type="function", function=dict(name="read", arguments='{"path":"a.c"}'))])
RESULT = dict(role="tool", content="int x;", tool_call_id="call-1")
TOOLS = [dict(type="function", function=dict(name="read", description="Read", parameters=dict(type="object")))]
FINAL = dict(role="assistant", content=[dict(type="thinking", thinking="done"), dict(type="text", text="完成")])
def attempt(root, final=FINAL, **receipt_changes):
"""A two-request GLM rollout recorded as pi/2 prefix deltas, plus its top-level receipt."""
run = root / "run-1"
(run / "trace").mkdir(parents=True)
events = [dict(type="trace_header", schema="embedded-rollout.pi/2"),
dict(type="model_payload", generation_id=0, delta=dict(field="messages", **{"from": 0}),
payload=dict(model="glm-5.3", messages=[SYSTEM, USER], tools=TOOLS)),
dict(type="turn_end", message=dict(role="assistant", content=[dict(type="toolCall", id="call-1")])),
dict(type="model_payload", generation_id=1, delta=dict(field="messages", **{"from": 2}),
payload=dict(model="glm-5.3", messages=[CALL, RESULT], tools=TOOLS)),
dict(type="turn_end", message=final),
dict(type="session_end", complete=True, closed=True)]
raw = "".join(json.dumps(event, ensure_ascii=False) + "\n" for event in events).encode()
(run / "trace/native.jsonl").write_bytes(raw)
receipt = dict(attempt="/elsewhere/run-1", status="PASS", cleanup="confirmed", contamination=[],
model_api="openai-completions", model_id="glm-5.3", image_id="sha256:" + "a" * 64,
task_id="rtc", task_digest="d" * 64,
pi_evidence=dict(status="VERIFIED", trace_sha256=hashlib.sha256(raw).hexdigest()))
receipt.update(receipt_changes)
(root / "receipt.json").write_text(json.dumps(receipt))
return root / "receipt.json"
class SftTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
def test_verified_rollout_becomes_one_ultradata_style_conversation(self):
row = sft.record(attempt(self.root))
self.assertEqual(set(row), {"uuid", "messages", "tools", "source", "domain", "rollout"})
self.assertEqual(row["messages"], [
dict(role="system", content="You are pi."), dict(role="user", content="fix rtc_client.c"),
dict(role="assistant", content="", reasoning_content="read first",
tool_calls=[dict(type="function", function=dict(name="read", arguments={"path": "a.c"}))]),
dict(role="tool", content="int x;"),
dict(role="assistant", content="完成", reasoning_content="done")])
self.assertEqual(row["tools"], TOOLS)
self.assertEqual((row["source"], row["domain"]), ("embedded-rollout:rtc", "Code_Agent"))
self.assertEqual(row["rollout"]["verdict"], "PASS")
self.assertTrue(row["uuid"].startswith("Code_Agent_"))
def test_batch_export_keeps_pass_and_fail_apart_and_lists_skips(self):
batch = self.root / "batch"
for name, changes in (("salty-a", {}), ("salty-b", dict(status="FAIL")),
("pi-c", dict(status="INFRA_ERROR")), ("salty-d", dict(contamination=["/data"]))):
(batch / name).mkdir(parents=True)
attempt(batch / name, **changes)
(batch / "report-x.json").write_text("{}")
output, summary = sft.export([batch], self.root / "export")
rows = {name: [json.loads(line) for line in (output / name).read_text().splitlines()]
for name in ("pass.jsonl", "fail.jsonl")}
self.assertEqual([row["rollout"]["verdict"] for row in rows["pass.jsonl"]], ["PASS"])
self.assertEqual([row["rollout"]["verdict"] for row in rows["fail.jsonl"]], ["FAIL"])
skipped = json.loads((output / "skipped.json").read_text())["skipped"]
self.assertEqual(sorted(Path(item["controller"]).name for item in skipped), ["pi-c", "salty-d"])
self.assertEqual((summary["passed"], summary["failed"]), (1, 1))
with self.assertRaises(FileExistsError):
sft.export([batch], output)
def test_unverified_contaminated_or_infra_rollouts_are_rejected(self):
for changes in (dict(pi_evidence=dict(status="UNVERIFIED")), dict(contamination=["/data/personal"]),
dict(status="INFRA_ERROR"), dict(cleanup="unknown"), dict(model_api="openai-responses")):
with self.subTest(changes=changes), tempfile.TemporaryDirectory() as temporary:
with self.assertRaises(ValueError):
sft.record(attempt(Path(temporary).resolve(), **changes))
def test_trace_must_match_receipt_and_end_with_a_reply(self):
receipt = attempt(self.root, pi_evidence=dict(status="VERIFIED", trace_sha256="0" * 64))
with self.assertRaisesRegex(ValueError, "differs"):
sft.record(receipt)
with tempfile.TemporaryDirectory() as temporary:
pending = dict(role="assistant", content=[dict(type="toolCall", id="call-2")])
with self.assertRaisesRegex(ValueError, "final assistant"):
sft.record(attempt(Path(temporary).resolve(), final=pending))
def test_cli_rejects_a_missing_batch_without_writing(self):
with contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(sft.main([str(self.root / "missing"), "--output", str(self.root / "out")]), 2)
self.assertFalse((self.root / "out").exists())
if __name__ == "__main__":
unittest.main()