data-gen-ebd/rollout/__main__.py

55 lines
3.0 KiB
Python

"""Run with python3 -m rollout."""
import argparse
import json
import sys
from pathlib import Path
from .core import calibrate, doctor, run
def main(argv=None):
parser = argparse.ArgumentParser(description="Embedded task calibration and rollout receipts")
commands = parser.add_subparsers(dest="command", required=True)
commands.add_parser("doctor", help="Check local runtime prerequisites")
calibration = commands.add_parser("calibrate", help="Check gold and negative fixtures")
calibration.add_argument("task", type=Path)
calibration.add_argument("--output", type=Path, default=Path("runs"))
executor = calibration.add_mutually_exclusive_group(required=True)
executor.add_argument("--trusted-local", action="store_true", help="Execute only trusted repository fixtures on this host")
executor.add_argument("--image", help="Docker image to resolve to a fixed local image ID")
attempt = commands.add_parser("run", help="Run a supplied Agent command in Docker and independently grade")
attempt.add_argument("task", type=Path)
attempt.add_argument("--output", type=Path, default=Path("runs"))
attempt.add_argument("--image", required=True)
attempt.add_argument("--agent-id", required=True, help="Versioned Agent/model identity for this attempt")
attempt.add_argument("--calibration", type=Path, required=True, help="Matching Docker calibration attempt directory")
attempt.add_argument("--agent-timeout", type=int, default=900, help="Agent wall time in seconds; independent of grading")
attempt.add_argument("--model-base-url", help="Non-secret HTTP(S) endpoint for the trusted model proxy")
attempt.add_argument("--agent-network", help="Existing dedicated Docker network; caller owns its cleanup")
attempt.add_argument("--agent", nargs=argparse.REMAINDER, required=True, help="Agent argv; place this option last")
args = parser.parse_args(argv)
try:
if args.command == "doctor":
result = doctor()
print(json.dumps(result, indent=2, ensure_ascii=False))
return 0 if result["status"] == "AVAILABLE" else 2
if args.command == "calibrate":
directory = calibrate(args.task, args.output, trusted_local=args.trusted_local, image=args.image)
else:
if not args.agent:
parser.error("--agent requires a command")
directory = run(args.task, args.output, args.image, args.agent, args.agent_id, args.calibration,
agent_timeout=args.agent_timeout, model_base_url=args.model_base_url,
agent_network=args.agent_network)
receipt = json.loads((directory / "receipt.json").read_text())
print(json.dumps({"attempt": str(directory.resolve()), "task_verdict": receipt["task_verdict"]}, indent=2))
return {"PASS": 0, "FAIL": 1}.get(receipt["task_verdict"], 2)
except (ValueError, RuntimeError, OSError) as error:
print("error: {}".format(error), file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())