55 lines
3.0 KiB
Python
55 lines
3.0 KiB
Python
"""Run with python3 -m rollout."""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from .core import calibrate, doctor, run
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description="Embedded task calibration and rollout receipts")
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
commands.add_parser("doctor", help="Check local runtime prerequisites")
|
|
calibration = commands.add_parser("calibrate", help="Check gold and negative fixtures")
|
|
calibration.add_argument("task", type=Path)
|
|
calibration.add_argument("--output", type=Path, default=Path("runs"))
|
|
executor = calibration.add_mutually_exclusive_group(required=True)
|
|
executor.add_argument("--trusted-local", action="store_true", help="Execute only trusted repository fixtures on this host")
|
|
executor.add_argument("--image", help="Docker image to resolve to a fixed local image ID")
|
|
attempt = commands.add_parser("run", help="Run a supplied Agent command in Docker and independently grade")
|
|
attempt.add_argument("task", type=Path)
|
|
attempt.add_argument("--output", type=Path, default=Path("runs"))
|
|
attempt.add_argument("--image", required=True)
|
|
attempt.add_argument("--agent-id", required=True, help="Versioned Agent/model identity for this attempt")
|
|
attempt.add_argument("--calibration", type=Path, required=True, help="Matching Docker calibration attempt directory")
|
|
attempt.add_argument("--agent-timeout", type=int, default=900, help="Agent wall time in seconds; independent of grading")
|
|
attempt.add_argument("--model-base-url", help="Non-secret HTTP(S) endpoint for the trusted model proxy")
|
|
attempt.add_argument("--agent-network", help="Existing dedicated Docker network; caller owns its cleanup")
|
|
attempt.add_argument("--agent", nargs=argparse.REMAINDER, required=True, help="Agent argv; place this option last")
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
if args.command == "doctor":
|
|
result = doctor()
|
|
print(json.dumps(result, indent=2, ensure_ascii=False))
|
|
return 0 if result["status"] == "AVAILABLE" else 2
|
|
if args.command == "calibrate":
|
|
directory = calibrate(args.task, args.output, trusted_local=args.trusted_local, image=args.image)
|
|
else:
|
|
if not args.agent:
|
|
parser.error("--agent requires a command")
|
|
directory = run(args.task, args.output, args.image, args.agent, args.agent_id, args.calibration,
|
|
agent_timeout=args.agent_timeout, model_base_url=args.model_base_url,
|
|
agent_network=args.agent_network)
|
|
receipt = json.loads((directory / "receipt.json").read_text())
|
|
print(json.dumps({"attempt": str(directory.resolve()), "task_verdict": receipt["task_verdict"]}, indent=2))
|
|
return {"PASS": 0, "FAIL": 1}.get(receipt["task_verdict"], 2)
|
|
except (ValueError, RuntimeError, OSError) as error:
|
|
print("error: {}".format(error), file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|