- Tasks may declare up to 16 candidate files; the whole template becomes the Agent workspace while only candidate files are frozen and graded. - Optional fixtures add more cheats or legal alternatives (five trials each); optional grader_files ship private tests next to verify.py. - Tasks without an argv oracle export call_type=verifier ground truth. - Upstream model routes live in one MODELS table. - One bounded no-follow reader, one exclusive writer and one SIGTERM helper replace per-module copies; drop unused validate_task. frame-decoder keeps its task digest, calibration receipts and exported query. Verified on host 36: internal network probe, 20-trial calibration and a real GLM rollout (PASS, evidence VERIFIED, cleanup confirmed). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
83 lines
3.9 KiB
Python
83 lines
3.9 KiB
Python
"""Export a frozen task and its trusted ground truth as one five-field JSONL row."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
from . import core
|
|
|
|
LANGUAGES = {".c": "c", ".h": "c", ".cc": "cpp", ".cpp": "cpp", ".hpp": "cpp", ".py": "python",
|
|
".md": "markdown", ".sh": "bash", ".json": "json"}
|
|
|
|
|
|
def _truth(task):
|
|
verifier = hashlib.sha256(task["_files"]["verify.py"]).hexdigest()
|
|
if not os.path.lexists(task["_root"] / "dataset.json"):
|
|
# No portable oracle: the protected grader package itself is the reference.
|
|
return dict(call_type="verifier", fn_name=None, task_digest=task["_digest"], verifier_digest=verifier)
|
|
snapshot = json.loads(core._read(task["_root"], "dataset.json"))
|
|
if not isinstance(snapshot, dict) or set(snapshot) != {
|
|
"task_digest", "verifier_digest", "ground_truth"}:
|
|
raise ValueError("dataset.json must contain frozen task/verifier identities and ground_truth")
|
|
if snapshot["task_digest"] != task["_digest"] or snapshot["verifier_digest"] != verifier:
|
|
raise ValueError("dataset ground truth does not match the frozen task/verifier")
|
|
truth = snapshot["ground_truth"]
|
|
if not isinstance(truth, dict) or set(truth) != {
|
|
"call_type", "fn_name", "inputs", "outputs", "exit_codes"}:
|
|
raise ValueError("ground_truth must contain the argv evaluation fields")
|
|
if truth["call_type"] != "argv" or truth["fn_name"] is not None:
|
|
raise ValueError("this task requires call_type=argv and fn_name=null")
|
|
inputs, outputs, exits = (truth[key] for key in ("inputs", "outputs", "exit_codes"))
|
|
if not all(isinstance(value, list) for value in (inputs, outputs, exits)) or not (
|
|
1 <= len(inputs) <= 1000 and len(inputs) == len(outputs) == len(exits)):
|
|
raise ValueError("ground truth must have 1..1000 equally sized case lists")
|
|
for arguments, output, code in zip(inputs, outputs, exits):
|
|
if not isinstance(arguments, list) or len(arguments) > 128 or any(
|
|
not isinstance(argument, str) or "\0" in argument or
|
|
len(argument.encode("utf-8")) > core.MAX_CHECKER_BYTES for argument in arguments):
|
|
raise ValueError("inputs must be bounded argv string arrays without NUL")
|
|
if not isinstance(output, str) or len(output.encode("utf-8")) > core.MAX_CHECKER_BYTES:
|
|
raise ValueError("outputs must be bounded stdout strings")
|
|
if type(code) is not int or code not in (0, 1, 2):
|
|
raise ValueError("exit_codes must contain integers 0, 1 or 2")
|
|
return truth
|
|
|
|
|
|
def record(task):
|
|
query = task["_files"]["prompt.md"].decode("utf-8")
|
|
for name, data in sorted(core._template(task).items()):
|
|
query += "\n## 初始 {}\n\n```{}\n{}\n```\n".format(
|
|
name, LANGUAGES.get(Path(name).suffix, ""), data.decode("utf-8"))
|
|
return dict(uuid="Code_" + task["_digest"], query=query, ground_truth=_truth(task),
|
|
source=task["source"], domain="Code")
|
|
|
|
|
|
def export(task_path, output_path):
|
|
task = core.load_task(task_path)
|
|
output = core._path(output_path)
|
|
if output == task["_root"] or task["_root"] in output.parents:
|
|
raise ValueError("dataset output must be outside the task directory")
|
|
data = (json.dumps(record(task), ensure_ascii=False, separators=(",", ":")) + "\n").encode("utf-8")
|
|
return core._create(output, data)
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description="Export one frozen task as a five-field JSONL row")
|
|
parser.add_argument("task", type=Path)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
output = export(args.task, args.output)
|
|
print(json.dumps(dict(dataset=str(output))))
|
|
return 0
|
|
except (ValueError, OSError) as error:
|
|
print("dataset export rejected: " + str(error), file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|