The Bos wrapper ran check.py and every candidate process as UID 65534, so a candidate could read the grader copy through /proc/<checker>/cwd and pass without implementing anything. check.py now runs as root on a 0700 grader copy; its subprocesses drop to 65534 and only the work copy and mkdtemp scratch directories are handed over. An audit hook ends the check as UNVERIFIED if candidate code would run inside the checker. 36 tasks run hidden tests inside the candidate process (tests.py style and S500_a2_426889b9a968); they carry isolation-blocked.txt and always grade UNVERIFIED. The importer adds the marker for any extra root *.py. All 74 Bos tasks get the new verify.py and a version bump. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
168 lines
9.0 KiB
Python
168 lines
9.0 KiB
Python
"""Convert one Bos-qiao factory environment into a task package without running its scripts."""
|
|
|
|
import argparse
|
|
import ast
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
|
|
from . import bos_verify, core
|
|
|
|
VERIFY = Path(__file__).resolve().with_name("bos_verify.py")
|
|
FACTORY = "bos-qiao/task_factory_500_20260914"
|
|
WORD = r"(?<![A-Za-z0-9_]){}(?![A-Za-z0-9_])" # ASCII boundaries: README text puts CJK right next to names.
|
|
|
|
|
|
def contract_gaps(public, hidden, candidates):
|
|
"""Names hidden tests take from candidate Python modules that the public prompt and template never mention."""
|
|
# shortcut: only module attributes and imports are checked; hidden behaviour rules still need human review.
|
|
names = set()
|
|
for module in {Path(name).stem for name in candidates if name.endswith(".py")}:
|
|
for alias in {module, *re.findall(r"import\s+%s\s+as\s+(\w+)" % re.escape(module), hidden)}:
|
|
names.update(re.findall(WORD.format(re.escape(alias)) + r"\.([A-Za-z_]\w*)", hidden))
|
|
for imported in re.findall(r"from\s+%s\s+import\s+([\w\s,()]+)" % re.escape(module), hidden):
|
|
names.update(re.findall(r"(\w+)(?:\s+as\s+\w+)?", imported))
|
|
return sorted(name for name in names - {"py", "__file__"} if not re.search(WORD.format(re.escape(name)), public))
|
|
|
|
|
|
def task_contract_gaps(task):
|
|
text = lambda data: data.decode("utf-8", "replace")
|
|
public = text(task["_files"]["prompt.md"]) + "".join(map(text, core._template(task).values()))
|
|
return contract_gaps(public, "".join(text(task["_files"][name]) for name in task["grader_files"]),
|
|
task["candidate_files"])
|
|
|
|
|
|
LOOP = ast.dump(ast.parse("for name,body in files.items():\n p=Path(name);p.parent.mkdir(parents=True,exist_ok=True)"
|
|
";p.write_text(body)").body[0])
|
|
|
|
|
|
def _python_writes(code):
|
|
"""Accept only literal Path(...).write_text(...) calls or the factory's literal files loop."""
|
|
body = ast.parse(code).body
|
|
if not body or ast.dump(body[0]) != ast.dump(ast.parse("from pathlib import Path").body[0]):
|
|
raise ValueError("unsupported fixture program")
|
|
writes, index = [], 1
|
|
while index < len(body):
|
|
node = body[index]
|
|
call = node.value if isinstance(node, ast.Expr) else None
|
|
if (isinstance(call, ast.Call) and isinstance(call.func, ast.Attribute) and call.func.attr == "write_text"
|
|
and isinstance(call.func.value, ast.Call) and getattr(call.func.value.func, "id", None) == "Path"
|
|
and len(call.func.value.args) == 1 and len(call.args) == 1 and not call.keywords):
|
|
writes.append((ast.literal_eval(call.func.value.args[0]), ast.literal_eval(call.args[0])))
|
|
elif (isinstance(node, ast.Assign) and ast.unparse(node.targets[0]) == "files" and index + 1 < len(body)
|
|
and ast.dump(body[index + 1]) == LOOP):
|
|
writes.extend(ast.literal_eval(node.value).items())
|
|
index += 1
|
|
else:
|
|
raise ValueError("unsupported fixture statement: " + ast.unparse(node)[:80])
|
|
index += 1
|
|
return writes
|
|
|
|
|
|
def _script(text, targets):
|
|
"""Read the target files a fixture script writes, without running it; reject anything else."""
|
|
files, lines, index = {}, text.split("\n"), 0
|
|
while index < len(lines):
|
|
line = lines[index].strip()
|
|
index += 1
|
|
if line in ("", "set -eu", "set -e") or re.fullmatch(r"mkdir -p '?[\w./-]+'?", line):
|
|
continue
|
|
match = (re.fullmatch(r"cat > ('?)([\w./-]+)\1 <<'([A-Za-z0-9_]+)'", line) or
|
|
re.fullmatch(r"python3 -B - <<'([A-Za-z0-9_]+)'", line))
|
|
if not match:
|
|
raise ValueError("unsupported fixture command: " + line[:80])
|
|
try:
|
|
end = lines.index(match.groups()[-1], index)
|
|
except ValueError:
|
|
raise ValueError("unterminated heredoc") from None
|
|
body = "".join(line + "\n" for line in lines[index:end])
|
|
index = end + 1
|
|
writes = [(match.group(2), body)] if line.startswith("cat") else _python_writes(body)
|
|
for name, content in writes:
|
|
name = name[2:] if isinstance(name, str) and name.startswith("./") else name
|
|
if name not in targets or name in files or not isinstance(content, str):
|
|
raise ValueError("fixture writes a non-target or repeated file: " + str(name))
|
|
files[name] = content
|
|
return files
|
|
|
|
|
|
def load(env):
|
|
env = core._path(env)
|
|
meta = json.loads(core._read(env, "task.json"))
|
|
targets = meta.get("final_targets")
|
|
if not isinstance(meta.get("env_id"), str) or not isinstance(meta.get("task"), str) or not isinstance(targets, list):
|
|
raise ValueError("Bos task.json lacks env_id, task or final_targets")
|
|
template = {entry.relative_to(env / "template").as_posix(): core._read(env / "template",
|
|
entry.relative_to(env / "template").as_posix())
|
|
for entry in sorted((env / "template").rglob("*")) if not entry.is_dir()}
|
|
if any(name not in template for name in targets):
|
|
raise ValueError("every final target must exist in the Bos template")
|
|
scripts = sorted(path.name for path in env.glob("*.sh"))
|
|
cheats = [name for name in scripts if name.startswith("cheat_")]
|
|
valid = [name for name in scripts if name.startswith("valid_")]
|
|
if "gold.sh" not in scripts or set(scripts) - set(cheats) - set(valid) - {"gold.sh"}:
|
|
raise ValueError("expected gold.sh plus valid_*.sh and cheat_*.sh fixtures")
|
|
noop = "cheat_exit_zero.sh" if "cheat_exit_zero.sh" in cheats else None
|
|
others = [name for name in cheats if name != noop]
|
|
if not others:
|
|
raise ValueError("a cheat fixture besides the no-op is required")
|
|
plan = dict(gold="gold.sh", template=None, noop=noop, cheat=others[0])
|
|
plan.update((name[:-3], name) for name in others[1:] + valid)
|
|
fixtures = {name: "PASS" if name == "gold" or name.startswith("valid_") else "FAIL" for name in plan}
|
|
files = {}
|
|
for fixture, script in plan.items():
|
|
written = _script(core._read(env, script).decode("utf-8"), targets) if script else {}
|
|
files.update((fixture + "/" + name, written[name].encode("utf-8") if name in written else template[name])
|
|
for name in targets)
|
|
grader = {path.name: core._read(env, path.name) for path in sorted(env.glob("*.py"))}
|
|
if "check.py" not in grader:
|
|
raise ValueError("Bos check.py is missing")
|
|
gaps = contract_gaps(meta["task"] + "".join(data.decode("utf-8", "replace") for data in template.values()),
|
|
"".join(data.decode("utf-8", "replace") for data in grader.values()), targets)
|
|
if gaps:
|
|
raise ValueError("hidden tests use names the public spec never mentions: " + ", ".join(gaps))
|
|
if len(grader) > 1:
|
|
# Factory checks hand these hidden tests to the process that imports or links the candidate.
|
|
grader[bos_verify.BLOCKED] = ("hidden tests in " + ", ".join(sorted(set(grader) - {"check.py"})) +
|
|
" run inside the candidate process; re-author the check before admission\n"
|
|
).encode("utf-8")
|
|
return meta, template, fixtures, plan, files, grader
|
|
|
|
|
|
def write(env, output, family):
|
|
meta, template, fixtures, plan, files, grader = load(env)
|
|
output = core._path(output)
|
|
output.mkdir()
|
|
task = dict(id=meta["env_id"], version="1", family=family, source=FACTORY + "/envs/" + meta["env_id"],
|
|
template="template", verifier="verify.py", candidate_files=meta["final_targets"],
|
|
timeout_seconds=120, fixtures=fixtures, grader_files=sorted(grader),
|
|
bos=dict(source_qid=meta.get("source_qid"), origin=meta.get("origin"), image=meta.get("image"),
|
|
budget_s=meta.get("budget_s"), fixture_scripts={k: v for k, v in plan.items() if v}))
|
|
prompt = meta["task"].rstrip() + "\n\n最终只提交并评分以下文件:" + "、".join(
|
|
"`" + name + "`" for name in meta["final_targets"]) + "。\n"
|
|
contents = {"task.json": (json.dumps(task, ensure_ascii=False, indent=2) + "\n").encode("utf-8"),
|
|
"prompt.md": prompt.encode("utf-8"), "verify.py": VERIFY.read_bytes(), **grader, **files,
|
|
**{"template/" + name: data for name, data in template.items()}}
|
|
core._write_tree(output, contents, 0o644, 0o755)
|
|
return core.load_task(output)
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description="Import one Bos-qiao environment as a task package")
|
|
parser.add_argument("env", type=Path)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--family", required=True)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
task = write(args.env, args.output, args.family)
|
|
print(json.dumps(dict(task=str(task["_root"]), digest=task["_digest"], fixtures=task["fixtures"])))
|
|
return 0
|
|
except (ValueError, OSError, UnicodeDecodeError) as error:
|
|
print("import rejected: " + str(error), file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|