data-gen-ebd/rollout/bos_import.py
amos de3755c14b fix: keep the Bos grader away from candidate processes
The Bos wrapper ran check.py and every candidate process as UID 65534, so a
candidate could read the grader copy through /proc/<checker>/cwd and pass
without implementing anything. check.py now runs as root on a 0700 grader
copy; its subprocesses drop to 65534 and only the work copy and mkdtemp
scratch directories are handed over. An audit hook ends the check as
UNVERIFIED if candidate code would run inside the checker.

36 tasks run hidden tests inside the candidate process (tests.py style and
S500_a2_426889b9a968); they carry isolation-blocked.txt and always grade
UNVERIFIED. The importer adds the marker for any extra root *.py. All 74 Bos
tasks get the new verify.py and a version bump.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 12:33:34 +08:00

168 lines
9.0 KiB
Python

"""Convert one Bos-qiao factory environment into a task package without running its scripts."""
import argparse
import ast
import json
from pathlib import Path
import re
import sys
from . import bos_verify, core
VERIFY = Path(__file__).resolve().with_name("bos_verify.py")
FACTORY = "bos-qiao/task_factory_500_20260914"
WORD = r"(?<![A-Za-z0-9_]){}(?![A-Za-z0-9_])" # ASCII boundaries: README text puts CJK right next to names.
def contract_gaps(public, hidden, candidates):
"""Names hidden tests take from candidate Python modules that the public prompt and template never mention."""
# shortcut: only module attributes and imports are checked; hidden behaviour rules still need human review.
names = set()
for module in {Path(name).stem for name in candidates if name.endswith(".py")}:
for alias in {module, *re.findall(r"import\s+%s\s+as\s+(\w+)" % re.escape(module), hidden)}:
names.update(re.findall(WORD.format(re.escape(alias)) + r"\.([A-Za-z_]\w*)", hidden))
for imported in re.findall(r"from\s+%s\s+import\s+([\w\s,()]+)" % re.escape(module), hidden):
names.update(re.findall(r"(\w+)(?:\s+as\s+\w+)?", imported))
return sorted(name for name in names - {"py", "__file__"} if not re.search(WORD.format(re.escape(name)), public))
def task_contract_gaps(task):
text = lambda data: data.decode("utf-8", "replace")
public = text(task["_files"]["prompt.md"]) + "".join(map(text, core._template(task).values()))
return contract_gaps(public, "".join(text(task["_files"][name]) for name in task["grader_files"]),
task["candidate_files"])
LOOP = ast.dump(ast.parse("for name,body in files.items():\n p=Path(name);p.parent.mkdir(parents=True,exist_ok=True)"
";p.write_text(body)").body[0])
def _python_writes(code):
"""Accept only literal Path(...).write_text(...) calls or the factory's literal files loop."""
body = ast.parse(code).body
if not body or ast.dump(body[0]) != ast.dump(ast.parse("from pathlib import Path").body[0]):
raise ValueError("unsupported fixture program")
writes, index = [], 1
while index < len(body):
node = body[index]
call = node.value if isinstance(node, ast.Expr) else None
if (isinstance(call, ast.Call) and isinstance(call.func, ast.Attribute) and call.func.attr == "write_text"
and isinstance(call.func.value, ast.Call) and getattr(call.func.value.func, "id", None) == "Path"
and len(call.func.value.args) == 1 and len(call.args) == 1 and not call.keywords):
writes.append((ast.literal_eval(call.func.value.args[0]), ast.literal_eval(call.args[0])))
elif (isinstance(node, ast.Assign) and ast.unparse(node.targets[0]) == "files" and index + 1 < len(body)
and ast.dump(body[index + 1]) == LOOP):
writes.extend(ast.literal_eval(node.value).items())
index += 1
else:
raise ValueError("unsupported fixture statement: " + ast.unparse(node)[:80])
index += 1
return writes
def _script(text, targets):
"""Read the target files a fixture script writes, without running it; reject anything else."""
files, lines, index = {}, text.split("\n"), 0
while index < len(lines):
line = lines[index].strip()
index += 1
if line in ("", "set -eu", "set -e") or re.fullmatch(r"mkdir -p '?[\w./-]+'?", line):
continue
match = (re.fullmatch(r"cat > ('?)([\w./-]+)\1 <<'([A-Za-z0-9_]+)'", line) or
re.fullmatch(r"python3 -B - <<'([A-Za-z0-9_]+)'", line))
if not match:
raise ValueError("unsupported fixture command: " + line[:80])
try:
end = lines.index(match.groups()[-1], index)
except ValueError:
raise ValueError("unterminated heredoc") from None
body = "".join(line + "\n" for line in lines[index:end])
index = end + 1
writes = [(match.group(2), body)] if line.startswith("cat") else _python_writes(body)
for name, content in writes:
name = name[2:] if isinstance(name, str) and name.startswith("./") else name
if name not in targets or name in files or not isinstance(content, str):
raise ValueError("fixture writes a non-target or repeated file: " + str(name))
files[name] = content
return files
def load(env):
env = core._path(env)
meta = json.loads(core._read(env, "task.json"))
targets = meta.get("final_targets")
if not isinstance(meta.get("env_id"), str) or not isinstance(meta.get("task"), str) or not isinstance(targets, list):
raise ValueError("Bos task.json lacks env_id, task or final_targets")
template = {entry.relative_to(env / "template").as_posix(): core._read(env / "template",
entry.relative_to(env / "template").as_posix())
for entry in sorted((env / "template").rglob("*")) if not entry.is_dir()}
if any(name not in template for name in targets):
raise ValueError("every final target must exist in the Bos template")
scripts = sorted(path.name for path in env.glob("*.sh"))
cheats = [name for name in scripts if name.startswith("cheat_")]
valid = [name for name in scripts if name.startswith("valid_")]
if "gold.sh" not in scripts or set(scripts) - set(cheats) - set(valid) - {"gold.sh"}:
raise ValueError("expected gold.sh plus valid_*.sh and cheat_*.sh fixtures")
noop = "cheat_exit_zero.sh" if "cheat_exit_zero.sh" in cheats else None
others = [name for name in cheats if name != noop]
if not others:
raise ValueError("a cheat fixture besides the no-op is required")
plan = dict(gold="gold.sh", template=None, noop=noop, cheat=others[0])
plan.update((name[:-3], name) for name in others[1:] + valid)
fixtures = {name: "PASS" if name == "gold" or name.startswith("valid_") else "FAIL" for name in plan}
files = {}
for fixture, script in plan.items():
written = _script(core._read(env, script).decode("utf-8"), targets) if script else {}
files.update((fixture + "/" + name, written[name].encode("utf-8") if name in written else template[name])
for name in targets)
grader = {path.name: core._read(env, path.name) for path in sorted(env.glob("*.py"))}
if "check.py" not in grader:
raise ValueError("Bos check.py is missing")
gaps = contract_gaps(meta["task"] + "".join(data.decode("utf-8", "replace") for data in template.values()),
"".join(data.decode("utf-8", "replace") for data in grader.values()), targets)
if gaps:
raise ValueError("hidden tests use names the public spec never mentions: " + ", ".join(gaps))
if len(grader) > 1:
# Factory checks hand these hidden tests to the process that imports or links the candidate.
grader[bos_verify.BLOCKED] = ("hidden tests in " + ", ".join(sorted(set(grader) - {"check.py"})) +
" run inside the candidate process; re-author the check before admission\n"
).encode("utf-8")
return meta, template, fixtures, plan, files, grader
def write(env, output, family):
meta, template, fixtures, plan, files, grader = load(env)
output = core._path(output)
output.mkdir()
task = dict(id=meta["env_id"], version="1", family=family, source=FACTORY + "/envs/" + meta["env_id"],
template="template", verifier="verify.py", candidate_files=meta["final_targets"],
timeout_seconds=120, fixtures=fixtures, grader_files=sorted(grader),
bos=dict(source_qid=meta.get("source_qid"), origin=meta.get("origin"), image=meta.get("image"),
budget_s=meta.get("budget_s"), fixture_scripts={k: v for k, v in plan.items() if v}))
prompt = meta["task"].rstrip() + "\n\n最终只提交并评分以下文件:" + "、".join(
"`" + name + "`" for name in meta["final_targets"]) + "。\n"
contents = {"task.json": (json.dumps(task, ensure_ascii=False, indent=2) + "\n").encode("utf-8"),
"prompt.md": prompt.encode("utf-8"), "verify.py": VERIFY.read_bytes(), **grader, **files,
**{"template/" + name: data for name, data in template.items()}}
core._write_tree(output, contents, 0o644, 0o755)
return core.load_task(output)
def main(argv=None):
parser = argparse.ArgumentParser(description="Import one Bos-qiao environment as a task package")
parser.add_argument("env", type=Path)
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--family", required=True)
args = parser.parse_args(argv)
try:
task = write(args.env, args.output, args.family)
print(json.dumps(dict(task=str(task["_root"]), digest=task["_digest"], fixtures=task["fixtures"])))
return 0
except (ValueError, OSError, UnicodeDecodeError) as error:
print("import rejected: " + str(error), file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())