The user wants 2000 tasks, most of them ones Qwen cannot solve. New batch tasks are now measured on their branch first (3 Qwen trials each), and only HARD or MEDIUM tasks reach main; ROUTINE ones stay on the batch branches. A session cut by context_limit or output_limit counts as a failure for difficulty (user decision 2026-11-03); the trajectory stays incomplete. - Batch 26 (route D, real upstream fixes in embedded open-source projects): 7 HARD of 20 (coreJSON, littlefs, CANopen, coreMQTT, LwM2M, Zephyr ring, lwIP SNTP). Most failures are Qwen exhausting its context in the larger workspaces. - Batch 25 (HWE mutation repairs): 1 HARD and 1 MEDIUM of 56. - Batch 27 (spec-dense implementation tasks): 2 MEDIUM of the first 10. Each task comes from the exact commit Qwen measured, and its digest matches the measured one. Failure reasons are not yet reviewed, so difficulty is the automatic label. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
131 lines
5.9 KiB
Python
131 lines
5.9 KiB
Python
"""Run an imported Bos-qiao check.py unchanged; candidate processes run as UID 65534."""
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
GRADER = Path(__file__).resolve().parent
|
|
BLOCKED = "isolation-blocked.txt"
|
|
GUARD = 86
|
|
# Runs the original check unchanged as the trusted user. In isolation mode every child process drops to
|
|
# UID 65534 and every mkdtemp scratch is handed to it; an audit hook stops the check if candidate code
|
|
# would run inside this process. Failing child processes also copy their stderr tail out for private triage.
|
|
CHECK = (
|
|
"import os, runpy, subprocess, sys, tempfile\n"
|
|
"isolated, sys.argv = sys.argv[1] == '1', sys.argv[2:]\n"
|
|
"untrusted = [os.path.realpath(sys.argv[1])]\n"
|
|
"def inside(path):\n"
|
|
" path = os.path.realpath(str(path))\n"
|
|
" return any(path == root or path.startswith(root + os.sep) for root in untrusted)\n"
|
|
"spawns = ('os.system', 'os.exec', 'os.spawn', 'os.fork', 'os.forkpty', 'os.posix_spawn') if isolated else ()\n"
|
|
"def guard(event, args):\n"
|
|
" if (event in spawns or event == 'exec' and inside(getattr(args[0], 'co_filename', '')) or\n"
|
|
" event == 'ctypes.dlopen' and args[0] and inside(args[0]) or\n"
|
|
" event == 'import' and any(inside(entry or '.') for entry in sys.path)):\n"
|
|
" sys.__stderr__.write('rollout-guard: ' + event + '\\n')\n"
|
|
" sys.__stderr__.flush()\n"
|
|
" os._exit(%d)\n"
|
|
"mkdtemp = tempfile.mkdtemp\n"
|
|
"def scratch(*args, **kwargs):\n"
|
|
" path = mkdtemp(*args, **kwargs)\n"
|
|
" untrusted.append(os.path.realpath(path))\n"
|
|
" if isolated:\n"
|
|
" os.chown(path, 65534, 65534)\n"
|
|
" return path\n"
|
|
"tempfile.mkdtemp = scratch\n"
|
|
"popen = subprocess.Popen.__init__\n"
|
|
"def drop(self, *args, **kwargs):\n"
|
|
" if isolated:\n"
|
|
" kwargs.update(user=65534, group=65534, extra_groups=[])\n"
|
|
" popen(self, *args, **kwargs)\n"
|
|
"subprocess.Popen.__init__ = drop\n"
|
|
"real = subprocess.run\n"
|
|
"def run(*args, **kwargs):\n"
|
|
" result = real(*args, **kwargs)\n"
|
|
" if result.returncode and result.stderr:\n"
|
|
" tail = result.stderr if isinstance(result.stderr, str) else result.stderr.decode('utf-8', 'replace')\n"
|
|
" sys.__stderr__.write(tail[-2000:] + '\\n')\n"
|
|
" return result\n"
|
|
"subprocess.run = run\n"
|
|
"sys.addaudithook(guard)\n"
|
|
"runpy.run_path(sys.argv[0], run_name='__main__')\n" % GUARD)
|
|
|
|
|
|
class UnsupportedEnvironment(RuntimeError):
|
|
pass
|
|
|
|
|
|
def candidate_user():
|
|
os.setgroups([])
|
|
os.setgid(65534)
|
|
os.setuid(65534)
|
|
|
|
|
|
def isolation():
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") != "uid65534":
|
|
return None
|
|
if sys.platform != "linux" or os.geteuid() != 0:
|
|
raise UnsupportedEnvironment("isolated grader requires a Linux root supervisor")
|
|
probe = (
|
|
"from pathlib import Path\n"
|
|
"try: Path('/grader/verify.py').read_bytes()\n"
|
|
"except PermissionError: raise SystemExit(0)\n"
|
|
"raise SystemExit(1)\n"
|
|
)
|
|
try:
|
|
result = subprocess.run([sys.executable, "-I", "-c", probe],
|
|
capture_output=True, timeout=10, preexec_fn=candidate_user)
|
|
except subprocess.SubprocessError as error:
|
|
raise UnsupportedEnvironment("candidate identity could not be isolated") from error
|
|
if result.returncode != 0:
|
|
raise UnsupportedEnvironment("candidate can read the private grader or isolation probe failed")
|
|
return candidate_user
|
|
|
|
|
|
def verify(candidate):
|
|
if (GRADER / BLOCKED).exists():
|
|
raise UnsupportedEnvironment("isolation blocked: " + (GRADER / BLOCKED).read_text().strip()[:200])
|
|
isolated = isolation() is not None
|
|
if shutil.which("gcc") is None:
|
|
raise UnsupportedEnvironment("C compiler unavailable")
|
|
with tempfile.TemporaryDirectory(prefix="bos-grade-") as temporary:
|
|
root = Path(temporary)
|
|
work, grader = root / "work", root / "grader"
|
|
shutil.copytree(candidate, work, symlinks=True)
|
|
shutil.copytree(GRADER, grader, ignore=lambda folder, names: ["verify.py"] if Path(folder) == GRADER else [])
|
|
if isolated:
|
|
os.chmod(root, 0o711)
|
|
os.chmod(grader, 0o700)
|
|
for path in (work, *work.rglob("*")):
|
|
os.lchown(path, 65534, 65534)
|
|
# shortcut: the check runs as root and follows symlinks a child plants in shared scratch directories;
|
|
# upgrade by running the check under its own unprivileged UID once a setuid helper is available.
|
|
completed = subprocess.run([sys.executable, "-I", "-B", "-c", CHECK, "1" if isolated else "0",
|
|
str(grader / "check.py"), str(work)],
|
|
cwd=root, capture_output=True, text=True, timeout=100)
|
|
# grade.stderr stays on the trusted side with the archive; it may quote hidden test lines.
|
|
sys.stderr.write(completed.stderr[-4000:])
|
|
if completed.returncode == GUARD and "rollout-guard: " in completed.stderr:
|
|
raise UnsupportedEnvironment("isolation blocked: the check would run candidate code in the grader process")
|
|
lines = completed.stdout.splitlines()
|
|
if completed.returncode == 0 and lines[-1:] == ["PASS"]:
|
|
return {"status": "PASS", "checks": 1}
|
|
return {"status": "FAIL", "checks": 1, "reason": (lines[-1] if lines else "no checker output")[:300]}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
result = verify(Path(sys.argv[1]))
|
|
except UnsupportedEnvironment as error:
|
|
result = {"status": "UNVERIFIED", "checks": 0, "reason": str(error)}
|
|
except subprocess.TimeoutExpired:
|
|
result = {"status": "FAIL", "checks": 1, "reason": "timeout"}
|
|
except Exception as error:
|
|
result = {"status": "INFRA_ERROR", "checks": 0, "reason": str(error)}
|
|
print(json.dumps(result, sort_keys=True))
|
|
sys.exit({"PASS": 0, "FAIL": 1}.get(result["status"], 2))
|