Route D (real fixes in embedded open-source projects) keeps producing the hard tasks: of 64 measured so far, 15 are HARD and 4 MEDIUM, mostly because Qwen exhausts its context in the larger real workspaces. This adds CANopen SRDO and ASCII timing, FreeRTOS DHCP, lwIP MQTT fragments, printf extremal exponents, Zephyr packet buffers and heatshrink (HARD), plus lwjson, RFM9x, SFUD and nanopb proto3 (MEDIUM). From batch 27 (spec-dense implementation tasks) come one HARD and one MEDIUM; that batch is winding down after 30 tasks gave 1 HARD and 3 MEDIUM. Each task comes from the commit Qwen measured, its digest matches, and it is still present at the batch branch head. Agent timeouts stay excluded: a pyserial attempt timed out on Qwen's own blocking read yet graded PASS. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
131 lines
5.9 KiB
Python
131 lines
5.9 KiB
Python
"""Run an imported Bos-qiao check.py unchanged; candidate processes run as UID 65534."""
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
GRADER = Path(__file__).resolve().parent
|
|
BLOCKED = "isolation-blocked.txt"
|
|
GUARD = 86
|
|
# Runs the original check unchanged as the trusted user. In isolation mode every child process drops to
|
|
# UID 65534 and every mkdtemp scratch is handed to it; an audit hook stops the check if candidate code
|
|
# would run inside this process. Failing child processes also copy their stderr tail out for private triage.
|
|
CHECK = (
|
|
"import os, runpy, subprocess, sys, tempfile\n"
|
|
"isolated, sys.argv = sys.argv[1] == '1', sys.argv[2:]\n"
|
|
"untrusted = [os.path.realpath(sys.argv[1])]\n"
|
|
"def inside(path):\n"
|
|
" path = os.path.realpath(str(path))\n"
|
|
" return any(path == root or path.startswith(root + os.sep) for root in untrusted)\n"
|
|
"spawns = ('os.system', 'os.exec', 'os.spawn', 'os.fork', 'os.forkpty', 'os.posix_spawn') if isolated else ()\n"
|
|
"def guard(event, args):\n"
|
|
" if (event in spawns or event == 'exec' and inside(getattr(args[0], 'co_filename', '')) or\n"
|
|
" event == 'ctypes.dlopen' and args[0] and inside(args[0]) or\n"
|
|
" event == 'import' and any(inside(entry or '.') for entry in sys.path)):\n"
|
|
" sys.__stderr__.write('rollout-guard: ' + event + '\\n')\n"
|
|
" sys.__stderr__.flush()\n"
|
|
" os._exit(%d)\n"
|
|
"mkdtemp = tempfile.mkdtemp\n"
|
|
"def scratch(*args, **kwargs):\n"
|
|
" path = mkdtemp(*args, **kwargs)\n"
|
|
" untrusted.append(os.path.realpath(path))\n"
|
|
" if isolated:\n"
|
|
" os.chown(path, 65534, 65534)\n"
|
|
" return path\n"
|
|
"tempfile.mkdtemp = scratch\n"
|
|
"popen = subprocess.Popen.__init__\n"
|
|
"def drop(self, *args, **kwargs):\n"
|
|
" if isolated:\n"
|
|
" kwargs.update(user=65534, group=65534, extra_groups=[])\n"
|
|
" popen(self, *args, **kwargs)\n"
|
|
"subprocess.Popen.__init__ = drop\n"
|
|
"real = subprocess.run\n"
|
|
"def run(*args, **kwargs):\n"
|
|
" result = real(*args, **kwargs)\n"
|
|
" if result.returncode and result.stderr:\n"
|
|
" tail = result.stderr if isinstance(result.stderr, str) else result.stderr.decode('utf-8', 'replace')\n"
|
|
" sys.__stderr__.write(tail[-2000:] + '\\n')\n"
|
|
" return result\n"
|
|
"subprocess.run = run\n"
|
|
"sys.addaudithook(guard)\n"
|
|
"runpy.run_path(sys.argv[0], run_name='__main__')\n" % GUARD)
|
|
|
|
|
|
class UnsupportedEnvironment(RuntimeError):
|
|
pass
|
|
|
|
|
|
def candidate_user():
|
|
os.setgroups([])
|
|
os.setgid(65534)
|
|
os.setuid(65534)
|
|
|
|
|
|
def isolation():
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") != "uid65534":
|
|
return None
|
|
if sys.platform != "linux" or os.geteuid() != 0:
|
|
raise UnsupportedEnvironment("isolated grader requires a Linux root supervisor")
|
|
probe = (
|
|
"from pathlib import Path\n"
|
|
"try: Path('/grader/verify.py').read_bytes()\n"
|
|
"except PermissionError: raise SystemExit(0)\n"
|
|
"raise SystemExit(1)\n"
|
|
)
|
|
try:
|
|
result = subprocess.run([sys.executable, "-I", "-c", probe],
|
|
capture_output=True, timeout=10, preexec_fn=candidate_user)
|
|
except subprocess.SubprocessError as error:
|
|
raise UnsupportedEnvironment("candidate identity could not be isolated") from error
|
|
if result.returncode != 0:
|
|
raise UnsupportedEnvironment("candidate can read the private grader or isolation probe failed")
|
|
return candidate_user
|
|
|
|
|
|
def verify(candidate):
|
|
if (GRADER / BLOCKED).exists():
|
|
raise UnsupportedEnvironment("isolation blocked: " + (GRADER / BLOCKED).read_text().strip()[:200])
|
|
isolated = isolation() is not None
|
|
if shutil.which("gcc") is None:
|
|
raise UnsupportedEnvironment("C compiler unavailable")
|
|
with tempfile.TemporaryDirectory(prefix="bos-grade-") as temporary:
|
|
root = Path(temporary)
|
|
work, grader = root / "work", root / "grader"
|
|
shutil.copytree(candidate, work, symlinks=True)
|
|
shutil.copytree(GRADER, grader, ignore=lambda folder, names: ["verify.py"] if Path(folder) == GRADER else [])
|
|
if isolated:
|
|
os.chmod(root, 0o711)
|
|
os.chmod(grader, 0o700)
|
|
for path in (work, *work.rglob("*")):
|
|
os.lchown(path, 65534, 65534)
|
|
# shortcut: the check runs as root and follows symlinks a child plants in shared scratch directories;
|
|
# upgrade by running the check under its own unprivileged UID once a setuid helper is available.
|
|
completed = subprocess.run([sys.executable, "-I", "-B", "-c", CHECK, "1" if isolated else "0",
|
|
str(grader / "check.py"), str(work)],
|
|
cwd=root, capture_output=True, text=True, timeout=100)
|
|
# grade.stderr stays on the trusted side with the archive; it may quote hidden test lines.
|
|
sys.stderr.write(completed.stderr[-4000:])
|
|
if completed.returncode == GUARD and "rollout-guard: " in completed.stderr:
|
|
raise UnsupportedEnvironment("isolation blocked: the check would run candidate code in the grader process")
|
|
lines = completed.stdout.splitlines()
|
|
if completed.returncode == 0 and lines[-1:] == ["PASS"]:
|
|
return {"status": "PASS", "checks": 1}
|
|
return {"status": "FAIL", "checks": 1, "reason": (lines[-1] if lines else "no checker output")[:300]}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
result = verify(Path(sys.argv[1]))
|
|
except UnsupportedEnvironment as error:
|
|
result = {"status": "UNVERIFIED", "checks": 0, "reason": str(error)}
|
|
except subprocess.TimeoutExpired:
|
|
result = {"status": "FAIL", "checks": 1, "reason": "timeout"}
|
|
except Exception as error:
|
|
result = {"status": "INFRA_ERROR", "checks": 0, "reason": str(error)}
|
|
print(json.dumps(result, sort_keys=True))
|
|
sys.exit({"PASS": 0, "FAIL": 1}.get(result["status"], 2))
|