amos 60319fadd5 Merge only Qwen-hard and -medium tasks: 11 from batches 25, 26 and 27
The user wants 2000 tasks, most of them ones Qwen cannot solve. New batch
tasks are now measured on their branch first (3 Qwen trials each), and only
HARD or MEDIUM tasks reach main; ROUTINE ones stay on the batch branches.
A session cut by context_limit or output_limit counts as a failure for
difficulty (user decision 2026-11-03); the trajectory stays incomplete.

- Batch 26 (route D, real upstream fixes in embedded open-source projects):
  7 HARD of 20 (coreJSON, littlefs, CANopen, coreMQTT, LwM2M, Zephyr ring,
  lwIP SNTP). Most failures are Qwen exhausting its context in the larger
  workspaces.
- Batch 25 (HWE mutation repairs): 1 HARD and 1 MEDIUM of 56.
- Batch 27 (spec-dense implementation tasks): 2 MEDIUM of the first 10.

Each task comes from the exact commit Qwen measured, and its digest matches
the measured one. Failure reasons are not yet reviewed, so difficulty is
the automatic label.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 10:51:08 +08:00

131 lines
5.9 KiB
Python

"""Run an imported Bos-qiao check.py unchanged; candidate processes run as UID 65534."""
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
GRADER = Path(__file__).resolve().parent
BLOCKED = "isolation-blocked.txt"
GUARD = 86
# Runs the original check unchanged as the trusted user. In isolation mode every child process drops to
# UID 65534 and every mkdtemp scratch is handed to it; an audit hook stops the check if candidate code
# would run inside this process. Failing child processes also copy their stderr tail out for private triage.
CHECK = (
"import os, runpy, subprocess, sys, tempfile\n"
"isolated, sys.argv = sys.argv[1] == '1', sys.argv[2:]\n"
"untrusted = [os.path.realpath(sys.argv[1])]\n"
"def inside(path):\n"
" path = os.path.realpath(str(path))\n"
" return any(path == root or path.startswith(root + os.sep) for root in untrusted)\n"
"spawns = ('os.system', 'os.exec', 'os.spawn', 'os.fork', 'os.forkpty', 'os.posix_spawn') if isolated else ()\n"
"def guard(event, args):\n"
" if (event in spawns or event == 'exec' and inside(getattr(args[0], 'co_filename', '')) or\n"
" event == 'ctypes.dlopen' and args[0] and inside(args[0]) or\n"
" event == 'import' and any(inside(entry or '.') for entry in sys.path)):\n"
" sys.__stderr__.write('rollout-guard: ' + event + '\\n')\n"
" sys.__stderr__.flush()\n"
" os._exit(%d)\n"
"mkdtemp = tempfile.mkdtemp\n"
"def scratch(*args, **kwargs):\n"
" path = mkdtemp(*args, **kwargs)\n"
" untrusted.append(os.path.realpath(path))\n"
" if isolated:\n"
" os.chown(path, 65534, 65534)\n"
" return path\n"
"tempfile.mkdtemp = scratch\n"
"popen = subprocess.Popen.__init__\n"
"def drop(self, *args, **kwargs):\n"
" if isolated:\n"
" kwargs.update(user=65534, group=65534, extra_groups=[])\n"
" popen(self, *args, **kwargs)\n"
"subprocess.Popen.__init__ = drop\n"
"real = subprocess.run\n"
"def run(*args, **kwargs):\n"
" result = real(*args, **kwargs)\n"
" if result.returncode and result.stderr:\n"
" tail = result.stderr if isinstance(result.stderr, str) else result.stderr.decode('utf-8', 'replace')\n"
" sys.__stderr__.write(tail[-2000:] + '\\n')\n"
" return result\n"
"subprocess.run = run\n"
"sys.addaudithook(guard)\n"
"runpy.run_path(sys.argv[0], run_name='__main__')\n" % GUARD)
class UnsupportedEnvironment(RuntimeError):
pass
def candidate_user():
os.setgroups([])
os.setgid(65534)
os.setuid(65534)
def isolation():
if os.environ.get("ROLLOUT_GRADE_ISOLATION") != "uid65534":
return None
if sys.platform != "linux" or os.geteuid() != 0:
raise UnsupportedEnvironment("isolated grader requires a Linux root supervisor")
probe = (
"from pathlib import Path\n"
"try: Path('/grader/verify.py').read_bytes()\n"
"except PermissionError: raise SystemExit(0)\n"
"raise SystemExit(1)\n"
)
try:
result = subprocess.run([sys.executable, "-I", "-c", probe],
capture_output=True, timeout=10, preexec_fn=candidate_user)
except subprocess.SubprocessError as error:
raise UnsupportedEnvironment("candidate identity could not be isolated") from error
if result.returncode != 0:
raise UnsupportedEnvironment("candidate can read the private grader or isolation probe failed")
return candidate_user
def verify(candidate):
if (GRADER / BLOCKED).exists():
raise UnsupportedEnvironment("isolation blocked: " + (GRADER / BLOCKED).read_text().strip()[:200])
isolated = isolation() is not None
if shutil.which("gcc") is None:
raise UnsupportedEnvironment("C compiler unavailable")
with tempfile.TemporaryDirectory(prefix="bos-grade-") as temporary:
root = Path(temporary)
work, grader = root / "work", root / "grader"
shutil.copytree(candidate, work, symlinks=True)
shutil.copytree(GRADER, grader, ignore=lambda folder, names: ["verify.py"] if Path(folder) == GRADER else [])
if isolated:
os.chmod(root, 0o711)
os.chmod(grader, 0o700)
for path in (work, *work.rglob("*")):
os.lchown(path, 65534, 65534)
# shortcut: the check runs as root and follows symlinks a child plants in shared scratch directories;
# upgrade by running the check under its own unprivileged UID once a setuid helper is available.
completed = subprocess.run([sys.executable, "-I", "-B", "-c", CHECK, "1" if isolated else "0",
str(grader / "check.py"), str(work)],
cwd=root, capture_output=True, text=True, timeout=100)
# grade.stderr stays on the trusted side with the archive; it may quote hidden test lines.
sys.stderr.write(completed.stderr[-4000:])
if completed.returncode == GUARD and "rollout-guard: " in completed.stderr:
raise UnsupportedEnvironment("isolation blocked: the check would run candidate code in the grader process")
lines = completed.stdout.splitlines()
if completed.returncode == 0 and lines[-1:] == ["PASS"]:
return {"status": "PASS", "checks": 1}
return {"status": "FAIL", "checks": 1, "reason": (lines[-1] if lines else "no checker output")[:300]}
if __name__ == "__main__":
try:
result = verify(Path(sys.argv[1]))
except UnsupportedEnvironment as error:
result = {"status": "UNVERIFIED", "checks": 0, "reason": str(error)}
except subprocess.TimeoutExpired:
result = {"status": "FAIL", "checks": 1, "reason": "timeout"}
except Exception as error:
result = {"status": "INFRA_ERROR", "checks": 0, "reason": str(error)}
print(json.dumps(result, sort_keys=True))
sys.exit({"PASS": 0, "FAIL": 1}.get(result["status"], 2))