One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
460 lines
20 KiB
Python
460 lines
20 KiB
Python
"""Independent probe observer. Candidate C++ is compiled and run only by child processes."""
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import uuid
|
|
import zlib
|
|
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
DEADLINE = 12.0
|
|
RECOVERY = 3.0
|
|
CLEANUP_CODE = "import shutil,sys; shutil.rmtree(sys.argv[1])"
|
|
ABSENCE_CODE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: raise SystemExit(0)\nraise SystemExit(1)\n"
|
|
STAGES = []
|
|
STAGE_FIELDS = (
|
|
"role", "requested_phase", "requested_argv", "requested_cwd", "requested_input_len",
|
|
"requested_input_sha256", "wrapper_rc", "wire_len", "wire_sha256", "validated",
|
|
"phase", "supervisor_pid", "supervisor_start", "leader_pid", "leader_start",
|
|
"uid", "gid", "groups", "elapsed", "timeout", "rc", "leader_rc",
|
|
"leader_reaped", "descendant_absent", "exceptions", "signaled", "adopted",
|
|
"argv", "cwd", "input_len", "input_sha256", "stdout_len", "stdout_sha256",
|
|
"stderr_len", "stderr_sha256",
|
|
)
|
|
|
|
|
|
def emit_evidence():
|
|
document = {"version": 1, "fields": STAGE_FIELDS,
|
|
"rows": [[record.get(field) for field in STAGE_FIELDS] for record in STAGES]}
|
|
raw = json.dumps(document, separators=(",", ":"), ensure_ascii=True).encode("ascii")
|
|
encoded = base64.b85encode(zlib.compress(raw, 9)).decode("ascii")
|
|
sys.stderr.write("STAGES1:" + encoded + "\n")
|
|
sys.stderr.flush()
|
|
return len(encoded) + 9
|
|
|
|
SUPERVISOR = r'''
|
|
import base64, ctypes, hashlib, json, math, os, pathlib, select, shutil, signal, subprocess, sys, time
|
|
|
|
def stat_of(pid):
|
|
try:
|
|
raw = pathlib.Path('/proc/%d/stat' % pid).read_text()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
end = raw.rfind(')')
|
|
if end < 0:
|
|
raise RuntimeError('bad proc stat')
|
|
items = raw[end+2:].split()
|
|
if len(items) < 20:
|
|
raise RuntimeError('short proc stat')
|
|
return {'pid':pid, 'state':items[0], 'ppid':int(items[1]), 'start':int(items[19])}
|
|
|
|
def table():
|
|
result = {}
|
|
for name in os.listdir('/proc'):
|
|
if not name.isdecimal():
|
|
continue
|
|
item = stat_of(int(name))
|
|
if item is not None:
|
|
result[item['pid']] = item
|
|
return result
|
|
|
|
def descendants(parent, snapshot):
|
|
own = stat_of(parent)
|
|
if own is None or own['state'] in ('Z','X'):
|
|
raise RuntimeError('supervisor absent')
|
|
selected = {parent: own['start']}
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for pid, item in snapshot.items():
|
|
if item['state'] in ('Z','X') or pid in selected:
|
|
continue
|
|
ancestor = item['ppid']
|
|
known = selected.get(ancestor)
|
|
ancestor_item = snapshot.get(ancestor)
|
|
if known is not None and ancestor_item is not None and ancestor_item['start'] == known and ancestor_item['state'] not in ('Z','X'):
|
|
selected[pid] = item['start']
|
|
changed = True
|
|
selected.pop(parent, None)
|
|
return selected
|
|
|
|
def fresh_kill(tokens):
|
|
signaled = []
|
|
for pid, start in tokens.items():
|
|
current = stat_of(pid)
|
|
if current is None or current['start'] != start or current['state'] in ('Z','X'):
|
|
continue
|
|
try:
|
|
os.kill(pid, signal.SIGKILL)
|
|
signaled.append([pid,start])
|
|
except ProcessLookupError:
|
|
pass
|
|
return signaled
|
|
|
|
def reap_adopted(leader):
|
|
found = []
|
|
if leader.poll() is None:
|
|
return found
|
|
while True:
|
|
try:
|
|
pid, status = os.waitpid(-1, os.WNOHANG)
|
|
except ChildProcessError:
|
|
break
|
|
if pid == 0:
|
|
break
|
|
if pid == leader.pid:
|
|
raise RuntimeError('leader reaped outside Popen')
|
|
found.append([pid,status])
|
|
return found
|
|
|
|
def execute(argv, cwd, input_bytes, deadline):
|
|
start = time.monotonic()
|
|
libc = ctypes.CDLL(None, use_errno=True)
|
|
if libc.prctl(36,1,0,0,0) != 0:
|
|
raise RuntimeError('subreaper setup failed')
|
|
leader = subprocess.Popen(argv,cwd=cwd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,start_new_session=True)
|
|
initial = stat_of(leader.pid)
|
|
if initial is None:
|
|
raise RuntimeError('leader vanished before identity capture')
|
|
output = b''
|
|
errors = b''
|
|
timed_out = False
|
|
exceptions = []
|
|
adopted = []
|
|
signaled = []
|
|
try:
|
|
try:
|
|
output, errors = leader.communicate(input=input_bytes, timeout=deadline)
|
|
except subprocess.TimeoutExpired as error:
|
|
timed_out = True
|
|
output = error.output or b''
|
|
errors = error.stderr or b''
|
|
except BaseException as error:
|
|
exceptions.append(type(error).__name__ + ':' + str(error))
|
|
finally:
|
|
until = time.monotonic() + 2.0
|
|
while True:
|
|
leader.poll()
|
|
try:
|
|
tokens = descendants(os.getpid(), table())
|
|
except BaseException as error:
|
|
exceptions.append('scan:' + type(error).__name__ + ':' + str(error))
|
|
tokens = {}
|
|
try:
|
|
signaled += fresh_kill(tokens)
|
|
except BaseException as error:
|
|
exceptions.append('signal:' + type(error).__name__ + ':' + str(error))
|
|
if leader.poll() is not None:
|
|
try:
|
|
adopted += reap_adopted(leader)
|
|
except BaseException as error:
|
|
exceptions.append('reap:' + type(error).__name__ + ':' + str(error))
|
|
if leader.poll() is not None and not tokens:
|
|
break
|
|
if time.monotonic() >= until:
|
|
exceptions.append('descendant cleanup deadline')
|
|
break
|
|
time.sleep(0.01)
|
|
try:
|
|
if leader.poll() is None:
|
|
leader.wait(timeout=0.1)
|
|
more_out, more_err = leader.communicate(timeout=0.1)
|
|
if not output:
|
|
output = more_out
|
|
if not errors:
|
|
errors = more_err
|
|
except BaseException as error:
|
|
exceptions.append('final communicate:' + type(error).__name__ + ':' + str(error))
|
|
try:
|
|
remaining = descendants(os.getpid(), table())
|
|
except BaseException as error:
|
|
exceptions.append('final scan:' + type(error).__name__ + ':' + str(error))
|
|
remaining = {'unknown':0}
|
|
leader_rc = leader.poll()
|
|
return {'phase':'execute','supervisor_pid':os.getpid(),
|
|
'supervisor_start':stat_of(os.getpid())['start'],'leader_pid':leader.pid,
|
|
'leader_start':initial['start'],'uid':os.getuid(),'gid':os.getgid(),
|
|
'groups':os.getgroups(),'elapsed':time.monotonic()-start,
|
|
'timeout':timed_out,'rc':124 if timed_out else leader_rc,
|
|
'leader_rc':leader_rc,'leader_reaped':leader_rc is not None,
|
|
'descendant_absent':not bool(remaining),'exceptions':exceptions,
|
|
'signaled':signaled,'adopted':adopted,
|
|
'argv':argv,'cwd':cwd,'input_len':len(input_bytes),
|
|
'input_sha256':hashlib.sha256(input_bytes).hexdigest(),
|
|
'stdout_len':len(output),'stdout_sha256':hashlib.sha256(output).hexdigest(),
|
|
'stderr_len':len(errors),'stderr_sha256':hashlib.sha256(errors).hexdigest(),
|
|
'stdout_b64':base64.b64encode(output).decode('ascii'),
|
|
'stderr_b64':base64.b64encode(errors).decode('ascii')}
|
|
|
|
def main():
|
|
config=json.loads(sys.argv[1])
|
|
mode=config['mode']
|
|
if mode not in ('execute','cleanup','absence'):
|
|
raise RuntimeError('invalid supervisor mode')
|
|
record=execute(config['argv'],config['cwd'],sys.stdin.buffer.read(),config['deadline'])
|
|
record['phase']=mode
|
|
record['nonce']=config['nonce']
|
|
sys.stdout.write(json.dumps(record,separators=(',',':'))+'\n')
|
|
|
|
if __name__=='__main__':
|
|
main()
|
|
'''
|
|
|
|
|
|
def exact_int(value, minimum=None):
|
|
if type(value) is not int:
|
|
return False
|
|
return minimum is None or value >= minimum
|
|
|
|
|
|
def run_supervisor(config, input_bytes=b"", role="unknown"):
|
|
config = dict(config, nonce=uuid.uuid4().hex)
|
|
stage = {"role": role, "requested_phase": config["mode"],
|
|
"requested_argv": config["argv"], "requested_cwd": config["cwd"],
|
|
"requested_input_len": len(input_bytes),
|
|
"requested_input_sha256": hashlib.sha256(input_bytes).hexdigest(),
|
|
"validated": False}
|
|
STAGES.append(stage)
|
|
command = [sys.executable, "-I", "-c", SUPERVISOR, json.dumps(config, separators=(",", ":"))]
|
|
result = subprocess.run(command, input=input_bytes, capture_output=True, timeout=DEADLINE + RECOVERY + 8)
|
|
raw = result.stdout
|
|
stage.update(wrapper_rc=result.returncode, wire_len=len(raw), wire_sha256=hashlib.sha256(raw).hexdigest())
|
|
if result.returncode != 0:
|
|
raise RuntimeError("supervisor rc=%s raw=%s" % (result.returncode, hashlib.sha256(raw).hexdigest()))
|
|
wire = json.loads(raw)
|
|
if not isinstance(wire, dict):
|
|
raise RuntimeError("supervisor wire is not an object")
|
|
stage.update({field: wire.get(field) for field in STAGE_FIELDS if field in wire})
|
|
phase = config["mode"]
|
|
if wire.get("phase") != phase:
|
|
raise RuntimeError("supervisor phase mismatch")
|
|
if wire.get("nonce") != config["nonce"]:
|
|
raise RuntimeError("supervisor nonce mismatch")
|
|
if not exact_int(wire.get("supervisor_pid"), 1) or not exact_int(wire.get("leader_pid"), 1):
|
|
raise RuntimeError("supervisor pid metadata")
|
|
if wire["leader_pid"] == wire["supervisor_pid"]:
|
|
raise RuntimeError("supervisor and leader identity collide")
|
|
if not exact_int(wire.get("supervisor_start"), 1) or not exact_int(wire.get("leader_start"), 1):
|
|
raise RuntimeError("leader start metadata")
|
|
if wire.get("argv") != config["argv"] or wire.get("cwd") != config["cwd"]:
|
|
raise RuntimeError("execution argv/cwd mismatch")
|
|
if not exact_int(wire.get("input_len"), 0) or wire["input_len"] != len(input_bytes) or wire.get("input_sha256") != hashlib.sha256(input_bytes).hexdigest():
|
|
raise RuntimeError("execution input mismatch")
|
|
uid = wire.get("uid")
|
|
gid = wire.get("gid")
|
|
groups = wire.get("groups")
|
|
if not exact_int(uid, 0) or not exact_int(gid, 0) or type(groups) is not list or any(not exact_int(x, 0) for x in groups):
|
|
raise RuntimeError("identity metadata")
|
|
expected_identity = ((65534, 65534, []) if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534"
|
|
else (os.getuid(), os.getgid(), os.getgroups()))
|
|
if (uid, gid, groups) != expected_identity:
|
|
raise RuntimeError("candidate identity mismatch")
|
|
elapsed = wire.get("elapsed")
|
|
if type(elapsed) not in (int, float) or not math.isfinite(elapsed) or elapsed < 0 or elapsed > config["deadline"] + RECOVERY:
|
|
raise RuntimeError("elapsed metadata")
|
|
if type(wire.get("timeout")) is not bool or type(wire.get("leader_reaped")) is not bool or type(wire.get("descendant_absent")) is not bool:
|
|
raise RuntimeError("execution boolean metadata")
|
|
if not exact_int(wire.get("rc")) or not exact_int(wire.get("leader_rc")) or not wire["leader_reaped"] or not wire["descendant_absent"]:
|
|
raise RuntimeError("leader or descendant metadata")
|
|
if wire["rc"] != (124 if wire["timeout"] else wire["leader_rc"]):
|
|
raise RuntimeError("timeout/leader rc mismatch")
|
|
if type(wire.get("exceptions")) is not list or wire["exceptions"] or wire["timeout"]:
|
|
raise RuntimeError("supervisor exception or timeout")
|
|
output = {}
|
|
for field in ("stdout", "stderr"):
|
|
length = wire.get(field + "_len")
|
|
digest = wire.get(field + "_sha256")
|
|
if not exact_int(length, 0) or type(digest) is not str or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest):
|
|
raise RuntimeError("IO metadata missing")
|
|
encoded = wire.get(field + "_b64")
|
|
if type(encoded) is not str:
|
|
raise RuntimeError(field + " wire missing")
|
|
data = base64.b64decode(encoded, validate=True)
|
|
if len(data) != length or hashlib.sha256(data).hexdigest() != digest:
|
|
raise RuntimeError(field + " metadata mismatch")
|
|
output[field] = data
|
|
stage["validated"] = True
|
|
return wire, output["stdout"], output["stderr"]
|
|
|
|
|
|
def expected_row(identifier, fault, token):
|
|
if fault == "A":
|
|
return "%d|unavailable|-|0|0|0|AX|-|-" % identifier
|
|
sent = ("ping:" + token).encode().hex()
|
|
if fault == "S":
|
|
return "%d|send_error|-|1|1|0|AOSXC|-|-" % identifier
|
|
if fault == "R":
|
|
return "%d|receive_error|-|1|1|0|AOSTRXC|%s|-" % (identifier, sent)
|
|
reply = ("pong:" + token).encode().hex()
|
|
return "%d|ok|%s|1|1|0|AOSTRDC|%s|%s" % (identifier, reply, sent, reply)
|
|
|
|
|
|
def expected_batch(rows):
|
|
counts = [len(rows), 0, 0, 0, 0, 0, 0, 0]
|
|
lines = []
|
|
for identifier, fault, token in rows:
|
|
lines.append(expected_row(identifier, fault, token))
|
|
if fault != "A":
|
|
counts[1] += 1
|
|
counts[2] += 1
|
|
counts[{"N": 4, "A": 5, "S": 6, "R": 7}[fault]] += 1
|
|
lines.append("batch|" + "|".join(map(str, counts)))
|
|
return ("\n".join(lines) + "\n").encode()
|
|
|
|
|
|
def decimal_token(raw, maximum):
|
|
if raw.startswith(b"+"):
|
|
raw = raw[1:]
|
|
if not raw or any(byte < 48 or byte > 57 for byte in raw):
|
|
raise ValueError("invalid ASCII decimal token")
|
|
significant = raw.lstrip(b"0") or b"0"
|
|
if len(significant) > len(str(maximum)):
|
|
raise ValueError("decimal value out of range")
|
|
number = int(significant)
|
|
if number > maximum:
|
|
raise ValueError("decimal value out of range")
|
|
return number
|
|
|
|
|
|
def output_equivalent(actual, expected):
|
|
def lines(data):
|
|
if not data.endswith(b"\n"):
|
|
raise ValueError("missing final line end")
|
|
rows = data[:-1].split(b"\n")
|
|
return [row[:-1] if row.endswith(b"\r") else row for row in rows]
|
|
|
|
actual_lines = lines(actual)
|
|
expected_lines = lines(expected)
|
|
if len(actual_lines) != len(expected_lines):
|
|
return False
|
|
for index, (got_line, want_line) in enumerate(zip(actual_lines, expected_lines)):
|
|
got = got_line.split(b"|")
|
|
want = want_line.split(b"|")
|
|
if len(got) != 9 or len(want) != 9:
|
|
return False
|
|
numeric = {1, 2, 3, 4, 5, 6, 7, 8} if index == len(expected_lines) - 1 else {0, 3, 4, 5}
|
|
maxima = {0: 100000, 3: 1, 4: 1, 5: 1}
|
|
for column, (value, target) in enumerate(zip(got, want)):
|
|
if column in numeric:
|
|
maximum = 128 if index == len(expected_lines) - 1 else maxima[column]
|
|
try:
|
|
if decimal_token(value, maximum) != decimal_token(target, maximum):
|
|
return False
|
|
except ValueError:
|
|
return False
|
|
elif value != target:
|
|
return False
|
|
return True
|
|
|
|
|
|
def input_batch(rows):
|
|
lines = [str(len(rows))]
|
|
for identifier, fault, token in rows:
|
|
lines.append("%d %s %s" % (identifier, fault, token.encode().hex()))
|
|
return ("\n".join(lines) + "\n").encode()
|
|
|
|
|
|
def cases():
|
|
generator = random.Random(750321)
|
|
alphabet = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789!@_"
|
|
groups = [[], [(11, "N", "alpha")], [(12, "S", "beta")], [(13, "R", "gamma")], [(14, "A", "delta")]]
|
|
for size in (4, 9, 18, 32):
|
|
rows = []
|
|
for index in range(size):
|
|
token = "".join(generator.choice(alphabet) for _ in range(generator.randrange(1, 65)))
|
|
rows.append((1000 + size * 100 + index, generator.choice("NASR"), token))
|
|
groups.append(rows)
|
|
groups.append([(9000 + index, "NASR"[index % 4], "Z" * (index % 64 + 1)) for index in range(64)])
|
|
return groups
|
|
|
|
|
|
def grade(candidate):
|
|
support = json.loads((HERE / "support.json").read_text())
|
|
source = (Path(candidate) / "src/probe.cpp").read_bytes()
|
|
scratch = Path(tempfile.mkdtemp(prefix="probe-grade-"))
|
|
good = False
|
|
problems = []
|
|
checks = 0
|
|
try:
|
|
work = scratch / "work"
|
|
(work / "src").mkdir(parents=True)
|
|
(work / "include").mkdir()
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534":
|
|
for directory in (work, work / "src", work / "include"):
|
|
os.chown(directory, 65534, 65534)
|
|
(work / "src/probe.cpp").write_bytes(source)
|
|
(work / "src/main.cpp").write_text(support["src/main.cpp"])
|
|
(work / "src/port.cpp").write_text(support["src/port.cpp"])
|
|
(work / "include/probe.hpp").write_text(support["include/probe.hpp"])
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534":
|
|
for path in (work, work / "src", work / "include", *work.rglob("*")):
|
|
os.chown(path, 65534, 65534)
|
|
compile_argv = ["g++", "-std=c++17", "-O2", "-Wall", "-Wextra", "-Wpedantic", "-Iinclude",
|
|
"src/main.cpp", "src/port.cpp", "src/probe.cpp", "-o", "probe-demo"]
|
|
config = {"mode": "execute", "argv": compile_argv, "cwd": str(work), "deadline": DEADLINE}
|
|
compile_wire, _, compile_err = run_supervisor(config, role="compile")
|
|
if compile_wire["rc"] != 0:
|
|
problems.append("compile rc=%d stderr=%s" % (compile_wire["rc"], compile_err.decode("utf-8", "replace")[-400:]))
|
|
else:
|
|
good = True
|
|
for rows in cases():
|
|
run_wire, output, error = run_supervisor({"mode": "execute", "argv": ["./probe-demo"],
|
|
"cwd": str(work), "deadline": DEADLINE},
|
|
input_batch(rows), role="candidate-%d" % (checks + 1))
|
|
checks += 1
|
|
if run_wire["rc"] != 0:
|
|
problems.append("candidate rc=%d stderr=%s" % (run_wire["rc"], error.decode("utf-8", "replace")[-300:]))
|
|
good = False
|
|
break
|
|
expected = expected_batch(rows)
|
|
if not output_equivalent(output, expected):
|
|
problems.append("normal exit observation mismatch case=%d actual_sha=%s expected_sha=%s" %
|
|
(checks, hashlib.sha256(output).hexdigest(), hashlib.sha256(expected).hexdigest()))
|
|
good = False
|
|
break
|
|
except BaseException as error:
|
|
problems.append("execution exception " + type(error).__name__ + ":" + str(error))
|
|
good = False
|
|
finally:
|
|
try:
|
|
cleanup, _, _ = run_supervisor({"mode": "cleanup", "argv": [sys.executable, "-I", "-c", CLEANUP_CODE, str(scratch)],
|
|
"cwd": "/tmp", "deadline": DEADLINE}, role="cleanup")
|
|
if cleanup["rc"] != 0:
|
|
problems.append("cleanup rc=%s" % cleanup["rc"])
|
|
good = False
|
|
except BaseException as error:
|
|
problems.append("cleanup exception " + type(error).__name__ + ":" + str(error))
|
|
good = False
|
|
try:
|
|
absence, _, _ = run_supervisor({"mode": "absence", "argv": [sys.executable, "-I", "-c", ABSENCE_CODE, str(scratch)],
|
|
"cwd": "/tmp", "deadline": DEADLINE}, role="absence")
|
|
if absence["rc"] != 0:
|
|
problems.append("absence rc=%s" % absence["rc"])
|
|
good = False
|
|
except BaseException as error:
|
|
problems.append("absence exception " + type(error).__name__ + ":" + str(error))
|
|
good = False
|
|
evidence_length = emit_evidence()
|
|
if evidence_length >= (3500 if good and not problems else 1990):
|
|
problems.append("stage evidence exceeds Bos retained tail")
|
|
good = False
|
|
if good and not problems:
|
|
print("PASS")
|
|
return 0
|
|
print("FAIL " + "; ".join(problems)[:1500])
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(grade(sys.argv[1]))
|