One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
989 lines
43 KiB
Python
989 lines
43 KiB
Python
"""Private, independent C11 behavior oracle for the bounded buffer task."""
|
|
|
|
import base64
|
|
import ctypes
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import zlib
|
|
|
|
|
|
SUPERVISOR = r'''
|
|
import base64, ctypes, hashlib, json, os, selectors, signal, subprocess, sys, time
|
|
|
|
def stat_of(pid):
|
|
try:
|
|
raw = open('/proc/%d/stat' % pid, 'rb').read()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
tail = raw[raw.rfind(b')') + 2:].split()
|
|
if len(tail) < 20:
|
|
raise RuntimeError('short proc stat')
|
|
return (tail[0].decode('ascii'), int(tail[1]), int(tail[19]),
|
|
int(tail[2]), int(tail[3]))
|
|
|
|
def table_now():
|
|
result = {}
|
|
for word in os.listdir('/proc'):
|
|
if not word.isdecimal():
|
|
continue
|
|
pid = int(word)
|
|
row = stat_of(pid)
|
|
if row is not None:
|
|
result[pid] = row
|
|
return result
|
|
|
|
def alive(row):
|
|
return row is not None and row[0] not in ('Z', 'X')
|
|
|
|
def closure(table, own, tracked):
|
|
roots = {own[0]: own[1]}
|
|
for pid, start in tracked.items():
|
|
row = table.get(pid)
|
|
if row is not None and row[2] == start and alive(row):
|
|
roots[pid] = start
|
|
output = {}
|
|
again = True
|
|
while again:
|
|
again = False
|
|
for pid, row in table.items():
|
|
if pid == own[0] or not alive(row):
|
|
continue
|
|
if row[1] in roots and pid not in output:
|
|
output[pid] = row[2]
|
|
roots[pid] = row[2]
|
|
again = True
|
|
return output
|
|
|
|
def fresh_signal(pid, start, errors):
|
|
try:
|
|
row = stat_of(pid)
|
|
if row is not None and row[2] == start and alive(row):
|
|
os.kill(pid, signal.SIGKILL)
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
pass
|
|
except Exception as exc:
|
|
errors.append('signal:%s:%s' % (type(exc).__name__, exc))
|
|
|
|
def drain(selector, buffers, delay):
|
|
for key, _ in selector.select(delay):
|
|
try:
|
|
data = os.read(key.fd, 65536)
|
|
except BlockingIOError:
|
|
continue
|
|
if data:
|
|
buffers[key.data].extend(data)
|
|
else:
|
|
selector.unregister(key.fileobj)
|
|
key.fileobj.close()
|
|
|
|
def reap_adopted(errors):
|
|
while True:
|
|
try:
|
|
pid, _ = os.waitpid(-1, os.WNOHANG)
|
|
except ChildProcessError:
|
|
return
|
|
except Exception as exc:
|
|
errors.append('reap:%s:%s' % (type(exc).__name__, exc))
|
|
return
|
|
if pid == 0:
|
|
return
|
|
|
|
def execute(argv, cwd, stdin_path, deadline):
|
|
errors = []
|
|
out, err = bytearray(), bytearray()
|
|
selector = selectors.DefaultSelector()
|
|
leader = None
|
|
leader_start = None
|
|
leader_parent = None
|
|
leader_pgid = None
|
|
leader_sid = None
|
|
input_length = None
|
|
input_sha256 = None
|
|
timed_out = False
|
|
beginning = time.monotonic()
|
|
own = (os.getpid(), stat_of(os.getpid())[2])
|
|
tracked = {}
|
|
stdin_file = None
|
|
try:
|
|
if ctypes.CDLL(None, use_errno=True).prctl(36, 1, 0, 0, 0) != 0:
|
|
raise OSError(ctypes.get_errno(), 'subreaper setup failed')
|
|
stdin_file = open(stdin_path, 'rb') if stdin_path != '-' else open(os.devnull, 'rb')
|
|
input_data = stdin_file.read()
|
|
input_length = len(input_data)
|
|
input_sha256 = hashlib.sha256(input_data).hexdigest()
|
|
stdin_file.seek(0)
|
|
leader = subprocess.Popen(argv, cwd=cwd, stdin=stdin_file,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
start_new_session=True)
|
|
row = stat_of(leader.pid)
|
|
if row is None:
|
|
raise RuntimeError('leader disappeared before start token')
|
|
leader_start = row[2]
|
|
leader_parent = row[1]
|
|
leader_pgid = row[3]
|
|
leader_sid = row[4]
|
|
tracked[leader.pid] = leader_start
|
|
for name, pipe in (('out', leader.stdout), ('err', leader.stderr)):
|
|
os.set_blocking(pipe.fileno(), False)
|
|
selector.register(pipe, selectors.EVENT_READ, name)
|
|
while True:
|
|
leader.poll()
|
|
table = table_now()
|
|
found = closure(table, own, tracked)
|
|
tracked.update(found)
|
|
elapsed = time.monotonic() - beginning
|
|
if elapsed >= deadline and not timed_out:
|
|
timed_out = True
|
|
if timed_out or leader.returncode is not None:
|
|
for pid, start in found.items():
|
|
fresh_signal(pid, start, errors)
|
|
drain(selector, {'out': out, 'err': err}, 0.03)
|
|
if leader.returncode is not None:
|
|
leader.wait()
|
|
reap_adopted(errors)
|
|
table = table_now()
|
|
found = closure(table, own, tracked)
|
|
tracked.update(found)
|
|
if not found and not selector.get_map():
|
|
break
|
|
if elapsed > deadline + 3.0:
|
|
errors.append('recovery_deadline')
|
|
break
|
|
except Exception as exc:
|
|
errors.append('execute:%s:%s' % (type(exc).__name__, exc))
|
|
finally:
|
|
if leader is not None:
|
|
recovery = time.monotonic() + 2.5
|
|
while time.monotonic() < recovery:
|
|
try:
|
|
leader.poll()
|
|
table = table_now()
|
|
found = closure(table, own, tracked)
|
|
tracked.update(found)
|
|
for pid, start in found.items():
|
|
fresh_signal(pid, start, errors)
|
|
drain(selector, {'out': out, 'err': err}, 0.01)
|
|
if leader.returncode is not None:
|
|
leader.wait()
|
|
reap_adopted(errors)
|
|
if not closure(table_now(), own, tracked) and not selector.get_map():
|
|
break
|
|
except Exception as exc:
|
|
errors.append('recovery:%s:%s' % (type(exc).__name__, exc))
|
|
break
|
|
if leader.returncode is None:
|
|
errors.append('leader_unreaped')
|
|
if stdin_file is not None:
|
|
stdin_file.close()
|
|
selector.close()
|
|
try:
|
|
absence = not closure(table_now(), own, tracked)
|
|
except Exception as exc:
|
|
errors.append('absence:%s:%s' % (type(exc).__name__, exc))
|
|
absence = False
|
|
elapsed = time.monotonic() - beginning
|
|
return {'supervisor_pid': own[0], 'supervisor_start': own[1],
|
|
'leader_pid': leader.pid if leader is not None else None,
|
|
'leader_start': leader_start,
|
|
'leader_parent': leader_parent, 'leader_pgid': leader_pgid,
|
|
'leader_sid': leader_sid, 'argv': argv, 'cwd': cwd,
|
|
'stdin_path': stdin_path, 'stdin_length': input_length,
|
|
'stdin_sha256': input_sha256,
|
|
'uid': os.getuid(), 'gid': os.getgid(), 'groups': os.getgroups(),
|
|
'elapsed': elapsed, 'deadline': deadline,
|
|
'timeout': timed_out, 'leader_rc': leader.returncode if leader is not None else None,
|
|
'rc': 124 if timed_out else (leader.returncode if leader is not None else None),
|
|
'leader_reaped': leader is not None and leader.returncode is not None,
|
|
'descendants_absent': absence, 'exceptions': errors,
|
|
'stdout_length': len(out), 'stderr_length': len(err),
|
|
'stdout_sha256': hashlib.sha256(out).hexdigest(),
|
|
'stderr_sha256': hashlib.sha256(err).hexdigest(),
|
|
'stdout_b64': base64.b64encode(out).decode('ascii'),
|
|
'stderr_b64': base64.b64encode(err).decode('ascii')}
|
|
|
|
try:
|
|
record = execute(json.loads(sys.argv[1]), sys.argv[2], sys.argv[3], float(sys.argv[4]))
|
|
sys.stdout.write(json.dumps(record, separators=(',', ':')) + '\n')
|
|
except Exception as exc:
|
|
sys.stderr.write('supervisor fatal %s: %s\n' % (type(exc).__name__, exc))
|
|
raise
|
|
'''
|
|
|
|
|
|
RESCUE = r'''
|
|
import json, os, signal, sys, time
|
|
target, start, parent, parent_start = map(int, sys.argv[1:5])
|
|
def row(pid):
|
|
try:
|
|
raw = open('/proc/%d/stat' % pid, 'rb').read()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
fields = raw[raw.rfind(b')') + 2:].split()
|
|
return (fields[0], int(fields[1]), int(fields[19]))
|
|
def live(item):
|
|
return item is not None and item[0] not in (b'Z', b'X')
|
|
if row(parent) is None or row(parent)[2] != parent_start:
|
|
raise SystemExit(2)
|
|
tracked = {target: start}
|
|
end = time.monotonic() + 4.0
|
|
adopted_seen = []
|
|
errors = []
|
|
while time.monotonic() < end:
|
|
table = {}
|
|
for word in os.listdir('/proc'):
|
|
if word.isdecimal():
|
|
pid = int(word)
|
|
item = row(pid)
|
|
if item is not None: table[pid] = item
|
|
roots = {pid for pid, token in tracked.items()
|
|
if pid in table and table[pid][2] == token and live(table[pid])}
|
|
for pid, item in table.items():
|
|
if pid == os.getpid() or pid == parent or not live(item):
|
|
continue
|
|
if item[1] == parent and item[2] >= start:
|
|
try:
|
|
owned = os.stat('/proc/%d' % pid).st_uid == os.getuid()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
owned = False
|
|
if owned:
|
|
roots.add(pid)
|
|
tracked[pid] = item[2]
|
|
adopted_seen.append((pid, item[2]))
|
|
found = set(roots)
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for pid, item in table.items():
|
|
if live(item) and item[1] in found and pid not in found:
|
|
found.add(pid)
|
|
tracked[pid] = item[2]
|
|
changed = True
|
|
for pid in found:
|
|
item = row(pid)
|
|
if live(item) and item[2] == tracked[pid]:
|
|
try: os.kill(pid, signal.SIGKILL)
|
|
except (FileNotFoundError, ProcessLookupError): pass
|
|
except Exception as exc: errors.append(type(exc).__name__)
|
|
if not found:
|
|
print(json.dumps({'absence': not errors, 'adopted_seen': adopted_seen,
|
|
'errors': errors, 'parent': parent,
|
|
'parent_start': parent_start}))
|
|
raise SystemExit(0 if not errors else 1)
|
|
time.sleep(0.02)
|
|
print(json.dumps({'absence': False, 'adopted_seen': adopted_seen,
|
|
'errors': errors, 'parent': parent,
|
|
'parent_start': parent_start}))
|
|
raise SystemExit(1)
|
|
'''
|
|
|
|
|
|
HARNESS = r'''
|
|
#include "dma_layout.h"
|
|
#include <inttypes.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
static dma_layout arena;
|
|
|
|
static void bytes_hex(const unsigned char *data, size_t length) {
|
|
for (size_t pos = 0; pos < length; ++pos) printf("%02x", data[pos]);
|
|
}
|
|
|
|
static size_t pointer_offset(const unsigned char *pointer) {
|
|
uintptr_t first = (uintptr_t)arena.raw;
|
|
uintptr_t value = (uintptr_t)pointer;
|
|
if (pointer == NULL || value < first || value >= first + DMA_RAW_BYTES) return 9999u;
|
|
return (size_t)(value - first);
|
|
}
|
|
|
|
static void dump(const char *name, dma_status status, size_t a, size_t b,
|
|
size_t c, size_t d, size_t e, const unsigned char *readback,
|
|
size_t readback_length) {
|
|
printf("%s %d %zu %zu %zu %zu %zu %zu %zu %zu ", name, (int)status,
|
|
a, b, c, d, e, arena.count, arena.cursor, dma_base_residue(&arena));
|
|
for (size_t index = 0; index < arena.count && index < DMA_MAX_SLOTS; ++index) {
|
|
const dma_slot *slot = dma_get_slot(&arena, index);
|
|
const unsigned char *pointer = dma_data(&arena, index);
|
|
if (slot == NULL) { printf("null;"); continue; }
|
|
printf("%zu:%zu:%zu:%u:%d:%zu:%zu;", slot->offset, slot->length,
|
|
slot->span, (unsigned)slot->tag, slot->active,
|
|
pointer_offset(pointer), pointer == NULL ? 9999u :
|
|
(size_t)((uintptr_t)pointer % DMA_QUANTUM));
|
|
}
|
|
printf(" ");
|
|
if (arena.prefix <= DMA_RAW_BYTES && arena.capacity <= DMA_RAW_BYTES - arena.prefix)
|
|
bytes_hex(arena.raw + arena.prefix, arena.capacity);
|
|
printf(" ");
|
|
if (readback == NULL) printf("-");
|
|
else bytes_hex(readback, readback_length);
|
|
putchar('\n');
|
|
}
|
|
|
|
static unsigned char pattern(size_t seed, size_t index) {
|
|
return (unsigned char)((seed + index * 17u + (index >> 1u)) & 255u);
|
|
}
|
|
|
|
int main(void) {
|
|
if (dma_init(&arena, 0, 256) != DMA_OK) return 3;
|
|
char line[256];
|
|
while (fgets(line, sizeof(line), stdin) != NULL) {
|
|
char op = 0;
|
|
size_t a = 0, b = 0, c = 0, d = 0, e = 0;
|
|
if (sscanf(line, " %c %zu %zu %zu %zu %zu", &op, &a, &b, &c, &d, &e) < 1) return 4;
|
|
if (op == 'I') {
|
|
dma_status status = dma_init(&arena, a, b);
|
|
dump("I", status, 0, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'R') {
|
|
size_t index = 777u;
|
|
unsigned char *pointer = (unsigned char *)(uintptr_t)1u;
|
|
dma_status status = dma_reserve(&arena, a, (unsigned char)b, &index, &pointer);
|
|
dump("R", status, index, pointer_offset(pointer),
|
|
(size_t)((uintptr_t)pointer % DMA_QUANTUM), 0, 0, NULL, 0);
|
|
} else if (op == 'W') {
|
|
unsigned char source[256];
|
|
for (size_t pos = 0; pos < c && pos < 256u; ++pos) source[pos] = pattern(d, pos);
|
|
dma_status status = dma_write(&arena, a, b, source, c);
|
|
dump("W", status, 0, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'D') {
|
|
unsigned char target[256];
|
|
memset(target, 0x7e, sizeof(target));
|
|
dma_status status = dma_read(&arena, a, b, target, c);
|
|
dump("D", status, 0, 0, 0, 0, 0, target, c <= 256u ? c : 256u);
|
|
} else if (op == 'F') {
|
|
dma_status status = dma_fill(&arena, a, (unsigned char)b);
|
|
dump("F", status, 0, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'X') {
|
|
dma_status status = dma_xor(&arena, a, (unsigned char)b);
|
|
dump("X", status, 0, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'C') {
|
|
uint32_t checksum = UINT32_C(0xdeadbeef);
|
|
dma_status status = dma_checksum(&arena, a, &checksum);
|
|
dump("C", status, checksum, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'S') {
|
|
dma_summary summary = {999u, 999u, 999u, 999u, UINT32_C(0xdeadbeef)};
|
|
dma_status status = dma_summarize(&arena, &summary);
|
|
dump("S", status, summary.count, summary.payload_bytes,
|
|
summary.occupied_bytes, summary.free_bytes, summary.checksum, NULL, 0);
|
|
} else if (op == 'Y') {
|
|
dma_status status = dma_copy(&arena, a, b, c, d, e);
|
|
dump("Y", status, 0, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'B') {
|
|
dma_status status = dma_swap(&arena, a, b);
|
|
dump("B", status, 0, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'Q') {
|
|
int result = 99;
|
|
dma_status status = dma_compare(&arena, a, b, &result);
|
|
dump("Q", status, result == 99 ? 99u : (size_t)(result + 1), 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'J') {
|
|
size_t position = 9999u;
|
|
dma_status status = dma_find(&arena, a, (unsigned char)b, &position);
|
|
dump("J", status, position, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'T') {
|
|
dma_status status = dma_set_tag(&arena, a, (unsigned char)b);
|
|
dump("T", status, 0, 0, 0, 0, 0, NULL, 0);
|
|
} else if (op == 'Z') {
|
|
dma_status status = dma_reset(&arena);
|
|
dump("Z", status, 0, 0, 0, 0, 0, NULL, 0);
|
|
} else return 5;
|
|
}
|
|
return ferror(stdin) ? 6 : 0;
|
|
}
|
|
'''
|
|
|
|
|
|
class CheckFailure(Exception):
|
|
pass
|
|
|
|
|
|
WIRE_AUDIT = []
|
|
STAGE_EVIDENCE = []
|
|
|
|
|
|
def _validated_fact(record):
|
|
"""Keep every validated execution fact while omitting duplicated raw byte payloads."""
|
|
fact = {key: value for key, value in record.items()
|
|
if key not in ("stdout_b64", "stderr_b64")}
|
|
fact["argv"] = _encode_argv(record["argv"])
|
|
fact["argv_encoding"] = "source-ref-v1"
|
|
return fact
|
|
|
|
|
|
def _encode_argv(argv):
|
|
"""Represent repeated source and JSON argv strings with reversible references."""
|
|
encoded = []
|
|
for item in argv:
|
|
if item == SUPERVISOR:
|
|
encoded.append({"ref": "SUPERVISOR", "length": len(item),
|
|
"sha256": hashlib.sha256(item.encode()).hexdigest()})
|
|
continue
|
|
if item.startswith("["):
|
|
try:
|
|
nested = json.loads(item)
|
|
except (ValueError, UnicodeDecodeError):
|
|
nested = None
|
|
if type(nested) is list and all(type(part) is str for part in nested):
|
|
encoded.append({"json_argv": _encode_argv(nested),
|
|
"length": len(item),
|
|
"sha256": hashlib.sha256(item.encode()).hexdigest()})
|
|
continue
|
|
encoded.append(item)
|
|
return encoded
|
|
|
|
|
|
def _exact_int(value, low=None):
|
|
return type(value) is int and (low is None or value >= low)
|
|
|
|
|
|
def _wire(record, wrapper_rc, deadline, identity, argv, cwd, stdin_path, input_facts):
|
|
if type(record) is not dict or wrapper_rc != 0:
|
|
raise CheckFailure("supervisor wrapper failed")
|
|
for key in ("supervisor_pid", "supervisor_start", "leader_pid", "leader_start"):
|
|
if not _exact_int(record.get(key), 1):
|
|
raise CheckFailure("invalid supervisor identity " + key)
|
|
if record["supervisor_pid"] == record["leader_pid"]:
|
|
raise CheckFailure("supervisor and leader identity overlap")
|
|
if (record["supervisor_pid"], record["supervisor_start"]) != identity:
|
|
raise CheckFailure("supervisor birth token mismatch")
|
|
if (not _exact_int(record.get("leader_parent"), 1) or
|
|
record["leader_parent"] != record["supervisor_pid"] or
|
|
not _exact_int(record.get("leader_pgid"), 1) or
|
|
record["leader_pgid"] != record["leader_pid"] or
|
|
not _exact_int(record.get("leader_sid"), 1) or
|
|
record["leader_sid"] != record["leader_pid"]):
|
|
raise CheckFailure("leader birth relationship mismatch")
|
|
if (type(record.get("argv")) is not list or
|
|
any(type(item) is not str for item in record["argv"]) or
|
|
record["argv"] != argv or record.get("cwd") != str(cwd) or
|
|
record.get("stdin_path") != str(stdin_path) or
|
|
not _exact_int(record.get("stdin_length"), 0) or
|
|
type(record.get("stdin_sha256")) is not str or
|
|
(record["stdin_length"], record["stdin_sha256"]) != input_facts):
|
|
raise CheckFailure("execution request or input mismatch")
|
|
expected_uid = 65534 if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534" else os.getuid()
|
|
expected_gid = 65534 if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534" else os.getgid()
|
|
if not _exact_int(record.get("uid"), 0) or record["uid"] != expected_uid:
|
|
raise CheckFailure("wrong child uid")
|
|
if not _exact_int(record.get("gid"), 0) or record["gid"] != expected_gid:
|
|
raise CheckFailure("wrong child gid")
|
|
groups = record.get("groups")
|
|
if type(groups) is not list or any(not _exact_int(group, 0) for group in groups):
|
|
raise CheckFailure("invalid child groups")
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534" and groups:
|
|
raise CheckFailure("candidate retained supplementary groups")
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") != "uid65534" and groups != os.getgroups():
|
|
raise CheckFailure("local child group mismatch")
|
|
elapsed = record.get("elapsed")
|
|
if type(elapsed) not in (int, float) or not math.isfinite(elapsed) or not 0 <= elapsed <= deadline + 5:
|
|
raise CheckFailure("invalid elapsed")
|
|
if type(record.get("deadline")) not in (int, float) or record["deadline"] != deadline:
|
|
raise CheckFailure("invalid deadline")
|
|
for key in ("timeout", "leader_reaped", "descendants_absent"):
|
|
if type(record.get(key)) is not bool:
|
|
raise CheckFailure("invalid " + key)
|
|
if not record["leader_reaped"] or not record["descendants_absent"]:
|
|
raise CheckFailure("process or descendants not reaped")
|
|
if type(record.get("exceptions")) is not list or any(type(item) is not str for item in record["exceptions"]):
|
|
raise CheckFailure("invalid supervisor exceptions")
|
|
if record["exceptions"]:
|
|
raise CheckFailure("supervisor exceptions: " + ";".join(record["exceptions"])[:120])
|
|
leader_rc = record.get("leader_rc")
|
|
rc = record.get("rc")
|
|
if not _exact_int(leader_rc) or not _exact_int(rc):
|
|
raise CheckFailure("invalid return code")
|
|
if record["timeout"]:
|
|
if rc != 124 or leader_rc == 0:
|
|
raise CheckFailure("inconsistent timeout")
|
|
elif rc != leader_rc:
|
|
raise CheckFailure("inconsistent leader rc")
|
|
output = []
|
|
for name in ("stdout", "stderr"):
|
|
length = record.get(name + "_length")
|
|
digest = record.get(name + "_sha256")
|
|
encoded = record.get(name + "_b64")
|
|
if not _exact_int(length, 0) or type(digest) is not str or type(encoded) is not str:
|
|
raise CheckFailure("missing raw IO metadata")
|
|
try:
|
|
data = base64.b64decode(encoded, validate=True)
|
|
except (ValueError, base64.binascii.Error) as exc:
|
|
raise CheckFailure("invalid raw IO encoding") from exc
|
|
if len(data) != length or hashlib.sha256(data).hexdigest() != digest:
|
|
raise CheckFailure("raw IO digest mismatch")
|
|
output.append(data)
|
|
return output
|
|
|
|
|
|
def _proc_start(pid):
|
|
raw = Path("/proc") / str(pid) / "stat"
|
|
data = raw.read_bytes()
|
|
fields = data[data.rfind(b")") + 2:].split()
|
|
if len(fields) < 20:
|
|
raise CheckFailure("short wrapper proc stat")
|
|
return int(fields[19])
|
|
|
|
|
|
def _reap_adopted():
|
|
reaped = []
|
|
end = time.monotonic() + 1.0
|
|
while time.monotonic() < end:
|
|
try:
|
|
pid, _ = os.waitpid(-1, os.WNOHANG)
|
|
except ChildProcessError:
|
|
break
|
|
if pid == 0:
|
|
break
|
|
reaped.append(pid)
|
|
return reaped
|
|
|
|
|
|
def _run_supervisor(argv, cwd, stdin_path, deadline, env=None, input_facts=None,
|
|
stage=None):
|
|
stage_fact = {"stage": stage}
|
|
STAGE_EVIDENCE.append(stage_fact)
|
|
wanted_argv = [str(item) for item in argv]
|
|
empty_facts = (0, hashlib.sha256(b"").hexdigest())
|
|
if input_facts is None:
|
|
if str(stdin_path) != "-":
|
|
raise CheckFailure("missing precomputed input fact")
|
|
input_facts = empty_facts
|
|
inner = [sys.executable, "-I", "-c", SUPERVISOR,
|
|
json.dumps(wanted_argv), str(cwd), str(stdin_path), str(deadline)]
|
|
outer_deadline = deadline + 4.0
|
|
outer = [sys.executable, "-I", "-c", SUPERVISOR,
|
|
json.dumps(inner), str(cwd), "-", str(outer_deadline)]
|
|
stage_fact["launch"] = {"argv": _encode_argv(outer),
|
|
"argv_encoding": "source-ref-v1",
|
|
"cwd": str(cwd), "deadline": outer_deadline,
|
|
"stdin": "inherited"}
|
|
process = subprocess.Popen(outer, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
env=env, cwd=str(cwd), start_new_session=True)
|
|
birth = (process.pid, _proc_start(process.pid))
|
|
parent_birth = (os.getpid(), _proc_start(os.getpid()))
|
|
try:
|
|
raw, raw_error = process.communicate(timeout=outer_deadline + 6.0)
|
|
except subprocess.TimeoutExpired as error:
|
|
rescue_attempts = []
|
|
rescue_confirmed = False
|
|
rescue_error = "same-identity rescue did not confirm absence"
|
|
recovered_stdout = error.stdout or b""
|
|
recovered_stderr = error.stderr or b""
|
|
for rescue_number in range(2):
|
|
rescue_fact = {"number": rescue_number + 1}
|
|
absence = False
|
|
try:
|
|
rescue = subprocess.run([sys.executable, "-I", "-c", RESCUE,
|
|
str(birth[0]), str(birth[1]),
|
|
str(parent_birth[0]), str(parent_birth[1])],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
cwd="/tmp", timeout=6.0)
|
|
rescue_fact.update(rc=rescue.returncode,
|
|
stdout_length=len(rescue.stdout),
|
|
stdout_sha256=hashlib.sha256(rescue.stdout).hexdigest(),
|
|
stderr_length=len(rescue.stderr),
|
|
stderr_sha256=hashlib.sha256(rescue.stderr).hexdigest())
|
|
try:
|
|
detail = json.loads(rescue.stdout)
|
|
rescue_fact["detail"] = detail
|
|
absence = (rescue.returncode == 0 and type(detail) is dict and
|
|
detail.get("absence") is True and
|
|
detail.get("parent") == parent_birth[0] and
|
|
detail.get("parent_start") == parent_birth[1])
|
|
except (ValueError, UnicodeDecodeError):
|
|
rescue_fact["error"] = "invalid rescue wire"
|
|
except Exception as exc:
|
|
rescue_fact["error"] = "%s: %s" % (type(exc).__name__, exc)
|
|
try:
|
|
recovered_stdout, recovered_stderr = process.communicate(timeout=0.8)
|
|
pipes_closed = True
|
|
except subprocess.TimeoutExpired:
|
|
pipes_closed = False
|
|
except Exception as exc:
|
|
pipes_closed = False
|
|
rescue_fact["pipe_error"] = "%s: %s" % (type(exc).__name__, exc)
|
|
try:
|
|
process.poll()
|
|
except Exception as exc:
|
|
rescue_fact["poll_error"] = "%s: %s" % (type(exc).__name__, exc)
|
|
rescue_fact["outer_rc_after"] = process.returncode
|
|
rescue_fact["pipes_closed_after"] = pipes_closed
|
|
rescue_attempts.append(rescue_fact)
|
|
if absence and process.returncode is not None and pipes_closed:
|
|
rescue_confirmed = True
|
|
rescue_error = None
|
|
break
|
|
if not rescue_confirmed:
|
|
try:
|
|
recovered_stdout, recovered_stderr = process.communicate(timeout=1.0)
|
|
except subprocess.TimeoutExpired:
|
|
process.stdout.close()
|
|
process.stderr.close()
|
|
rescue_error += "; outer pipes still held"
|
|
except Exception as exc:
|
|
process.stdout.close()
|
|
process.stderr.close()
|
|
rescue_error += "; outer pipe exception %s" % exc
|
|
try:
|
|
process.poll()
|
|
if process.returncode is not None:
|
|
process.wait(timeout=1.0)
|
|
else:
|
|
rescue_error = (rescue_error or "") + "; outer leader unreaped"
|
|
except Exception as exc:
|
|
rescue_error = (rescue_error or "") + "; wait exception %s" % exc
|
|
reaped = _reap_adopted() if process.returncode is not None else []
|
|
wrapper = {"pid": birth[0], "start": birth[1],
|
|
"rc": process.returncode,
|
|
"stdout_length": len(recovered_stdout),
|
|
"stdout_sha256": hashlib.sha256(recovered_stdout).hexdigest(),
|
|
"stderr_length": len(recovered_stderr),
|
|
"stderr_sha256": hashlib.sha256(recovered_stderr).hexdigest(),
|
|
"outer_timeout": True, "rescue_attempts": rescue_attempts,
|
|
"rescue_error": rescue_error, "adopted_reaped": reaped}
|
|
WIRE_AUDIT.append(wrapper)
|
|
stage_fact["wrapper"] = wrapper
|
|
raise CheckFailure("outer supervisor timeout; " + (rescue_error or "rescue confirmed"))
|
|
# Preserve the actual wrapper observation before parsing; a malformed wire cannot
|
|
# be reconstructed as a successful inner stage.
|
|
wrapper = {"pid": birth[0], "start": birth[1], "rc": process.returncode,
|
|
"stdout_length": len(raw),
|
|
"stdout_sha256": hashlib.sha256(raw).hexdigest(),
|
|
"stderr_length": len(raw_error),
|
|
"stderr_sha256": hashlib.sha256(raw_error).hexdigest()}
|
|
WIRE_AUDIT.append(wrapper)
|
|
stage_fact["wrapper"] = wrapper
|
|
try:
|
|
outer_record = json.loads(raw)
|
|
except (ValueError, UnicodeDecodeError) as exc:
|
|
raise CheckFailure("invalid supervisor wire " + json.dumps(wrapper)) from exc
|
|
inner_raw, _ = _wire(outer_record, process.returncode, outer_deadline,
|
|
birth, inner, cwd, "-", empty_facts)
|
|
stage_fact["outer"] = _validated_fact(outer_record)
|
|
inner_identity = (outer_record["leader_pid"], outer_record["leader_start"])
|
|
try:
|
|
record = json.loads(inner_raw)
|
|
except (ValueError, UnicodeDecodeError) as exc:
|
|
raise CheckFailure("invalid inner wire " + json.dumps(wrapper)) from exc
|
|
output = _wire(record, outer_record["leader_rc"], deadline,
|
|
inner_identity, wanted_argv, cwd, stdin_path, input_facts)
|
|
stage_fact["inner"] = _validated_fact(record)
|
|
return record, output
|
|
|
|
|
|
def _fnv(data):
|
|
state = 2166136261
|
|
for value in data:
|
|
state = ((state ^ value) * 16777619) & 0xffffffff
|
|
return state
|
|
|
|
|
|
class Model:
|
|
def __init__(self):
|
|
self.init(0, 256)
|
|
|
|
def init(self, prefix, capacity):
|
|
self.raw = bytearray(384)
|
|
self.prefix = prefix
|
|
self.capacity = capacity
|
|
self.cursor = 0
|
|
self.slots = []
|
|
|
|
def observe(self, name, status, aux=(0, 0, 0, 0, 0), readback=None):
|
|
records = []
|
|
for slot in self.slots:
|
|
offset, length, span, tag = slot
|
|
absolute = self.prefix + offset
|
|
records.append(f"{offset}:{length}:{span}:{tag}:1:{absolute}:{absolute % 32};")
|
|
slot_wire = "".join(records)
|
|
read_wire = "-" if readback is None else bytes(readback).hex()
|
|
values = [name, str(status), *(str(x) for x in aux), str(len(self.slots)),
|
|
str(self.cursor), str(self.prefix), slot_wire,
|
|
self.raw[self.prefix:self.prefix + self.capacity].hex(), read_wire]
|
|
return " ".join(values)
|
|
|
|
def apply(self, command):
|
|
fields = command.split()
|
|
op = fields[0]
|
|
values = [int(part) for part in fields[1:]]
|
|
if op == "I":
|
|
prefix, capacity = values
|
|
if prefix >= 32 or capacity > 256:
|
|
return self.observe(op, 1)
|
|
self.init(prefix, capacity)
|
|
return self.observe(op, 0)
|
|
if op == "R":
|
|
length, tag = values
|
|
if length == 0:
|
|
return self.observe(op, 1, (777, 9999, 1, 0, 0))
|
|
if len(self.slots) >= 12:
|
|
return self.observe(op, 2, (777, 9999, 1, 0, 0))
|
|
start = self.cursor + (-(self.prefix + self.cursor)) % 32
|
|
span = ((length + 31) // 32) * 32
|
|
if start + span > self.capacity:
|
|
return self.observe(op, 2, (777, 9999, 1, 0, 0))
|
|
self.raw[self.prefix + start:self.prefix + start + span] = b"\0" * span
|
|
index = len(self.slots)
|
|
self.slots.append((start, length, span, tag & 255))
|
|
self.cursor = start + span
|
|
return self.observe(op, 0, (index, self.prefix + start, 0, 0, 0))
|
|
if op == "Z":
|
|
self.raw[self.prefix:self.prefix + self.capacity] = b"\0" * self.capacity
|
|
self.cursor = 0
|
|
self.slots.clear()
|
|
return self.observe(op, 0)
|
|
if op == "S":
|
|
payload = sum(slot[1] for slot in self.slots)
|
|
occupied = sum(slot[2] for slot in self.slots)
|
|
sequence = bytearray()
|
|
for offset, length, _, tag in self.slots:
|
|
sequence.append(tag)
|
|
sequence.extend(self.raw[self.prefix + offset:self.prefix + offset + length])
|
|
return self.observe(op, 0, (len(self.slots), payload, occupied,
|
|
self.capacity - self.cursor, _fnv(sequence)))
|
|
if op == "Y":
|
|
target, target_offset, source, source_offset, length = values
|
|
if target >= len(self.slots) or source >= len(self.slots):
|
|
return self.observe(op, 3)
|
|
to_slot = self.slots[target]
|
|
from_slot = self.slots[source]
|
|
if (target_offset > to_slot[1] or source_offset > from_slot[1] or
|
|
length > to_slot[1] - target_offset or length > from_slot[1] - source_offset):
|
|
return self.observe(op, 4)
|
|
to_address = self.prefix + to_slot[0] + target_offset
|
|
from_address = self.prefix + from_slot[0] + source_offset
|
|
moved = bytes(self.raw[from_address:from_address + length])
|
|
self.raw[to_address:to_address + length] = moved
|
|
return self.observe(op, 0)
|
|
if op == "B":
|
|
first, second = values
|
|
if first >= len(self.slots) or second >= len(self.slots):
|
|
return self.observe(op, 3)
|
|
left = self.slots[first]
|
|
right = self.slots[second]
|
|
if left[1] != right[1]:
|
|
return self.observe(op, 4)
|
|
a = self.prefix + left[0]
|
|
b = self.prefix + right[0]
|
|
before = bytes(self.raw[a:a + left[1]])
|
|
self.raw[a:a + left[1]] = self.raw[b:b + right[1]]
|
|
self.raw[b:b + right[1]] = before
|
|
return self.observe(op, 0)
|
|
if op == "Q":
|
|
first, second = values
|
|
if first >= len(self.slots) or second >= len(self.slots):
|
|
return self.observe(op, 3, (99, 0, 0, 0, 0))
|
|
left = self.slots[first]
|
|
right = self.slots[second]
|
|
a = bytes(self.raw[self.prefix + left[0]:self.prefix + left[0] + left[1]])
|
|
b = bytes(self.raw[self.prefix + right[0]:self.prefix + right[0] + right[1]])
|
|
compare_wire = 0 if a < b else 2 if a > b else 1
|
|
return self.observe(op, 0, (compare_wire, 0, 0, 0, 0))
|
|
if op == "J":
|
|
index, value = values
|
|
if index >= len(self.slots):
|
|
return self.observe(op, 3, (9999, 0, 0, 0, 0))
|
|
slot = self.slots[index]
|
|
data = self.raw[self.prefix + slot[0]:self.prefix + slot[0] + slot[1]]
|
|
try:
|
|
position = data.index(value & 255)
|
|
except ValueError:
|
|
return self.observe(op, 5, (9999, 0, 0, 0, 0))
|
|
return self.observe(op, 0, (position, 0, 0, 0, 0))
|
|
if op == "T":
|
|
index, tag = values
|
|
if index >= len(self.slots):
|
|
return self.observe(op, 3)
|
|
offset, length, span, _ = self.slots[index]
|
|
self.slots[index] = (offset, length, span, tag & 255)
|
|
return self.observe(op, 0)
|
|
index = values[0]
|
|
if index >= len(self.slots):
|
|
if op == "C":
|
|
return self.observe(op, 3, (0xdeadbeef, 0, 0, 0, 0))
|
|
if op == "D":
|
|
return self.observe(op, 3, readback=b"\x7e" * values[2])
|
|
return self.observe(op, 3)
|
|
offset, length, _, tag = self.slots[index]
|
|
first = self.prefix + offset
|
|
if op == "F":
|
|
self.raw[first:first + length] = bytes([values[1] & 255]) * length
|
|
return self.observe(op, 0)
|
|
if op == "X":
|
|
for pos in range(length):
|
|
self.raw[first + pos] ^= values[1] & 255
|
|
return self.observe(op, 0)
|
|
if op == "C":
|
|
return self.observe(op, 0, (_fnv(self.raw[first:first + length]), 0, 0, 0, 0))
|
|
if op in ("W", "D"):
|
|
suboffset, count = values[1:3]
|
|
valid = suboffset <= length and count <= length - suboffset
|
|
if op == "W":
|
|
if valid:
|
|
seed = values[3]
|
|
for pos in range(count):
|
|
self.raw[first + suboffset + pos] = (seed + pos * 17 + (pos >> 1)) & 255
|
|
return self.observe(op, 0 if valid else 4)
|
|
readback = bytearray(b"\x7e" * count)
|
|
if valid:
|
|
readback[:] = self.raw[first + suboffset:first + suboffset + count]
|
|
return self.observe(op, 0 if valid else 4, readback=readback)
|
|
raise AssertionError(op)
|
|
|
|
|
|
def _cases():
|
|
commands = ["I 0 256", "R 17 9", "W 0 0 17 93", "D 0 0 17",
|
|
"C 0", "R 32 255", "F 1 170", "X 1 15", "R 17 12",
|
|
"W 2 0 17 41", "Y 2 3 0 1 10", "B 0 2", "Q 0 2",
|
|
"J 2 93", "T 2 240", "S", "Y 0 4 0 0 10",
|
|
"Q 0 1", "J 0 255", "B 0 1", "Y 0 17 1 0 1", "Z",
|
|
"I 16 80", "R 1 1", "R 7 2", "S", "I 3 32", "R 1 4",
|
|
"I 0 31", "R 1 5", "I 0 33", "R 33 6",
|
|
"I 1 63", "R 1 7", "W 0 1 1 42", "D 0 0 2", "S"]
|
|
rng = random.Random(14371)
|
|
for _ in range(16):
|
|
prefix = rng.randrange(32)
|
|
capacity = rng.randrange(32, 257)
|
|
commands.append(f"I {prefix} {capacity}")
|
|
for slot in range(4):
|
|
length = rng.randrange(1, 70)
|
|
commands.append(f"R {length} {rng.randrange(256)}")
|
|
commands.append("S")
|
|
commands.append(f"C {slot}")
|
|
commands.extend(["I 0 256", "R 32 13", "W 0 32 1 11", "D 0 32 1",
|
|
"W 0 31 1 11", "D 0 31 1", "C 19", "Z", "S"])
|
|
return commands
|
|
|
|
|
|
def _safe_candidate_copy(candidate, destination):
|
|
for relative in ("include/dma_layout.h", "src/dma_layout.c", "src/dma_probe.c"):
|
|
origin = candidate / relative
|
|
entry = origin.lstat()
|
|
if not stat.S_ISREG(entry.st_mode):
|
|
raise CheckFailure("candidate file is not regular")
|
|
target = destination / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
descriptor = os.open(origin, os.O_RDONLY | os.O_NOFOLLOW)
|
|
with os.fdopen(descriptor, "rb") as opened:
|
|
target.write_bytes(opened.read())
|
|
target.chmod(0o644)
|
|
|
|
|
|
def _prepare_owned_directory(path):
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534":
|
|
os.chown(path, 65534, 65534)
|
|
|
|
|
|
def check(candidate):
|
|
libc = ctypes.CDLL(None, use_errno=True)
|
|
if libc.prctl(36, 1, 0, 0, 0) != 0:
|
|
print("FAIL")
|
|
return 1
|
|
scratch = Path(tempfile.mkdtemp(prefix="dma-layout-"))
|
|
outcome = None
|
|
reason = None
|
|
phases = {}
|
|
try:
|
|
work = scratch / "work"
|
|
work.mkdir(mode=0o755)
|
|
(work / "include").mkdir(mode=0o755)
|
|
(work / "src").mkdir(mode=0o755)
|
|
_safe_candidate_copy(candidate, work)
|
|
(work / "harness.c").write_text(HARNESS)
|
|
(work / "harness.c").chmod(0o644)
|
|
commands = _cases()
|
|
input_bytes = ("\n".join(commands) + "\n").encode()
|
|
(work / "input.txt").write_bytes(input_bytes)
|
|
(work / "input.txt").chmod(0o644)
|
|
input_facts = (len(input_bytes), hashlib.sha256(input_bytes).hexdigest())
|
|
for folder in (work / "include", work / "src", work):
|
|
_prepare_owned_directory(folder)
|
|
expected_model = Model()
|
|
expected = ("\n".join(expected_model.apply(line) for line in commands) + "\n").encode()
|
|
env = os.environ.copy()
|
|
env["GRADER_PROBE_PATH"] = str(Path(__file__).resolve())
|
|
compile_argv = ["gcc", "-std=c11", "-O2", "-Wall", "-Wextra", "-Wpedantic",
|
|
"-I", str(work / "include"), str(work / "src/dma_layout.c"),
|
|
str(work / "harness.c"), "-o", str(work / "observe")]
|
|
build, (build_out, build_err) = _run_supervisor(compile_argv, work, "-", 20.0,
|
|
env, stage="compile")
|
|
phases["compile"] = (build["rc"], build["timeout"], build["stdout_length"],
|
|
build["stdout_sha256"], build["stderr_length"], build["stderr_sha256"])
|
|
if build["timeout"] or build["rc"] != 0:
|
|
reason = "compile rc=%s timeout=%s" % (build["rc"], build["timeout"])
|
|
else:
|
|
observed, (stdout, stderr) = _run_supervisor([str(work / "observe")],
|
|
work, work / "input.txt", 20.0, env,
|
|
input_facts, stage="candidate")
|
|
phases["candidate"] = (observed["rc"], observed["timeout"],
|
|
observed["stdout_length"], observed["stdout_sha256"],
|
|
observed["stderr_length"], observed["stderr_sha256"])
|
|
if observed["timeout"] or observed["rc"] != 0:
|
|
reason = "candidate rc=%s timeout=%s" % (observed["rc"], observed["timeout"])
|
|
elif stdout != expected:
|
|
reason = "normal output mismatch, actual=%s expected=%s" % (
|
|
hashlib.sha256(stdout).hexdigest()[:12], hashlib.sha256(expected).hexdigest()[:12])
|
|
else:
|
|
outcome = True
|
|
except Exception as exc:
|
|
reason = "checker exception %s: %s" % (type(exc).__name__, exc)
|
|
finally:
|
|
# Every scratch, including a failed preparation, has identity-bound cleanup.
|
|
clean_code = "import shutil,sys; shutil.rmtree(sys.argv[1])"
|
|
absent_code = ("import os,sys\ntry: os.lstat(sys.argv[1])\n"
|
|
"except FileNotFoundError: raise SystemExit(0)\n"
|
|
"raise SystemExit(1)\n")
|
|
try:
|
|
clean, _ = _run_supervisor([sys.executable, "-I", "-c", clean_code,
|
|
str(scratch)], Path("/tmp"), "-", 8.0,
|
|
stage="cleanup")
|
|
phases["cleanup"] = (clean["rc"], clean["timeout"],
|
|
clean["stdout_length"], clean["stdout_sha256"],
|
|
clean["stderr_length"], clean["stderr_sha256"])
|
|
if clean["rc"] != 0 or clean["timeout"]:
|
|
outcome = False
|
|
reason = (reason or "") + " cleanup rc=%s timeout=%s" % (clean["rc"], clean["timeout"])
|
|
except Exception as exc:
|
|
outcome = False
|
|
reason = (reason or "") + " cleanup exception %s" % exc
|
|
try:
|
|
absent, _ = _run_supervisor([sys.executable, "-I", "-c", absent_code,
|
|
str(scratch)], Path("/tmp"), "-", 8.0,
|
|
stage="absence")
|
|
phases["absence"] = (absent["rc"], absent["timeout"],
|
|
absent["stdout_length"], absent["stdout_sha256"],
|
|
absent["stderr_length"], absent["stderr_sha256"])
|
|
if absent["rc"] != 0 or absent["timeout"]:
|
|
outcome = False
|
|
reason = (reason or "") + " absence rc=%s timeout=%s" % (absent["rc"], absent["timeout"])
|
|
except Exception as exc:
|
|
outcome = False
|
|
reason = (reason or "") + " absence exception %s" % exc
|
|
evidence = {"version": 1, "scratch": str(scratch), "stages": STAGE_EVIDENCE,
|
|
"phases": phases, "reason": reason}
|
|
packed = json.dumps(evidence, sort_keys=True, separators=(",", ":")).encode()
|
|
encoded = base64.b64encode(zlib.compress(packed, 9)).decode("ascii")
|
|
print("dma-check " + json.dumps({"codec": "zlib+base64", "bytes": len(packed),
|
|
"sha256": hashlib.sha256(packed).hexdigest(),
|
|
"payload": encoded}, separators=(",", ":")),
|
|
file=sys.stderr)
|
|
if outcome is True and reason is None:
|
|
print("PASS")
|
|
return 0
|
|
print("FAIL")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(check(Path(sys.argv[1])))
|