amos 9d633e9e67 Batch 15 (20261023-C-hwe-scale-15): 320 HWE-derived repair tasks
One user-space repair task per eligible HWE debug-diag / compile-deploy base
ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content
filter. Each ships a public spec with a check-to-basis table, at least two
defects with single-fix fixtures, a symptom-masking fixture and an alternative
correct implementation. Claude re-checked all 320: structure, the HWE quality
items and trace6 Docker calibration receipts against current digests
(320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120
ad-hoc family names; the next batch maps them to the canonical list.
The batch's 1,256-line progress log stays in its factory report and the
36-local branch batch15-hwe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 01:55:56 +08:00

754 lines
35 KiB
Python

"""Private independent model and isolated C++ observer for the shared value service."""
import base64
import hashlib
import json
import math
import os
from pathlib import Path
import random
import re
import shutil
import stat
import subprocess
import sys
import tempfile
import time
FILES = ('include/resource_store.hpp', 'src/resource_store.cpp',
'src/resource_batch.cpp', 'src/resource_console.cpp')
OBSERVER = '#include "resource_store.hpp"\n#include <iostream>\nint main() { return shared::run_console(std::cin, std::cout); }\n'
GXX = ['g++', '-std=c++17', '-O2', '-Wall', '-Wextra', '-pedantic', '-Iinclude',
'observer.cpp', 'src/resource_store.cpp', 'src/resource_batch.cpp',
'src/resource_console.cpp', '-o', 'observe']
BUILD_DEADLINE = 25
RUN_DEADLINE = 15
CLEAN_DEADLINE = 12
ABSENCE_DEADLINE = 8
REAP_MARGIN = 3
class Model:
def __init__(self):
self.reset()
def reset(self):
self.handles = {}
self.objects = {}
self.history = []
self.epoch = 1
def free(self, used, maximum):
return next((i for i in range(1, maximum + 1) if i not in used), None)
def create(self, owner, value):
if owner not in range(4) or value not in range(1000001): return (-22, None)
handle, object_id = self.free(self.handles, 16), self.free(self.objects, 8)
if handle is None or object_id is None: return (-24, None)
self.objects[object_id] = [value, self.epoch]
self.epoch += 1
self.handles[handle] = (object_id, owner)
return (0, handle)
def valid(self, handle):
return handle in self.handles
def retain(self, handle, owner):
if not self.valid(handle): return (-9, None)
if owner not in range(4): return (-22, None)
new = self.free(self.handles, 16)
if new is None: return (-24, None)
self.handles[new] = (self.handles[handle][0], owner)
return (0, new)
def clone(self, handle, owner):
if not self.valid(handle): return (-9, None)
if owner not in range(4): return (-22, None)
new, object_id = self.free(self.handles, 16), self.free(self.objects, 8)
if new is None or object_id is None: return (-24, None)
value = self.objects[self.handles[handle][0]][0]
self.objects[object_id] = [value, self.epoch]
self.epoch += 1
self.handles[new] = (object_id, owner)
return (0, new)
def same(self, first, second):
if not self.valid(first) or not self.valid(second): return (-9, None)
return (0, int(self.handles[first][0] == self.handles[second][0]))
def release(self, handle):
if not self.valid(handle): return (-9, [0, 0, 0, 0])
object_id, owner = self.handles.pop(handle)
if any(other[0] == object_id for other in self.handles.values()):
return (0, [0, 0, 0, 0])
value = self.objects.pop(object_id)[0]
event = [len(self.history) + 1, object_id, value, owner]
self.history.append(event)
return (0, event)
def transfer(self, handle, owner):
if not self.valid(handle): return -9
if owner not in range(4): return -22
self.handles[handle] = (self.handles[handle][0], owner)
return 0
def set(self, handle, value):
if not self.valid(handle): return -9
if value not in range(1000001): return -22
self.objects[self.handles[handle][0]][0] = value
return 0
def add(self, handle, delta):
if not self.valid(handle): return -9
if abs(delta) > 1000000: return -22
record = self.objects[self.handles[handle][0]]
if not 0 <= record[0] + delta <= 1000000: return -34
record[0] += delta
return 0
def view(self, handle, nonce):
if not self.valid(handle): return [-9] + [nonce] * 6
object_id, owner = self.handles[handle]
value, epoch = self.objects[object_id]
refs = sum(pair[0] == object_id for pair in self.handles.values())
return [0, handle, object_id, owner, value, refs, epoch]
def owner_sum(self, owner, nonce):
if owner not in range(4): return [-22, nonce]
return [0, sum(self.objects[obj][0] for obj, held in self.handles.values() if held == owner)]
def retain_owner(self, owner, target):
if owner not in range(4) or target not in range(4): return [-22, 0]
sources = [h for h, pair in sorted(self.handles.items()) if pair[1] == owner]
if len(sources) > 16 - len(self.handles): return [-24, 0]
output = [self.retain(h, target)[1] for h in sources]
return [0, len(output)] + output
def release_owner(self, owner):
if owner not in range(4): return [-22, 0]
sources = [h for h, pair in sorted(self.handles.items()) if pair[1] == owner]
events = [self.release(h)[1] for h in sources]
nonzero = [event for event in events if event[0]]
return [0, len(nonzero)] + [field for event in nonzero for field in event]
def transfer_owner(self, owner, target, nonce):
if owner not in range(4) or target not in range(4): return [-22, nonce]
if owner == target: return [0, 0]
sources = [h for h, pair in sorted(self.handles.items()) if pair[1] == owner]
for h in sources: self.transfer(h, target)
return [0, len(sources)]
def add_owner(self, owner, delta, nonce):
if owner not in range(4) or abs(delta) > 1000000: return [-22, nonce]
objects = {obj for obj, held in self.handles.values() if held == owner}
if any(not 0 <= self.objects[obj][0] + delta <= 1000000 for obj in objects):
return [-34, nonce]
for obj in objects: self.objects[obj][0] += delta
return [0, len(objects)]
def summary(self, owner, nonce):
if owner not in range(4): return [-22] + [nonce] * 6
pairs = [obj for obj, held in self.handles.values() if held == owner]
distinct = set(pairs)
values = [self.objects[obj][0] for obj in distinct]
oldest = min((self.objects[obj][1] for obj in distinct), default=0)
return [0, len(pairs), len(distinct), sum(self.objects[obj][0] for obj in pairs),
min(values, default=0), max(values, default=0), oldest]
def compare_exchange(self, handle, expected, desired, nonce):
if not self.valid(handle): return [-9, nonce, nonce]
if not 0 <= expected <= 1000000 or not 0 <= desired <= 1000000:
return [-22, nonce, nonce]
record = self.objects[self.handles[handle][0]]
old = record[0]
matched = int(old == expected)
if matched: record[0] = desired
return [0, matched, old]
def rotate(self, steps, nonce):
if steps not in range(4): return [-22, nonce]
if steps == 0: return [0, 0]
size = len(self.handles)
self.handles = {handle: (obj, (owner + steps) % 4)
for handle, (obj, owner) in self.handles.items()}
return [0, size]
def release_object(self, handle):
if not self.valid(handle): return [-9, 0]
object_id = self.handles[handle][0]
related = [h for h, pair in sorted(self.handles.items()) if pair[0] == object_id]
result = [self.release(h)[1] for h in related]
events = [event for event in result if event[0]]
return [0, len(events)] + [field for event in events for field in event]
def totals(self):
return [len(self.handles), len(self.objects)] + [sum(pair[1] == owner for pair in self.handles.values()) for owner in range(4)] + [len(self.history)]
def operation(self, command):
pieces = re.split(r'[ \t\r\v\f]+', command.strip(' \t\r\v\f'))
op = pieces[0]
a = list(map(int, pieces[1:]))
if op == 'C':
rc, out = self.create(*a[:2]); return [rc, out if rc == 0 else a[2]]
if op == 'R':
rc, out = self.retain(*a[:2]); return [rc, out if rc == 0 else a[2]]
if op == 'N':
rc, out = self.clone(*a[:2]); return [rc, out if rc == 0 else a[2]]
if op == 'E':
rc, out = self.same(*a[:2]); return [rc, out if rc == 0 else a[2]]
if op == 'L':
rc, event = self.release(a[0]); return [rc] + event
if op == 'T': return [self.transfer(*a)]
if op == 'S': return [self.set(*a)]
if op == 'A': return [self.add(*a)]
if op == 'V': return self.view(*a)
if op == 'O': return self.owner_sum(*a)
if op == 'B': return self.retain_owner(*a)
if op == 'X': return self.release_owner(*a)
if op == 'M': return self.transfer_owner(*a)
if op == 'D': return self.add_owner(*a)
if op == 'Q': return self.summary(*a)
if op == 'J': return self.compare_exchange(*a)
if op == 'Y': return self.rotate(*a)
if op == 'F': return self.release_object(*a)
if op == 'K': return self.totals()
if op == 'H': return [len(self.history)] + [field for event in self.history for field in event]
if op == 'Z': self.reset(); return [0]
raise ValueError(op)
def cases():
model = Model()
rows = []
rng = random.Random(0xCC010)
def add(command, label):
rows.append((command, [command.split()[0]] + model.operation(command), label))
for command, label in [
('K','empty'), ('C\t0\v42\f991','create'), ('R 1 1 993','share'),
('V 1 995','initial-reference'), ('V 2 997','new-reference'),
('L 1','nonfinal-release'), ('V 2 999','survivor-value'),
('A 2 8','survivor-write'), ('L 2','final-release'), ('H','event-history'),
('C 0 17 1001','recreate'), ('T 1 2','handoff'), ('K','handoff-counts'),
('X 2','handoff-batch-release'), ('K','handoff-final'), ('H','handoff-history'),
('Z','reset'), ('C 3 100 1003','clone-start'), ('N 1 1 1005','clone'),
('E 1 2 1007','clone-independent'), ('R 1 2 1009','retain-original'),
('E 1 3 1011','same-object'), ('S 3 29','shared-write'),
('V 1 1013','shared-read'), ('V 2 1015','clone-value'),
('O 2 1017','owner-sum'), ('B 2 3','batch-retain'),
('D 3 7 1019','batch-add'), ('V 1 1021','batch-shared'),
('M 3 0 1023','batch-transfer'), ('K','batch-totals'),
('X 0','batch-release'), ('H','batch-events'),
('Q\t2\r1025','owner-summary'), ('J 1 29 81 1027','cas-change'),
('J 1 29 82 1029','cas-mismatch'), ('Y 2 1031','owner-rotate'),
('K','rotated-counts'), ('F 1','object-release'), ('H','object-events')]: add(command,label)
for iteration in range(8):
add('Z', 'random-reset')
for step in range(45):
choice = rng.randrange(17)
h = rng.randrange(-1, 18)
owner = rng.randrange(-1, 5)
value = rng.choice([0, 1, 999999, 1000000, -1, 1000001, rng.randrange(1000001)])
delta = rng.choice([-1000001, -1000000, -1, 0, 1, 1000000, 1000001, rng.randrange(-100, 101)])
nonce = rng.randrange(2000, 10000)
command = [f'C {owner} {value} {nonce}', f'R {h} {owner} {nonce}',
f'N {h} {owner} {nonce}', f'L {h}', f'T {h} {owner}',
f'S {h} {value}', f'A {h} {delta}', f'V {h} {nonce}',
f'B {owner} {rng.randrange(4)}', f'X {owner}',
f'M {owner} {rng.randrange(4)} {nonce}',
f'D {owner} {delta} {nonce}', f'O {owner} {nonce}',
f'Q {owner} {nonce}', f'J {h} {value} {rng.randrange(1000001)} {nonce}',
f'Y {rng.randrange(-1,5)} {nonce}', f'F {h}'][choice]
add(command, 'random-'+str(iteration))
if step % 6 == 0: add('K', 'random-totals')
if step % 11 == 0: add('H', 'random-history')
add('Z','capacity-reset')
for i in range(8): add(f'C {i%4} {i*91} {11000+i}', 'object-fill')
add('C 1 77 11010','object-capacity')
for i in range(1,9): add(f'R {i} {(i+1)%4} {11010+i}', 'handle-fill')
add('R 1 0 11020','handle-capacity')
add('B 0 1','batch-atomic-capacity')
add('K','capacity-no-effect')
add('Z','final-reset')
return rows
def parse_output(raw, expected):
lines = raw.split(b'\n')
if lines[-1] != b'': return 'missing final newline'
lines.pop()
if len(lines) != len(expected): return f'row count {len(lines)} expected {len(expected)}'
for index, (line, (_, want, label)) in enumerate(zip(lines, expected)):
line = line.strip(b' \t\r')
if not line: return f'empty row {index} {label}'
fields = re.split(rb'[ \t\r]+', line)
if fields[0] != want[0].encode('ascii') or len(fields) != len(want):
return f'shape row {index} {label}'
for part, number in zip(fields[1:], want[1:]):
if re.fullmatch(rb'[+-]?[0-9]+', part) is None:
return f'nondecimal row {index} {label}'
negative = part.startswith(b'-')
digits = part.lstrip(b'+-').lstrip(b'0') or b'0'
if len(digits) > 20: return f'out-of-range row {index} {label}'
observed = int(digits) * (-1 if negative else 1)
if observed != number: return f'value row {index} {label}'
return None
SUPERVISOR = r'''
import base64, ctypes, hashlib, json, os, selectors, signal, subprocess, sys, time
PR_SET_CHILD_SUBREAPER = 36
libc = ctypes.CDLL(None, use_errno=True)
if libc.prctl(PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'PR_SET_CHILD_SUBREAPER failed')
def proc(pid):
try:
raw = open('/proc/%d/stat' % pid, 'rb').read()
except (FileNotFoundError, ProcessLookupError):
return None
right = raw.rfind(b')')
if right < 0: raise RuntimeError('malformed proc stat')
parts = raw[right + 2:].split()
return (int(parts[1]), int(parts[19]), parts[0], int(parts[2]))
def table():
result = {}
for name in os.listdir('/proc'):
if not name.isdigit(): continue
pid = int(name)
record = proc(pid)
if record is not None: result[pid] = record
return result
def descendants(snapshot, supervisor, known):
live = {pid: row for pid, row in snapshot.items() if row[2] not in (b'Z', b'X')}
roots = {supervisor}
for pid, start in known.items():
row = live.get(pid)
if row is not None and row[1] == start: roots.add(pid)
found = {}
changed = True
while changed:
changed = False
for pid, row in live.items():
if pid != supervisor and pid not in found and row[0] in roots:
found[pid] = row
roots.add(pid)
changed = True
return found
def fresh_signal(pid, start, signum, errors):
try:
row = proc(pid)
if row is None or row[1] != start or row[2] in (b'Z', b'X'):
return
os.kill(pid, signum)
except (FileNotFoundError, ProcessLookupError):
return
except Exception as error:
errors.append('signal:%s:%s' % (type(error).__name__, str(error)[:100]))
def reap_adopted(snapshot, own_pid, leader_pid, errors):
for pid, row in snapshot.items():
if pid == leader_pid or row[0] != own_pid: continue
try: os.waitpid(pid, os.WNOHANG)
except (ChildProcessError, ProcessLookupError): pass
except Exception as error:
errors.append('reap:%s:%s' % (type(error).__name__, str(error)[:100]))
def finish_children(leader, leader_start, tracked, errors, stop):
leader_rc = None
reaped = False
last = {}
while time.monotonic() < stop:
leader_rc = leader.poll()
if leader_rc is not None: reaped = True
try:
snapshot = table()
reap_adopted(snapshot, os.getpid(), leader.pid, errors)
present = descendants(snapshot, os.getpid(), tracked)
except Exception as error:
errors.append('reap-scan:%s:%s' % (type(error).__name__, str(error)[:100]))
fresh_signal(leader.pid, leader_start, signal.SIGKILL, errors)
time.sleep(0.02)
continue
for pid, row in present.items(): tracked[pid] = row[1]
for pid, row in present.items():
fresh_signal(pid, row[1], signal.SIGKILL, errors)
if not reaped:
fresh_signal(leader.pid, leader_start, signal.SIGKILL, errors)
for pid, row in present.items():
if pid != leader.pid and row[0] == os.getpid():
try: os.waitpid(pid, os.WNOHANG)
except (ChildProcessError, ProcessLookupError): pass
last = present
if reaped and not present: break
time.sleep(0.02)
return leader_rc, reaped, last
def run(request):
errors = []
beginning = time.monotonic()
supervisor_record = proc(os.getpid())
if supervisor_record is None: raise RuntimeError('supervisor start unavailable')
supervisor_start = supervisor_record[1]
deadline = float(request['deadline'])
argv = request['argv']
data = base64.b64decode(request['input'], validate=True)
gate_read, gate_write = os.pipe()
wrapper = ('import json,os,sys; fd=int(sys.argv[1]); argv=json.loads(sys.argv[2]); '
'os.read(fd,1); os.close(fd); os.execvp(argv[0],argv)')
leader = None
leader_start = 0
tracked = {}
out = bytearray()
err = bytearray()
timed_out = False
reaped = False
leader_rc = None
absent = False
pg_absent = False
selector = selectors.DefaultSelector()
try:
leader = subprocess.Popen([sys.executable, '-I', '-c', wrapper, str(gate_read), json.dumps(argv)],
cwd=request['cwd'], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
pass_fds=(gate_read,), start_new_session=True)
os.close(gate_read)
gate_read = -1
row = proc(leader.pid)
if row is None: raise RuntimeError('leader start unavailable')
leader_start = row[1]
tracked[leader.pid] = leader_start
os.write(gate_write, b'x')
os.close(gate_write)
gate_write = -1
for stream, event in ((leader.stdout, 'stdout'), (leader.stderr, 'stderr')):
os.set_blocking(stream.fileno(), False)
selector.register(stream, selectors.EVENT_READ, event)
os.set_blocking(leader.stdin.fileno(), False)
selector.register(leader.stdin, selectors.EVENT_WRITE, 'stdin')
input_at = 0
post_exit = None
while True:
now = time.monotonic()
leader_rc = leader.poll()
if leader_rc is not None:
reaped = True
if post_exit is None: post_exit = now
snapshot = table()
reap_adopted(snapshot, os.getpid(), leader.pid, errors)
present = descendants(snapshot, os.getpid(), tracked)
for pid, item in present.items(): tracked[pid] = item[1]
if now >= beginning + deadline and not reaped: timed_out = True
if timed_out or reaped:
for pid, item in present.items():
fresh_signal(pid, item[1], signal.SIGKILL, errors)
if not reaped: fresh_signal(leader.pid, leader_start, signal.SIGKILL, errors)
for pid, item in present.items():
if pid != leader.pid and item[0] == os.getpid():
try: os.waitpid(pid, os.WNOHANG)
except (ChildProcessError, ProcessLookupError): pass
for key, _ in selector.select(0.02):
stream = key.fileobj
if key.data == 'stdin':
if input_at == len(data):
selector.unregister(stream); stream.close()
else:
try: amount = os.write(stream.fileno(), data[input_at:input_at + 65536])
except BrokenPipeError:
selector.unregister(stream); stream.close()
else: input_at += amount
else:
try: chunk = os.read(stream.fileno(), 65536)
except BlockingIOError: chunk = None
if chunk == b'': selector.unregister(stream); stream.close()
elif chunk is not None:
(out if key.data == 'stdout' else err).extend(chunk)
if reaped and not present and not selector.get_map(): break
if now > beginning + deadline + 3:
errors.append('reap-deadline')
break
leader_rc, reaped, _ = finish_children(leader, leader_start, tracked, errors,
beginning + deadline + 3)
snapshot = table()
reap_adopted(snapshot, os.getpid(), leader.pid, errors)
present = descendants(snapshot, os.getpid(), tracked)
absent = not present
pg_absent = not any(row[3] == leader.pid and row[2] not in (b'Z', b'X')
for row in snapshot.values())
except Exception as error:
errors.append('supervisor:%s:%s' % (type(error).__name__, str(error)[:120]))
finally:
for fd in (gate_read, gate_write):
if fd >= 0:
try: os.close(fd)
except OSError: pass
if leader is not None and leader_start:
try:
leader_rc, reaped, _ = finish_children(leader, leader_start, tracked, errors,
beginning + deadline + 3)
snapshot = table()
present = descendants(snapshot, os.getpid(), tracked)
absent = not present
pg_absent = not any(row[3] == leader.pid and row[2] not in (b'Z', b'X')
for row in snapshot.values())
except Exception as error:
errors.append('final-reap:%s:%s' % (type(error).__name__, str(error)[:100]))
selector.close()
elapsed = time.monotonic() - beginning
output = bytes(out)
stderr = bytes(err)
return dict(stage=request['stage'], nonce=request['nonce'], argv=argv, cwd=request['cwd'],
deadline=request['deadline'], stdin_len=len(data),
stdin_sha=hashlib.sha256(data).hexdigest(),
supervisor_pid=os.getpid(), supervisor_start=supervisor_start,
leader_pid=leader.pid if leader else 0,
leader_start=leader_start, uid=os.getuid(), gid=os.getgid(), groups=os.getgroups(),
leader_rc=leader_rc, rc=124 if timed_out else leader_rc, reaped=reaped,
timeout=timed_out, descendants_absent=absent, pg_absent=pg_absent,
exceptions=errors, elapsed=elapsed,
stdout_len=len(output), stdout_sha=hashlib.sha256(output).hexdigest(),
stderr_len=len(stderr), stderr_sha=hashlib.sha256(stderr).hexdigest(),
stdout=base64.b64encode(output).decode('ascii'),
stderr=base64.b64encode(stderr).decode('ascii'))
try:
request = json.loads(sys.stdin.buffer.read())
wire = run(request)
sys.stdout.write(json.dumps(wire, separators=(',', ':')))
except Exception as error:
sys.stderr.write('supervisor fatal: %s: %s\n' % (type(error).__name__, error))
sys.exit(2)
'''
def exact_int(value):
return type(value) is int
def process_start(pid):
with open('/proc/%d/stat' % pid, 'rb') as stream:
raw = stream.read()
right = raw.rfind(b')')
if right < 0: raise RuntimeError('supervisor proc stat malformed')
fields = raw[right + 2:].split()
if fields[0] in (b'Z', b'X'): raise RuntimeError('supervisor exited before handshake')
start = int(fields[19])
if start <= 0: raise RuntimeError('supervisor proc start invalid')
return start
def stage(label, argv, cwd, data, deadline, identity):
nonce = os.urandom(16).hex()
stdin_sha = hashlib.sha256(data).hexdigest()
request = json.dumps(dict(stage=label, nonce=nonce, argv=argv, cwd=str(cwd),
input=base64.b64encode(data).decode('ascii'),
deadline=deadline), separators=(',', ':')).encode('ascii')
wrapper = subprocess.Popen([sys.executable, '-I', '-c', SUPERVISOR],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
wrapper_start = process_start(wrapper.pid)
try:
raw, wrapper_stderr = wrapper.communicate(input=request,
timeout=deadline + REAP_MARGIN + 4)
except subprocess.TimeoutExpired as error:
raise RuntimeError('supervisor outer timeout pid=%d stage=%s partial_len=%d' %
(wrapper.pid, label, len(error.output or b'')))
wire_sha = hashlib.sha256(raw).hexdigest()
print('WIRE stage=%s pid=%d rc=%d len=%d sha=%s stderr_len=%d stderr_sha=%s' %
(label, wrapper.pid, wrapper.returncode, len(raw), wire_sha,
len(wrapper_stderr), hashlib.sha256(wrapper_stderr).hexdigest()), file=sys.stderr)
if wrapper.returncode != 0:
raise RuntimeError('supervisor rc %d wire_len=%d wire_sha=%s stderr=%s' %
(wrapper.returncode, len(raw), wire_sha,
wrapper_stderr[-180:].decode('utf-8','replace')))
try:
meta = json.loads(raw)
except Exception as error:
raise RuntimeError('supervisor wire invalid len=%d sha=%s %s' %
(len(raw), wire_sha, error))
required_ints = ('supervisor_pid','supervisor_start','leader_pid','leader_start',
'uid','gid','stdin_len','stdout_len','stderr_len')
if not isinstance(meta, dict) or any(not exact_int(meta.get(key)) for key in required_ints):
raise RuntimeError('supervisor numeric metadata absent or mistyped')
if any(meta[key] <= 0 for key in ('supervisor_pid','supervisor_start','leader_pid','leader_start')):
raise RuntimeError('supervisor PID/start invalid')
if (meta['supervisor_pid'] != wrapper.pid or meta['supervisor_start'] != wrapper_start or
meta['supervisor_pid'] == meta['leader_pid']):
raise RuntimeError('supervisor PID binding invalid')
if any(meta[key] < 0 for key in ('stdin_len','stdout_len','stderr_len')):
raise RuntimeError('supervisor IO length invalid')
if meta.get('stage') != label or type(meta.get('stage')) is not str:
raise RuntimeError('supervisor stage mismatch')
if meta.get('nonce') != nonce or type(meta.get('nonce')) is not str:
raise RuntimeError('supervisor nonce mismatch')
if type(meta.get('argv')) is not list or any(type(x) is not str for x in meta['argv']) or meta['argv'] != argv:
raise RuntimeError('supervisor argv mismatch')
if type(meta.get('cwd')) is not str or meta['cwd'] != str(cwd):
raise RuntimeError('supervisor cwd mismatch')
if type(meta.get('deadline')) not in (int,float) or meta['deadline'] != deadline:
raise RuntimeError('supervisor deadline mismatch')
if meta['stdin_len'] != len(data) or type(meta.get('stdin_sha')) is not str or meta['stdin_sha'] != stdin_sha:
raise RuntimeError('supervisor stdin mismatch')
if meta['uid'] != identity[0] or meta['gid'] != identity[1]:
raise RuntimeError('supervisor identity mismatch')
groups = meta.get('groups')
if type(groups) is not list or any(not exact_int(x) for x in groups) or groups != identity[2]:
raise RuntimeError('supervisor groups mismatch')
for field in ('timeout','reaped','descendants_absent','pg_absent'):
if type(meta.get(field)) is not bool: raise RuntimeError('supervisor boolean missing '+field)
if type(meta.get('exceptions')) is not list or any(type(x) is not str for x in meta['exceptions']):
raise RuntimeError('supervisor exceptions invalid')
elapsed = meta.get('elapsed')
if type(elapsed) not in (int, float) or not math.isfinite(elapsed) or not 0 <= elapsed <= deadline + REAP_MARGIN + 1:
raise RuntimeError('supervisor elapsed invalid')
leader_rc, rc = meta.get('leader_rc'), meta.get('rc')
if not exact_int(leader_rc) or not exact_int(rc):
raise RuntimeError('supervisor return code missing')
if rc != (124 if meta['timeout'] else leader_rc):
raise RuntimeError('supervisor return code inconsistent')
if meta['timeout'] and leader_rc == 0:
raise RuntimeError('supervisor timeout and success conflict')
output = {}
for kind in ('stdout', 'stderr'):
if type(meta.get(kind)) is not str or type(meta.get(kind+'_sha')) is not str:
raise RuntimeError('supervisor IO metadata missing '+kind)
try: decoded = base64.b64decode(meta[kind], validate=True)
except Exception: raise RuntimeError('supervisor IO encoding invalid '+kind)
if len(decoded) != meta[kind+'_len'] or hashlib.sha256(decoded).hexdigest() != meta[kind+'_sha']:
raise RuntimeError('supervisor IO digest invalid '+kind)
output[kind] = decoded
good = (not meta['timeout'] and meta['reaped'] and meta['descendants_absent']
and meta['pg_absent'] and not meta['exceptions'])
return meta, output, good
def candidate_bytes(candidate, relative):
parts = Path(relative).parts
if not parts or any(part in ('', '.', '..') for part in parts):
raise ValueError('unsafe candidate path')
opened = []
try:
directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
current = os.open(candidate, directory_flags)
opened.append(current)
for part in parts[:-1]:
current = os.open(part, directory_flags, dir_fd=current)
opened.append(current)
file_flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC
descriptor = os.open(parts[-1], file_flags, dir_fd=current)
opened.append(descriptor)
info = os.fstat(descriptor)
if not stat.S_ISREG(info.st_mode) or info.st_size > 1048576:
raise ValueError('candidate must be regular and at most 1 MiB')
chunks = []
remaining = info.st_size + 1
while remaining:
chunk = os.read(descriptor, min(65536, remaining))
if not chunk: break
chunks.append(chunk)
remaining -= len(chunk)
raw = b''.join(chunks)
if len(raw) != info.st_size:
raise ValueError('candidate changed during read')
return raw
finally:
for descriptor in reversed(opened):
os.close(descriptor)
def main(candidate):
rows = cases()
input_bytes = ('\n'.join(row[0] for row in rows) + '\n').encode('ascii')
scratch = Path(tempfile.mkdtemp(prefix='shared-value-'))
source = scratch / 'src'
identity = (65534, 65534, []) if os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534' else (os.getuid(), os.getgid(), os.getgroups())
phase = 'prepare'
failure = None
stages = {}
clean_ok = False
absence_ok = False
try:
source.mkdir()
os.chmod(source, 0o755)
if identity[0] == 65534: os.chown(source, 65534, 65534)
(source / 'include').mkdir()
os.chmod(source / 'include', 0o755)
if identity[0] == 65534: os.chown(source / 'include', 65534, 65534)
(source / 'src').mkdir()
os.chmod(source / 'src', 0o755)
if identity[0] == 65534: os.chown(source / 'src', 65534, 65534)
for rel in FILES:
target = source / rel
target.write_bytes(candidate_bytes(candidate, rel))
(source / 'observer.cpp').write_text(OBSERVER)
for rel in (*FILES, 'observer.cpp'):
os.chmod(source / rel, 0o644)
phase = 'compile'
meta, io, good = stage('compile', GXX, source, b'', BUILD_DEADLINE, identity)
stages['compile'] = meta
stages['compile']['stderr_sample'] = io['stderr'][-300:].decode('utf-8', 'replace')
if not good or meta['rc'] != 0:
failure = 'compile rc=%s timeout=%s exceptions=%s' % (meta['rc'],meta['timeout'],meta['exceptions'])
else:
phase = 'candidate'
meta, io, good = stage('candidate', [str(source / 'observe')], source, input_bytes, RUN_DEADLINE, identity)
stages['candidate'] = meta
stages['candidate']['stderr_sample'] = io['stderr'][-300:].decode('utf-8', 'replace')
if not good or meta['rc'] != 0:
failure = 'candidate rc=%s timeout=%s exceptions=%s' % (meta['rc'],meta['timeout'],meta['exceptions'])
else:
phase = 'compare'
failure = parse_output(io['stdout'], rows)
except Exception as error:
failure = '%s %s: %s' % (phase, type(error).__name__, str(error))
finally:
try:
meta, io, good = stage('cleanup', ['rm','-rf','--',str(scratch)], '/', b'', CLEAN_DEADLINE, identity)
stages['cleanup'] = meta
stages['cleanup']['stderr_sample'] = io['stderr'][-300:].decode('utf-8', 'replace')
clean_ok = good and meta['rc'] == 0
if not clean_ok:
failure = (failure or '') + ' cleanup rc=%s timeout=%s exceptions=%s' % (meta['rc'],meta['timeout'],meta['exceptions'])
except Exception as error:
failure = (failure or '') + ' cleanup wire '+type(error).__name__+':'+str(error)
try:
absence = ('import os,sys\ntry: os.lstat(sys.argv[1])\n'
'except FileNotFoundError: raise SystemExit(0)\n'
'except Exception: raise SystemExit(2)\n'
'raise SystemExit(1)\n')
meta, io, good = stage('absence', [sys.executable,'-I','-c',absence,str(scratch)], '/', b'', ABSENCE_DEADLINE, identity)
stages['absence'] = meta
stages['absence']['stderr_sample'] = io['stderr'][-300:].decode('utf-8', 'replace')
absence_ok = good and meta['rc'] == 0
if not absence_ok:
failure = (failure or '') + ' absence rc=%s timeout=%s exceptions=%s' % (meta['rc'],meta['timeout'],meta['exceptions'])
except Exception as error:
failure = (failure or '') + ' absence wire '+type(error).__name__+':'+str(error)
columns = ('stage','nonce','supervisor_pid','supervisor_start','leader_pid',
'leader_start','uid','gid','groups','argv','cwd','deadline',
'stdin_len','stdin_sha','elapsed','leader_rc','rc','timeout',
'reaped','descendants_absent','pg_absent','exceptions',
'stdout_len','stdout_sha','stderr_len','stderr_sha')
rows = [[item[key] for key in columns] for item in stages.values()]
summary = {'phase':phase,'failure':failure,'columns':columns,'rows':rows}
print(json.dumps(summary, separators=(',',':')), file=sys.stderr)
if (failure is None and clean_ok and absence_ok and
set(stages) == {'compile','candidate','cleanup','absence'} and
all(not item['exceptions'] for item in stages.values())):
print('PASS')
return 0
print('FAIL '+(failure or 'incomplete cleanup/absence')[:240])
return 1
if __name__ == '__main__':
sys.exit(main(Path(sys.argv[1])))