One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
584 lines
37 KiB
Python
584 lines
37 KiB
Python
import base64
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import zlib
|
|
|
|
if hasattr(sys, "set_int_max_str_digits"):
|
|
sys.set_int_max_str_digits(0)
|
|
|
|
SUPERVISOR = r'''
|
|
import base64, ctypes, hashlib, json, os, selectors, signal, subprocess, sys, time
|
|
|
|
RECLAIM = 3.0
|
|
|
|
def stat_token(pid):
|
|
try:
|
|
with open('/proc/%d/stat' % pid, 'rb') as stream:
|
|
raw = stream.read()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
fields = raw[raw.rfind(b')') + 2:].split()
|
|
return (pid, int(fields[19]), fields[0].decode('ascii'), int(fields[1]), int(fields[2]))
|
|
|
|
def proc_table():
|
|
result = {}
|
|
for name in os.listdir('/proc'):
|
|
if name.isascii() and name.isdecimal():
|
|
token = stat_token(int(name))
|
|
if token:
|
|
result[token[0]] = token
|
|
return result
|
|
|
|
def discover(table, supervisor, tracked):
|
|
current = table.get(supervisor[0])
|
|
roots = set()
|
|
if current and current[1] == supervisor[1] and current[2] not in ('Z', 'X'):
|
|
roots.add(supervisor[0])
|
|
for pid, start in tracked.items():
|
|
current = table.get(pid)
|
|
if current and current[1] == start and current[2] not in ('Z', 'X'):
|
|
roots.add(pid)
|
|
descendants = set(roots)
|
|
while True:
|
|
added = {pid for pid, item in table.items() if item[3] in descendants}
|
|
combined = descendants | added
|
|
if combined == descendants:
|
|
break
|
|
descendants = combined
|
|
descendants.discard(supervisor[0])
|
|
return {pid: table[pid][1] for pid in descendants}
|
|
|
|
def fresh_signal(pid, start):
|
|
# Signal authorization is deliberately refreshed for each PID, after poll/wait.
|
|
current = stat_token(pid)
|
|
if current and current[1] == start and current[2] not in ('Z', 'X'):
|
|
try:
|
|
os.kill(pid, signal.SIGKILL)
|
|
return True
|
|
except ProcessLookupError:
|
|
return False
|
|
return False
|
|
|
|
def main():
|
|
request = json.loads(sys.stdin.buffer.read())
|
|
started = time.monotonic()
|
|
deadline = request['deadline']
|
|
supervisor = stat_token(os.getpid())
|
|
child = None
|
|
leader_start = None
|
|
leader_group = None
|
|
tracked = {}
|
|
adopted = []
|
|
errors = []
|
|
timed_out = False
|
|
output = bytearray()
|
|
diagnostic = bytearray()
|
|
sent = 0
|
|
selector = selectors.DefaultSelector()
|
|
offered = base64.b64decode(request['input'], validate=True)
|
|
|
|
def poll_own():
|
|
if child is not None:
|
|
return child.poll()
|
|
return None
|
|
|
|
def scan_reap():
|
|
poll_own()
|
|
table = proc_table()
|
|
found = discover(table, supervisor, tracked)
|
|
tracked.update(found)
|
|
for pid, start in list(tracked.items()):
|
|
if child is not None and pid == child.pid:
|
|
continue
|
|
item = table.get(pid)
|
|
if not item or item[1] != start or item[3] != supervisor[0]:
|
|
continue
|
|
try:
|
|
reaped, status = os.waitpid(pid, os.WNOHANG)
|
|
if reaped:
|
|
adopted.append([pid, start, status])
|
|
except ChildProcessError:
|
|
pass
|
|
return table, found
|
|
|
|
try:
|
|
libc = ctypes.CDLL(None, use_errno=True)
|
|
if libc.prctl(36, 1, 0, 0, 0) != 0:
|
|
raise OSError(ctypes.get_errno(), 'subreaper')
|
|
child = subprocess.Popen(request['argv'], cwd=request['cwd'], stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True)
|
|
leader = stat_token(child.pid)
|
|
if leader is None:
|
|
raise RuntimeError('leader identity unavailable')
|
|
leader_start, leader_group = leader[1], leader[4]
|
|
tracked[child.pid] = leader_start
|
|
for stream, label in ((child.stdout, 'out'), (child.stderr, 'err')):
|
|
os.set_blocking(stream.fileno(), False)
|
|
selector.register(stream, selectors.EVENT_READ, label)
|
|
if offered:
|
|
os.set_blocking(child.stdin.fileno(), False)
|
|
selector.register(child.stdin, selectors.EVENT_WRITE, 'in')
|
|
else:
|
|
child.stdin.close()
|
|
while True:
|
|
poll_own()
|
|
scan_reap()
|
|
if child.returncode is not None and not selector.get_map():
|
|
break
|
|
if time.monotonic() - started >= deadline:
|
|
timed_out = True
|
|
break
|
|
for key, mask in selector.select(0.02):
|
|
stream, label = key.fileobj, key.data
|
|
if label == 'in':
|
|
try:
|
|
count = os.write(stream.fileno(), offered[sent:sent + 65536])
|
|
sent += count
|
|
except BrokenPipeError:
|
|
count = 0
|
|
if sent == len(offered) or count == 0:
|
|
selector.unregister(stream)
|
|
stream.close()
|
|
else:
|
|
try:
|
|
data = os.read(stream.fileno(), 65536)
|
|
except BlockingIOError:
|
|
continue
|
|
if data:
|
|
(output if label == 'out' else diagnostic).extend(data)
|
|
else:
|
|
selector.unregister(stream)
|
|
stream.close()
|
|
except BaseException as error:
|
|
errors.append(type(error).__name__ + ':' + str(error))
|
|
finally:
|
|
stop = time.monotonic() + RECLAIM
|
|
while time.monotonic() < stop:
|
|
try:
|
|
poll_own()
|
|
table, found = scan_reap()
|
|
current = dict(found)
|
|
for pid, start in tracked.items():
|
|
token = table.get(pid)
|
|
if token and token[1] == start and token[2] not in ('Z', 'X'):
|
|
current[pid] = start
|
|
for pid, start in current.items():
|
|
poll_own()
|
|
fresh_signal(pid, start)
|
|
poll_own()
|
|
table, found = scan_reap()
|
|
remaining = [pid for pid, start in tracked.items()
|
|
if pid in table and table[pid][1] == start]
|
|
if not remaining and (child is None or child.returncode is not None):
|
|
break
|
|
except BaseException as error:
|
|
errors.append(type(error).__name__ + ':' + str(error))
|
|
time.sleep(0.01)
|
|
# Drain the pipes after killing owned descendants. Closing them is bounded.
|
|
for key in list(selector.get_map().values()):
|
|
stream, label = key.fileobj, key.data
|
|
if label != 'in':
|
|
try:
|
|
while time.monotonic() < stop:
|
|
data = os.read(stream.fileno(), 65536)
|
|
if not data:
|
|
break
|
|
(output if label == 'out' else diagnostic).extend(data)
|
|
except BlockingIOError:
|
|
pass
|
|
except BaseException as error:
|
|
errors.append(type(error).__name__ + ':' + str(error))
|
|
try:
|
|
selector.unregister(stream)
|
|
stream.close()
|
|
except BaseException as error:
|
|
errors.append(type(error).__name__ + ':' + str(error))
|
|
selector.close()
|
|
poll_own()
|
|
table = proc_table()
|
|
current = discover(table, supervisor, tracked)
|
|
remaining = {pid: start for pid, start in tracked.items()
|
|
if pid in table and table[pid][1] == start}
|
|
remaining.update(current)
|
|
group_remaining = [pid for pid, start in remaining.items()
|
|
if pid in table and table[pid][1] == start and table[pid][4] == leader_group]
|
|
rc = child.returncode if child is not None else None
|
|
meta = dict(schema=1, stage=request['stage'], supervisor_pid=supervisor[0], supervisor_start=supervisor[1],
|
|
leader_pid=child.pid if child else None, leader_start=leader_start,
|
|
uid=os.geteuid(), gid=os.getegid(), groups=os.getgroups(), deadline=deadline,
|
|
elapsed=time.monotonic() - started, leader_rc=rc,
|
|
control_rc=124 if timed_out else (125 if errors else rc), timeout=timed_out,
|
|
leader_reaped=child is not None and rc is not None,
|
|
descendants_absent=not remaining, process_group_absent=not group_remaining,
|
|
adopted=adopted, tracked=sorted([pid, start] for pid, start in tracked.items()), errors=errors,
|
|
input_offered_length=len(offered), input_offered_sha256=hashlib.sha256(offered).hexdigest(),
|
|
input_sent_length=sent, stdout_length=len(output), stdout_sha256=hashlib.sha256(output).hexdigest(),
|
|
stderr_length=len(diagnostic), stderr_sha256=hashlib.sha256(diagnostic).hexdigest())
|
|
wire = dict(meta=meta, stdout=base64.b64encode(output).decode('ascii'),
|
|
stderr=base64.b64encode(diagnostic).decode('ascii'))
|
|
sys.stdout.write(json.dumps(wire, separators=(',', ':')))
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
'''
|
|
|
|
COMPILE_DEADLINE = 20
|
|
CANDIDATE_DEADLINE = 8
|
|
CLEANUP_DEADLINE = 5
|
|
ABSENCE_DEADLINE = 3
|
|
RECLAIM_ALLOWANCE = 4
|
|
SUPPORT = {'edge_service.h': '#ifndef EDGE_SERVICE_H\n#define EDGE_SERVICE_H\n#include <stddef.h>\n#include <stdint.h>\n\nenum { EDGE_PORTS = 4, EDGE_QUEUE = 64, EDGE_BATCH = 16 };\nenum edge_mode { EDGE_NONE, EDGE_RISING, EDGE_FALLING, EDGE_BOTH };\nenum device_kind { DEVICE_ENCODER, DEVICE_BUTTON_LOW, DEVICE_BUTTON_HIGH, DEVICE_LEVEL };\ntypedef struct {\n uint64_t sequence;\n int level;\n} edge_event;\ntypedef struct {\n int claimed;\n int configured;\n int level;\n int edge;\n edge_event queue[EDGE_QUEUE];\n size_t head;\n size_t count;\n uint64_t fed;\n uint64_t matched;\n uint64_t reads;\n} backend_port;\ntypedef struct {\n backend_port ports[EDGE_PORTS];\n uint64_t serial;\n} edge_backend;\ntypedef struct {\n int edge;\n int level;\n size_t queued;\n uint64_t fed;\n uint64_t matched;\n uint64_t reads;\n} backend_view;\ntypedef struct {\n int active;\n int kind;\n size_t batch;\n uint64_t delivered;\n uint64_t rising;\n uint64_t falling;\n uint64_t sequence_sum;\n uint64_t fold;\n uint64_t last_sequence;\n int last_level;\n} edge_session;\ntypedef struct {\n edge_backend *backend;\n edge_session sessions[EDGE_PORTS];\n} edge_service;\n\nvoid backend_init(edge_backend *backend);\nint backend_claim(edge_backend *backend, int port, int initial);\nint backend_configure(edge_backend *backend, int port, int mode);\nint backend_release(edge_backend *backend, int port);\nint backend_feed(edge_backend *backend, int port, int level);\nint backend_read(edge_backend *backend, int port, edge_event *events, size_t capacity);\nint backend_snapshot(const edge_backend *backend, int port, backend_view *view);\nint edge_select_mode(int kind);\nvoid service_init(edge_service *service, edge_backend *backend);\nint service_open(edge_service *service, int port, int kind, int initial, size_t batch);\nint service_close(edge_service *service, int port);\nint service_feed(edge_service *service, int port, int level);\nint service_drain(edge_service *service, int port, size_t budget);\nint service_snapshot(const edge_service *service, int port, edge_session *view);\n#endif\n', 'backend.c': '#include "edge_service.h"\n#include <errno.h>\n#include <string.h>\n\nstatic int port_valid(int port)\n{\n return port >= 0 && port < EDGE_PORTS;\n}\n\nstatic int level_valid(int level)\n{\n return level == 0 || level == 1;\n}\n\nstatic int mode_valid(int mode)\n{\n return mode >= EDGE_NONE && mode <= EDGE_BOTH;\n}\n\nstatic backend_port *claimed_port(edge_backend *backend, int port)\n{\n if (!backend || !port_valid(port)) {\n return NULL;\n }\n if (!backend->ports[port].claimed) {\n return NULL;\n }\n return &backend->ports[port];\n}\n\nstatic const backend_port *const_claimed_port(const edge_backend *backend, int port)\n{\n if (!backend || !port_valid(port)) {\n return NULL;\n }\n if (!backend->ports[port].claimed) {\n return NULL;\n }\n return &backend->ports[port];\n}\n\nstatic int matches(int mode, int before, int after)\n{\n if (before == after) {\n return 0;\n }\n switch (mode) {\n case EDGE_RISING:\n return after == 1;\n case EDGE_FALLING:\n return after == 0;\n case EDGE_BOTH:\n return 1;\n default:\n return 0;\n }\n}\n\nvoid backend_init(edge_backend *backend)\n{\n if (backend) {\n memset(backend, 0, sizeof(*backend));\n }\n}\n\nint backend_claim(edge_backend *backend, int port, int initial)\n{\n backend_port *slot;\n if (!backend || !port_valid(port)) {\n return -EINVAL;\n }\n if (!level_valid(initial)) {\n return -EINVAL;\n }\n slot = &backend->ports[port];\n if (slot->claimed) {\n return -EBUSY;\n }\n memset(slot, 0, sizeof(*slot));\n slot->claimed = 1;\n slot->level = initial;\n return 0;\n}\n\nint backend_configure(edge_backend *backend, int port, int mode)\n{\n backend_port *slot = claimed_port(backend, port);\n if (!slot) {\n return -ENOENT;\n }\n if (!mode_valid(mode)) {\n return -EINVAL;\n }\n slot->edge = mode;\n slot->configured = 1;\n return 0;\n}\n\nint backend_release(edge_backend *backend, int port)\n{\n backend_port *slot = claimed_port(backend, port);\n if (!slot) {\n return -ENOENT;\n }\n memset(slot, 0, sizeof(*slot));\n return 0;\n}\n\nint backend_feed(edge_backend *backend, int port, int level)\n{\n backend_port *slot = claimed_port(backend, port);\n int selected;\n size_t tail;\n if (!slot || !slot->configured) {\n return -ENOENT;\n }\n if (!level_valid(level)) {\n return -EINVAL;\n }\n selected = matches(slot->edge, slot->level, level);\n if (selected && slot->count == EDGE_QUEUE) {\n return -ENOSPC;\n }\n backend->serial++;\n slot->fed++;\n slot->level = level;\n if (!selected) {\n return 0;\n }\n tail = (slot->head + slot->count) % EDGE_QUEUE;\n slot->queue[tail].sequence = backend->serial;\n slot->queue[tail].level = level;\n slot->count++;\n slot->matched++;\n return 1;\n}\n\nint backend_read(edge_backend *backend, int port, edge_event *events, size_t capacity)\n{\n backend_port *slot = claimed_port(backend, port);\n size_t count;\n size_t first;\n if (!slot || !slot->configured) {\n return -ENOENT;\n }\n if (!events || capacity == 0 || capacity > EDGE_BATCH) {\n return -EINVAL;\n }\n slot->reads++;\n count = slot->count < capacity ? slot->count : capacity;\n if (count == 0) {\n return 0;\n }\n first = EDGE_QUEUE - slot->head;\n if (first > count) {\n first = count;\n }\n memcpy(events, slot->queue + slot->head, first * sizeof(*events));\n if (first < count) {\n memcpy(events + first, slot->queue, (count - first) * sizeof(*events));\n }\n slot->head = (slot->head + count) % EDGE_QUEUE;\n slot->count -= count;\n return (int)count;\n}\n\nint backend_snapshot(const edge_backend *backend, int port, backend_view *view)\n{\n const backend_port *slot = const_claimed_port(backend, port);\n if (!slot || !slot->configured) {\n return -ENOENT;\n }\n if (!view) {\n return -EINVAL;\n }\n view->edge = slot->edge;\n view->level = slot->level;\n view->queued = slot->count;\n view->fed = slot->fed;\n view->matched = slot->matched;\n view->reads = slot->reads;\n return 0;\n}\n', 'console.c': '#include "edge_service.h"\n#include <errno.h>\n#include <inttypes.h>\n#include <limits.h>\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n\nstatic int parse_number(const char *text, int *out)\n{\n const unsigned char *scan = (const unsigned char *)text;\n char *end;\n long value;\n if (*scan == \'+\' || *scan == \'-\') {\n ++scan;\n }\n if (!*scan) {\n return 0;\n }\n for (; *scan; ++scan) {\n if (*scan < \'0\' || *scan > \'9\') {\n return 0;\n }\n }\n errno = 0;\n value = strtol(text, &end, 10);\n if (errno || *end || value < INT_MIN || value > INT_MAX) {\n return 0;\n }\n *out = (int)value;\n return 1;\n}\n\nstatic size_t tokenize(char *line, char **parts, size_t capacity)\n{\n char *token = strtok(line, " \\t\\r\\n");\n size_t count = 0;\n while (token) {\n if (count == capacity) {\n return capacity + 1;\n }\n parts[count++] = token;\n token = strtok(NULL, " \\t\\r\\n");\n }\n return count;\n}\n\nstatic int read_line(FILE *stream, char *line, size_t capacity)\n{\n size_t length = 0;\n int byte;\n while ((byte = fgetc(stream)) != EOF && byte != \'\\n\') {\n if (byte == 0 || byte > 127 || length + 1 >= capacity) {\n return -1;\n }\n line[length++] = (char)byte;\n }\n if (ferror(stream)) {\n return -1;\n }\n if (byte == EOF && length == 0) {\n return 0;\n }\n line[length] = 0;\n return 1;\n}\n\nstatic void print_session(edge_service *service, int port)\n{\n edge_session view;\n int rc = service_snapshot(service, port, &view);\n printf("S %d", rc);\n if (rc == 0) {\n printf(" %d %zu %" PRIu64 " %" PRIu64 " %" PRIu64,\n view.kind, view.batch, view.delivered, view.rising, view.falling);\n printf(" %" PRIu64 " %" PRIu64 " %" PRIu64 " %d",\n view.sequence_sum, view.fold, view.last_sequence, view.last_level);\n }\n putchar(\'\\n\');\n}\n\nstatic void print_backend(edge_backend *backend, int port)\n{\n backend_view view;\n int rc = backend_snapshot(backend, port, &view);\n printf("B %d", rc);\n if (rc == 0) {\n printf(" %d %d %zu %" PRIu64 " %" PRIu64,\n view.edge, view.level, view.queued, view.fed, view.matched);\n }\n putchar(\'\\n\');\n}\n\nstatic int execute(edge_service *service, edge_backend *backend, char **parts, size_t count)\n{\n int values[4];\n size_t index;\n size_t required;\n if (strlen(parts[0]) != 1) {\n return 0;\n }\n switch (parts[0][0]) {\n case \'O\':\n required = 5;\n break;\n case \'F\':\n case \'D\':\n required = 3;\n break;\n case \'S\':\n case \'B\':\n case \'C\':\n required = 2;\n break;\n default:\n return 0;\n }\n if (count != required) {\n return 0;\n }\n for (index = 1; index < count; ++index) {\n if (!parse_number(parts[index], &values[index - 1])) {\n return 0;\n }\n }\n switch (parts[0][0]) {\n case \'O\':\n printf("O %d\\n", service_open(service, values[0], values[1], values[2], (size_t)values[3]));\n break;\n case \'F\':\n printf("F %d\\n", service_feed(service, values[0], values[1]));\n break;\n case \'D\':\n printf("D %d\\n", service_drain(service, values[0], (size_t)values[1]));\n break;\n case \'S\':\n print_session(service, values[0]);\n break;\n case \'B\':\n print_backend(backend, values[0]);\n break;\n case \'C\':\n printf("C %d\\n", service_close(service, values[0]));\n break;\n }\n return 1;\n}\n\nint main(void)\n{\n edge_backend backend;\n edge_service service;\n char line[256];\n char *parts[6];\n int line_status;\n backend_init(&backend);\n service_init(&service, &backend);\n while ((line_status = read_line(stdin, line, sizeof(line))) > 0) {\n size_t count;\n count = tokenize(line, parts, 6);\n if (count == 0) {\n continue;\n }\n if (count > 6 || !execute(&service, &backend, parts, count)) {\n fputs("invalid command\\n", stderr);\n return 2;\n }\n }\n if (line_status < 0) {\n fputs("invalid input line\\n", stderr);\n return 2;\n }\n return 0;\n}\n'}
|
|
|
|
|
|
def digest(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def identity_int(value, minimum=0):
|
|
return type(value) is int and value >= minimum
|
|
|
|
|
|
def validate_meta(meta, stage, deadline, identity, output, diagnostic, offered):
|
|
required = {'schema', 'stage', 'supervisor_pid', 'supervisor_start', 'leader_pid', 'leader_start',
|
|
'uid', 'gid', 'groups', 'deadline', 'elapsed', 'leader_rc', 'control_rc', 'timeout',
|
|
'leader_reaped', 'descendants_absent', 'process_group_absent', 'adopted', 'tracked', 'errors',
|
|
'input_offered_length', 'input_offered_sha256', 'input_sent_length', 'stdout_length',
|
|
'stdout_sha256', 'stderr_length', 'stderr_sha256'}
|
|
if type(meta) is not dict or set(meta) != required:
|
|
raise ValueError('metadata fields')
|
|
if type(meta['schema']) is not int or meta['schema'] != 1 or meta['stage'] != stage:
|
|
raise ValueError('metadata stage')
|
|
for key in ('supervisor_pid', 'supervisor_start', 'leader_pid', 'leader_start'):
|
|
if not identity_int(meta[key], 1):
|
|
raise ValueError('metadata process identity ' + key)
|
|
if meta['supervisor_pid'] == meta['leader_pid']:
|
|
raise ValueError('metadata duplicate process identity')
|
|
if (not identity_int(meta['uid']) or not identity_int(meta['gid']) or
|
|
type(meta['groups']) is not list or any(not identity_int(item) for item in meta['groups']) or
|
|
(meta['uid'], meta['gid'], meta['groups']) != identity):
|
|
raise ValueError('metadata execution identity')
|
|
if type(meta['deadline']) is not int or meta['deadline'] != deadline:
|
|
raise ValueError('metadata deadline')
|
|
elapsed = meta['elapsed']
|
|
if (type(elapsed) not in (int, float) or not math.isfinite(elapsed) or
|
|
not 0 <= elapsed <= deadline + RECLAIM_ALLOWANCE):
|
|
raise ValueError('metadata elapsed')
|
|
if any(type(meta[key]) is not bool for key in ('timeout', 'leader_reaped', 'descendants_absent', 'process_group_absent')):
|
|
raise ValueError('metadata boolean')
|
|
if type(meta['leader_rc']) is not int or type(meta['control_rc']) is not int:
|
|
raise ValueError('metadata return code')
|
|
if (not -64 <= meta['leader_rc'] <= 255 or not -64 <= meta['control_rc'] <= 255 or
|
|
not meta['leader_reaped']):
|
|
raise ValueError('metadata leader result')
|
|
if type(meta['errors']) is not list or any(type(item) is not str for item in meta['errors']):
|
|
raise ValueError('metadata errors')
|
|
expected_rc = 124 if meta['timeout'] else (125 if meta['errors'] else meta['leader_rc'])
|
|
if meta['control_rc'] != expected_rc:
|
|
raise ValueError('metadata controlled return code')
|
|
if meta['timeout'] and meta['elapsed'] < deadline:
|
|
raise ValueError('metadata early timeout')
|
|
if type(meta['tracked']) is not list or type(meta['adopted']) is not list:
|
|
raise ValueError('metadata descendants')
|
|
tokens = set()
|
|
for token in meta['tracked']:
|
|
if type(token) is not list or len(token) != 2 or any(not identity_int(item, 1) for item in token):
|
|
raise ValueError('metadata tracked token')
|
|
if token[0] == meta['supervisor_pid'] or tuple(token) in tokens:
|
|
raise ValueError('metadata tracked identity')
|
|
tokens.add(tuple(token))
|
|
if (meta['leader_pid'], meta['leader_start']) not in tokens:
|
|
raise ValueError('metadata untracked leader')
|
|
for token in meta['adopted']:
|
|
if (type(token) is not list or len(token) != 3 or any(not identity_int(item, 1) for item in token[:2]) or
|
|
not identity_int(token[2]) or tuple(token[:2]) not in tokens or token[0] == meta['leader_pid']):
|
|
raise ValueError('metadata adopted identity')
|
|
for key, data in (('stdout', output), ('stderr', diagnostic), ('input_offered', offered)):
|
|
if not identity_int(meta[key + '_length']) or meta[key + '_length'] != len(data):
|
|
raise ValueError('metadata IO length ' + key)
|
|
if type(meta[key + '_sha256']) is not str or meta[key + '_sha256'] != digest(data):
|
|
raise ValueError('metadata IO digest ' + key)
|
|
if not identity_int(meta['input_sent_length']) or meta['input_sent_length'] > len(offered):
|
|
raise ValueError('metadata sent length')
|
|
|
|
|
|
def run_stage(stage, argv, cwd, deadline, offered, records):
|
|
request = dict(stage=stage, argv=argv, cwd=cwd, deadline=deadline,
|
|
input=base64.b64encode(offered).decode('ascii'))
|
|
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
|
|
identity = (65534, 65534, []) if isolated else (os.geteuid(), os.getegid(), os.getgroups())
|
|
record = dict(stage=stage, argv=argv, cwd=cwd, deadline=deadline)
|
|
records.append(record)
|
|
child = subprocess.Popen([sys.executable, '-I', '-c', SUPERVISOR], stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
try:
|
|
wire, wrapper_diagnostic = child.communicate(json.dumps(request).encode('ascii'), timeout=deadline + 7)
|
|
except subprocess.TimeoutExpired:
|
|
record.update(wrapper_timeout=True, wrapper_rc=child.poll())
|
|
for stream in (child.stdin, child.stdout, child.stderr):
|
|
if stream and not stream.closed:
|
|
stream.close()
|
|
raise RuntimeError('supervisor outer deadline')
|
|
record.update(wrapper_timeout=False, wrapper_rc=child.returncode, wire_length=len(wire), wire_sha256=digest(wire),
|
|
wrapper_stderr_length=len(wrapper_diagnostic), wrapper_stderr_sha256=digest(wrapper_diagnostic))
|
|
if child.returncode != 0:
|
|
raise ValueError(stage + ' supervisor nonzero')
|
|
def unique_object(pairs):
|
|
result = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError('duplicate wire key')
|
|
result[key] = value
|
|
return result
|
|
parsed = json.loads(wire, object_pairs_hook=unique_object)
|
|
if type(parsed) is not dict or set(parsed) != {'meta', 'stdout', 'stderr'}:
|
|
raise ValueError(stage + ' wire fields')
|
|
output = base64.b64decode(parsed['stdout'], validate=True)
|
|
diagnostic = base64.b64decode(parsed['stderr'], validate=True)
|
|
meta = parsed['meta']
|
|
record['meta'] = meta
|
|
validate_meta(meta, stage, deadline, identity, output, diagnostic, offered)
|
|
record['validated'] = True
|
|
record['ok'] = (meta['leader_rc'] == 0 and meta['control_rc'] == 0 and not meta['timeout'] and
|
|
meta['leader_reaped'] and meta['descendants_absent'] and meta['process_group_absent'] and
|
|
not meta['errors'] and meta['input_sent_length'] == len(offered))
|
|
return record['ok'], output
|
|
|
|
|
|
class Model:
|
|
def __init__(self):
|
|
self.serial = 0
|
|
self.ports = {}
|
|
|
|
def command(self, op, *args):
|
|
port = args[0]
|
|
state = self.ports.get(port)
|
|
if op == 'O':
|
|
_, kind, initial, batch = args
|
|
if port not in range(4) or kind not in range(4) or initial not in (0, 1) or not 1 <= batch <= 16:
|
|
return ['O', -22]
|
|
if state is not None:
|
|
return ['O', -16]
|
|
self.ports[port] = dict(kind=kind, initial=initial, batch=batch, level=initial, edge=[3, 2, 1, 0][kind],
|
|
fed=0, matched=0, queue=[], delivered=0, rising=0, falling=0,
|
|
sequence_sum=0, fold=0, last_sequence=0, last_level=initial)
|
|
return ['O', 0]
|
|
if state is None:
|
|
return [op, -2]
|
|
if op == 'F':
|
|
level = args[1]
|
|
if level not in (0, 1):
|
|
return ['F', -22]
|
|
selected = level != state['level'] and (state['edge'] == 3 or
|
|
state['edge'] == 1 and level == 1 or state['edge'] == 2 and level == 0)
|
|
if selected and len(state['queue']) == 64:
|
|
return ['F', -28]
|
|
self.serial += 1
|
|
state['fed'] += 1
|
|
state['level'] = level
|
|
if selected:
|
|
state['matched'] += 1
|
|
state['queue'].append((self.serial, level))
|
|
return ['F', int(selected)]
|
|
if op == 'D':
|
|
budget = args[1]
|
|
if not 0 <= budget <= 4096:
|
|
return ['D', -22]
|
|
count = min(budget, len(state['queue']))
|
|
consumed, state['queue'] = state['queue'][:count], state['queue'][count:]
|
|
for sequence, level in consumed:
|
|
state['delivered'] += 1
|
|
state['rising' if level else 'falling'] += 1
|
|
state['sequence_sum'] = (state['sequence_sum'] + sequence) % (1 << 64)
|
|
state['fold'] = (state['fold'] * 257 + sequence * 2 + level) % (1 << 64)
|
|
state['last_sequence'], state['last_level'] = sequence, level
|
|
return ['D', count]
|
|
if op == 'S':
|
|
return ['S', 0] + [state[key] for key in ('kind', 'batch', 'delivered', 'rising', 'falling',
|
|
'sequence_sum', 'fold', 'last_sequence', 'last_level')]
|
|
if op == 'B':
|
|
return ['B', 0, state['edge'], state['level'], len(state['queue']), state['fed'], state['matched']]
|
|
if op == 'C':
|
|
del self.ports[port]
|
|
return ['C', 0]
|
|
raise ValueError('model operation')
|
|
|
|
|
|
def scenarios():
|
|
commands = []
|
|
expectations = []
|
|
criteria = []
|
|
model = Model()
|
|
|
|
def add(clause, op, *args):
|
|
commands.append(op + ' ' + ' '.join(str(value) for value in args))
|
|
expectations.append(model.command(op, *args))
|
|
criteria.append(clause)
|
|
|
|
# Public default scenario and a separate button-only multi-event batch scenario.
|
|
for port, kind, initial, batch in ((0, 0, 0, 1), (1, 1, 1, 4), (2, 2, 0, 16), (3, 3, 1, 3)):
|
|
add('E04/E05', 'O', port, kind, initial, batch)
|
|
for level in (initial, 1 - initial, initial, 1 - initial, initial, initial):
|
|
add('E07/E08/E09', 'F', port, level)
|
|
add('E08/E09', 'B', port)
|
|
for budget in (0, 4096, 1, 2, 0):
|
|
add('E11', 'D', port, budget)
|
|
add('E12', 'S', port)
|
|
add('E08/E09', 'B', port)
|
|
for port in range(4):
|
|
add('E04', 'O', port, 1, 0, 4)
|
|
add('E04', 'O', port, 1, 0, 0)
|
|
add('E07', 'F', port, 2)
|
|
add('E11', 'D', port, 4097)
|
|
add('E11', 'D', port, -1)
|
|
add('E12', 'S', port)
|
|
add('E13', 'C', port)
|
|
add('E06', 'S', port)
|
|
add('E06', 'F', port, 3)
|
|
add('E06', 'D', port, 4097)
|
|
add('E06', 'B', port)
|
|
add('E06', 'C', port)
|
|
for args in ((-1, 0, 0, 1), (4, 0, 0, 1), (0, -1, 0, 1), (0, 4, 0, 1),
|
|
(0, 0, 2, 1), (0, 0, 0, 0), (0, 0, 0, 17)):
|
|
add('E04', 'O', *args)
|
|
# Force wraparound, full queue, repeated levels while full, and recovery.
|
|
add('E05', 'O', 0, 0, 0, 16)
|
|
for index in range(64):
|
|
add('E07/E08/E09', 'F', 0, 1 - index % 2)
|
|
add('E09', 'F', 0, 0)
|
|
add('E09', 'F', 0, 1)
|
|
add('E09', 'B', 0)
|
|
add('E11', 'D', 0, 37)
|
|
for index in range(37):
|
|
add('E07/E08/E09', 'F', 0, 1 - index % 2)
|
|
add('E11', 'D', 0, 4096)
|
|
add('E12', 'S', 0)
|
|
add('E13', 'C', 0)
|
|
rng = random.Random(0x45444745)
|
|
for cycle in range(8):
|
|
for port in range(4):
|
|
add('E04/E05/E13', 'O', port, (port + cycle) % 4, rng.randrange(2), rng.choice((1, 2, 3, 7, 16)))
|
|
for step in range(140):
|
|
port = rng.randrange(4)
|
|
action = rng.randrange(5)
|
|
if action < 3:
|
|
add('E07/E08/E09', 'F', port, rng.randrange(2))
|
|
elif action == 3:
|
|
add('E11/E12', 'D', port, rng.choice((0, 1, 2, 3, 17, 65, 4096)))
|
|
else:
|
|
add('E12', 'S', port)
|
|
add('E08/E09', 'B', port)
|
|
for port in range(4):
|
|
add('E11/E12', 'D', port, 4096)
|
|
add('E12', 'S', port)
|
|
add('E08/E09', 'B', port)
|
|
add('E13', 'C', port)
|
|
return ('\n'.join(commands) + '\n').encode('ascii'), expectations, criteria
|
|
|
|
|
|
def parse_observations(raw):
|
|
# Whitespace and decimal syntax are explicitly defined in E16.
|
|
if any(value > 127 for value in raw):
|
|
raise ValueError('non-ASCII stdout')
|
|
tokens = re.split(rb'[ \t\r\n]+', raw.strip(b' \t\r\n')) if raw.strip(b' \t\r\n') else []
|
|
rows = []
|
|
index = 0
|
|
sizes = {b'O': 2, b'F': 2, b'D': 2, b'C': 2}
|
|
while index < len(tokens):
|
|
label = tokens[index]
|
|
if label not in (b'O', b'F', b'D', b'C', b'S', b'B') or index + 1 >= len(tokens):
|
|
raise ValueError('stdout label/shape')
|
|
if not re.fullmatch(rb'[+-]?[0-9]+', tokens[index + 1]):
|
|
raise ValueError('stdout decimal token')
|
|
rc = int(tokens[index + 1])
|
|
count = sizes.get(label, (11 if label == b'S' else 7) if rc == 0 else 2)
|
|
if index + count > len(tokens):
|
|
raise ValueError('stdout fields')
|
|
numeric = tokens[index + 1:index + count]
|
|
if any(not re.fullmatch(rb'[+-]?[0-9]+', token) for token in numeric):
|
|
raise ValueError('stdout decimal token')
|
|
rows.append([label.decode('ascii')] + [int(token) for token in numeric])
|
|
index += count
|
|
return rows
|
|
|
|
|
|
def main():
|
|
records = []
|
|
exceptions = []
|
|
scratch = None
|
|
success = False
|
|
cleanup_ok = False
|
|
absence_ok = False
|
|
failure = 'not executed'
|
|
rows = 0
|
|
input_data, expected, criteria = scenarios()
|
|
try:
|
|
scratch = tempfile.mkdtemp(prefix='edge-policy-')
|
|
project = Path(scratch) / 'project'
|
|
project.mkdir(mode=0o777)
|
|
project.chmod(0o777)
|
|
for name, content in SUPPORT.items():
|
|
(project / name).write_text(content)
|
|
for name in ('service.c', 'policy.c'):
|
|
(project / name).write_bytes((Path(sys.argv[1]) / name).read_bytes())
|
|
# All candidate input and public support is ready before the first child.
|
|
compile_argv = ['gcc', '-std=c11', '-O2', '-Wall', '-Wextra', '-pedantic',
|
|
str(project / 'service.c'), str(project / 'policy.c'),
|
|
str(project / 'backend.c'), str(project / 'console.c'), '-o', str(project / 'edge-run')]
|
|
compiled, unused = run_stage('compile', compile_argv, str(project), COMPILE_DEADLINE, b'', records)
|
|
if not compiled:
|
|
failure = 'compile did not complete successfully'
|
|
else:
|
|
executed, raw = run_stage('candidate', [str(project / 'edge-run')], str(project),
|
|
CANDIDATE_DEADLINE, input_data, records)
|
|
if not executed:
|
|
failure = 'candidate execution rejected'
|
|
else:
|
|
actual = parse_observations(raw)
|
|
rows = len(actual)
|
|
mismatch = next((index for index, (left, right) in enumerate(zip(actual, expected)) if left != right), None)
|
|
if mismatch is not None:
|
|
failure = 'behavior row %d (%s): expected %s observed %s' % (mismatch, criteria[mismatch], expected[mismatch], actual[mismatch])
|
|
elif len(actual) != len(expected):
|
|
failure = 'behavior row count expected %d observed %d' % (len(expected), len(actual))
|
|
else:
|
|
success = True
|
|
failure = ''
|
|
except BaseException as error:
|
|
exceptions.append(type(error).__name__ + ':' + str(error))
|
|
failure = 'checker exception ' + type(error).__name__ + ':' + str(error)
|
|
finally:
|
|
if scratch is not None:
|
|
try:
|
|
cleanup_code = 'import shutil,sys;shutil.rmtree(sys.argv[1])'
|
|
cleanup_ok, unused = run_stage('cleanup', [sys.executable, '-I', '-c', cleanup_code, scratch],
|
|
'/tmp', CLEANUP_DEADLINE, b'', records)
|
|
except BaseException as error:
|
|
exceptions.append('cleanup:' + type(error).__name__ + ':' + str(error))
|
|
try:
|
|
absence_code = ('import os,sys\ntry: os.lstat(sys.argv[1])\n'
|
|
'except FileNotFoundError: raise SystemExit(0)\nraise SystemExit(1)\n')
|
|
absence_ok, unused = run_stage('absence', [sys.executable, '-I', '-c', absence_code, scratch],
|
|
'/tmp', ABSENCE_DEADLINE, b'', records)
|
|
except BaseException as error:
|
|
exceptions.append('absence:' + type(error).__name__ + ':' + str(error))
|
|
summary = dict(records=records, exceptions=exceptions, success=success, cleanup_ok=cleanup_ok,
|
|
absence_ok=absence_ok, failure=failure, observed_rows=rows, expected_rows=len(expected),
|
|
input_offered_length=len(input_data), input_offered_sha256=digest(input_data))
|
|
packed = base64.b64encode(zlib.compress(json.dumps(summary, separators=(',', ':')).encode(), 9)).decode('ascii')
|
|
sys.stderr.write('EDGE_META ' + packed + '\n')
|
|
passed = success and cleanup_ok and absence_ok and not exceptions
|
|
print('PASS' if passed else 'FAIL ' + failure[:250])
|
|
return 0 if passed else 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|