amos 9d633e9e67 Batch 15 (20261023-C-hwe-scale-15): 320 HWE-derived repair tasks
One user-space repair task per eligible HWE debug-diag / compile-deploy base
ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content
filter. Each ships a public spec with a check-to-basis table, at least two
defects with single-fix fixtures, a symptom-masking fixture and an alternative
correct implementation. Claude re-checked all 320: structure, the HWE quality
items and trace6 Docker calibration receipts against current digests
(320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120
ad-hoc family names; the next batch maps them to the canonical list.
The batch's 1,256-line progress log stays in its factory report and the
36-local branch batch15-hwe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 01:55:56 +08:00

429 lines
45 KiB
Python

import base64
import bz2
import hashlib
import json
import math
import os
from pathlib import Path
import random
import subprocess
import sys
import tempfile
import time
RECOVERY_SECONDS = 3
RECORD_WIRE = False
SUPERVISOR = r'''
import base64,bz2,ctypes,hashlib,json,os,signal,subprocess,sys,time
cfg=json.loads(sys.argv[1]); start=time.monotonic(); errors=[]; tracked={}; signals={}; waits={}
leader=None; leader_start=None; output=b''; diagnostic=b''; timed_out=False; drained=False
reaped=False; leader_rc=None; remaining=[]; pg_absent=False
def fault(op, error):
errors.append([op,type(error).__name__,str(error)])
def exact(pid):
try:
with open('/proc/%d/stat'%pid,'rb') as stream: data=stream.read()
words=data[data.rfind(b')')+2:].split()
return {'pid':pid,'state':words[0].decode('ascii'),'ppid':int(words[1]),'pg':int(words[2]),'start':int(words[19])}
except (FileNotFoundError,ProcessLookupError): return None
def table_now():
result={}
for name in os.listdir('/proc'):
if name.isascii() and name.isdigit():
value=exact(int(name))
if value is not None: result[value['pid']]=value
return result
def closure(table, supervisor, remembered):
roots={supervisor}
for pid,old in remembered.items():
current=table.get(pid)
if current and current['start']==old['start'] and current['state'] not in ('Z','X'):
roots.add(pid)
found={}
changed=True
while changed:
changed=False
for pid,current in table.items():
if pid!=supervisor and pid not in found and (current['ppid'] in roots or pid in roots):
found[pid]=current; roots.add(pid); changed=True
return found
def discover():
table=table_now(); found=closure(table,os.getpid(),tracked)
for pid,current in found.items():
if pid not in tracked or tracked[pid]['start']!=current['start']:
tracked[pid]=dict(current)
return table,found
def send_fresh(pid, token):
current=exact(pid)
if current is None or current['start']!=token['start'] or current['state'] in ('Z','X'): return
os.kill(pid,signal.SIGKILL)
signals[(pid,token['start'])]=signals.get((pid,token['start']),0)+1
def own_wait():
global reaped,leader_rc
if leader is not None:
value=leader.poll()
if value is not None:
leader_rc=leader.wait(timeout=0)
reaped=True
def adopted_wait():
if leader is not None and not reaped: return
while True:
try: pid,status=os.waitpid(-1,os.WNOHANG)
except ChildProcessError: break
if pid==0: break
waits[pid]=os.waitstatus_to_exitcode(status)
supervisor_start=exact(os.getpid())['start']
try:
libc=ctypes.CDLL(None,use_errno=True)
if libc.prctl(36,1,0,0,0)!=0: raise OSError(ctypes.get_errno(),'subreaper')
leader=subprocess.Popen(cfg['argv'],cwd=cfg['cwd'],stdin=subprocess.PIPE,
stdout=subprocess.PIPE,stderr=subprocess.PIPE,start_new_session=True)
info=exact(leader.pid)
if info is None: raise RuntimeError('leader identity disappeared before capture')
leader_start=info['start']; tracked[leader.pid]=info
payload=sys.stdin.buffer.read()
first=True
while True:
try:
output,diagnostic=leader.communicate(input=payload if first else None,timeout=0.04)
drained=True; own_wait(); break
except subprocess.TimeoutExpired:
first=False
own_wait()
discover()
if time.monotonic()-start>=cfg['deadline']:
timed_out=True; break
except BaseException as error:
fault('execute',error)
finally:
recovery_end=time.monotonic()+cfg['recovery']
while True:
try: own_wait()
except BaseException as error: fault('own_wait',error)
found={}
try: _,found=discover()
except BaseException as error: fault('discover',error)
for pid,token in list(found.items()):
try: send_fresh(pid,token)
except (FileNotFoundError,ProcessLookupError): pass
except BaseException as error: fault('signal',error)
try: own_wait()
except BaseException as error: fault('own_wait',error)
try: adopted_wait()
except BaseException as error: fault('adopted_wait',error)
if leader is not None and not drained:
try:
output,diagnostic=leader.communicate(timeout=0.03)
drained=True; own_wait()
except subprocess.TimeoutExpired: pass
except BaseException as error: fault('drain',error)
try:
table,found=discover()
remaining=[[p,v['start'],v['state']] for p,v in found.items()]
pg_absent=leader is not None and not any(v['pg']==leader.pid for v in table.values())
except BaseException as error:
fault('absence',error); remaining=[['unknown']]; pg_absent=False
if reaped and drained and not remaining and pg_absent: break
if time.monotonic()>=recovery_end:
fault('recovery',RuntimeError('deadline with unconfirmed closure')); break
time.sleep(0.01)
meta={'phase':cfg['phase'],'nonce':cfg['nonce'],'argv':cfg['argv'],
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
'supervisor_pid':os.getpid(),'supervisor_start':supervisor_start,
'leader_pid':None if leader is None else leader.pid,'leader_start':leader_start,
'leader_rc':leader_rc,'leader_reaped':reaped,'timeout':timed_out,
'controlled_rc':124 if timed_out else leader_rc,'elapsed':time.monotonic()-start,
'deadline':cfg['deadline'],'recovery':cfg['recovery'],
'pg_absent':pg_absent,'remaining':remaining,'exceptions':errors,
'adopted_waits':[[p,None,rc] for p,rc in waits.items()],
'descendants':[[p,t['start'],signals.get((p,t['start']),0),None]
for p,t in tracked.items() if leader is None or p!=leader.pid],
'stdout_len':len(output),'stdout_sha256':hashlib.sha256(output).hexdigest(),
'stderr_len':len(diagnostic),'stderr_sha256':hashlib.sha256(diagnostic).hexdigest()}
wire={'meta':meta,'stdout':base64.b64encode(bz2.compress(output)).decode('ascii'),
'stderr':base64.b64encode(bz2.compress(diagnostic)).decode('ascii')}
sys.stdout.write(json.dumps(wire,separators=(',',':'))+'\n')
'''
CLEANUP = "import shutil,sys; shutil.rmtree(sys.argv[1])"
ABSENCE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: sys.exit(0)\nsys.exit(1)"
def identity_int(value, minimum=0):
return type(value) is int and value >= minimum
def validate_meta(meta, config, out, err, expected_rc=0):
required = {'phase','nonce','argv','uid','gid','groups','supervisor_pid','supervisor_start',
'leader_pid','leader_start','leader_rc','leader_reaped','timeout','controlled_rc',
'elapsed','deadline','recovery','pg_absent','remaining','exceptions','descendants',
'stdout_len','stdout_sha256','stderr_len','stderr_sha256','adopted_waits'}
if type(meta) is not dict or not required <= meta.keys():
raise ValueError('missing stage metadata')
if meta['phase'] != config['phase'] or meta['nonce'] != config['nonce'] or meta['argv'] != config['argv']:
raise ValueError('stage binding conflict')
for name in ('uid','gid','supervisor_pid','supervisor_start','leader_pid','leader_start'):
if not identity_int(meta[name], 1 if name not in ('uid','gid') else 0):
raise ValueError('invalid identity '+name)
if meta['supervisor_pid'] == meta['leader_pid']:
raise ValueError('supervisor and leader identical')
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
expected_identity = (65534,65534,[]) if isolated else (os.getuid(),os.getgid(),os.getgroups())
if type(meta['groups']) is not list or any(type(x) is not int or x < 0 for x in meta['groups']):
raise ValueError('invalid groups')
if (meta['uid'],meta['gid'],meta['groups']) != expected_identity:
raise ValueError('execution identity conflict')
if type(meta['deadline']) is not int or meta['deadline'] != config['deadline']:
raise ValueError('deadline conflict')
if type(meta['recovery']) is not int or meta['recovery'] != RECOVERY_SECONDS:
raise ValueError('recovery conflict')
elapsed = meta['elapsed']
if type(elapsed) not in (int,float) or not math.isfinite(elapsed) or not 0 <= elapsed <= config['deadline']+RECOVERY_SECONDS+2:
raise ValueError('invalid elapsed')
for name in ('timeout','leader_reaped','pg_absent'):
if type(meta[name]) is not bool:
raise ValueError('invalid boolean '+name)
for name in ('leader_rc','controlled_rc'):
if type(meta[name]) is not int:
raise ValueError('invalid return code '+name)
if meta['controlled_rc'] != (124 if meta['timeout'] else meta['leader_rc']):
raise ValueError('return code conflict')
for name in ('remaining','exceptions','descendants','adopted_waits'):
if type(meta[name]) is not list:
raise ValueError('invalid list '+name)
for row in meta['adopted_waits']:
if type(row) is not list or len(row) != 3 or not identity_int(row[0],1) or row[1] is not None or type(row[2]) is not int:
raise ValueError('invalid adopted wait identity')
for row in meta['descendants']:
if type(row) is not list or len(row) != 4 or not all(identity_int(x,1) for x in row[:2]) or not identity_int(row[2]) or (row[3] is not None and type(row[3]) is not int):
raise ValueError('invalid descendant record')
for name,data in (('stdout',out),('stderr',err)):
if not identity_int(meta[name+'_len']) or meta[name+'_len'] != len(data):
raise ValueError('invalid IO length')
if type(meta[name+'_sha256']) is not str or meta[name+'_sha256'] != hashlib.sha256(data).hexdigest():
raise ValueError('invalid IO digest')
if meta['exceptions'] or meta['remaining'] or not meta['leader_reaped'] or not meta['pg_absent']:
raise ValueError('stage recovery or exception failure')
if meta['timeout'] or meta['leader_rc'] != expected_rc or meta['controlled_rc'] != expected_rc:
raise ValueError('controlled timeout' if meta['timeout'] else 'leader nonzero exit')
def execute_stage(phase, argv, cwd, deadline, payload, evidence, expected_rc=0):
nonce = os.urandom(8).hex()
config = dict(phase=phase,argv=argv,cwd=str(cwd),deadline=deadline,recovery=RECOVERY_SECONDS,nonce=nonce)
wrapper_argv = [sys.executable,'-I','-c',SUPERVISOR,json.dumps(config,separators=(',',':'))]
raw = subprocess.run(wrapper_argv,input=payload,capture_output=True,timeout=deadline+RECOVERY_SECONDS+5)
# Preserve actual wrapper facts before touching or trusting its wire protocol.
entry = dict(phase=phase,nonce=nonce,wrapper_rc=raw.returncode,
wire_len=len(raw.stdout),wire_sha256=hashlib.sha256(raw.stdout).hexdigest(),
wrapper_stderr_len=len(raw.stderr),wrapper_stderr_sha256=hashlib.sha256(raw.stderr).hexdigest())
if RECORD_WIRE:
entry['wire_ascii'] = raw.stdout.decode('ascii')
entry['wrapper_stderr_bz2'] = base64.b64encode(bz2.compress(raw.stderr)).decode('ascii')
evidence.append(entry)
if raw.returncode != 0:
raise ValueError(phase+' supervisor nonzero')
wire = json.loads(raw.stdout)
out = bz2.decompress(base64.b64decode(wire['stdout'],validate=True))
err = bz2.decompress(base64.b64decode(wire['stderr'],validate=True))
meta = wire['meta']
if not RECORD_WIRE: entry['meta'] = meta
validate_meta(meta,config,out,err,expected_rc)
return out,err
FIXED = {'Makefile': 'CC = gcc\nCFLAGS = -std=c11 -O2 -Wall -Wextra -Werror -pedantic\nSOURCES = src/main.c src/plan.c src/format_port.c src/transport.c src/publisher.c\nHEADERS = include/topic_runtime.h\n.PHONY: all clean demo recovery\nall: topic-replay\ntopic-replay: $(SOURCES) $(HEADERS)\n\t$(CC) $(CFLAGS) -Iinclude $(SOURCES) -o $@\ndemo: topic-replay\n\t./topic-replay examples/normal.txt\nrecovery: topic-replay\n\t./topic-replay --batch 2 --fail-at 3 examples/recovery.txt\nclean:\n\trm -f topic-replay\n', 'examples/recovery.txt': 'P east temperature 10\nP west humidity 11\nP north vibration 12\nP south pressure 13\nF\nP recovery temperature 14\n', 'examples/normal.txt': 'P edge01 temperature 0\nP edge01 humidity 1\nP edge02 temperature 2\nP edge03 vibration 3\nF\nP edge01 temperature 4\nP edge01 humidity 5\n', 'include/topic_runtime.h': '#ifndef TOPIC_RUNTIME_H\n#define TOPIC_RUNTIME_H\n#include <stddef.h>\n#include <stdint.h>\n#include <stdio.h>\n#define TOPIC_EVENT_LIMIT 2000\n#define TOPIC_COMPONENT_LIMIT 96\n#define TOPIC_BATCH_LIMIT 16\n#define TOPIC_QUEUE_LIMIT 32\n#define TOPIC_ALLOCATION_LIMIT 2048\n\ntypedef enum {\n EVENT_PUBLISH,\n EVENT_FLUSH\n} EventKind;\n\ntypedef struct {\n EventKind kind;\n char device[TOPIC_COMPONENT_LIMIT + 1];\n char channel[TOPIC_COMPONENT_LIMIT + 1];\n uint32_t sequence;\n} TopicEvent;\n\ntypedef struct {\n TopicEvent events[TOPIC_EVENT_LIMIT];\n size_t count;\n size_t batch;\n size_t fail_at;\n} TopicPlan;\n\ntypedef struct {\n char *address;\n size_t size;\n unsigned long token;\n int live;\n} Allocation;\n\ntypedef struct {\n char *address;\n size_t length;\n unsigned long token;\n size_t ordinal;\n} PendingDelivery;\n\ntypedef struct {\n size_t attempts;\n size_t allocated;\n size_t allocation_failures;\n size_t released;\n size_t live_blocks;\n size_t live_bytes;\n size_t release_errors;\n size_t enqueued;\n size_t delivered;\n size_t lifetime_errors;\n size_t queue_errors;\n} TopicAudit;\n\ntypedef struct {\n Allocation allocations[TOPIC_ALLOCATION_LIMIT];\n size_t allocation_count;\n unsigned long next_token;\n size_t fail_at;\n PendingDelivery queue[TOPIC_QUEUE_LIMIT];\n size_t queued;\n TopicAudit audit;\n FILE *output;\n int first_delivery;\n} TopicRuntime;\n\nvoid topic_runtime_init(TopicRuntime *runtime, size_t fail_at, FILE *output);\nint topic_format(TopicRuntime *runtime, char **output, const char *device,\n const char *channel, uint32_t sequence);\nint topic_release(TopicRuntime *runtime, char *address);\nAllocation *topic_allocation(TopicRuntime *runtime, const char *address);\nint topic_enqueue(TopicRuntime *runtime, char *address);\nvoid topic_complete(TopicRuntime *runtime);\nvoid topic_observer_open(TopicRuntime *runtime);\nvoid topic_observer_close(TopicRuntime *runtime, int publisher_rc);\nvoid topic_runtime_dispose(TopicRuntime *runtime);\nint topic_read_plan(FILE *input, TopicPlan *plan, char *error, size_t capacity);\nint topic_parse_options(int argc, char **argv, TopicPlan *plan,\n const char **path, char *error, size_t capacity);\nint publisher_run(TopicRuntime *runtime, const TopicPlan *plan);\n#endif\n', 'docs/contract.md': "# Topic replay public contract v1\n\n## C1 Build and scope\nThe offline C11 application is built by `make` with gcc; there are no downloaded dependencies.\n`Makefile`, include/topic_runtime.h, src/main.c, src/plan.c, src/format_port.c and src/transport.c are fixed public support. Official grading rebuilds exactly these files plus the submitted src/publisher.c using `gcc -std=c11 -O2 -Wall -Wextra -Werror -pedantic -Iinclude`.\nOnly src/publisher.c is submitted. It implements `int publisher_run(TopicRuntime *, const TopicPlan *)` using the public runtime ports. All actual publication, formatting and resource operations must go through those ports; directly modifying runtime accounting/queue/allocation state, printing substitute observations or inspecting private graders is outside the project API.\nThis deterministic deferred transport models a broker handoff without MQTT, networking, physical boards, GNU asprintf or RSS measurements.\n\n## C2 Input and public replay commands\n`./topic-replay examples/normal.txt` is the default demo: batch=4, fail-at=0. It contains no allocation failure.\nThe independent optional recovery example is `./topic-replay --batch 2 --fail-at 3 examples/recovery.txt`.\nGeneral invocation is `./topic-replay [--batch N] [--fail-at K] PATH`; batch N is 1..16 (default 4), K is 0..2000 (default 0). K=0 disables injected failures; otherwise exactly the Kth topic-format allocation attempt fails. If fewer attempts occur it never fails.\nInput consists of at most 2000 events, with optional blank lines or lines whose first token starts with #. Each nonempty event is either `P DEVICE CHANNEL SEQUENCE` or `F`. Tokens are separated by space, TAB or CR; LF ends a line. Each line has at most 8190 bytes excluding LF. No embedded NUL, VT or FF is valid.\nDEVICE and CHANNEL are 1..96 characters from ASCII A-Z, a-z, 0-9, underscore and hyphen, case sensitive. SEQUENCE is a decimal integer 0..1000000000; ASCII digits, optional leading + and arbitrary leading zeroes are allowed within the line bound. Options accept the same unsigned decimal spelling within their stated bounds.\n`F` completes all pending deliveries and ends the current batch; consecutive F and empty inputs are valid. End of input also completes all pending deliveries. P and F ordering is preserved.\nFor multiple independent replay sessions, `./topic-replay --suite PATH` reads rows `N K INPUT_PATH` and emits an array of session observations. Paths have no whitespace, each row is at most 1022 bytes excluding LF, N and K obey the same option bounds. Empty LF/CR lines and lines beginning # are ignored. Every session starts with fresh resource and transport state; suite rows are processed in order. An empty suite is valid.\nValid runs exit 0, including injected allocation failures; invalid CLI/input or I/O failure exits 2. stderr may contain arbitrary diagnostic bytes and is not part of the observation protocol.\n\n## C3 Topic formatting and failure recovery\nFor every P, call topic_format once with that event's exact device/channel/sequence. A successful result is the dynamically allocated NUL-terminated ASCII topic `fleet/DEVICE/telemetry/CHANNEL/SEQUENCE`, using canonical unsigned decimal sequence without leading zeroes.\nThe return value is the topic byte length excluding NUL; caller receives a unique allocation token and owns its returned char buffer. Failure returns -1 and writes NULL to the output pointer. The fixed port records real malloc/free operations, including the terminating NUL in resource byte accounting; no fixed-size replacement or bypass is permitted.\nA failed allocation affects only that P: skip its enqueue and continue processing every later event. It must not discard pending successful publications. No retries for the failed event, and no extra format calls. All valid replay sessions return publisher_rc=0.\n\n## C4 Deferred handoff and ownership\nEach successfully formatted P is enqueued exactly once using topic_enqueue; successful enqueue borrows the caller's buffer. Completion reads the actual original buffer later. The allocation must remain live with unchanged bytes until topic_complete finishes the delivery; topic_enqueue does not copy ownership.\nThe fixed transport records delivery ordinal (1-based successful enqueue count), alive (integer 1 if the same allocation token remains live, otherwise 0), and topic as lowercase hex of the bytes it actually reads. It reports an empty string for a dead token, never dereferencing freed memory.\nThe batch size is an upper bound on pending successful publications. Complete before exceeding N, at every F, and at EOF. Earlier completion is allowed, including after every P. Thus implementations may use a batch-owned pointer array or immediately complete each publication.\nThe caller releases every successful allocation exactly once with topic_release after its completion. topic_release(NULL) is allowed and has no effect. Double release, untracked pointers and releasing queued buffers early are erroneous. No pending deliveries, live allocations or live bytes may remain when publisher_run returns.\n\n## C5 Raw observer and audit\nSingle-session stdout is one JSON object; suite stdout is one JSON array of those objects. JSON whitespace and object key order are unrestricted; the complete stdout must decode as one JSON value. Integer fields are JSON integers, not booleans. No other stdout text is permitted.\nEach session object has deliveries, publisher_rc and audit. deliveries is the ordered array of {ordinal,alive,topic}; publisher_rc is the actual return code from publisher_run. The fixed observer reports facts, including failure facts, rather than a success verdict.\naudit has attempts, allocated, allocation_failures, released, live_blocks, live_bytes, release_errors, enqueued, delivered, lifetime_errors, queue_errors and pending. attempts equals the number of P events; allocated equals attempts minus the single injected failure if reached; allocation_failures is 0 or 1 accordingly. released, enqueued and delivered equal allocated. All other audit values are zero on a conforming run. Each delivery has alive=1 and exact topic contents from C3 in P order, excluding failed events.\nObserver disposal frees leftover blocks only after emitting the audit, so disposal cannot hide leaks. There are no wall-clock, peak-allocation, stderr-size or RSS pass thresholds. Only final ownership, actual publication content/order, failure continuation and the public limits above are assessed.\n", 'src/format_port.c': '#include "topic_runtime.h"\n#include <inttypes.h>\n#include <stdlib.h>\n#include <string.h>\n\nvoid topic_runtime_init(TopicRuntime *runtime, size_t fail_at, FILE *output)\n{\n memset(runtime, 0, sizeof(*runtime));\n runtime->fail_at = fail_at;\n runtime->output = output;\n runtime->next_token = 1;\n runtime->first_delivery = 1;\n}\n\nAllocation *topic_allocation(TopicRuntime *runtime, const char *address)\n{\n size_t index = runtime->allocation_count;\n while (index > 0) {\n Allocation *entry = &runtime->allocations[--index];\n if (entry->address == address && entry->live) {\n return entry;\n }\n }\n return NULL;\n}\n\nstatic int format_size(const char *device, const char *channel, uint32_t sequence)\n{\n return snprintf(NULL, 0, "fleet/%s/telemetry/%s/%" PRIu32,\n device, channel, sequence);\n}\n\nstatic char *allocate_topic(TopicRuntime *runtime, size_t bytes)\n{\n char *address;\n Allocation *entry;\n runtime->audit.attempts++;\n if (runtime->fail_at != 0 && runtime->audit.attempts == runtime->fail_at) {\n runtime->audit.allocation_failures++;\n return NULL;\n }\n if (runtime->allocation_count == TOPIC_ALLOCATION_LIMIT) {\n runtime->audit.allocation_failures++;\n return NULL;\n }\n address = malloc(bytes);\n if (address == NULL) {\n runtime->audit.allocation_failures++;\n return NULL;\n }\n entry = &runtime->allocations[runtime->allocation_count++];\n entry->address = address;\n entry->size = bytes;\n entry->token = runtime->next_token++;\n entry->live = 1;\n runtime->audit.allocated++;\n runtime->audit.live_blocks++;\n runtime->audit.live_bytes += bytes;\n return address;\n}\n\nint topic_format(TopicRuntime *runtime, char **output, const char *device,\n const char *channel, uint32_t sequence)\n{\n int count;\n char *buffer;\n if (output == NULL) {\n return -1;\n }\n *output = NULL;\n if (device == NULL || channel == NULL) {\n return -1;\n }\n count = format_size(device, channel, sequence);\n if (count < 0) {\n return -1;\n }\n buffer = allocate_topic(runtime, (size_t)count + 1);\n if (buffer == NULL) {\n return -1;\n }\n if (snprintf(buffer, (size_t)count + 1, "fleet/%s/telemetry/%s/%" PRIu32,\n device, channel, sequence) != count) {\n topic_release(runtime, buffer);\n return -1;\n }\n *output = buffer;\n return count;\n}\n\nint topic_release(TopicRuntime *runtime, char *address)\n{\n Allocation *entry;\n if (address == NULL) {\n return 0;\n }\n entry = topic_allocation(runtime, address);\n if (entry == NULL) {\n runtime->audit.release_errors++;\n return -1;\n }\n entry->live = 0;\n runtime->audit.released++;\n runtime->audit.live_blocks--;\n runtime->audit.live_bytes -= entry->size;\n free(entry->address);\n return 0;\n}\n\nvoid topic_runtime_dispose(TopicRuntime *runtime)\n{\n size_t index;\n for (index = 0; index < runtime->allocation_count; index++) {\n Allocation *entry = &runtime->allocations[index];\n if (entry->live) {\n free(entry->address);\n entry->live = 0;\n }\n }\n runtime->queued = 0;\n}\n', 'src/plan.c': '#include "topic_runtime.h"\n#include <stdlib.h>\n#include <string.h>\n\nstatic int report_error(char *error, size_t capacity, const char *text)\n{\n if (capacity > 0) {\n snprintf(error, capacity, "%s", text);\n }\n return -1;\n}\n\nstatic int horizontal(unsigned char byte)\n{\n return byte == \' \' || byte == \'\\t\' || byte == \'\\r\';\n}\n\nstatic char *token_next(char **cursor)\n{\n char *start;\n while (horizontal((unsigned char)**cursor)) {\n (*cursor)++;\n }\n if (**cursor == \'\\0\' || **cursor == \'\\n\') {\n return NULL;\n }\n start = *cursor;\n while (**cursor && **cursor != \'\\n\' && !horizontal((unsigned char)**cursor)) {\n (*cursor)++;\n }\n if (**cursor) {\n *(*cursor)++ = \'\\0\';\n }\n return start;\n}\n\nstatic int decimal(const char *text, size_t limit, size_t *result)\n{\n size_t value = 0;\n const unsigned char *cursor = (const unsigned char *)text;\n if (*cursor == \'+\') {\n cursor++;\n }\n if (*cursor == \'\\0\') {\n return -1;\n }\n while (*cursor) {\n unsigned digit;\n if (*cursor < \'0\' || *cursor > \'9\') {\n return -1;\n }\n digit = (unsigned)(*cursor++ - \'0\');\n if (value > limit / 10 || (value == limit / 10 && digit > limit % 10)) {\n return -1;\n }\n value = value * 10 + digit;\n }\n *result = value;\n return 0;\n}\n\nstatic int component(const char *text)\n{\n size_t length = 0;\n const unsigned char *cursor = (const unsigned char *)text;\n while (*cursor) {\n unsigned char byte = *cursor++;\n if (!((byte >= \'a\' && byte <= \'z\') || (byte >= \'A\' && byte <= \'Z\') ||\n (byte >= \'0\' && byte <= \'9\') || byte == \'_\' || byte == \'-\')) {\n return 0;\n }\n length++;\n }\n return length > 0 && length <= TOPIC_COMPONENT_LIMIT;\n}\n\nstatic int append_line(char *line, TopicPlan *plan, char *error, size_t capacity)\n{\n char *cursor = line;\n char *operation = token_next(&cursor);\n char *device;\n char *channel;\n char *sequence;\n size_t number;\n TopicEvent *event;\n if (operation == NULL || operation[0] == \'#\') {\n return 0;\n }\n if (plan->count == TOPIC_EVENT_LIMIT) {\n return report_error(error, capacity, "event limit exceeded");\n }\n event = &plan->events[plan->count];\n memset(event, 0, sizeof(*event));\n if (strcmp(operation, "F") == 0) {\n if (token_next(&cursor) != NULL) {\n return report_error(error, capacity, "extra flush fields");\n }\n event->kind = EVENT_FLUSH;\n plan->count++;\n return 0;\n }\n if (strcmp(operation, "P") != 0) {\n return report_error(error, capacity, "unknown operation");\n }\n device = token_next(&cursor);\n channel = token_next(&cursor);\n sequence = token_next(&cursor);\n if (device == NULL || channel == NULL || sequence == NULL || token_next(&cursor)) {\n return report_error(error, capacity, "publish requires three fields");\n }\n if (!component(device) || !component(channel)) {\n return report_error(error, capacity, "invalid topic component");\n }\n if (decimal(sequence, 1000000000, &number) != 0) {\n return report_error(error, capacity, "invalid sequence");\n }\n event->kind = EVENT_PUBLISH;\n strcpy(event->device, device);\n strcpy(event->channel, channel);\n event->sequence = (uint32_t)number;\n plan->count++;\n return 0;\n}\n\nint topic_read_plan(FILE *input, TopicPlan *plan, char *error, size_t capacity)\n{\n char line[8192];\n size_t length = 0;\n int byte;\n plan->count = 0;\n while ((byte = fgetc(input)) != EOF) {\n if (byte == \'\\0\') {\n return report_error(error, capacity, "NUL in input line");\n }\n if (byte == \'\\n\') {\n line[length] = \'\\0\';\n if (append_line(line, plan, error, capacity) != 0) {\n return -1;\n }\n length = 0;\n continue;\n }\n if (length == sizeof(line) - 2) {\n return report_error(error, capacity, "input line too long");\n }\n line[length++] = (char)byte;\n }\n if (ferror(input)) {\n return report_error(error, capacity, "input read failed");\n }\n if (length != 0) {\n line[length] = \'\\0\';\n if (append_line(line, plan, error, capacity) != 0) {\n return -1;\n }\n }\n return 0;\n}\n\nint topic_parse_options(int argc, char **argv, TopicPlan *plan,\n const char **path, char *error, size_t capacity)\n{\n int index;\n plan->batch = 4;\n plan->fail_at = 0;\n *path = NULL;\n for (index = 1; index < argc; index++) {\n if (strcmp(argv[index], "--batch") == 0 || strcmp(argv[index], "--fail-at") == 0) {\n int is_batch = strcmp(argv[index], "--batch") == 0;\n size_t number;\n if (++index == argc || decimal(argv[index], is_batch ? 16 : 2000, &number) != 0) {\n return report_error(error, capacity, "invalid option value");\n }\n if (is_batch && number == 0) {\n return report_error(error, capacity, "batch must be positive");\n }\n if (is_batch) {\n plan->batch = number;\n } else {\n plan->fail_at = number;\n }\n } else {\n if (*path != NULL || argv[index][0] == \'-\') {\n return report_error(error, capacity, "expected one input path");\n }\n *path = argv[index];\n }\n }\n if (*path == NULL) {\n return report_error(error, capacity, "input path required");\n }\n return 0;\n}\n', 'src/main.c': '#include "topic_runtime.h"\n#include <stdlib.h>\n#include <string.h>\n\nstatic int run_one(int argc, char **argv)\n{\n TopicPlan *plan = calloc(1, sizeof(*plan));\n TopicRuntime *runtime = calloc(1, sizeof(*runtime));\n const char *path = NULL;\n char error[128];\n FILE *input;\n int result;\n if (plan == NULL || runtime == NULL) {\n fputs("initialization allocation failed\\n", stderr);\n free(plan);\n free(runtime);\n return 2;\n }\n if (topic_parse_options(argc, argv, plan, &path, error, sizeof(error)) != 0) {\n fprintf(stderr, "options: %s\\n", error);\n free(plan);\n free(runtime);\n return 2;\n }\n input = fopen(path, "rb");\n if (input == NULL) {\n perror(path);\n free(plan);\n free(runtime);\n return 2;\n }\n result = topic_read_plan(input, plan, error, sizeof(error));\n if (fclose(input) != 0) {\n fputs("close input failed\\n", stderr);\n free(plan);\n free(runtime);\n return 2;\n }\n if (result != 0) {\n fprintf(stderr, "input: %s\\n", error);\n free(plan);\n free(runtime);\n return 2;\n }\n topic_runtime_init(runtime, plan->fail_at, stdout);\n topic_observer_open(runtime);\n result = publisher_run(runtime, plan);\n topic_observer_close(runtime, result);\n topic_runtime_dispose(runtime);\n free(runtime);\n free(plan);\n return 0;\n}\n\nstatic int run_suite_row(char *line, size_t *count)\n{\n char batch[1024];\n char failure[1024];\n char input[1024];\n char extra;\n char *arguments[6];\n size_t index;\n if (line[0] == \'#\') {\n return 0;\n }\n for (index = 0; line[index] == \'\\r\'; index++) {\n }\n if (line[index] == \'\\0\') {\n return 0;\n }\n if (sscanf(line, "%1023s %1023s %1023s %c", batch, failure, input, &extra) != 3) {\n fputs("invalid suite row\\n", stderr);\n return 2;\n }\n if ((*count)++ > 0) {\n fputc(\',\', stdout);\n }\n arguments[0] = "topic-replay";\n arguments[1] = "--batch";\n arguments[2] = batch;\n arguments[3] = "--fail-at";\n arguments[4] = failure;\n arguments[5] = input;\n return run_one(6, arguments);\n}\n\nstatic int run_suite(const char *path)\n{\n FILE *suite = fopen(path, "rb");\n char line[1024];\n size_t length = 0;\n size_t count = 0;\n int result = 0;\n int byte;\n if (suite == NULL) {\n perror(path);\n return 2;\n }\n fputs("[", stdout);\n while ((byte = fgetc(suite)) != EOF) {\n if (byte == \'\\0\') {\n fputs("NUL in suite row\\n", stderr);\n result = 2;\n break;\n }\n if (byte == \'\\n\') {\n line[length] = \'\\0\';\n result = run_suite_row(line, &count);\n length = 0;\n if (result != 0) {\n break;\n }\n continue;\n }\n if (length == sizeof(line) - 2) {\n fputs("suite row too long\\n", stderr);\n result = 2;\n break;\n }\n line[length++] = (char)byte;\n }\n if (result == 0 && length != 0) {\n line[length] = \'\\0\';\n result = run_suite_row(line, &count);\n }\n if (ferror(suite)) {\n fputs("suite read failed\\n", stderr);\n result = 2;\n }\n if (fclose(suite) != 0) {\n fputs("suite close failed\\n", stderr);\n result = 2;\n }\n fputs("]\\n", stdout);\n return result;\n}\n\nint main(int argc, char **argv)\n{\n if (argc == 3 && strcmp(argv[1], "--suite") == 0) {\n return run_suite(argv[2]);\n }\n return run_one(argc, argv);\n}\n', 'src/transport.c': '#include "topic_runtime.h"\n#include <string.h>\n\nstatic void print_hex(FILE *stream, const unsigned char *data, size_t count)\n{\n static const char digits[] = "0123456789abcdef";\n size_t index;\n fputc(\'"\', stream);\n for (index = 0; index < count; index++) {\n fputc(digits[data[index] >> 4], stream);\n fputc(digits[data[index] & 15], stream);\n }\n fputc(\'"\', stream);\n}\n\nvoid topic_observer_open(TopicRuntime *runtime)\n{\n fputs("{\\"deliveries\\":[", runtime->output);\n}\n\nint topic_enqueue(TopicRuntime *runtime, char *address)\n{\n Allocation *allocation = topic_allocation(runtime, address);\n PendingDelivery *pending;\n if (allocation == NULL) {\n runtime->audit.lifetime_errors++;\n return -1;\n }\n if (runtime->queued >= TOPIC_QUEUE_LIMIT) {\n runtime->audit.queue_errors++;\n return -1;\n }\n pending = &runtime->queue[runtime->queued++];\n pending->address = address;\n pending->length = strlen(address);\n pending->token = allocation->token;\n pending->ordinal = ++runtime->audit.enqueued;\n return 0;\n}\n\nstatic void observe_delivery(TopicRuntime *runtime, const PendingDelivery *pending)\n{\n Allocation *allocation = topic_allocation(runtime, pending->address);\n int live = allocation != NULL && allocation->token == pending->token;\n if (!runtime->first_delivery) {\n fputc(\',\', runtime->output);\n }\n runtime->first_delivery = 0;\n fprintf(runtime->output, "{\\"ordinal\\":%zu,\\"alive\\":%d,\\"topic\\":",\n pending->ordinal, live);\n if (live && pending->length < allocation->size) {\n print_hex(runtime->output, (const unsigned char *)pending->address,\n pending->length);\n } else {\n fputs("\\"\\"", runtime->output);\n }\n fputc(\'}\', runtime->output);\n if (!live) {\n runtime->audit.lifetime_errors++;\n }\n runtime->audit.delivered++;\n}\n\nvoid topic_complete(TopicRuntime *runtime)\n{\n size_t index;\n for (index = 0; index < runtime->queued; index++) {\n observe_delivery(runtime, &runtime->queue[index]);\n }\n runtime->queued = 0;\n}\n\nstatic void audit_field(FILE *output, const char *name, size_t value)\n{\n fprintf(output, ",\\"%s\\":%zu", name, value);\n}\n\nvoid topic_observer_close(TopicRuntime *runtime, int publisher_rc)\n{\n const TopicAudit *audit = &runtime->audit;\n FILE *output = runtime->output;\n fprintf(output, "],\\"publisher_rc\\":%d,\\"audit\\":{\\"attempts\\":%zu",\n publisher_rc, audit->attempts);\n audit_field(output, "allocated", audit->allocated);\n audit_field(output, "allocation_failures", audit->allocation_failures);\n audit_field(output, "released", audit->released);\n audit_field(output, "live_blocks", audit->live_blocks);\n audit_field(output, "live_bytes", audit->live_bytes);\n audit_field(output, "release_errors", audit->release_errors);\n audit_field(output, "enqueued", audit->enqueued);\n audit_field(output, "delivered", audit->delivered);\n audit_field(output, "lifetime_errors", audit->lifetime_errors);\n audit_field(output, "queue_errors", audit->queue_errors);\n audit_field(output, "pending", runtime->queued);\n fputs("}}\\n", output);\n}\n'}
import random
def cases():
result = []
result.append((4, 0, [("P", "edge01", "temperature", i) for i in range(100)]))
result.append((2, 3, [("P", x, "humidity", i) for i, x in enumerate(["east","west","north","south","recovery"])]))
result.extend([(1,0,[]),(16,1,[("P","a","z",0)]),(4,0,[("F",),("F",)])])
rng = random.Random(19420261025)
for index in range(18):
batch = 1 + index % 16
failure = [0,1,2,17,2000][index % 5]
events = []
for seq in range(31 + index * 3):
if rng.randrange(7) == 0:
events.append(("F",))
events.append(("P", "D_" + str(rng.randrange(101)), "c-" + str(rng.randrange(89)), rng.choice([0,1,9,10,999999999,1000000000,rng.randrange(1000000001)])))
result.append((batch,failure,events))
result.append((16,1999,[("P","d"*96,"C"*96,i) for i in range(2000)]))
result.append((3,2,[("P","000","_",0),("F",),("P","A","B",8),("F",),("P","after","failure",1000000000)]))
return result
def expected(case):
_, failure, events = case
deliveries = []
attempts = 0
for event in events:
if event[0] != "P":
continue
attempts += 1
if failure and attempts == failure:
continue
_, device, channel, sequence = event
topic = ("fleet/%s/telemetry/%s/%d" % (device,channel,sequence)).encode("ascii").hex()
deliveries.append(dict(ordinal=len(deliveries)+1,alive=1,topic=topic))
allocated = len(deliveries)
audit = dict(attempts=attempts,allocated=allocated,allocation_failures=attempts-allocated,
released=allocated,live_blocks=0,live_bytes=0,release_errors=0,
enqueued=allocated,delivered=allocated,lifetime_errors=0,queue_errors=0,pending=0)
return dict(deliveries=deliveries,publisher_rc=0,audit=audit)
def compare(actual, wanted, path='root'):
if type(actual) is not type(wanted):
raise ValueError(path+' type mismatch')
if isinstance(wanted,dict):
if actual.keys()!=wanted.keys():
raise ValueError(path+' fields mismatch')
for name in wanted:
compare(actual[name],wanted[name],path+'.'+name)
elif isinstance(wanted,list):
if len(actual)!=len(wanted):
raise ValueError(path+' length '+str(len(actual))+' expected '+str(len(wanted)))
for index,(a,w) in enumerate(zip(actual,wanted)):
compare(a,w,path+'['+str(index)+']')
elif actual!=wanted:
raise ValueError(path+' observed '+repr(actual)[:100]+' expected '+repr(wanted)[:100])
def prepare(scratch, candidate):
# Every source, suite and replay input is written before the first subprocess.
for name,content in FIXED.items():
target=scratch/name
target.parent.mkdir(parents=True,exist_ok=True)
target.write_text(content)
target=scratch/'src/publisher.c'
target.write_bytes((candidate/'src/publisher.c').read_bytes())
scenarios=cases()
rows=[]
for index,case in enumerate(scenarios):
batch,failure,events=case
path=scratch/('input-%02d.txt'%index)
if index==1:
# This valid path makes the entire suite row exactly 1022 bytes.
parent=scratch.joinpath(*(['d'*200]*4))
final_length=1018-len(str(parent))-1
if not 1<=final_length<=255:
raise ValueError('suite boundary path construction failed')
path=parent/('p'*final_length)
path.parent.mkdir(parents=True,exist_ok=True)
text=[]
for ordinal,event in enumerate(events):
if event[0]=='F':
text.append('F')
else:
_,device,channel,sequence=event
# Legal spelling variation exercises the public parser without changing the model.
prefix='P\t%s %s\t'%(device,channel)
if index==24 and ordinal==0:
number='+'+'0'*(8190-len(prefix)-1-len(str(sequence)))+str(sequence)
if len(prefix+number)!=8190:
raise ValueError('input boundary construction failed')
else:
number=('+'+'0'*5000+str(sequence)) if index==24 else str(sequence)
text.append(prefix+number)
path.write_text('\n'.join(text)+'\n')
batch_text=('+'+'0'*400+str(batch)) if index==0 else str(batch)
fail_text=('+'+'0'*400+str(failure)) if index==0 else str(failure)
row='%s %s %s'%(batch_text,fail_text,path)
if index==1 and len(row)!=1022:
raise ValueError('suite row boundary construction failed')
rows.append(row)
(scratch/'suite.txt').write_text('\n'.join(rows))
(scratch/'invalid-plan-nul.txt').write_bytes(b'P a b 1\0P c d 2\n')
(scratch/'invalid-suite-nul.txt').write_bytes(('1 0 %s'%path).encode()+b'\0garbage\n')
(scratch/'invalid-plan-length.txt').write_bytes(b'P a b '+b'0'*8185+b'\n')
(scratch/'invalid-suite-length.txt').write_bytes(b'1 0 '+b'x'*1019+b'\n')
# Preparation ends before execute_stage, whose child owns the entire scratch tree.
return [expected(case) for case in scenarios]
def main(candidate):
evidence=[]
faults=[]
scratch=None
business_ok=False
cleanup_ok=False
absence_ok=False
try:
scratch=Path(tempfile.mkdtemp(prefix='topic-format-'))
try:
wanted=prepare(scratch,candidate)
finally:
# No child has run yet: finish handoff even if material preparation failed.
if os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534':
for path in scratch.rglob('*'):
os.chown(path,65534,65534)
argv=['gcc','-std=c11','-O2','-Wall','-Wextra','-Werror','-pedantic','-Iinclude',
'src/main.c','src/plan.c','src/format_port.c','src/transport.c','src/publisher.c',
'-o','topic-replay']
execute_stage('compiler',argv,scratch,20,b'',evidence)
output,diagnostic=execute_stage('candidate',[str(scratch/'topic-replay'),'--suite',str(scratch/'suite.txt')],
scratch,12,b'',evidence)
# Strict JSON consumes the full stdout; whitespace need not be single-line.
actual=json.loads(output)
compare(actual,wanted)
for name,mode in (('invalid-plan-nul.txt','input'),
('invalid-suite-nul.txt','suite'),
('invalid-plan-length.txt','input'),
('invalid-suite-length.txt','suite')):
path=scratch/name
argv=[str(scratch/'topic-replay'),'--suite',str(path)] if mode=='suite' else [str(scratch/'topic-replay'),str(path)]
execute_stage(name,argv,scratch,12,b'',evidence,expected_rc=2)
business_ok=True
except BaseException as error:
faults.append(['business',type(error).__name__,str(error)])
finally:
if scratch is not None:
try:
execute_stage('cleanup',[sys.executable,'-I','-c',CLEANUP,str(scratch)],'/tmp',8,b'',evidence)
cleanup_ok=True
except BaseException as error:
faults.append(['cleanup',type(error).__name__,str(error)])
try:
execute_stage('absence',[sys.executable,'-I','-c',ABSENCE,str(scratch)],'/tmp',8,b'',evidence)
absence_ok=True
except BaseException as error:
faults.append(['absence',type(error).__name__,str(error)])
record=dict(stages=evidence,exceptions=faults,business_ok=business_ok,cleanup_ok=cleanup_ok,
absence_ok=absence_ok,raw_body_saved=False,
raw_body_policy='wire/inner IO bodies not persisted; recorded lengths and SHA256 are from actual captured bytes')
marker=base64.b64encode(bz2.compress(json.dumps(record,separators=(',',':')).encode())).decode()
print('TOPIC_EXEC_V1:'+marker,file=sys.stderr)
if business_ok and cleanup_ok and absence_ok and not faults:
print('PASS')
return 0
print('FAIL '+repr(faults)[:250])
return 1
if __name__=='__main__':
sys.exit(main(Path(sys.argv[1])))