One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1025 lines
48 KiB
Python
1025 lines
48 KiB
Python
"""Independent C11 row observer and Linux candidate-identity process supervisor."""
|
|
|
|
import base64
|
|
import ctypes
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import selectors
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
|
|
HEADERS = {'layout.h': '#ifndef LAYOUT_H\n#define LAYOUT_H\n\n#include <stddef.h>\n#include <stdint.h>\n\n#define LAYOUT_MAX_SLOTS 8u\n#define LAYOUT_MAX_ROW 128u\n#define LAYOUT_OK 0\n#define LAYOUT_EINVAL -22\n#define LAYOUT_ERANGE -34\n\ntypedef struct {\n _Alignas(128) unsigned char bytes[256];\n size_t used;\n uint32_t generation;\n} LayoutSlot;\n\ntypedef struct {\n LayoutSlot slots[LAYOUT_MAX_SLOTS];\n size_t width;\n size_t count;\n uint32_t revision;\n} LayoutBank;\n\ntypedef struct {\n size_t occupied;\n size_t total_bytes;\n size_t longest;\n uint32_t revision;\n} LayoutStats;\n\nint layout_init(LayoutBank *bank, size_t width, size_t count);\nint layout_clear(LayoutBank *bank, size_t index);\nint layout_store(LayoutBank *bank, size_t index, const unsigned char *data, size_t length);\nint layout_fill(LayoutBank *bank, size_t index, unsigned char value, size_t length);\nint layout_patch(LayoutBank *bank, size_t index, size_t offset,\n const unsigned char *data, size_t length);\nint layout_insert(LayoutBank *bank, size_t index, size_t offset,\n const unsigned char *data, size_t length);\nint layout_erase(LayoutBank *bank, size_t index, size_t offset, size_t length);\nint layout_fetch(const LayoutBank *bank, size_t index, unsigned char *out,\n size_t capacity, size_t *length);\nint layout_view(const LayoutBank *bank, size_t index, const unsigned char **data,\n size_t *length);\nint layout_copy(LayoutBank *bank, size_t source, size_t destination);\nint layout_swap(LayoutBank *bank, size_t left, size_t right);\nint layout_stats(const LayoutBank *bank, LayoutStats *stats);\nuint32_t layout_hash(const unsigned char *data, size_t length);\nint layout_slot_hash(const LayoutBank *bank, size_t index, uint32_t *hash);\n\n#endif\n', 'journal.h': '#ifndef JOURNAL_H\n#define JOURNAL_H\n\n#include "layout.h"\n\n#define JOURNAL_MAX_OPS 16u\n\ntypedef enum {\n JOURNAL_PUT = 1,\n JOURNAL_FILL = 2,\n JOURNAL_COPY = 3,\n JOURNAL_SWAP = 4,\n JOURNAL_CLEAR = 5,\n JOURNAL_PATCH = 6,\n JOURNAL_INSERT = 7,\n JOURNAL_ERASE = 8\n} JournalKind;\n\ntypedef struct {\n JournalKind kind;\n size_t first;\n size_t second;\n size_t length;\n unsigned char value;\n unsigned char data[LAYOUT_MAX_ROW];\n} JournalOp;\n\ntypedef struct {\n JournalOp ops[JOURNAL_MAX_OPS];\n size_t count;\n} Journal;\n\nvoid journal_reset(Journal *journal);\nint journal_put(Journal *journal, size_t index, const unsigned char *data, size_t length);\nint journal_fill(Journal *journal, size_t index, unsigned char value, size_t length);\nint journal_copy(Journal *journal, size_t source, size_t destination);\nint journal_swap(Journal *journal, size_t left, size_t right);\nint journal_clear(Journal *journal, size_t index);\nint journal_patch(Journal *journal, size_t index, size_t offset,\n const unsigned char *data, size_t length);\nint journal_insert(Journal *journal, size_t index, size_t offset,\n const unsigned char *data, size_t length);\nint journal_erase(Journal *journal, size_t index, size_t offset, size_t length);\nint journal_commit(LayoutBank *bank, Journal *journal);\nsize_t journal_count(const Journal *journal);\n\n#endif\n'}
|
|
OBSERVER = r'''
|
|
#include "layout.h"
|
|
#include "journal.h"
|
|
#include <inttypes.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
static int digit(char c) {
|
|
if (c >= '0' && c <= '9') return c - '0';
|
|
if (c >= 'a' && c <= 'f') return c - 'a' + 10;
|
|
if (c >= 'A' && c <= 'F') return c - 'A' + 10;
|
|
return -1;
|
|
}
|
|
static int decode(const char *text, unsigned char *out, size_t *length) {
|
|
size_t i, n;
|
|
if (strcmp(text, "-") == 0) { *length = 0; return 1; }
|
|
n = strlen(text);
|
|
if (n % 2 || n / 2 > 128) return 0;
|
|
for (i = 0; i < n / 2; i++) {
|
|
int a = digit(text[2*i]), b = digit(text[2*i+1]);
|
|
if (a < 0 || b < 0) return 0;
|
|
out[i] = (unsigned char)(16*a+b);
|
|
}
|
|
*length = n / 2;
|
|
return 1;
|
|
}
|
|
static void hex(const unsigned char *data, size_t length) {
|
|
static const char d[] = "0123456789abcdef";
|
|
size_t i;
|
|
for (i = 0; i < length; i++) {
|
|
putchar(d[data[i] >> 4]); putchar(d[data[i] & 15]);
|
|
}
|
|
}
|
|
static void snapshot(const LayoutBank *bank, const Journal *j, int rc) {
|
|
LayoutStats stats;
|
|
size_t i;
|
|
int sr = layout_stats(bank, &stats);
|
|
printf("{\"rc\":%d,\"queued\":%zu,\"stats_rc\":%d,\"stats\":[%zu,%zu,%zu,%" PRIu32 "],\"rows\":[",
|
|
rc, journal_count(j), sr, stats.occupied, stats.total_bytes,
|
|
stats.longest, stats.revision);
|
|
for (i = 0; i < bank->count; i++) {
|
|
const unsigned char *view = NULL;
|
|
unsigned char fetched[128];
|
|
unsigned char short_buffer[1] = {0xa7};
|
|
size_t used = 0, got = 999;
|
|
size_t short_got = 999;
|
|
uint32_t hash = 0;
|
|
int vr = layout_view(bank, i, &view, &used);
|
|
int fr = layout_fetch(bank, i, fetched, sizeof(fetched), &got);
|
|
int short_rc = layout_fetch(bank, i, short_buffer, used ? used - 1u : 0u, &short_got);
|
|
int hr = layout_slot_hash(bank, i, &hash);
|
|
size_t offset = view ? (size_t)((uintptr_t)view % bank->width) : 999u;
|
|
if (i) putchar(',');
|
|
printf("[%d,%zu,%zu,%d,%zu,%d,%" PRIu32 ",\"", vr, offset, used, fr, got, hr, hash);
|
|
if (vr == 0 && view && used <= bank->width) hex(view, used);
|
|
printf("\",\"");
|
|
if (fr == 0 && got <= sizeof(fetched)) hex(fetched, got);
|
|
printf("\",%d,%zu,%u]", short_rc, short_got, (unsigned)short_buffer[0]);
|
|
}
|
|
puts("]}");
|
|
}
|
|
int main(void) {
|
|
LayoutBank bank;
|
|
Journal journal;
|
|
char line[1024], payload[300];
|
|
size_t a, b, length;
|
|
unsigned char bytes[128];
|
|
int rc;
|
|
if (layout_init(&bank, 64u, 4u) != 0) return 3;
|
|
journal_reset(&journal);
|
|
while (fgets(line, sizeof(line), stdin)) {
|
|
a = b = length = 0;
|
|
payload[0] = 0;
|
|
if (sscanf(line, "I %zu %zu", &a, &b) == 2) {
|
|
rc = layout_init(&bank, a, b);
|
|
if (rc == 0) journal_reset(&journal);
|
|
} else if (sscanf(line, "P %zu %299s", &a, payload) == 2 && decode(payload, bytes, &length)) {
|
|
rc = journal_put(&journal, a, bytes, length);
|
|
} else if (sscanf(line, "T %zu %299s", &a, payload) == 2 && decode(payload, bytes, &length)) {
|
|
rc = layout_store(&bank, a, bytes, length);
|
|
} else if (sscanf(line, "F %zu %zu %zu", &a, &b, &length) == 3 && b <= 255) {
|
|
rc = journal_fill(&journal, a, (unsigned char)b, length);
|
|
} else if (sscanf(line, "U %zu %zu %zu", &a, &b, &length) == 3 && b <= 255) {
|
|
rc = layout_fill(&bank, a, (unsigned char)b, length);
|
|
} else if (sscanf(line, "N %zu %zu", &a, &b) == 2) {
|
|
rc = layout_store(&bank, a, NULL, b);
|
|
} else if (sscanf(line, "A %zu %zu %299s", &a, &b, payload) == 3 && decode(payload, bytes, &length)) {
|
|
rc = journal_patch(&journal, a, b, bytes, length);
|
|
} else if (sscanf(line, "E %zu %zu %299s", &a, &b, payload) == 3 && decode(payload, bytes, &length)) {
|
|
rc = journal_insert(&journal, a, b, bytes, length);
|
|
} else if (sscanf(line, "Q %zu %zu %zu", &a, &b, &length) == 3) {
|
|
rc = journal_erase(&journal, a, b, length);
|
|
} else if (sscanf(line, "G %zu %zu %299s", &a, &b, payload) == 3 && decode(payload, bytes, &length)) {
|
|
rc = layout_patch(&bank, a, b, bytes, length);
|
|
} else if (sscanf(line, "H %zu %zu %299s", &a, &b, payload) == 3 && decode(payload, bytes, &length)) {
|
|
rc = layout_insert(&bank, a, b, bytes, length);
|
|
} else if (sscanf(line, "J %zu %zu %zu", &a, &b, &length) == 3) {
|
|
rc = layout_erase(&bank, a, b, length);
|
|
} else if (sscanf(line, "C %zu %zu", &a, &b) == 2) {
|
|
rc = journal_copy(&journal, a, b);
|
|
} else if (sscanf(line, "Y %zu %zu", &a, &b) == 2) {
|
|
rc = layout_copy(&bank, a, b);
|
|
} else if (sscanf(line, "S %zu %zu", &a, &b) == 2) {
|
|
rc = journal_swap(&journal, a, b);
|
|
} else if (sscanf(line, "Z %zu %zu", &a, &b) == 2) {
|
|
rc = layout_swap(&bank, a, b);
|
|
} else if (sscanf(line, "D %zu", &a) == 1) {
|
|
rc = journal_clear(&journal, a);
|
|
} else if (sscanf(line, "X %zu", &a) == 1) {
|
|
rc = layout_clear(&bank, a);
|
|
} else if (line[0] == 'B') {
|
|
rc = journal_commit(&bank, &journal);
|
|
} else if (line[0] == 'R') {
|
|
journal_reset(&journal);
|
|
rc = 0;
|
|
} else {
|
|
fprintf(stderr, "invalid observer input: %s", line);
|
|
return 4;
|
|
}
|
|
snapshot(&bank, &journal, rc);
|
|
}
|
|
return ferror(stdin) ? 5 : 0;
|
|
}
|
|
'''
|
|
|
|
SUPERVISOR = r'''
|
|
import base64, ctypes, hashlib, json, os, pathlib, selectors, signal, subprocess, sys, time
|
|
|
|
def stat(pid):
|
|
try:
|
|
raw = pathlib.Path('/proc/%d/stat' % pid).read_text()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
close = raw.rfind(')')
|
|
fields = raw[close+2:].split()
|
|
return {'pid':pid, 'state':fields[0], 'ppid':int(fields[1]),
|
|
'pgrp':int(fields[2]), 'start':int(fields[19])}
|
|
|
|
def table():
|
|
result = {}
|
|
for name in os.listdir('/proc'):
|
|
if name.isdecimal():
|
|
item = stat(int(name))
|
|
if item is not None: result[item['pid']] = item
|
|
return result
|
|
|
|
def descendants(snapshot, own, tracked):
|
|
roots = {own}
|
|
for pid,start in tracked.items():
|
|
item = snapshot.get(pid)
|
|
if item and item['start'] == start and item['state'] not in ('Z','X'):
|
|
roots.add(pid)
|
|
found = roots - {own}
|
|
changed = True
|
|
while changed:
|
|
changed = False
|
|
for pid,item in snapshot.items():
|
|
if (pid != own and item['state'] not in ('Z','X') and
|
|
item['ppid'] in roots | found and pid not in found):
|
|
found.add(pid); changed = True
|
|
return {pid:snapshot[pid] for pid in found if snapshot[pid]['state'] not in ('Z','X')}
|
|
|
|
def fresh_kill(pid, token, errors):
|
|
try:
|
|
now = stat(pid)
|
|
if now and now['start'] == token and now['state'] not in ('Z','X'):
|
|
os.kill(pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
return
|
|
except Exception as exc:
|
|
errors.append('signal:%s:%s' % (type(exc).__name__, exc))
|
|
|
|
def fresh_kill_group(pid, token, errors):
|
|
try:
|
|
now = stat(pid)
|
|
if now and now['start'] == token and now['state'] not in ('Z','X') and now['pgrp'] == pid:
|
|
os.killpg(pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
return
|
|
except Exception as exc:
|
|
errors.append('signal-group:%s:%s' % (type(exc).__name__, exc))
|
|
|
|
def group_absent(snapshot, group):
|
|
return not any(item['pgrp'] == group and item['state'] not in ('Z','X')
|
|
for item in snapshot.values())
|
|
|
|
def pump(proc, send, limit):
|
|
selector = selectors.DefaultSelector()
|
|
output = [bytearray(), bytearray()]
|
|
sent = 0
|
|
for channel,index in ((proc.stdout,0),(proc.stderr,1)):
|
|
if channel is None or channel.closed: continue
|
|
os.set_blocking(channel.fileno(),False)
|
|
selector.register(channel,selectors.EVENT_READ,index)
|
|
if proc.stdin is not None and not proc.stdin.closed:
|
|
if send:
|
|
os.set_blocking(proc.stdin.fileno(),False)
|
|
selector.register(proc.stdin,selectors.EVENT_WRITE,2)
|
|
else:
|
|
proc.stdin.close()
|
|
expired = False
|
|
try:
|
|
while True:
|
|
proc.poll()
|
|
if not selector.get_map() and proc.returncode is not None: break
|
|
remaining = limit-time.monotonic()
|
|
if remaining <= 0:
|
|
expired = True; break
|
|
for key,mask in selector.select(min(remaining,.1)):
|
|
if key.data == 2:
|
|
try:
|
|
n=os.write(key.fileobj.fileno(),send[sent:sent+65536])
|
|
sent += n
|
|
except BrokenPipeError:
|
|
sent = min(sent,len(send))
|
|
selector.unregister(key.fileobj); key.fileobj.close()
|
|
except BlockingIOError:
|
|
pass
|
|
else:
|
|
if sent == len(send):
|
|
selector.unregister(key.fileobj); key.fileobj.close()
|
|
else:
|
|
try: data=os.read(key.fileobj.fileno(),65536)
|
|
except BlockingIOError: continue
|
|
if data: output[key.data].extend(data)
|
|
else:
|
|
selector.unregister(key.fileobj); key.fileobj.close()
|
|
finally:
|
|
selector.close()
|
|
return bytes(output[0]),bytes(output[1]),sent,expired
|
|
|
|
def reap_adopted(errors):
|
|
while True:
|
|
try:
|
|
pid,status = os.waitpid(-1, os.WNOHANG)
|
|
if pid == 0: return
|
|
except ChildProcessError:
|
|
return
|
|
except Exception as exc:
|
|
errors.append('reap:%s:%s' % (type(exc).__name__, exc)); return
|
|
|
|
def run(request):
|
|
started = time.monotonic()
|
|
own = os.getpid()
|
|
errors = []
|
|
leader = None
|
|
leader_stat = None
|
|
out = b''; err = b''; timed_out = False; sent = 0; input_bytes = b''
|
|
tracked = {}
|
|
own_stat = stat(own)
|
|
if ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) != 0:
|
|
errors.append('subreaper-enable-failed')
|
|
deadline = float(request['deadline'])
|
|
grace = float(request.get('grace', 2.0))
|
|
limit = started + deadline
|
|
try:
|
|
leader = subprocess.Popen(request['argv'], cwd=request.get('cwd'),
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
start_new_session=True)
|
|
leader_stat = stat(leader.pid)
|
|
if leader_stat is None: errors.append('leader-stat-missing')
|
|
else:
|
|
tracked[leader.pid] = leader_stat['start']
|
|
if leader_stat['ppid'] != own or leader_stat['pgrp'] != leader.pid:
|
|
errors.append('leader-parent-or-group-mismatch')
|
|
input_bytes = base64.b64decode(request.get('input_b64',''), validate=True)
|
|
try:
|
|
out,err,sent,timed_out = pump(leader,input_bytes,limit)
|
|
except Exception as exc:
|
|
errors.append('pump:%s:%s' % (type(exc).__name__,exc))
|
|
finally:
|
|
leader.poll()
|
|
# Subreaper retains escaped descendants, including setsid children with closed stdio.
|
|
# Sweep until the leader is reaped and the live descendant closure is empty.
|
|
finish = time.monotonic() + grace
|
|
while True:
|
|
snap = table()
|
|
found = descendants(snap, own, tracked)
|
|
for pid,item in found.items(): tracked[pid] = item['start']
|
|
if leader.poll() is not None:
|
|
reap_adopted(errors)
|
|
if leader.returncode is not None and not found:
|
|
break
|
|
if leader_stat is not None:
|
|
fresh_kill_group(leader.pid, leader_stat['start'], errors)
|
|
for pid,item in found.items(): fresh_kill(pid,item['start'],errors)
|
|
if leader.returncode is None:
|
|
fresh_kill(leader.pid,tracked.get(leader.pid,-1),errors)
|
|
leader.poll()
|
|
if time.monotonic() >= finish:
|
|
errors.append('descendant-grace-expired'); break
|
|
time.sleep(.01)
|
|
if timed_out:
|
|
try:
|
|
more_out,more_err,_,still_open = pump(leader,b'',finish)
|
|
out += more_out; err += more_err
|
|
if still_open: errors.append('pipe-remains-open')
|
|
except Exception as exc:
|
|
errors.append('post-timeout:%s:%s' % (type(exc).__name__,exc))
|
|
leader.poll()
|
|
if leader.returncode is not None: reap_adopted(errors)
|
|
final_table = table()
|
|
final = descendants(final_table, own, tracked)
|
|
absent = not final
|
|
pg_absent = group_absent(final_table, leader.pid)
|
|
if final: errors.append('live-descendants:'+','.join(map(str,sorted(final))))
|
|
if not pg_absent: errors.append('live-process-group')
|
|
real_rc = leader.returncode
|
|
return {'supervisor_pid':own,'supervisor_start':own_stat['start'],
|
|
'leader_pid':leader.pid,'leader_start':leader_stat['start'] if leader_stat else None,
|
|
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
|
|
'real_leader_rc':real_rc,'rc':124 if timed_out else real_rc,
|
|
'timeout':timed_out,'leader_reaped':real_rc is not None,
|
|
'descendants_absent':absent,'pg_absent':pg_absent,
|
|
'elapsed':time.monotonic()-started,'exceptions':errors,
|
|
'request':{'argv':request['argv'],'cwd':request.get('cwd'),
|
|
'deadline':deadline,'input_len':len(input_bytes),
|
|
'input_sha256':hashlib.sha256(input_bytes).hexdigest()},
|
|
'stdin_written_len':sent,
|
|
'stdin_written_sha256':hashlib.sha256(input_bytes[:sent]).hexdigest(),
|
|
'stdout_len':len(out),'stdout_sha256':hashlib.sha256(out).hexdigest(),
|
|
'stderr_len':len(err),'stderr_sha256':hashlib.sha256(err).hexdigest(),
|
|
'stdout_b64':base64.b64encode(out).decode('ascii'),
|
|
'stderr_b64':base64.b64encode(err).decode('ascii')}
|
|
except BaseException as exc:
|
|
errors.append('supervisor:%s:%s' % (type(exc).__name__,exc))
|
|
# Keep the same owned-process sweep on an exceptional path.
|
|
if leader is not None:
|
|
finish = time.monotonic()+grace
|
|
while time.monotonic()<finish:
|
|
try:
|
|
snap=table(); found=descendants(snap,own,tracked)
|
|
if leader_stat is not None: fresh_kill_group(leader.pid,leader_stat['start'],errors)
|
|
for pid,item in found.items(): tracked[pid]=item['start']; fresh_kill(pid,item['start'],errors)
|
|
leader.poll()
|
|
if leader.returncode is not None: reap_adopted(errors)
|
|
if leader.returncode is not None and not found: break
|
|
except Exception as cleanup_exc:
|
|
errors.append('exception-sweep:%s:%s' % (type(cleanup_exc).__name__,cleanup_exc))
|
|
time.sleep(.01)
|
|
return {'supervisor_pid':own,'supervisor_start':own_stat['start'] if own_stat else None,
|
|
'leader_pid':leader.pid if leader else None,
|
|
'leader_start':tracked.get(leader.pid) if leader else None,
|
|
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
|
|
'real_leader_rc':leader.returncode if leader else None,'rc':None,
|
|
'timeout':False,'leader_reaped':leader is not None and leader.returncode is not None,
|
|
'descendants_absent':False,'pg_absent':False,
|
|
'elapsed':time.monotonic()-started,'exceptions':errors,
|
|
'request':{'argv':request.get('argv'),'cwd':request.get('cwd'),
|
|
'deadline':deadline,'input_len':len(input_bytes),
|
|
'input_sha256':hashlib.sha256(input_bytes).hexdigest()},
|
|
'stdin_written_len':sent,
|
|
'stdin_written_sha256':hashlib.sha256(input_bytes[:sent]).hexdigest(),
|
|
'stdout_len':len(out),'stdout_sha256':hashlib.sha256(out).hexdigest(),
|
|
'stderr_len':len(err),'stderr_sha256':hashlib.sha256(err).hexdigest(),
|
|
'stdout_b64':base64.b64encode(out).decode('ascii'),
|
|
'stderr_b64':base64.b64encode(err).decode('ascii')}
|
|
|
|
request=json.loads(sys.stdin.buffer.read())
|
|
result=run(request)
|
|
sys.stdout.write(json.dumps(result,separators=(',',':')))
|
|
'''
|
|
|
|
KILLER = r'''
|
|
import json, os, pathlib, signal, sys, time
|
|
target=int(sys.argv[1]); start=int(sys.argv[2]); inodes=set(sys.argv[3].split(','))
|
|
def stat(pid):
|
|
try: raw=pathlib.Path('/proc/%d/stat'%pid).read_text()
|
|
except (FileNotFoundError,ProcessLookupError): return None
|
|
fields=raw[raw.rfind(')')+2:].split()
|
|
return {'pid':pid,'state':fields[0],'ppid':int(fields[1]),'start':int(fields[19])}
|
|
def table():
|
|
result={}
|
|
for name in os.listdir('/proc'):
|
|
if name.isdecimal():
|
|
pid=int(name); item=stat(pid)
|
|
if item is not None: result[pid]=item
|
|
return result
|
|
def owns_pipe(pid):
|
|
try:
|
|
if os.stat('/proc/%d'%pid).st_uid != os.getuid(): return False
|
|
for fd in os.listdir('/proc/%d/fd'%pid):
|
|
try:
|
|
if os.readlink('/proc/%d/fd/%s'%(pid,fd)) in inodes: return True
|
|
except (FileNotFoundError,ProcessLookupError,PermissionError): pass
|
|
except (FileNotFoundError,ProcessLookupError,PermissionError): pass
|
|
return False
|
|
def owned(snapshot):
|
|
roots=set()
|
|
item=snapshot.get(target)
|
|
if item and item['start']==start and item['state'] not in ('Z','X'): roots.add(target)
|
|
for pid,item in snapshot.items():
|
|
if pid!=os.getpid() and item['state'] not in ('Z','X') and owns_pipe(pid): roots.add(pid)
|
|
found=set(roots)
|
|
changed=True
|
|
while changed:
|
|
changed=False
|
|
for pid,item in snapshot.items():
|
|
if pid!=os.getpid() and item['state'] not in ('Z','X') and item['ppid'] in found and pid not in found:
|
|
found.add(pid); changed=True
|
|
return {pid:snapshot[pid]['start'] for pid in found}
|
|
errors=[]; remaining=[]; limit=time.monotonic()+3.0
|
|
signal.alarm(4)
|
|
while time.monotonic()<limit:
|
|
found=owned(table())
|
|
if not found: break
|
|
for pid,token in sorted(found.items(),reverse=True):
|
|
try:
|
|
fresh=stat(pid)
|
|
if fresh and fresh['start']==token and fresh['state'] not in ('Z','X'):
|
|
os.kill(pid,signal.SIGKILL)
|
|
except ProcessLookupError: pass
|
|
except Exception as exc: errors.append(type(exc).__name__+':'+str(exc))
|
|
time.sleep(.02)
|
|
remaining=sorted(owned(table()))
|
|
print(json.dumps({'remaining':remaining,'errors':errors,'uid':os.getuid(),
|
|
'gid':os.getgid(),'groups':os.getgroups()},separators=(',',':')))
|
|
'''
|
|
|
|
ABSENCE = ("import os,sys\n"
|
|
"try: os.lstat(sys.argv[1])\n"
|
|
"except FileNotFoundError: raise SystemExit(0)\n"
|
|
"raise SystemExit(1)\n")
|
|
CLEANUP = "import shutil,sys; shutil.rmtree(sys.argv[1])"
|
|
IMAGE_UID = 65534
|
|
COMPILER_DEADLINE = 8.0
|
|
CANDIDATE_DEADLINE = 10.0
|
|
CLEANUP_DEADLINE = 5.0
|
|
ABSENCE_DEADLINE = 5.0
|
|
GRACE = 2.0
|
|
|
|
|
|
def exact_int(value, minimum=None):
|
|
return type(value) is int and (minimum is None or value >= minimum)
|
|
|
|
|
|
def digest(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def process_stat(pid):
|
|
try:
|
|
raw = Path("/proc/%d/stat" % pid).read_text()
|
|
status = Path("/proc/%d/status" % pid).read_text()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
fields = raw[raw.rfind(")") + 2:].split()
|
|
properties = {}
|
|
for line in status.splitlines():
|
|
if line.startswith(("Uid:", "Gid:", "Groups:")):
|
|
key, values = line.split(":", 1)
|
|
properties[key] = [int(part) for part in values.split()]
|
|
if not all(key in properties for key in ("Uid", "Gid", "Groups")):
|
|
raise ValueError("incomplete wrapper /proc status")
|
|
return dict(pid=pid, state=fields[0], ppid=int(fields[1]), pgrp=int(fields[2]),
|
|
start=int(fields[19]), uid=properties["Uid"][0],
|
|
gid=properties["Gid"][0], groups=properties["Groups"])
|
|
|
|
|
|
def exchange(proc, send, deadline):
|
|
selector = selectors.DefaultSelector()
|
|
output = [bytearray(), bytearray()]
|
|
sent = 0
|
|
for channel, index in ((proc.stdout, 0), (proc.stderr, 1)):
|
|
if channel is None or channel.closed:
|
|
continue
|
|
os.set_blocking(channel.fileno(), False)
|
|
selector.register(channel, selectors.EVENT_READ, index)
|
|
if proc.stdin is not None and not proc.stdin.closed:
|
|
if send:
|
|
os.set_blocking(proc.stdin.fileno(), False)
|
|
selector.register(proc.stdin, selectors.EVENT_WRITE, 2)
|
|
else:
|
|
proc.stdin.close()
|
|
expired = False
|
|
try:
|
|
while True:
|
|
proc.poll()
|
|
if not selector.get_map() and proc.returncode is not None:
|
|
break
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
expired = True
|
|
break
|
|
for key, _ in selector.select(min(remaining, 0.1)):
|
|
if key.data == 2:
|
|
try:
|
|
count = os.write(key.fileobj.fileno(), send[sent:sent + 65536])
|
|
sent += count
|
|
except BrokenPipeError:
|
|
selector.unregister(key.fileobj)
|
|
key.fileobj.close()
|
|
except BlockingIOError:
|
|
pass
|
|
else:
|
|
if sent == len(send):
|
|
selector.unregister(key.fileobj)
|
|
key.fileobj.close()
|
|
else:
|
|
try:
|
|
chunk = os.read(key.fileobj.fileno(), 65536)
|
|
except BlockingIOError:
|
|
continue
|
|
if chunk:
|
|
output[key.data].extend(chunk)
|
|
else:
|
|
selector.unregister(key.fileobj)
|
|
key.fileobj.close()
|
|
finally:
|
|
selector.close()
|
|
return bytes(output[0]), bytes(output[1]), sent, expired
|
|
|
|
|
|
def same_uid_reap(wrapper, start, pipes, who):
|
|
argv = [sys.executable, "-I", "-c", KILLER, str(wrapper.pid), str(start),
|
|
",".join("pipe:[%d]" % inode for inode in pipes)]
|
|
killer = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, start_new_session=True)
|
|
out, err, _, expired = exchange(killer, b"", time.monotonic() + 5.0)
|
|
facts = dict(pid=killer.pid, rc=killer.returncode, timeout=expired,
|
|
stdout_len=len(out), stdout_sha256=digest(out),
|
|
stderr_len=len(err), stderr_sha256=digest(err))
|
|
try:
|
|
parsed = json.loads(out)
|
|
facts["result"] = parsed
|
|
facts["confirmed"] = (not expired and killer.returncode == 0 and
|
|
type(parsed) is dict and parsed.get("remaining") == [] and
|
|
parsed.get("errors") == [] and
|
|
type(parsed.get("uid")) is int and parsed["uid"] == who[0] and
|
|
type(parsed.get("gid")) is int and parsed["gid"] == who[1] and
|
|
type(parsed.get("groups")) is list and
|
|
all(type(x) is int for x in parsed["groups"]) and
|
|
parsed["groups"] == who[2])
|
|
except (ValueError, TypeError):
|
|
facts["confirmed"] = False
|
|
return facts
|
|
|
|
|
|
def stage(argv, cwd, data, deadline, expected_identity):
|
|
request = dict(argv=argv, cwd=cwd, input_b64=base64.b64encode(data).decode("ascii"),
|
|
deadline=deadline, grace=GRACE)
|
|
request_bytes = json.dumps(request, separators=(",", ":")).encode()
|
|
started = time.monotonic()
|
|
wrapper = subprocess.Popen([sys.executable, "-I", "-c", SUPERVISOR],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, start_new_session=True)
|
|
observed_error = None
|
|
try:
|
|
observed = process_stat(wrapper.pid)
|
|
except Exception as exc:
|
|
observed = None
|
|
observed_error = "%s: %s" % (type(exc).__name__, exc)
|
|
pipe_inodes = []
|
|
pipe_observation_errors = []
|
|
for channel in (wrapper.stdin, wrapper.stdout, wrapper.stderr):
|
|
try:
|
|
pipe_inodes.append(os.fstat(channel.fileno()).st_ino)
|
|
except Exception as exc:
|
|
pipe_observation_errors.append("%s: %s" % (type(exc).__name__, exc))
|
|
wrapper_exchange_error = None
|
|
try:
|
|
wire, wrapper_err, written, wrapper_timeout = exchange(
|
|
wrapper, request_bytes, started + deadline + GRACE + 4.0)
|
|
except BaseException as exc:
|
|
wire, wrapper_err, written, wrapper_timeout = b"", b"", 0, True
|
|
wrapper_exchange_error = "%s: %s" % (type(exc).__name__, exc)
|
|
reaper = None
|
|
if wrapper_timeout:
|
|
try:
|
|
reaper = same_uid_reap(wrapper, observed["start"] if observed else -1,
|
|
pipe_inodes, expected_identity)
|
|
except BaseException as exc:
|
|
reaper = dict(confirmed=False, error="%s: %s" % (type(exc).__name__, exc))
|
|
if wrapper.stdin is not None and not wrapper.stdin.closed:
|
|
wrapper.stdin.close()
|
|
try:
|
|
more_wire, more_err, _, still_open = exchange(wrapper, b"", time.monotonic() + 2.0)
|
|
wire += more_wire
|
|
wrapper_err += more_err
|
|
if still_open:
|
|
reaper["confirmed"] = False
|
|
reaper["pipe_remained_open"] = True
|
|
except BaseException as exc:
|
|
reaper["confirmed"] = False
|
|
reaper["drain_error"] = "%s: %s" % (type(exc).__name__, exc)
|
|
wrapper.poll()
|
|
for channel in (wrapper.stdin, wrapper.stdout, wrapper.stderr):
|
|
if channel is not None and not channel.closed:
|
|
channel.close()
|
|
facts = dict(argv=argv, cwd=cwd, deadline=deadline,
|
|
wrapper_pid=wrapper.pid,
|
|
wrapper_start=observed["start"] if observed else None,
|
|
wrapper_observation_error=observed_error,
|
|
pipe_observation_errors=pipe_observation_errors,
|
|
wrapper_exchange_error=wrapper_exchange_error,
|
|
wrapper_rc=wrapper.returncode, wrapper_timeout=wrapper_timeout,
|
|
wrapper_elapsed=time.monotonic() - started,
|
|
wrapper_input_len=len(request_bytes), wrapper_input_sha256=digest(request_bytes),
|
|
wrapper_input_written_len=written,
|
|
wrapper_input_written_sha256=digest(request_bytes[:written]),
|
|
reaper=reaper, wire_len=len(wire), wire_sha256=digest(wire),
|
|
wrapper_stderr_len=len(wrapper_err),
|
|
wrapper_stderr_sha256=digest(wrapper_err))
|
|
if wrapper_timeout or pipe_observation_errors:
|
|
reason = "outer supervisor deadline" if wrapper_timeout and not wrapper_exchange_error else "outer supervisor transport failure"
|
|
return dict(facts, good=False, reason=reason, out=b"", err=b"")
|
|
if wrapper.returncode != 0:
|
|
return dict(facts, good=False, reason="supervisor nonzero", out=b"", err=b"")
|
|
if (observed is None or observed["ppid"] != os.getpid() or
|
|
observed["pgrp"] != wrapper.pid or
|
|
(observed["uid"], observed["gid"], observed["groups"]) != expected_identity or
|
|
written != len(request_bytes)):
|
|
return dict(facts, good=False, reason="wrapper identity/input incomplete", out=b"", err=b"")
|
|
try:
|
|
meta = json.loads(wire)
|
|
if type(meta) is not dict:
|
|
raise ValueError("wire is not an object")
|
|
out = base64.b64decode(meta["stdout_b64"], validate=True)
|
|
err = base64.b64decode(meta["stderr_b64"], validate=True)
|
|
required_ints = ("supervisor_pid", "supervisor_start", "leader_pid", "leader_start",
|
|
"uid", "gid", "stdout_len", "stderr_len", "stdin_written_len")
|
|
if any(not exact_int(meta.get(key), 1 if key.endswith(("pid", "start")) else 0)
|
|
for key in required_ints):
|
|
raise ValueError("missing or invalid numeric identity/IO")
|
|
if (meta["supervisor_pid"] != wrapper.pid or meta["supervisor_start"] != observed["start"] or
|
|
meta["leader_pid"] == wrapper.pid):
|
|
raise ValueError("supervisor/leader PID/start mismatch")
|
|
if meta["uid"] != expected_identity[0] or meta["gid"] != expected_identity[1]:
|
|
raise ValueError("wrong uid/gid")
|
|
groups = meta.get("groups")
|
|
if type(groups) is not list or any(type(group) is not int for group in groups):
|
|
raise ValueError("invalid supplementary groups")
|
|
if groups != expected_identity[2]:
|
|
raise ValueError("wrong supplementary groups")
|
|
elapsed = meta.get("elapsed")
|
|
if type(elapsed) not in (int, float) or not math.isfinite(elapsed) or not 0 <= elapsed <= deadline + GRACE + 1:
|
|
raise ValueError("invalid elapsed")
|
|
if type(meta.get("timeout")) is not bool or type(meta.get("leader_reaped")) is not bool:
|
|
raise ValueError("invalid timeout/reaped")
|
|
if type(meta.get("descendants_absent")) is not bool or type(meta.get("pg_absent")) is not bool:
|
|
raise ValueError("invalid absence")
|
|
if not exact_int(meta.get("real_leader_rc")) or not exact_int(meta.get("rc")):
|
|
raise ValueError("invalid real/controlled rc")
|
|
if meta["rc"] != (124 if meta["timeout"] else meta["real_leader_rc"]):
|
|
raise ValueError("inconsistent rc")
|
|
if type(meta.get("exceptions")) is not list or any(type(x) is not str for x in meta["exceptions"]):
|
|
raise ValueError("invalid exception list")
|
|
if meta["stdout_len"] != len(out) or meta["stderr_len"] != len(err):
|
|
raise ValueError("IO length mismatch")
|
|
if meta.get("stdout_sha256") != digest(out) or meta.get("stderr_sha256") != digest(err):
|
|
raise ValueError("IO digest mismatch")
|
|
echoed = meta.get("request")
|
|
wanted = dict(argv=argv, cwd=cwd, deadline=deadline,
|
|
input_len=len(data), input_sha256=digest(data))
|
|
if not same_json(echoed, wanted):
|
|
raise ValueError("request echo mismatch")
|
|
if meta["stdin_written_len"] != len(data) or meta.get("stdin_written_sha256") != digest(data):
|
|
raise ValueError("candidate stdin pipe write incomplete")
|
|
good = (not meta["timeout"] and meta["leader_reaped"] and
|
|
meta["descendants_absent"] and meta["pg_absent"] and not meta["exceptions"])
|
|
reason = "ok" if good else "timeout, live process, or supervisor exception"
|
|
return dict(facts, good=good, reason=reason, meta=meta, out=out, err=err)
|
|
except (KeyError, ValueError, TypeError) as exc:
|
|
return dict(facts, good=False, reason="invalid wire: " + str(exc), out=b"", err=b"")
|
|
|
|
|
|
class Model:
|
|
def __init__(self):
|
|
self.width = 64
|
|
self.rows = [b""] * 4
|
|
self.revision = 0
|
|
self.queue = []
|
|
|
|
def action(self, kind, a=0, b=0, payload=b""):
|
|
if kind == "I":
|
|
if a not in (64, 128) or not 1 <= b <= 8:
|
|
return -22
|
|
self.width, self.rows, self.revision, self.queue = a, [b""] * b, 0, []
|
|
return 0
|
|
if kind == "R":
|
|
self.queue = []
|
|
return 0
|
|
if kind == "N":
|
|
if not 0 <= a < len(self.rows):
|
|
return -22
|
|
if b > self.width:
|
|
return -34
|
|
return -22 if b else self._apply("T", a, 0, b"")
|
|
if kind == "B":
|
|
saved_rows, saved_revision = self.rows[:], self.revision
|
|
for op in self.queue:
|
|
rc = self._apply(*op)
|
|
if rc:
|
|
self.rows, self.revision = saved_rows, saved_revision
|
|
self.queue = []
|
|
return rc
|
|
self.queue = []
|
|
return 0
|
|
if kind in "PFCSD AEQ".replace(" ", ""):
|
|
if len(self.queue) == 16:
|
|
return -34
|
|
if kind in "PFAEQ" and len(payload) > 128:
|
|
return -34
|
|
self.queue.append((kind, a, b, payload))
|
|
return 0
|
|
return self._apply(kind, a, b, payload)
|
|
|
|
def _apply(self, kind, a, b, payload):
|
|
if kind in ("P", "T", "F", "U", "D", "X", "A", "E", "Q", "G", "H", "J") and not 0 <= a < len(self.rows):
|
|
return -22
|
|
if kind in ("C", "Y", "S", "Z") and (not 0 <= a < len(self.rows) or not 0 <= b < len(self.rows)):
|
|
return -22
|
|
if kind in ("P", "T", "F", "U") and len(payload) > self.width:
|
|
return -34
|
|
if kind in ("P", "T", "F", "U"):
|
|
self.rows[a] = payload
|
|
self.revision += 1
|
|
elif kind in ("A", "G", "E", "H", "Q", "J"):
|
|
old = self.rows[a]
|
|
if b > len(old):
|
|
return -34
|
|
if kind in ("A", "G", "Q", "J") and len(payload) > len(old) - b:
|
|
return -34
|
|
if kind in ("E", "H") and len(payload) > self.width - len(old):
|
|
return -34
|
|
if kind in ("A", "G"):
|
|
self.rows[a] = old[:b] + payload + old[b + len(payload):]
|
|
elif kind in ("E", "H"):
|
|
self.rows[a] = old[:b] + payload + old[b:]
|
|
else:
|
|
self.rows[a] = old[:b] + old[b + len(payload):]
|
|
self.revision += 1
|
|
elif kind in ("D", "X"):
|
|
self.rows[a] = b""
|
|
self.revision += 1
|
|
elif kind in ("C", "Y"):
|
|
self.rows[b] = self.rows[a]
|
|
self.revision += 1
|
|
elif kind in ("S", "Z"):
|
|
self.rows[a], self.rows[b] = self.rows[b], self.rows[a]
|
|
self.revision += 1 if a == b else 2
|
|
else:
|
|
raise ValueError("unsupported model action")
|
|
return 0
|
|
|
|
def observation(self, rc):
|
|
lengths = [len(row) for row in self.rows]
|
|
rows = []
|
|
for row in self.rows:
|
|
value = 2166136261
|
|
for byte in row:
|
|
value = ((value ^ byte) * 16777619) & 0xffffffff
|
|
short_rc = -34 if row else 0
|
|
short_got = 999 if row else 0
|
|
rows.append([0, 0, len(row), 0, len(row), 0, value, row.hex(), row.hex(),
|
|
short_rc, short_got, 0xa7])
|
|
return dict(rc=rc, queued=len(self.queue), stats_rc=0,
|
|
stats=[sum(x > 0 for x in lengths), sum(lengths), max(lengths), self.revision],
|
|
rows=rows)
|
|
|
|
|
|
def test_vectors():
|
|
rng = random.Random(0xCC004)
|
|
model = Model()
|
|
commands = []
|
|
expected = []
|
|
|
|
def add(kind, a=0, b=0, payload=b""):
|
|
if kind in ("P", "T"):
|
|
line = "%s %d %s" % (kind, a, payload.hex() or "-")
|
|
elif kind in ("A", "E", "G", "H"):
|
|
line = "%s %d %d %s" % (kind, a, b, payload.hex() or "-")
|
|
elif kind in ("Q", "J"):
|
|
line = "%s %d %d %d" % (kind, a, b, len(payload))
|
|
elif kind in ("F", "U"):
|
|
line = "%s %d %d %d" % (kind, a, b, len(payload))
|
|
payload = bytes([b]) * len(payload)
|
|
elif kind == "I" or kind in ("C", "Y", "S", "Z", "N"):
|
|
line = "%s %d %d" % (kind, a, b)
|
|
elif kind in ("D", "X"):
|
|
line = "%s %d" % (kind, a)
|
|
else:
|
|
line = kind
|
|
rc = model.action(kind, a, b, payload)
|
|
commands.append(line)
|
|
expected.append(model.observation(rc))
|
|
|
|
add("T", 0, payload=bytes(range(64)))
|
|
add("T", 1, payload=b"\xff")
|
|
add("Y", 0, 2)
|
|
add("Z", 1, 2)
|
|
add("P", 3, payload=bytes(range(64, 128)))
|
|
add("B")
|
|
add("D", 0)
|
|
add("B")
|
|
add("N", 1, 1)
|
|
add("N", 1, 0)
|
|
add("I", 128, 4)
|
|
add("P", 0, payload=bytes(range(128)))
|
|
add("B")
|
|
add("G", 0, 60, payload=b"\xfa\xfb\xfc\xfd")
|
|
add("J", 0, 1, payload=b"x" * 2)
|
|
add("H", 0, 2, payload=b"\x7a\x7b")
|
|
add("A", 0, 10, payload=b"\x11\x12")
|
|
add("E", 0, 0, payload=b"\x13\x14")
|
|
add("Q", 0, 6, payload=b"x" * 3)
|
|
add("B")
|
|
add("A", 0, 200, payload=b"\x00")
|
|
add("B")
|
|
add("P", 1, payload=bytes(reversed(range(128))))
|
|
add("C", 0, 2)
|
|
add("S", 1, 2)
|
|
add("B")
|
|
add("F", 3, 0xA5, b"x" * 128)
|
|
add("B")
|
|
add("P", 9, payload=b"bad")
|
|
add("D", 0)
|
|
add("B")
|
|
add("I", 7, 4)
|
|
for width in (64, 128):
|
|
add("I", width, 6)
|
|
for size in (0, 1, width - 1, width) + ((width + 1,) if width < 128 else ()):
|
|
payload = bytes(rng.randrange(256) for _ in range(size))
|
|
add("T", size % 6, payload=payload)
|
|
add("P", (size + 1) % 6, payload=payload)
|
|
add("B")
|
|
for _ in range(12):
|
|
a = rng.randrange(6)
|
|
b = rng.randrange(6)
|
|
size = rng.randrange(width + 1)
|
|
payload = bytes(rng.randrange(256) for _ in range(size))
|
|
add("P", a, payload=payload)
|
|
add("C", a, b)
|
|
add("B")
|
|
add("U", b, rng.randrange(256), payload=b"x" * size)
|
|
add("Z", a, b)
|
|
add("T", 4, payload=bytes(range(width)))
|
|
add("G", 4, width - 1, payload=b"\xed")
|
|
add("J", 4, width - 2, payload=b"x")
|
|
add("H", 4, width - 1, payload=b"\xab")
|
|
add("E", 4, 0, payload=b"\xcd")
|
|
add("A", 4, 1, payload=b"\xef")
|
|
add("Q", 4, 0, payload=b"x" * 2)
|
|
add("B")
|
|
add("G", 4, width, payload=b"\x09")
|
|
add("J", 4, width, payload=b"x")
|
|
add("H", 4, width, payload=b"x" * 3)
|
|
add("G", 4, width - 1, payload=b"")
|
|
add("P", 99, payload=b"invalid")
|
|
add("B")
|
|
add("X", 99)
|
|
add("F", 0, 7, payload=b"x" * 129)
|
|
add("U", 0, 7, payload=b"x" * 129)
|
|
for _ in range(17):
|
|
add("F", 0, 0, payload=b"")
|
|
add("B")
|
|
return ("\n".join(commands) + "\n").encode(), expected
|
|
|
|
|
|
def same_json(actual, wanted):
|
|
if type(actual) is not type(wanted):
|
|
return False
|
|
if isinstance(wanted, dict):
|
|
return actual.keys() == wanted.keys() and all(same_json(actual[k], v) for k, v in wanted.items())
|
|
if isinstance(wanted, list):
|
|
return len(actual) == len(wanted) and all(same_json(a, w) for a, w in zip(actual, wanted))
|
|
return actual == wanted
|
|
|
|
|
|
def check_output(data, expected):
|
|
try:
|
|
lines = data.decode("utf-8").splitlines()
|
|
if len(lines) != len(expected):
|
|
return "row count %d versus %d" % (len(lines), len(expected))
|
|
for index, (line, wanted) in enumerate(zip(lines, expected)):
|
|
actual = json.loads(line)
|
|
if not same_json(actual, wanted):
|
|
return "observation %d differs" % index
|
|
except (UnicodeError, ValueError, TypeError) as exc:
|
|
return "invalid observation: " + str(exc)
|
|
return None
|
|
|
|
|
|
def identity():
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534":
|
|
return (IMAGE_UID, IMAGE_UID, [])
|
|
return (os.getuid(), os.getgid(), os.getgroups())
|
|
|
|
|
|
def prepare(root, candidate):
|
|
work = root / "work"
|
|
work.mkdir(mode=0o700)
|
|
for name in ("layout.c", "journal.c"):
|
|
source = candidate / name
|
|
with source.open("rb") as stream:
|
|
data = stream.read(1024 * 1024 + 1)
|
|
if len(data) > 1024 * 1024:
|
|
raise ValueError("candidate source too large")
|
|
(work / name).write_bytes(data)
|
|
for name, content in HEADERS.items():
|
|
(work / name).write_text(content)
|
|
(work / "observer.c").write_text(OBSERVER)
|
|
if os.environ.get("ROLLOUT_GRADE_ISOLATION") == "uid65534":
|
|
os.chown(work, IMAGE_UID, IMAGE_UID)
|
|
for path in work.iterdir():
|
|
os.chown(path, IMAGE_UID, IMAGE_UID)
|
|
return work
|
|
|
|
|
|
def compact(stage_name, result):
|
|
meta = result.get("meta", {})
|
|
fields = dict(stage=stage_name, argv=result.get("argv"), cwd=result.get("cwd"),
|
|
deadline=result.get("deadline"), good=result.get("good"), reason=result.get("reason"),
|
|
wrapper_rc=result.get("wrapper_rc"), wire_len=result.get("wire_len"),
|
|
wire_sha256=result.get("wire_sha256"), rc=meta.get("rc"),
|
|
wrapper_pid=result.get("wrapper_pid"), wrapper_start=result.get("wrapper_start"),
|
|
wrapper_timeout=result.get("wrapper_timeout"),
|
|
wrapper_exchange_error=result.get("wrapper_exchange_error"),
|
|
pipe_observation_errors=result.get("pipe_observation_errors"),
|
|
wrapper_input_written_len=result.get("wrapper_input_written_len"),
|
|
reaper=result.get("reaper"),
|
|
real_rc=meta.get("real_leader_rc"), timeout=meta.get("timeout"),
|
|
leader_reaped=meta.get("leader_reaped"),
|
|
leader_pid=meta.get("leader_pid"), leader_start=meta.get("leader_start"),
|
|
supervisor_pid=meta.get("supervisor_pid"),
|
|
supervisor_start=meta.get("supervisor_start"), elapsed=meta.get("elapsed"),
|
|
uid=meta.get("uid"), gid=meta.get("gid"), groups=meta.get("groups"),
|
|
stdin_written_len=meta.get("stdin_written_len"),
|
|
stdin_written_sha256=meta.get("stdin_written_sha256"),
|
|
stdout_len=meta.get("stdout_len"), stdout_sha256=meta.get("stdout_sha256"),
|
|
stderr_len=meta.get("stderr_len"), stderr_sha256=meta.get("stderr_sha256"),
|
|
absent=meta.get("descendants_absent"), pg_absent=meta.get("pg_absent"),
|
|
exceptions=meta.get("exceptions"),
|
|
stderr_tail=result.get("err", b"")[-300:].decode("utf-8", "replace"))
|
|
if fields["reaper"] is None:
|
|
del fields["reaper"]
|
|
if fields["wrapper_exchange_error"] is None:
|
|
del fields["wrapper_exchange_error"]
|
|
if not fields["pipe_observation_errors"]:
|
|
del fields["pipe_observation_errors"]
|
|
if not fields["stderr_tail"]:
|
|
del fields["stderr_tail"]
|
|
if fields["reason"] == "ok":
|
|
del fields["reason"]
|
|
print(json.dumps(fields, separators=(",", ":"), ensure_ascii=True), file=sys.stderr)
|
|
|
|
|
|
def main(candidate):
|
|
scratch = None
|
|
failure = None
|
|
stages = {}
|
|
who = identity()
|
|
try:
|
|
scratch = Path(tempfile.mkdtemp(prefix="layout-grade-"))
|
|
work = prepare(scratch, candidate)
|
|
compiler = ["gcc", "-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "-pedantic",
|
|
"layout.c", "journal.c", "observer.c", "-o", "observer"]
|
|
stages["compiler"] = stage(compiler, str(work), b"", COMPILER_DEADLINE, who)
|
|
compact("compiler", stages["compiler"])
|
|
if not stages["compiler"]["good"]:
|
|
failure = "compiler supervision: " + stages["compiler"]["reason"]
|
|
elif stages["compiler"]["meta"]["rc"] != 0:
|
|
failure = "compiler rc %d" % stages["compiler"]["meta"]["rc"]
|
|
if failure is None:
|
|
input_data, expected = test_vectors()
|
|
stages["candidate"] = stage([str(work / "observer")], str(work), input_data,
|
|
CANDIDATE_DEADLINE, who)
|
|
compact("candidate", stages["candidate"])
|
|
if not stages["candidate"]["good"]:
|
|
failure = "candidate supervision: " + stages["candidate"]["reason"]
|
|
elif stages["candidate"]["meta"]["rc"] != 0:
|
|
failure = "candidate rc %d" % stages["candidate"]["meta"]["rc"]
|
|
else:
|
|
failure = check_output(stages["candidate"]["out"], expected)
|
|
except BaseException as exc:
|
|
failure = "checker exception: %s: %s" % (type(exc).__name__, exc)
|
|
finally:
|
|
if scratch is not None:
|
|
try:
|
|
stages["cleanup"] = stage([sys.executable, "-I", "-c", CLEANUP, str(scratch)],
|
|
None, b"", CLEANUP_DEADLINE, who)
|
|
compact("cleanup", stages["cleanup"])
|
|
except BaseException as exc:
|
|
failure = (failure or "") + " cleanup wrapper exception: " + str(exc)
|
|
try:
|
|
stages["absence"] = stage([sys.executable, "-I", "-c", ABSENCE, str(scratch)],
|
|
None, b"", ABSENCE_DEADLINE, who)
|
|
compact("absence", stages["absence"])
|
|
except BaseException as exc:
|
|
failure = (failure or "") + " absence wrapper exception: " + str(exc)
|
|
for name in ("cleanup", "absence"):
|
|
result = stages.get(name)
|
|
if result is None or not result["good"] or result["meta"]["rc"] != 0:
|
|
failure = (failure or "") + " %s not confirmed" % name
|
|
if failure:
|
|
print("FAIL: " + failure)
|
|
return 1
|
|
print("PASS")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(Path(sys.argv[1])))
|