droboticflow/scripts/drosync-real-bos-matrix.py

734 lines
39 KiB
Python

#!/usr/bin/env python3
"""Native CLI/daemon acceptance against deployed POC and real BOS only.
Private access configuration and BOS secrets never enter evidence. Every data
key belongs to one dedicated longtaotest run. No SQLite writes or mock servers.
"""
from __future__ import annotations
import argparse
import concurrent.futures
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import re
import signal
import subprocess
import sys
import time
from datetime import datetime, timezone, timedelta
import urllib.request
import zipfile
REPO = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("poc_acceptance", REPO / "scripts/drosync-poc-acceptance.py")
api_module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = api_module
spec.loader.exec_module(api_module)
sys.path.insert(0, str(REPO / "scripts/lib"))
def safe(value):
if isinstance(value, dict):
return {k: "[REDACTED]" if any(s in k.lower() for s in ("secret", "password", "token", "authorization", "cookie")) else safe(v) for k, v in value.items()}
if isinstance(value, list):
return [safe(v) for v in value]
if isinstance(value, str):
value = re.sub(r"(?i)Bearer\s+[^\s\"']+", "Bearer [REDACTED]", value)
value = re.sub(r"bce-auth-v1/[^\s\"']+", "[REDACTED BCE SIGNATURE]", value)
value = re.sub(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", "[REDACTED JWT]", value)
return re.sub(r"(https?://[^\s?\"']+)\?[^\s\"']+", r"\1?[REDACTED]", value)
return value
def save(path, value):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
temp = path.with_suffix(path.suffix + ".tmp")
temp.write_text(json.dumps(safe(value), ensure_ascii=False, indent=2) + "\n")
temp.replace(path)
def wait_for(fn, seconds=180, interval=1):
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
last = fn()
if last:
return last
except (ConnectionError, OSError, ValueError, RuntimeError):
pass
time.sleep(interval)
raise AssertionError(f"condition not reached after {seconds}s; last={safe(last)}")
def local_get(url):
return urllib.request.build_opener(urllib.request.ProxyHandler({})).open(url, timeout=5).read()
def pattern(path, size, seed=0, done=True):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
# Actual allocated file, not a sparse placeholder. Digest is computed during
# fixture generation; independent downloads must match this immutable value.
block = hashlib.shake_256(str(seed).encode()).digest(1024 * 1024)
digest = hashlib.md5()
remaining = size
with path.open("wb") as stream:
while remaining:
part = block[:min(remaining, len(block))]
stream.write(part)
digest.update(part)
remaining -= len(part)
os.utime(path, (time.time() - 180, time.time() - 180))
if done:
path.with_name(path.name + ".done").touch()
return {"size": size, "md5sum": digest.hexdigest()}
class Native:
def __init__(self, suite, name, concurrency=4, level="debug", baseline=False, log_max=33554432, fallback=False):
self.process = None
try:
self._initialize(suite, name, concurrency, level, baseline, log_max, fallback)
except BaseException:
self.stop()
raise
def _initialize(self, suite, name, concurrency, level, baseline, log_max, fallback):
if getattr(suite.args, "suffix", ""):
name = suite.args.suffix + "-" + name
self.suite, self.name = suite, name
self.credentials = dict(suite.access)
self.root = suite.work / name
self.root.mkdir(parents=True, exist_ok=True)
self.source = self.root / "capture"
self.source.mkdir(exist_ok=True)
self.state = self.root / "s"
self.config = self.root / "environment"
base = REPO / (".codex-artifacts/log-enhance27/baseline" if baseline else "tools/drosync/target/debug")
self.cli, self.daemon = str(base / "drosync"), str(base / "drosync-daemon")
self.evidence = suite.evidence / name
self.evidence.mkdir(parents=True, exist_ok=True)
primary = self.root / "primary-log-directory" if fallback else self.root / "daemon.log"
if fallback:
primary.mkdir(exist_ok=True)
self.config.write_text(f"DROSYNC_DATA_DIR={self.state}\nDROSYNC_SECRET_FILE={self.root / 'device-token'}\nDROSYNC_LOG_PATH={primary}\nDROSYNC_LOG_FALLBACK_PATH={self.root / 'fallback.log'}\nDROSYNC_LOG_LEVEL={level}\nDROSYNC_FILE_CONCURRENCY={concurrency}\nDROSYNC_PUT_CONCURRENCY={concurrency}\nDROSYNC_PART_CONCURRENCY={concurrency}\nDROSYNC_MAPPING_CONCURRENCY=4\nDROSYNC_HASH_CONCURRENCY=4\nDROSYNC_LOG_MAX_BYTES={log_max}\nDROSYNC_LOG_KEEP_FILES=3\n")
self.config.chmod(0o600)
self.level, self.mapping, self.process = level, None, None
self.commands = []
self.expected = {}
self.start()
self.call("pause")
self.call("auth", "--server", suite.access["baseURL"], "--access-key-id", suite.access["accessKeyId"], "--secret-stdin", stdin=suite.access["secret"])
self.call("version")
self.call("doctor")
result = self.call("mapping", "add", str(self.source), "--space", suite.access["spaceId"], "--remote-prefix", name, "--require-done")
self.mapping = result.get("id", result.get("mapping", {}).get("id"))
assert self.mapping, result
def start(self):
output = (self.evidence / "process.txt").open("ab")
self.process = subprocess.Popen([self.daemon, "-c", str(self.config)], stdout=output, stderr=output, start_new_session=True, env=self.suite.env)
output.close()
wait_for(lambda: self.call("status", record=False) if self.process.poll() is None else None, 20, .2)
def call(self, *args, stdin="", accepted=(0,), timeout=600, record=True):
command = [self.cli, "-c", str(self.config), "--json", *map(str, args)]
start = time.monotonic()
p = subprocess.run(command, input=stdin, capture_output=True, text=True, timeout=timeout, env=self.suite.env)
try:
value = json.loads(p.stdout)
except ValueError:
value = {"stdout": p.stdout[-1500:], "stderr": p.stderr[-1500:]}
if record:
self.commands.append({"command": command, "exit": p.returncode, "seconds": time.monotonic() - start, "result": safe(value)})
save(self.evidence / "commands.json", self.commands)
if p.returncode not in accepted:
raise RuntimeError(f"{args[0]} exit={p.returncode}: {safe(value)}")
return value
def add(self, relative, size, seed=0, done=True):
item = pattern(self.source / relative, size, seed, done)
self.expected[relative] = item
self.suite.known[self.suite.access["rootPrefix"].strip("/") + "/" + self.name + "/" + relative] = item
self.suite.save_known()
return item
def sync(self, accepted=(0,), timeout=600):
self.call("resume")
result = self.call("sync", self.mapping, accepted=(*accepted, 20, 50), timeout=timeout)
if result.get("errorCode") == "busy" or (isinstance(result.get("error"), str) and ("busy" in result["error"] or "already has an active sync" in result["error"])):
self.wait_complete()
result = self.call("sync", self.mapping, accepted=(*accepted, 50), timeout=timeout)
return result
def status(self):
return self.call("status", record=False)
def wait_complete(self, count=None, seconds=600):
def check():
report = self.report()
values = report.get("counts", {})
if values.get("missing", 0) == 0 and values.get("running", 0) == 0 and values.get("completed", 0) >= (count or 1):
return report
return None
return wait_for(check, seconds, 2)
def report(self, date=None):
return self.call("episodes", "--mapping", self.mapping, "--date", date or self.suite.today, "--limit", "1000", record=False)
def text(self):
return "\n".join(p.read_text(errors="replace") for p in sorted(self.root.glob("*.log*")) if p.is_file())
def snapshot(self):
time.sleep(.3)
text = self.text()
for secret in (self.credentials["secret"], self.credentials.get("adminSession", {}).get("password", "")):
assert not secret or secret not in text, "secret leaked into daemon log"
(self.evidence / "daemon.txt").write_text(text)
try:
from drosync_real_evidence import analyze_logs
save(self.evidence / "log-analysis.json", analyze_logs(text))
except ImportError:
pass
save(self.evidence / "status.json", self.status())
save(self.evidence / "report.json", self.report())
def stop(self, hard=False):
if self.process and self.process.poll() is None:
self.process.send_signal(signal.SIGKILL if hard else signal.SIGTERM)
try:
self.process.wait(8)
except subprocess.TimeoutExpired:
self.process.kill()
self.process.wait()
def close(self):
try:
self.call("pause")
self.snapshot()
finally:
self.stop()
class Suite:
def __init__(self, args):
self.args = args
self.access = json.loads(Path(args.access_file).read_text())
assert self.access["bucket"] == "testxyz"
assert self.access["rootPrefix"].strip("/") == "longtaotest/20261013T100437Z"
self.work, self.evidence = Path(args.workdir), Path(args.evidence_dir)
if not self.work.resolve().is_relative_to(Path(args.access_file).resolve().parent):
raise ValueError("generated fixtures must remain inside the private acceptance directory")
self.work.mkdir(parents=True, exist_ok=True)
self.work.chmod(0o700)
self.evidence.mkdir(parents=True, exist_ok=True)
self.env = {k: v for k, v in os.environ.items() if not k.startswith(("DROSYNC_", "UBT_"))}
now = datetime.now(timezone(timedelta(hours=8)))
self.today, self.yesterday = now.strftime("%Y_%m_%d"), (now - timedelta(days=1)).strftime("%Y_%m_%d")
self.known, self.results = {}, []
def save_known(self):
save(self.evidence / "known-files.json", self.known)
def management(self):
api = api_module.API(self.access["baseURL"])
session = self.access["adminSession"]
api.request("POST", "/api/auth/login", {"email": session["email"], "password": session["password"]})
return api
def case(self, name, fn):
start = time.monotonic()
print(json.dumps({"event": "case_start", "case": name}), flush=True)
try:
value = fn()
result = {"name": name, "status": "PASS", "seconds": time.monotonic() - start, "details": value}
except Exception as error:
result = {"name": name, "status": "FAIL", "seconds": time.monotonic() - start, "error": str(error)}
self.results.append(result)
save(self.evidence / "results.json", self.results)
print(json.dumps(safe({k: v for k, v in result.items() if k != "details"}), ensure_ascii=False), flush=True)
return result
def verify(self, client, relatives=None, suffix="verify"):
files = [{"key": self.access["rootPrefix"].strip("/") + "/" + client.name + "/" + p, **v} for p, v in client.expected.items() if relatives is None or p in relatives]
secret = subprocess.run(["kubectl", "--kubeconfig", self.args.kubeconfig, "-n", "droboticflow", "get", "secret", "droboticflow-bos-workspace", "-o", "json"], capture_output=True, check=True).stdout
import base64
values = json.loads(secret)["data"]
payload = {"endpoint": "https://bj.bcebos.com", "bucket": "testxyz", "prefix": self.access["rootPrefix"].strip("/") + "/" + client.name + "/", "accessKeyId": base64.b64decode(values["accessKeyId"]).decode(), "secretAccessKey": base64.b64decode(values["secretAccessKey"]).decode(), "files": files}
p = subprocess.run([self.args.verify_bin], input=json.dumps(payload), capture_output=True, text=True, timeout=900)
payload.clear()
value = json.loads(p.stdout)
save(client.evidence / (suffix + ".json"), value)
assert p.returncode == 0 and value["status"] == "PASS", safe(value)
return value
def stability(self):
c = Native(self, "stability", level="info")
cycles = []
start = time.monotonic()
try:
c.call("resume")
for index in range(self.args.duration // self.args.interval):
target = start + index * self.args.interval
while time.monotonic() < target:
time.sleep(min(30, target - time.monotonic()))
relative = f"{self.today}/stability/episode_{index:03d}/data.bin"
c.add(relative, 1024 * 1024, index)
report = c.wait_complete(index + 1, 180)
cycles.append({"cycle": index, "atSeconds": time.monotonic() - start, "counts": report["counts"]})
save(c.evidence / "cycles.json", cycles)
print(json.dumps({"event": "stability_cycle", **cycles[-1]}), flush=True)
while time.monotonic() - start < self.args.duration:
time.sleep(min(30, self.args.duration - (time.monotonic() - start)))
verification = self.verify(c)
c.snapshot()
assert 'event="request_completed"' not in c.text(), "INFO emitted successful requests"
assert len(cycles) == self.args.duration // self.args.interval
return {"durationSeconds": time.monotonic() - start, "cycles": cycles, "version": c.call("version"), "verification": verification}
finally:
c.close()
def run_suite(self):
cases = {"bulk": ("1003-small-and-two-multipart-boundaries", self.bulk), "reports": ("marker-report-late-date-conflict-delete-clean", self.reports), "clean": ("fresh-receipt-clean-and-mutation-protection", self.clean), "lifecycle": ("pause-resume-and-explicit-abort", self.lifecycle), "crash": ("multipart-sigkill-recovery", self.crash), "race": ("same-target-two-device-race", self.race), "commit": ("real-commit-wait-30-90-150", self.commit_wait), "lost": ("real-control-confirmation-response-lost", self.lost), "connection": ("owned-control-forward-loss", self.connection_loss), "logging": ("real-log-rotation-and-fallback", self.logging)}
for key in self.args.case or cases:
name, fn = cases[key]
self.case(name, fn)
def bulk(self):
c = Native(self, "bulk", level="debug")
try:
for i in range(1003):
c.add(f"{self.today}/bulk/episode_many/data_{i:04d}.bin", 65536, i)
for name, size in [("empty.bin", 0), ("中文 空格.txt", 337), ("large257.bin", 257 << 20), ("large321.bin", 321 << 20)]:
c.add(f"{self.today}/bulk/episode_{name.split('.')[0]}/{name}", size, size)
pattern(c.source / ".hidden.bin", 8)
pattern(c.source / "ignored.tmp", 8)
(c.source / "ignored-symlink").symlink_to(next(c.source.rglob("data_0000.bin")))
begin = time.monotonic()
result = c.sync(timeout=1200)
seconds = time.monotonic() - begin
assert result.get("session", {}).get("status") == "completed", result
assert result.get("itemCount") == 1007, result
verification = self.verify(c)
assert verification["verifiedCount"] == 1007
assert verification["unexpectedObjects"] == 0, verification
before = len(c.status()["sessions"])
repeat = c.sync()
assert repeat.get("confirmedFilesSkipped") == 1007, repeat
assert len(c.status()["sessions"]) == before
assert repeat.get("session") is None
c.snapshot()
text = c.text()
assert 'endpoint_kind="bos_put"' in text and 'event="request_completed"' in text
assert "part_number" in text or "multipart" in text
return {"files": 1007, "bytes": sum(v["size"] for v in c.expected.values()), "syncSeconds": seconds, "sessionId": result["session"]["id"], "repeat": repeat, "verification": verification}
finally:
c.close()
def web(self, c, port):
ps = subprocess.run(["ps", "-axo", "pid=,command="], capture_output=True, text=True, check=True).stdout
for line in ps.splitlines():
if "drosync -c /private/tmp/drl-100437/" in line and " web " in line and f"--port {port}" in line:
os.kill(int(line.strip().split()[0]), signal.SIGTERM)
time.sleep(.3)
output = (c.evidence / "web.txt").open("ab")
p = subprocess.Popen([c.cli, "-c", str(c.config), "web", "--mapping", c.mapping, "--port", str(port)], stdout=output, stderr=output, env=self.env, start_new_session=True)
output.close()
def ready():
value = json.loads(local_get(f"http://127.0.0.1:{port}/api/report?date={self.today}"))
report = value.get("report", value)
return value if report.get("localRoot") == str(c.source) else None
wait_for(ready, 20)
return p
def reports(self):
c = Native(self, "reports", level="debug")
try:
first = f"{self.today}/report/episode_mix/ready.bin"
pending = f"{self.today}/report/episode_mix/pending.bin"
yesterday = f"{self.yesterday}/report/episode_late/data.bin"
c.add(first, 1 << 20, 1)
c.add(pending, 1 << 20, 2, False)
c.sync()
report = c.report()
save(c.evidence / "pending-report.json", report)
assert report["counts"]["missing"] == 1 and report["counts"]["completed"] == 0, report
assert report["items"][0]["totalFiles"] == 2 and report["items"][0]["localSize"] == 2 << 20
assert ".done" in report["items"][0]["reason"] and "pending.bin" in report["items"][0]["reason"], report
self.verify(c, [first], "pending-bos")
(c.source / pending).with_name("pending.bin.done").touch()
c.add(yesterday, 1 << 20, 3)
completed = c.sync()
assert c.report()["counts"]["missing"] == 0
assert c.report(self.yesterday)["counts"]["completed"] == 1
self.verify(c)
original = c.expected[first]
pattern(c.source / first, 1 << 20, 99)
conflict = c.sync(accepted=(0, 30))
conflict_report = c.report()
save(c.evidence / "conflict-report.json", conflict_report)
assert conflict_report["counts"]["missing"] > 0 and "冲突" in json.dumps(conflict_report, ensure_ascii=False), conflict_report
assert c.expected[first] == original
self.verify(c, [first], "conflict-bos-original")
(c.source / pending).unlink()
deleted = c.report()
save(c.evidence / "deleted-report.json", deleted)
self.verify(c, [pending], "deleted-bos-preserved")
c.web_process = self.web(c, 19341)
web_value = json.loads(local_get(f"http://127.0.0.1:19341/api/report?date={self.today}"))
save(c.evidence / "web-conflict-report.json", web_value)
web_report = web_value.get("report", web_value)
assert web_report["totalLocalEpisodes"] == deleted["counts"]["total"]
assert web_report["totalMissingEpisodes"] == deleted["counts"]["missing"]
assert web_report["totalCloudEpisodes"] == deleted["counts"]["completed"]
assert any("冲突" in warning for warning in web_report["warnings"])
for fmt in ("csv", "xlsx"):
c.call("report", "--mapping", c.mapping, "--date", self.today, "--format", fmt, "--output", str(c.evidence / fmt))
assert list((c.evidence / "csv").glob("*.csv"))
xlsx = next((c.evidence / "xlsx").glob("*.xlsx"))
assert zipfile.ZipFile(xlsx).testzip() is None
# Restore only generated source contents; immutable BOS expectations
# remain the accepted originals throughout conflict/delete checks.
pattern(c.source / first, 1 << 20, 1)
pattern(c.source / pending, 1 << 20, 2)
c.sync()
clean_session = completed.get("session", {}).get("id")
assert clean_session
# Change one file that belongs to this completed Session: cleaning
# must protect it while deleting its unchanged sibling.
pattern(c.source / yesterday, 1 << 20, 999)
preview = c.call("clean", c.mapping, "--session", clean_session, accepted=(0, 50))
save(c.evidence / "clean-preview.json", preview)
assert (c.source / first).exists() and (c.source / pending).exists()
cleaned = c.call("clean", c.mapping, "--session", clean_session, "--execute", accepted=(0, 50))
save(c.evidence / "clean-executed.json", cleaned)
assert cleaned["deletedFiles"] == 0 and cleaned["blockedFiles"] >= 1, cleaned
assert (c.source / pending).exists()
assert (c.source / yesterday).exists() and (c.source / first).exists()
self.verify(c)
c.snapshot()
return {"pending": report["counts"], "conflict": conflict_report["counts"], "delete": deleted["counts"], "clean": cleaned, "web": "http://127.0.0.1:19341/", "sessionId": clean_session}
finally:
c.close()
def clean(self):
c = Native(self, "clean")
try:
first = f"{self.today}/clean/episode/first.bin"
second = f"{self.today}/clean/episode/second.bin"
c.add(first, 1 << 20, 1)
c.add(second, 1 << 20, 2)
result = c.sync()
sid = result["session"]["id"]
c.call("pause")
pattern(c.source / second, 1 << 20, 999)
preview = c.call("clean", c.mapping, "--session", sid, accepted=(0, 50))
assert preview["candidateFiles"] == 1 and preview["blockedFiles"] == 1, preview
assert (c.source / first).exists()
actual = c.call("clean", c.mapping, "--session", sid, "--execute", accepted=(0, 50))
assert actual["deletedFiles"] == 1 and actual["blockedFiles"] == 1, actual
assert not (c.source / first).exists() and (c.source / second).exists()
verification = self.verify(c)
repeat = c.call("clean", c.mapping, "--session", sid, "--execute", accepted=(0, 50))
assert repeat["deletedFiles"] == 0 and repeat["alreadyCleanedFiles"] == 1, repeat
c.snapshot()
return {"preview": preview, "execute": actual, "repeat": repeat, "verification": verification}
finally:
c.close()
def lifecycle(self):
c = Native(self, "lifecycle", concurrency=1)
try:
for i in range(8):
c.add(f"{self.today}/lifecycle/episode_{i:02d}/data.bin", 16 << 20, i)
with concurrent.futures.ThreadPoolExecutor() as pool:
transfer = pool.submit(c.sync)
wait_for(lambda: 'event="file_upload_start"' in c.text(), 90, .1)
paused = c.call("pause")
result = transfer.result(300)
assert paused["paused"] is True
assert result.get("paused") is True, result
sid = result.get("sessionId") or next(s["id"] for s in c.status()["sessions"] if s["status"] not in ("completed", "aborted"))
resumed = c.sync()
assert any(s["id"] == sid and s["status"] == "completed" for s in c.status()["sessions"]), resumed
self.verify(c)
# New source item creates a separate owned session for cancellation.
c.add(f"{self.today}/lifecycle/episode_abort/data.bin", 65 << 20, 888)
before = c.text().count('event="file_upload_start"')
with concurrent.futures.ThreadPoolExecutor() as pool:
transfer = pool.submit(c.sync, (0, 20, 30, 40, 50))
wait_for(lambda: c.text().count('event="file_upload_start"') > before, 90, .1)
current = next(s for s in c.status()["sessions"] if s["status"] not in ("completed", "aborted", "failed", "expired", "conflicted"))
aborted = c.call("abort", current["id"])
after_abort = transfer.result(300)
assert (c.source / f"{self.today}/lifecycle/episode_abort/data.bin").exists()
cloud, _ = self.management().request("GET", "/api/object-sync/sessions/" + current["id"])
assert cloud.get("session", cloud)["status"] == "aborted", cloud
save(c.evidence / "cloud-aborted.json", cloud)
c.snapshot()
return {"pausedSession": sid, "resumed": resumed, "abortSession": current["id"], "abort": aborted, "transferAfterAbort": after_abort}
finally:
c.close()
def crash(self):
c = Native(self, "crash", concurrency=1)
try:
c.add(f"{self.today}/crash/episode_large/data.bin", 769 << 20, 123)
with concurrent.futures.ThreadPoolExecutor() as pool:
transfer = pool.submit(c.sync, (0, 10, 20, 30, 40, 50))
wait_for(lambda: 'event="request_completed" endpoint_kind="bos_put"' in c.text(), 180, .1)
sid = next(s["id"] for s in c.status()["sessions"] if s["status"] not in ("completed", "aborted"))
c.stop(hard=True)
killed = transfer.result(20)
time.sleep(1)
c.start()
result = c.sync(timeout=900)
assert result.get("session", {}).get("id") == sid or any(s["id"] == sid and s["status"] == "completed" for s in c.status()["sessions"]), result
verification = self.verify(c)
c.snapshot()
text = c.text()
recoveries = [int(v) for v in re.findall(r'event="multipart_recovery" completed_parts=(\d+)', text)]
assert any(v > 0 for v in recoveries), recoveries
first_part_puts = [line for line in text.splitlines() if 'part_number=1 ' in line and 'event="request_completed" endpoint_kind="bos_put"' in line]
assert len(first_part_puts) == 1, "confirmed first part was retransmitted"
return {"sessionId": sid, "postKill": killed, "resumed": result, "verification": verification}
finally:
c.close()
def race(self):
primary = dict(self.access)
gate = None
if self.args.plan_barrier:
from drosync_real_relay import RealResponseGate
gate = RealResponseGate(primary["baseURL"], self.evidence / "real-planned-response-gate.jsonl", port=19443, mode="mapping_plan").start()
self.access = {**primary, "baseURL": gate.base_url if gate else primary["baseURL"]}
a = Native(self, "race-a")
b = Native(self, "race-b")
self.access = primary
try:
b.call("mapping", "remove", b.mapping)
mapping = b.call("mapping", "add", str(b.source), "--space", primary["spaceId"], "--remote-prefix", a.name, "--require-done")
b.mapping = mapping.get("id", mapping.get("mapping", {}).get("id"))
relative = f"{self.today}/race/episode_001/data.bin"
a.add(relative, 4 << 20, 4)
b.add(relative, 4 << 20, 5)
with concurrent.futures.ThreadPoolExecutor() as pool:
if gate:
# B obtains an unchanged real planned response before any
# writer creates the key. Let A commit and free its lease,
# then deliver B's already obtained stale plan response.
gate.hold_mapping_id = b.mapping
task_b = pool.submit(b.sync, (0, 20, 30, 40, 50))
wait_for(gate.held.is_set, 60, .1)
outcome_a = a.sync((0, 20, 30, 40, 50))
gate.release()
outcomes = [outcome_a, task_b.result(300)]
else:
tasks = [pool.submit(c.sync, (0, 20, 30, 40)) for c in (a, b)]
outcomes = [t.result(600) for t in tasks]
# A closed Session can include skipped/aborted files. Count formal
# committed objects, not merely the Session lifecycle status.
winners = [i for i, value in enumerate(outcomes) if value.get("session", {}).get("counts", {}).get("committed") == 1]
assert len(winners) == 1, outcomes
loser = 1 - winners[0]
assert outcomes[loser].get("session", {}).get("counts", {}).get("committed", 0) == 0
assert (a, b)[loser].report()["counts"]["missing"] == 1
if winners[0] == 1:
a.expected[relative] = b.expected[relative]
self.known[self.access["rootPrefix"] + "/" + a.name + "/" + relative] = b.expected[relative]
self.save_known()
verification = self.verify(a)
a.snapshot()
b.snapshot()
if gate:
losing_text = (a, b)[loser].text()
assert len(re.findall(r'event="request_failed" endpoint_kind="bos_put"[^\n]*status=412 ', losing_text)) == 1, losing_text[-4000:]
assert 'event="file_sync_retry"' not in losing_text and 'event="upload_retry_backoff"' not in losing_text
return {"outcomes": outcomes, "winningClient": winners[0], "scope": "same Space; stale real planned response released after winner commit" if gate else "same Space race", "realBarrier": bool(gate), "verification": verification}
finally:
a.close()
b.close()
self.access = primary
if gate:
gate.release()
gate.close()
def scale(self, replicas):
deployment = self.access["coordinatorDeployment"]
assert deployment == "lt-bos-coordinator-100437"
command = ["kubectl", "--kubeconfig", self.args.kubeconfig, "-n", "droboticflow", "scale", "deployment/" + deployment, "--replicas=" + str(replicas)]
subprocess.run(command, check=True, capture_output=True)
if replicas:
subprocess.run(["kubectl", "--kubeconfig", self.args.kubeconfig, "-n", "droboticflow", "rollout", "status", "deployment/" + deployment, "--timeout=90s"], check=True, capture_output=True)
def commit_wait(self):
from drosync_real_relay import RealResponseGate
gate = RealResponseGate(self.access["baseURL"], self.evidence / "real-response-gate.jsonl").start()
actual_url = self.access["baseURL"]
self.access["baseURL"] = gate.base_url
c = Native(self, "commit-wait")
self.access["baseURL"] = actual_url
try:
c.add(f"{self.today}/commit/episode/data.bin", 1 << 20, 32)
with concurrent.futures.ThreadPoolExecutor() as pool:
transfer = pool.submit(c.sync, (0,), 900)
wait_for(gate.held.is_set, 120, .1)
try:
wait_for(lambda: len(re.findall(r'event="commit_wait"', c.text())) >= 3, 230, 1)
notices = [int(v) for v in re.findall(r'event="commit_wait"[^\n]*elapsed_ms=(\d+)', c.text())]
assert len(notices) == 3 and all(abs(v - target) < 2500 for v, target in zip(notices, [30000, 90000, 150000])), notices
finally:
gate.release()
result = transfer.result(300)
assert result.get("session", {}).get("status") == "completed", result
self.verify(c)
c.snapshot()
return {"fault": "delayed delivery of unchanged real POC terminal progress responses; BOS uploaded directly; no synthetic HTTP responses", "noticesMs": notices, "result": result}
finally:
self.access["baseURL"] = actual_url
gate.close()
c.close()
def connection_loss(self):
c = Native(self, "connection-loss", concurrency=1)
try:
return self._connection_loss(c)
finally:
c.close()
def _connection_loss(self, c):
forward = None
try:
c.add(f"{self.today}/connection/episode_001/data.bin", 8 << 20, 60)
before = c.text().count('event="file_upload_start"')
with concurrent.futures.ThreadPoolExecutor() as pool:
transfer = pool.submit(c.sync, (0, 10, 20, 30, 40, 50))
wait_for(lambda: c.text().count('event="file_upload_start"') > before, 120, .1)
# Identify only the port-forward whose args match this new API.
ps = subprocess.run(["ps", "-axo", "pid=,command="], capture_output=True, text=True, check=True).stdout
pids = [int(line.strip().split()[0]) for line in ps.splitlines() if "kubectl" in line and "port-forward" in line and "19441:3001" in line and "lt-bos-api-100437" in line]
assert len(pids) == 1, "owned port-forward not uniquely found"
os.kill(pids[0], signal.SIGTERM)
time.sleep(45)
killed = transfer.result(300)
assert 'event="request_failed"' in c.text()
finally:
log = (self.evidence / "forward-restart.txt").open("ab")
forward = subprocess.Popen(["kubectl", "--kubeconfig", self.args.kubeconfig, "-n", "droboticflow", "port-forward", "service/lt-bos-api-100437", "19441:3001"], stdout=log, stderr=log, start_new_session=True)
log.close()
wait_for(lambda: local_get(self.access["baseURL"] + "/api/health"), 30)
try:
resumed = c.sync(timeout=600)
wait_for(lambda: any(s["status"] == "completed" for s in c.status()["sessions"]), 60)
verification = self.verify(c)
c.snapshot()
return {"fault": "owned control connection unavailable for 45 seconds; BOS unchanged", "interrupted": killed, "resumed": resumed, "verification": verification, "forwardPid": forward.pid}
finally:
c.close()
def lost(self):
from drosync_real_relay import RealResponseGate
original_url = self.access["baseURL"]
gate = RealResponseGate(original_url, self.evidence / "real-lost-confirmation.jsonl", port=19444, mode="lost_completion").start()
self.access["baseURL"] = gate.base_url
c = Native(self, "lost-response")
self.access["baseURL"] = original_url
try:
c.add(f"{self.today}/lost/episode_001/data.bin", 1 << 20, 430)
result = c.sync()
assert result.get("session", {}).get("counts", {}).get("committed") == 1, result
assert gate.dropped.is_set(), "real upstream successful confirmation was not dropped"
verification = self.verify(c)
c.snapshot()
text = c.text()
assert text.count('event="request_completed" endpoint_kind="bos_put"') == 1
assert 'event="request_retry"' in text
return {"scope": "real POC confirmation response lost after real BOS PUT; idempotent control retry, not provider fault", "result": result, "verification": verification, "successfulPutCalls": 1}
finally:
self.access["baseURL"] = original_url
gate.close()
c.close()
def logging(self):
values = []
for name, fallback in [("rotation", False), ("fallback", True)]:
c = Native(self, name, level="debug", log_max=32768, fallback=fallback)
try:
for i in range(32):
c.add(f"{self.today}/logging/episode_{i:03d}/data.bin", 65536, i)
result = c.sync()
self.verify(c)
c.snapshot()
logs = [str(p.name) for p in c.root.glob("*.log*")]
assert any("fallback.log" in p for p in logs) if fallback else len(logs) > 1, logs
assert 'event="request_completed"' in c.text()
values.append({"name": name, "logs": logs, "sessionId": result.get("session", {}).get("id")})
finally:
c.close()
return values
def performance(self):
measurements = []
# All nine uploads use new keys, the same deterministic 256MiB content,
# the same host, debug build profile and real backend. 1/4 concurrency
# and pre-enhancement INFO are interleaved to reduce temporal drift.
for repetition in range(3):
modes = [("new4", 4, False), ("old4", 4, True), ("new1", 1, False)]
if repetition % 2:
modes.reverse()
for name, concurrency, baseline in modes:
c = Native(self, f"perf-{repetition}-{name}", concurrency=concurrency, level="info", baseline=baseline)
try:
for i in range(32):
c.add(f"{self.today}/performance/episode_{i:03d}/data.bin", 8 << 20, i)
before = time.monotonic()
result = c.sync(timeout=900)
elapsed = time.monotonic() - before
assert result.get("session", {}).get("status") == "completed", result
c.snapshot()
assert 'event="request_completed"' not in c.text()
verified = self.verify(c)
measurements.append({"name": name, "repetition": repetition, "seconds": elapsed, "bytes": 256 << 20, "mibPerSecond": 256 / elapsed, "logBytes": len(c.text().encode()), "version": c.call("version"), "verification": {k: verified[k] for k in ("verifiedCount", "verifiedBytes", "status")}})
save(self.evidence / "measurements.json", measurements)
print(json.dumps({"event": "performance", **measurements[-1]}, ensure_ascii=False), flush=True)
finally:
c.close()
import statistics
medians = {mode: statistics.median(v["seconds"] for v in measurements if v["name"] == mode) for mode in ("new4", "old4", "new1")}
delta = medians["new4"] / medians["old4"] - 1
self.results.append({"name": "paired-real-BOS-INFO-performance", "status": "PASS" if delta <= .05 else "FAIL", "mediansSeconds": medians, "infoOverheadFraction": delta, "measurements": measurements})
save(self.evidence / "results.json", self.results)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--access-file", required=True)
parser.add_argument("--workdir", required=True)
parser.add_argument("--evidence-dir", required=True)
parser.add_argument("--verify-bin", required=True)
parser.add_argument("--kubeconfig", default=str(Path.home() / ".config/droboticflow/kubeconfigs/poc.config"))
parser.add_argument("--mode", choices=["stability", "suite", "performance"], required=True)
parser.add_argument("--duration", type=int, default=7200)
parser.add_argument("--interval", type=int, default=600)
parser.add_argument("--case", action="append", choices=["bulk", "reports", "clean", "lifecycle", "crash", "race", "commit", "lost", "connection", "logging"])
parser.add_argument("--suffix", default="", help="Use a fresh case prefix for a focused rerun")
parser.add_argument("--plan-barrier", action="store_true", help="Deliver an unchanged real planned response after another client commits; no mocked responses")
args = parser.parse_args()
suite = Suite(args)
if args.mode == "stability":
suite.case("two-hour-auto-sync", suite.stability)
elif args.mode == "suite":
suite.run_suite()
else:
suite.performance()
return 1 if any(v["status"] == "FAIL" for v in suite.results) else 0
if __name__ == "__main__":
sys.exit(main())