forked from longtao.wu/droboticflow
734 lines
39 KiB
Python
734 lines
39 KiB
Python
#!/usr/bin/env python3
|
|
"""Native CLI/daemon acceptance against deployed POC and real BOS only.
|
|
|
|
Private access configuration and BOS secrets never enter evidence. Every data
|
|
key belongs to one dedicated longtaotest run. No SQLite writes or mock servers.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone, timedelta
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
REPO = Path(__file__).resolve().parents[1]
|
|
spec = importlib.util.spec_from_file_location("poc_acceptance", REPO / "scripts/drosync-poc-acceptance.py")
|
|
api_module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = api_module
|
|
spec.loader.exec_module(api_module)
|
|
sys.path.insert(0, str(REPO / "scripts/lib"))
|
|
|
|
|
|
def safe(value):
|
|
if isinstance(value, dict):
|
|
return {k: "[REDACTED]" if any(s in k.lower() for s in ("secret", "password", "token", "authorization", "cookie")) else safe(v) for k, v in value.items()}
|
|
if isinstance(value, list):
|
|
return [safe(v) for v in value]
|
|
if isinstance(value, str):
|
|
value = re.sub(r"(?i)Bearer\s+[^\s\"']+", "Bearer [REDACTED]", value)
|
|
value = re.sub(r"bce-auth-v1/[^\s\"']+", "[REDACTED BCE SIGNATURE]", value)
|
|
value = re.sub(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", "[REDACTED JWT]", value)
|
|
return re.sub(r"(https?://[^\s?\"']+)\?[^\s\"']+", r"\1?[REDACTED]", value)
|
|
return value
|
|
|
|
|
|
def save(path, value):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temp = path.with_suffix(path.suffix + ".tmp")
|
|
temp.write_text(json.dumps(safe(value), ensure_ascii=False, indent=2) + "\n")
|
|
temp.replace(path)
|
|
|
|
|
|
def wait_for(fn, seconds=180, interval=1):
|
|
deadline = time.monotonic() + seconds
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
last = fn()
|
|
if last:
|
|
return last
|
|
except (ConnectionError, OSError, ValueError, RuntimeError):
|
|
pass
|
|
time.sleep(interval)
|
|
raise AssertionError(f"condition not reached after {seconds}s; last={safe(last)}")
|
|
|
|
|
|
def local_get(url):
|
|
return urllib.request.build_opener(urllib.request.ProxyHandler({})).open(url, timeout=5).read()
|
|
|
|
|
|
def pattern(path, size, seed=0, done=True):
|
|
path = Path(path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
# Actual allocated file, not a sparse placeholder. Digest is computed during
|
|
# fixture generation; independent downloads must match this immutable value.
|
|
block = hashlib.shake_256(str(seed).encode()).digest(1024 * 1024)
|
|
digest = hashlib.md5()
|
|
remaining = size
|
|
with path.open("wb") as stream:
|
|
while remaining:
|
|
part = block[:min(remaining, len(block))]
|
|
stream.write(part)
|
|
digest.update(part)
|
|
remaining -= len(part)
|
|
os.utime(path, (time.time() - 180, time.time() - 180))
|
|
if done:
|
|
path.with_name(path.name + ".done").touch()
|
|
return {"size": size, "md5sum": digest.hexdigest()}
|
|
|
|
|
|
class Native:
|
|
def __init__(self, suite, name, concurrency=4, level="debug", baseline=False, log_max=33554432, fallback=False):
|
|
self.process = None
|
|
try:
|
|
self._initialize(suite, name, concurrency, level, baseline, log_max, fallback)
|
|
except BaseException:
|
|
self.stop()
|
|
raise
|
|
|
|
def _initialize(self, suite, name, concurrency, level, baseline, log_max, fallback):
|
|
if getattr(suite.args, "suffix", ""):
|
|
name = suite.args.suffix + "-" + name
|
|
self.suite, self.name = suite, name
|
|
self.credentials = dict(suite.access)
|
|
self.root = suite.work / name
|
|
self.root.mkdir(parents=True, exist_ok=True)
|
|
self.source = self.root / "capture"
|
|
self.source.mkdir(exist_ok=True)
|
|
self.state = self.root / "s"
|
|
self.config = self.root / "environment"
|
|
base = REPO / (".codex-artifacts/log-enhance27/baseline" if baseline else "tools/drosync/target/debug")
|
|
self.cli, self.daemon = str(base / "drosync"), str(base / "drosync-daemon")
|
|
self.evidence = suite.evidence / name
|
|
self.evidence.mkdir(parents=True, exist_ok=True)
|
|
primary = self.root / "primary-log-directory" if fallback else self.root / "daemon.log"
|
|
if fallback:
|
|
primary.mkdir(exist_ok=True)
|
|
self.config.write_text(f"DROSYNC_DATA_DIR={self.state}\nDROSYNC_SECRET_FILE={self.root / 'device-token'}\nDROSYNC_LOG_PATH={primary}\nDROSYNC_LOG_FALLBACK_PATH={self.root / 'fallback.log'}\nDROSYNC_LOG_LEVEL={level}\nDROSYNC_FILE_CONCURRENCY={concurrency}\nDROSYNC_PUT_CONCURRENCY={concurrency}\nDROSYNC_PART_CONCURRENCY={concurrency}\nDROSYNC_MAPPING_CONCURRENCY=4\nDROSYNC_HASH_CONCURRENCY=4\nDROSYNC_LOG_MAX_BYTES={log_max}\nDROSYNC_LOG_KEEP_FILES=3\n")
|
|
self.config.chmod(0o600)
|
|
self.level, self.mapping, self.process = level, None, None
|
|
self.commands = []
|
|
self.expected = {}
|
|
self.start()
|
|
self.call("pause")
|
|
self.call("auth", "--server", suite.access["baseURL"], "--access-key-id", suite.access["accessKeyId"], "--secret-stdin", stdin=suite.access["secret"])
|
|
self.call("version")
|
|
self.call("doctor")
|
|
result = self.call("mapping", "add", str(self.source), "--space", suite.access["spaceId"], "--remote-prefix", name, "--require-done")
|
|
self.mapping = result.get("id", result.get("mapping", {}).get("id"))
|
|
assert self.mapping, result
|
|
|
|
def start(self):
|
|
output = (self.evidence / "process.txt").open("ab")
|
|
self.process = subprocess.Popen([self.daemon, "-c", str(self.config)], stdout=output, stderr=output, start_new_session=True, env=self.suite.env)
|
|
output.close()
|
|
wait_for(lambda: self.call("status", record=False) if self.process.poll() is None else None, 20, .2)
|
|
|
|
def call(self, *args, stdin="", accepted=(0,), timeout=600, record=True):
|
|
command = [self.cli, "-c", str(self.config), "--json", *map(str, args)]
|
|
start = time.monotonic()
|
|
p = subprocess.run(command, input=stdin, capture_output=True, text=True, timeout=timeout, env=self.suite.env)
|
|
try:
|
|
value = json.loads(p.stdout)
|
|
except ValueError:
|
|
value = {"stdout": p.stdout[-1500:], "stderr": p.stderr[-1500:]}
|
|
if record:
|
|
self.commands.append({"command": command, "exit": p.returncode, "seconds": time.monotonic() - start, "result": safe(value)})
|
|
save(self.evidence / "commands.json", self.commands)
|
|
if p.returncode not in accepted:
|
|
raise RuntimeError(f"{args[0]} exit={p.returncode}: {safe(value)}")
|
|
return value
|
|
|
|
def add(self, relative, size, seed=0, done=True):
|
|
item = pattern(self.source / relative, size, seed, done)
|
|
self.expected[relative] = item
|
|
self.suite.known[self.suite.access["rootPrefix"].strip("/") + "/" + self.name + "/" + relative] = item
|
|
self.suite.save_known()
|
|
return item
|
|
|
|
def sync(self, accepted=(0,), timeout=600):
|
|
self.call("resume")
|
|
result = self.call("sync", self.mapping, accepted=(*accepted, 20, 50), timeout=timeout)
|
|
if result.get("errorCode") == "busy" or (isinstance(result.get("error"), str) and ("busy" in result["error"] or "already has an active sync" in result["error"])):
|
|
self.wait_complete()
|
|
result = self.call("sync", self.mapping, accepted=(*accepted, 50), timeout=timeout)
|
|
return result
|
|
|
|
def status(self):
|
|
return self.call("status", record=False)
|
|
|
|
def wait_complete(self, count=None, seconds=600):
|
|
def check():
|
|
report = self.report()
|
|
values = report.get("counts", {})
|
|
if values.get("missing", 0) == 0 and values.get("running", 0) == 0 and values.get("completed", 0) >= (count or 1):
|
|
return report
|
|
return None
|
|
return wait_for(check, seconds, 2)
|
|
|
|
def report(self, date=None):
|
|
return self.call("episodes", "--mapping", self.mapping, "--date", date or self.suite.today, "--limit", "1000", record=False)
|
|
|
|
def text(self):
|
|
return "\n".join(p.read_text(errors="replace") for p in sorted(self.root.glob("*.log*")) if p.is_file())
|
|
|
|
def snapshot(self):
|
|
time.sleep(.3)
|
|
text = self.text()
|
|
for secret in (self.credentials["secret"], self.credentials.get("adminSession", {}).get("password", "")):
|
|
assert not secret or secret not in text, "secret leaked into daemon log"
|
|
(self.evidence / "daemon.txt").write_text(text)
|
|
try:
|
|
from drosync_real_evidence import analyze_logs
|
|
save(self.evidence / "log-analysis.json", analyze_logs(text))
|
|
except ImportError:
|
|
pass
|
|
save(self.evidence / "status.json", self.status())
|
|
save(self.evidence / "report.json", self.report())
|
|
|
|
def stop(self, hard=False):
|
|
if self.process and self.process.poll() is None:
|
|
self.process.send_signal(signal.SIGKILL if hard else signal.SIGTERM)
|
|
try:
|
|
self.process.wait(8)
|
|
except subprocess.TimeoutExpired:
|
|
self.process.kill()
|
|
self.process.wait()
|
|
|
|
def close(self):
|
|
try:
|
|
self.call("pause")
|
|
self.snapshot()
|
|
finally:
|
|
self.stop()
|
|
|
|
|
|
class Suite:
|
|
def __init__(self, args):
|
|
self.args = args
|
|
self.access = json.loads(Path(args.access_file).read_text())
|
|
assert self.access["bucket"] == "testxyz"
|
|
assert self.access["rootPrefix"].strip("/") == "longtaotest/20261013T100437Z"
|
|
self.work, self.evidence = Path(args.workdir), Path(args.evidence_dir)
|
|
if not self.work.resolve().is_relative_to(Path(args.access_file).resolve().parent):
|
|
raise ValueError("generated fixtures must remain inside the private acceptance directory")
|
|
self.work.mkdir(parents=True, exist_ok=True)
|
|
self.work.chmod(0o700)
|
|
self.evidence.mkdir(parents=True, exist_ok=True)
|
|
self.env = {k: v for k, v in os.environ.items() if not k.startswith(("DROSYNC_", "UBT_"))}
|
|
now = datetime.now(timezone(timedelta(hours=8)))
|
|
self.today, self.yesterday = now.strftime("%Y_%m_%d"), (now - timedelta(days=1)).strftime("%Y_%m_%d")
|
|
self.known, self.results = {}, []
|
|
|
|
def save_known(self):
|
|
save(self.evidence / "known-files.json", self.known)
|
|
|
|
def management(self):
|
|
api = api_module.API(self.access["baseURL"])
|
|
session = self.access["adminSession"]
|
|
api.request("POST", "/api/auth/login", {"email": session["email"], "password": session["password"]})
|
|
return api
|
|
|
|
def case(self, name, fn):
|
|
start = time.monotonic()
|
|
print(json.dumps({"event": "case_start", "case": name}), flush=True)
|
|
try:
|
|
value = fn()
|
|
result = {"name": name, "status": "PASS", "seconds": time.monotonic() - start, "details": value}
|
|
except Exception as error:
|
|
result = {"name": name, "status": "FAIL", "seconds": time.monotonic() - start, "error": str(error)}
|
|
self.results.append(result)
|
|
save(self.evidence / "results.json", self.results)
|
|
print(json.dumps(safe({k: v for k, v in result.items() if k != "details"}), ensure_ascii=False), flush=True)
|
|
return result
|
|
|
|
def verify(self, client, relatives=None, suffix="verify"):
|
|
files = [{"key": self.access["rootPrefix"].strip("/") + "/" + client.name + "/" + p, **v} for p, v in client.expected.items() if relatives is None or p in relatives]
|
|
secret = subprocess.run(["kubectl", "--kubeconfig", self.args.kubeconfig, "-n", "droboticflow", "get", "secret", "droboticflow-bos-workspace", "-o", "json"], capture_output=True, check=True).stdout
|
|
import base64
|
|
values = json.loads(secret)["data"]
|
|
payload = {"endpoint": "https://bj.bcebos.com", "bucket": "testxyz", "prefix": self.access["rootPrefix"].strip("/") + "/" + client.name + "/", "accessKeyId": base64.b64decode(values["accessKeyId"]).decode(), "secretAccessKey": base64.b64decode(values["secretAccessKey"]).decode(), "files": files}
|
|
p = subprocess.run([self.args.verify_bin], input=json.dumps(payload), capture_output=True, text=True, timeout=900)
|
|
payload.clear()
|
|
value = json.loads(p.stdout)
|
|
save(client.evidence / (suffix + ".json"), value)
|
|
assert p.returncode == 0 and value["status"] == "PASS", safe(value)
|
|
return value
|
|
|
|
def stability(self):
|
|
c = Native(self, "stability", level="info")
|
|
cycles = []
|
|
start = time.monotonic()
|
|
try:
|
|
c.call("resume")
|
|
for index in range(self.args.duration // self.args.interval):
|
|
target = start + index * self.args.interval
|
|
while time.monotonic() < target:
|
|
time.sleep(min(30, target - time.monotonic()))
|
|
relative = f"{self.today}/stability/episode_{index:03d}/data.bin"
|
|
c.add(relative, 1024 * 1024, index)
|
|
report = c.wait_complete(index + 1, 180)
|
|
cycles.append({"cycle": index, "atSeconds": time.monotonic() - start, "counts": report["counts"]})
|
|
save(c.evidence / "cycles.json", cycles)
|
|
print(json.dumps({"event": "stability_cycle", **cycles[-1]}), flush=True)
|
|
while time.monotonic() - start < self.args.duration:
|
|
time.sleep(min(30, self.args.duration - (time.monotonic() - start)))
|
|
verification = self.verify(c)
|
|
c.snapshot()
|
|
assert 'event="request_completed"' not in c.text(), "INFO emitted successful requests"
|
|
assert len(cycles) == self.args.duration // self.args.interval
|
|
return {"durationSeconds": time.monotonic() - start, "cycles": cycles, "version": c.call("version"), "verification": verification}
|
|
finally:
|
|
c.close()
|
|
|
|
def run_suite(self):
|
|
cases = {"bulk": ("1003-small-and-two-multipart-boundaries", self.bulk), "reports": ("marker-report-late-date-conflict-delete-clean", self.reports), "clean": ("fresh-receipt-clean-and-mutation-protection", self.clean), "lifecycle": ("pause-resume-and-explicit-abort", self.lifecycle), "crash": ("multipart-sigkill-recovery", self.crash), "race": ("same-target-two-device-race", self.race), "commit": ("real-commit-wait-30-90-150", self.commit_wait), "lost": ("real-control-confirmation-response-lost", self.lost), "connection": ("owned-control-forward-loss", self.connection_loss), "logging": ("real-log-rotation-and-fallback", self.logging)}
|
|
for key in self.args.case or cases:
|
|
name, fn = cases[key]
|
|
self.case(name, fn)
|
|
|
|
def bulk(self):
|
|
c = Native(self, "bulk", level="debug")
|
|
try:
|
|
for i in range(1003):
|
|
c.add(f"{self.today}/bulk/episode_many/data_{i:04d}.bin", 65536, i)
|
|
for name, size in [("empty.bin", 0), ("中文 空格.txt", 337), ("large257.bin", 257 << 20), ("large321.bin", 321 << 20)]:
|
|
c.add(f"{self.today}/bulk/episode_{name.split('.')[0]}/{name}", size, size)
|
|
pattern(c.source / ".hidden.bin", 8)
|
|
pattern(c.source / "ignored.tmp", 8)
|
|
(c.source / "ignored-symlink").symlink_to(next(c.source.rglob("data_0000.bin")))
|
|
begin = time.monotonic()
|
|
result = c.sync(timeout=1200)
|
|
seconds = time.monotonic() - begin
|
|
assert result.get("session", {}).get("status") == "completed", result
|
|
assert result.get("itemCount") == 1007, result
|
|
verification = self.verify(c)
|
|
assert verification["verifiedCount"] == 1007
|
|
assert verification["unexpectedObjects"] == 0, verification
|
|
before = len(c.status()["sessions"])
|
|
repeat = c.sync()
|
|
assert repeat.get("confirmedFilesSkipped") == 1007, repeat
|
|
assert len(c.status()["sessions"]) == before
|
|
assert repeat.get("session") is None
|
|
c.snapshot()
|
|
text = c.text()
|
|
assert 'endpoint_kind="bos_put"' in text and 'event="request_completed"' in text
|
|
assert "part_number" in text or "multipart" in text
|
|
return {"files": 1007, "bytes": sum(v["size"] for v in c.expected.values()), "syncSeconds": seconds, "sessionId": result["session"]["id"], "repeat": repeat, "verification": verification}
|
|
finally:
|
|
c.close()
|
|
|
|
def web(self, c, port):
|
|
ps = subprocess.run(["ps", "-axo", "pid=,command="], capture_output=True, text=True, check=True).stdout
|
|
for line in ps.splitlines():
|
|
if "drosync -c /private/tmp/drl-100437/" in line and " web " in line and f"--port {port}" in line:
|
|
os.kill(int(line.strip().split()[0]), signal.SIGTERM)
|
|
time.sleep(.3)
|
|
output = (c.evidence / "web.txt").open("ab")
|
|
p = subprocess.Popen([c.cli, "-c", str(c.config), "web", "--mapping", c.mapping, "--port", str(port)], stdout=output, stderr=output, env=self.env, start_new_session=True)
|
|
output.close()
|
|
def ready():
|
|
value = json.loads(local_get(f"http://127.0.0.1:{port}/api/report?date={self.today}"))
|
|
report = value.get("report", value)
|
|
return value if report.get("localRoot") == str(c.source) else None
|
|
wait_for(ready, 20)
|
|
return p
|
|
|
|
def reports(self):
|
|
c = Native(self, "reports", level="debug")
|
|
try:
|
|
first = f"{self.today}/report/episode_mix/ready.bin"
|
|
pending = f"{self.today}/report/episode_mix/pending.bin"
|
|
yesterday = f"{self.yesterday}/report/episode_late/data.bin"
|
|
c.add(first, 1 << 20, 1)
|
|
c.add(pending, 1 << 20, 2, False)
|
|
c.sync()
|
|
report = c.report()
|
|
save(c.evidence / "pending-report.json", report)
|
|
assert report["counts"]["missing"] == 1 and report["counts"]["completed"] == 0, report
|
|
assert report["items"][0]["totalFiles"] == 2 and report["items"][0]["localSize"] == 2 << 20
|
|
assert ".done" in report["items"][0]["reason"] and "pending.bin" in report["items"][0]["reason"], report
|
|
self.verify(c, [first], "pending-bos")
|
|
(c.source / pending).with_name("pending.bin.done").touch()
|
|
c.add(yesterday, 1 << 20, 3)
|
|
completed = c.sync()
|
|
assert c.report()["counts"]["missing"] == 0
|
|
assert c.report(self.yesterday)["counts"]["completed"] == 1
|
|
self.verify(c)
|
|
original = c.expected[first]
|
|
pattern(c.source / first, 1 << 20, 99)
|
|
conflict = c.sync(accepted=(0, 30))
|
|
conflict_report = c.report()
|
|
save(c.evidence / "conflict-report.json", conflict_report)
|
|
assert conflict_report["counts"]["missing"] > 0 and "冲突" in json.dumps(conflict_report, ensure_ascii=False), conflict_report
|
|
assert c.expected[first] == original
|
|
self.verify(c, [first], "conflict-bos-original")
|
|
(c.source / pending).unlink()
|
|
deleted = c.report()
|
|
save(c.evidence / "deleted-report.json", deleted)
|
|
self.verify(c, [pending], "deleted-bos-preserved")
|
|
c.web_process = self.web(c, 19341)
|
|
web_value = json.loads(local_get(f"http://127.0.0.1:19341/api/report?date={self.today}"))
|
|
save(c.evidence / "web-conflict-report.json", web_value)
|
|
web_report = web_value.get("report", web_value)
|
|
assert web_report["totalLocalEpisodes"] == deleted["counts"]["total"]
|
|
assert web_report["totalMissingEpisodes"] == deleted["counts"]["missing"]
|
|
assert web_report["totalCloudEpisodes"] == deleted["counts"]["completed"]
|
|
assert any("冲突" in warning for warning in web_report["warnings"])
|
|
for fmt in ("csv", "xlsx"):
|
|
c.call("report", "--mapping", c.mapping, "--date", self.today, "--format", fmt, "--output", str(c.evidence / fmt))
|
|
assert list((c.evidence / "csv").glob("*.csv"))
|
|
xlsx = next((c.evidence / "xlsx").glob("*.xlsx"))
|
|
assert zipfile.ZipFile(xlsx).testzip() is None
|
|
# Restore only generated source contents; immutable BOS expectations
|
|
# remain the accepted originals throughout conflict/delete checks.
|
|
pattern(c.source / first, 1 << 20, 1)
|
|
pattern(c.source / pending, 1 << 20, 2)
|
|
c.sync()
|
|
clean_session = completed.get("session", {}).get("id")
|
|
assert clean_session
|
|
# Change one file that belongs to this completed Session: cleaning
|
|
# must protect it while deleting its unchanged sibling.
|
|
pattern(c.source / yesterday, 1 << 20, 999)
|
|
preview = c.call("clean", c.mapping, "--session", clean_session, accepted=(0, 50))
|
|
save(c.evidence / "clean-preview.json", preview)
|
|
assert (c.source / first).exists() and (c.source / pending).exists()
|
|
cleaned = c.call("clean", c.mapping, "--session", clean_session, "--execute", accepted=(0, 50))
|
|
save(c.evidence / "clean-executed.json", cleaned)
|
|
assert cleaned["deletedFiles"] == 0 and cleaned["blockedFiles"] >= 1, cleaned
|
|
assert (c.source / pending).exists()
|
|
assert (c.source / yesterday).exists() and (c.source / first).exists()
|
|
self.verify(c)
|
|
c.snapshot()
|
|
return {"pending": report["counts"], "conflict": conflict_report["counts"], "delete": deleted["counts"], "clean": cleaned, "web": "http://127.0.0.1:19341/", "sessionId": clean_session}
|
|
finally:
|
|
c.close()
|
|
|
|
def clean(self):
|
|
c = Native(self, "clean")
|
|
try:
|
|
first = f"{self.today}/clean/episode/first.bin"
|
|
second = f"{self.today}/clean/episode/second.bin"
|
|
c.add(first, 1 << 20, 1)
|
|
c.add(second, 1 << 20, 2)
|
|
result = c.sync()
|
|
sid = result["session"]["id"]
|
|
c.call("pause")
|
|
pattern(c.source / second, 1 << 20, 999)
|
|
preview = c.call("clean", c.mapping, "--session", sid, accepted=(0, 50))
|
|
assert preview["candidateFiles"] == 1 and preview["blockedFiles"] == 1, preview
|
|
assert (c.source / first).exists()
|
|
actual = c.call("clean", c.mapping, "--session", sid, "--execute", accepted=(0, 50))
|
|
assert actual["deletedFiles"] == 1 and actual["blockedFiles"] == 1, actual
|
|
assert not (c.source / first).exists() and (c.source / second).exists()
|
|
verification = self.verify(c)
|
|
repeat = c.call("clean", c.mapping, "--session", sid, "--execute", accepted=(0, 50))
|
|
assert repeat["deletedFiles"] == 0 and repeat["alreadyCleanedFiles"] == 1, repeat
|
|
c.snapshot()
|
|
return {"preview": preview, "execute": actual, "repeat": repeat, "verification": verification}
|
|
finally:
|
|
c.close()
|
|
|
|
def lifecycle(self):
|
|
c = Native(self, "lifecycle", concurrency=1)
|
|
try:
|
|
for i in range(8):
|
|
c.add(f"{self.today}/lifecycle/episode_{i:02d}/data.bin", 16 << 20, i)
|
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
transfer = pool.submit(c.sync)
|
|
wait_for(lambda: 'event="file_upload_start"' in c.text(), 90, .1)
|
|
paused = c.call("pause")
|
|
result = transfer.result(300)
|
|
assert paused["paused"] is True
|
|
assert result.get("paused") is True, result
|
|
sid = result.get("sessionId") or next(s["id"] for s in c.status()["sessions"] if s["status"] not in ("completed", "aborted"))
|
|
resumed = c.sync()
|
|
assert any(s["id"] == sid and s["status"] == "completed" for s in c.status()["sessions"]), resumed
|
|
self.verify(c)
|
|
# New source item creates a separate owned session for cancellation.
|
|
c.add(f"{self.today}/lifecycle/episode_abort/data.bin", 65 << 20, 888)
|
|
before = c.text().count('event="file_upload_start"')
|
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
transfer = pool.submit(c.sync, (0, 20, 30, 40, 50))
|
|
wait_for(lambda: c.text().count('event="file_upload_start"') > before, 90, .1)
|
|
current = next(s for s in c.status()["sessions"] if s["status"] not in ("completed", "aborted", "failed", "expired", "conflicted"))
|
|
aborted = c.call("abort", current["id"])
|
|
after_abort = transfer.result(300)
|
|
assert (c.source / f"{self.today}/lifecycle/episode_abort/data.bin").exists()
|
|
cloud, _ = self.management().request("GET", "/api/object-sync/sessions/" + current["id"])
|
|
assert cloud.get("session", cloud)["status"] == "aborted", cloud
|
|
save(c.evidence / "cloud-aborted.json", cloud)
|
|
c.snapshot()
|
|
return {"pausedSession": sid, "resumed": resumed, "abortSession": current["id"], "abort": aborted, "transferAfterAbort": after_abort}
|
|
finally:
|
|
c.close()
|
|
|
|
def crash(self):
|
|
c = Native(self, "crash", concurrency=1)
|
|
try:
|
|
c.add(f"{self.today}/crash/episode_large/data.bin", 769 << 20, 123)
|
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
transfer = pool.submit(c.sync, (0, 10, 20, 30, 40, 50))
|
|
wait_for(lambda: 'event="request_completed" endpoint_kind="bos_put"' in c.text(), 180, .1)
|
|
sid = next(s["id"] for s in c.status()["sessions"] if s["status"] not in ("completed", "aborted"))
|
|
c.stop(hard=True)
|
|
killed = transfer.result(20)
|
|
time.sleep(1)
|
|
c.start()
|
|
result = c.sync(timeout=900)
|
|
assert result.get("session", {}).get("id") == sid or any(s["id"] == sid and s["status"] == "completed" for s in c.status()["sessions"]), result
|
|
verification = self.verify(c)
|
|
c.snapshot()
|
|
text = c.text()
|
|
recoveries = [int(v) for v in re.findall(r'event="multipart_recovery" completed_parts=(\d+)', text)]
|
|
assert any(v > 0 for v in recoveries), recoveries
|
|
first_part_puts = [line for line in text.splitlines() if 'part_number=1 ' in line and 'event="request_completed" endpoint_kind="bos_put"' in line]
|
|
assert len(first_part_puts) == 1, "confirmed first part was retransmitted"
|
|
return {"sessionId": sid, "postKill": killed, "resumed": result, "verification": verification}
|
|
finally:
|
|
c.close()
|
|
|
|
def race(self):
|
|
primary = dict(self.access)
|
|
gate = None
|
|
if self.args.plan_barrier:
|
|
from drosync_real_relay import RealResponseGate
|
|
gate = RealResponseGate(primary["baseURL"], self.evidence / "real-planned-response-gate.jsonl", port=19443, mode="mapping_plan").start()
|
|
self.access = {**primary, "baseURL": gate.base_url if gate else primary["baseURL"]}
|
|
a = Native(self, "race-a")
|
|
b = Native(self, "race-b")
|
|
self.access = primary
|
|
try:
|
|
b.call("mapping", "remove", b.mapping)
|
|
mapping = b.call("mapping", "add", str(b.source), "--space", primary["spaceId"], "--remote-prefix", a.name, "--require-done")
|
|
b.mapping = mapping.get("id", mapping.get("mapping", {}).get("id"))
|
|
relative = f"{self.today}/race/episode_001/data.bin"
|
|
a.add(relative, 4 << 20, 4)
|
|
b.add(relative, 4 << 20, 5)
|
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
if gate:
|
|
# B obtains an unchanged real planned response before any
|
|
# writer creates the key. Let A commit and free its lease,
|
|
# then deliver B's already obtained stale plan response.
|
|
gate.hold_mapping_id = b.mapping
|
|
task_b = pool.submit(b.sync, (0, 20, 30, 40, 50))
|
|
wait_for(gate.held.is_set, 60, .1)
|
|
outcome_a = a.sync((0, 20, 30, 40, 50))
|
|
gate.release()
|
|
outcomes = [outcome_a, task_b.result(300)]
|
|
else:
|
|
tasks = [pool.submit(c.sync, (0, 20, 30, 40)) for c in (a, b)]
|
|
outcomes = [t.result(600) for t in tasks]
|
|
# A closed Session can include skipped/aborted files. Count formal
|
|
# committed objects, not merely the Session lifecycle status.
|
|
winners = [i for i, value in enumerate(outcomes) if value.get("session", {}).get("counts", {}).get("committed") == 1]
|
|
assert len(winners) == 1, outcomes
|
|
loser = 1 - winners[0]
|
|
assert outcomes[loser].get("session", {}).get("counts", {}).get("committed", 0) == 0
|
|
assert (a, b)[loser].report()["counts"]["missing"] == 1
|
|
if winners[0] == 1:
|
|
a.expected[relative] = b.expected[relative]
|
|
self.known[self.access["rootPrefix"] + "/" + a.name + "/" + relative] = b.expected[relative]
|
|
self.save_known()
|
|
verification = self.verify(a)
|
|
a.snapshot()
|
|
b.snapshot()
|
|
if gate:
|
|
losing_text = (a, b)[loser].text()
|
|
assert len(re.findall(r'event="request_failed" endpoint_kind="bos_put"[^\n]*status=412 ', losing_text)) == 1, losing_text[-4000:]
|
|
assert 'event="file_sync_retry"' not in losing_text and 'event="upload_retry_backoff"' not in losing_text
|
|
return {"outcomes": outcomes, "winningClient": winners[0], "scope": "same Space; stale real planned response released after winner commit" if gate else "same Space race", "realBarrier": bool(gate), "verification": verification}
|
|
finally:
|
|
a.close()
|
|
b.close()
|
|
self.access = primary
|
|
if gate:
|
|
gate.release()
|
|
gate.close()
|
|
|
|
def scale(self, replicas):
|
|
deployment = self.access["coordinatorDeployment"]
|
|
assert deployment == "lt-bos-coordinator-100437"
|
|
command = ["kubectl", "--kubeconfig", self.args.kubeconfig, "-n", "droboticflow", "scale", "deployment/" + deployment, "--replicas=" + str(replicas)]
|
|
subprocess.run(command, check=True, capture_output=True)
|
|
if replicas:
|
|
subprocess.run(["kubectl", "--kubeconfig", self.args.kubeconfig, "-n", "droboticflow", "rollout", "status", "deployment/" + deployment, "--timeout=90s"], check=True, capture_output=True)
|
|
|
|
def commit_wait(self):
|
|
from drosync_real_relay import RealResponseGate
|
|
gate = RealResponseGate(self.access["baseURL"], self.evidence / "real-response-gate.jsonl").start()
|
|
actual_url = self.access["baseURL"]
|
|
self.access["baseURL"] = gate.base_url
|
|
c = Native(self, "commit-wait")
|
|
self.access["baseURL"] = actual_url
|
|
try:
|
|
c.add(f"{self.today}/commit/episode/data.bin", 1 << 20, 32)
|
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
transfer = pool.submit(c.sync, (0,), 900)
|
|
wait_for(gate.held.is_set, 120, .1)
|
|
try:
|
|
wait_for(lambda: len(re.findall(r'event="commit_wait"', c.text())) >= 3, 230, 1)
|
|
notices = [int(v) for v in re.findall(r'event="commit_wait"[^\n]*elapsed_ms=(\d+)', c.text())]
|
|
assert len(notices) == 3 and all(abs(v - target) < 2500 for v, target in zip(notices, [30000, 90000, 150000])), notices
|
|
finally:
|
|
gate.release()
|
|
result = transfer.result(300)
|
|
assert result.get("session", {}).get("status") == "completed", result
|
|
self.verify(c)
|
|
c.snapshot()
|
|
return {"fault": "delayed delivery of unchanged real POC terminal progress responses; BOS uploaded directly; no synthetic HTTP responses", "noticesMs": notices, "result": result}
|
|
finally:
|
|
self.access["baseURL"] = actual_url
|
|
gate.close()
|
|
c.close()
|
|
|
|
def connection_loss(self):
|
|
c = Native(self, "connection-loss", concurrency=1)
|
|
try:
|
|
return self._connection_loss(c)
|
|
finally:
|
|
c.close()
|
|
|
|
def _connection_loss(self, c):
|
|
forward = None
|
|
try:
|
|
c.add(f"{self.today}/connection/episode_001/data.bin", 8 << 20, 60)
|
|
before = c.text().count('event="file_upload_start"')
|
|
with concurrent.futures.ThreadPoolExecutor() as pool:
|
|
transfer = pool.submit(c.sync, (0, 10, 20, 30, 40, 50))
|
|
wait_for(lambda: c.text().count('event="file_upload_start"') > before, 120, .1)
|
|
# Identify only the port-forward whose args match this new API.
|
|
ps = subprocess.run(["ps", "-axo", "pid=,command="], capture_output=True, text=True, check=True).stdout
|
|
pids = [int(line.strip().split()[0]) for line in ps.splitlines() if "kubectl" in line and "port-forward" in line and "19441:3001" in line and "lt-bos-api-100437" in line]
|
|
assert len(pids) == 1, "owned port-forward not uniquely found"
|
|
os.kill(pids[0], signal.SIGTERM)
|
|
time.sleep(45)
|
|
killed = transfer.result(300)
|
|
assert 'event="request_failed"' in c.text()
|
|
finally:
|
|
log = (self.evidence / "forward-restart.txt").open("ab")
|
|
forward = subprocess.Popen(["kubectl", "--kubeconfig", self.args.kubeconfig, "-n", "droboticflow", "port-forward", "service/lt-bos-api-100437", "19441:3001"], stdout=log, stderr=log, start_new_session=True)
|
|
log.close()
|
|
wait_for(lambda: local_get(self.access["baseURL"] + "/api/health"), 30)
|
|
try:
|
|
resumed = c.sync(timeout=600)
|
|
wait_for(lambda: any(s["status"] == "completed" for s in c.status()["sessions"]), 60)
|
|
verification = self.verify(c)
|
|
c.snapshot()
|
|
return {"fault": "owned control connection unavailable for 45 seconds; BOS unchanged", "interrupted": killed, "resumed": resumed, "verification": verification, "forwardPid": forward.pid}
|
|
finally:
|
|
c.close()
|
|
|
|
def lost(self):
|
|
from drosync_real_relay import RealResponseGate
|
|
original_url = self.access["baseURL"]
|
|
gate = RealResponseGate(original_url, self.evidence / "real-lost-confirmation.jsonl", port=19444, mode="lost_completion").start()
|
|
self.access["baseURL"] = gate.base_url
|
|
c = Native(self, "lost-response")
|
|
self.access["baseURL"] = original_url
|
|
try:
|
|
c.add(f"{self.today}/lost/episode_001/data.bin", 1 << 20, 430)
|
|
result = c.sync()
|
|
assert result.get("session", {}).get("counts", {}).get("committed") == 1, result
|
|
assert gate.dropped.is_set(), "real upstream successful confirmation was not dropped"
|
|
verification = self.verify(c)
|
|
c.snapshot()
|
|
text = c.text()
|
|
assert text.count('event="request_completed" endpoint_kind="bos_put"') == 1
|
|
assert 'event="request_retry"' in text
|
|
return {"scope": "real POC confirmation response lost after real BOS PUT; idempotent control retry, not provider fault", "result": result, "verification": verification, "successfulPutCalls": 1}
|
|
finally:
|
|
self.access["baseURL"] = original_url
|
|
gate.close()
|
|
c.close()
|
|
|
|
def logging(self):
|
|
values = []
|
|
for name, fallback in [("rotation", False), ("fallback", True)]:
|
|
c = Native(self, name, level="debug", log_max=32768, fallback=fallback)
|
|
try:
|
|
for i in range(32):
|
|
c.add(f"{self.today}/logging/episode_{i:03d}/data.bin", 65536, i)
|
|
result = c.sync()
|
|
self.verify(c)
|
|
c.snapshot()
|
|
logs = [str(p.name) for p in c.root.glob("*.log*")]
|
|
assert any("fallback.log" in p for p in logs) if fallback else len(logs) > 1, logs
|
|
assert 'event="request_completed"' in c.text()
|
|
values.append({"name": name, "logs": logs, "sessionId": result.get("session", {}).get("id")})
|
|
finally:
|
|
c.close()
|
|
return values
|
|
|
|
def performance(self):
|
|
measurements = []
|
|
# All nine uploads use new keys, the same deterministic 256MiB content,
|
|
# the same host, debug build profile and real backend. 1/4 concurrency
|
|
# and pre-enhancement INFO are interleaved to reduce temporal drift.
|
|
for repetition in range(3):
|
|
modes = [("new4", 4, False), ("old4", 4, True), ("new1", 1, False)]
|
|
if repetition % 2:
|
|
modes.reverse()
|
|
for name, concurrency, baseline in modes:
|
|
c = Native(self, f"perf-{repetition}-{name}", concurrency=concurrency, level="info", baseline=baseline)
|
|
try:
|
|
for i in range(32):
|
|
c.add(f"{self.today}/performance/episode_{i:03d}/data.bin", 8 << 20, i)
|
|
before = time.monotonic()
|
|
result = c.sync(timeout=900)
|
|
elapsed = time.monotonic() - before
|
|
assert result.get("session", {}).get("status") == "completed", result
|
|
c.snapshot()
|
|
assert 'event="request_completed"' not in c.text()
|
|
verified = self.verify(c)
|
|
measurements.append({"name": name, "repetition": repetition, "seconds": elapsed, "bytes": 256 << 20, "mibPerSecond": 256 / elapsed, "logBytes": len(c.text().encode()), "version": c.call("version"), "verification": {k: verified[k] for k in ("verifiedCount", "verifiedBytes", "status")}})
|
|
save(self.evidence / "measurements.json", measurements)
|
|
print(json.dumps({"event": "performance", **measurements[-1]}, ensure_ascii=False), flush=True)
|
|
finally:
|
|
c.close()
|
|
import statistics
|
|
medians = {mode: statistics.median(v["seconds"] for v in measurements if v["name"] == mode) for mode in ("new4", "old4", "new1")}
|
|
delta = medians["new4"] / medians["old4"] - 1
|
|
self.results.append({"name": "paired-real-BOS-INFO-performance", "status": "PASS" if delta <= .05 else "FAIL", "mediansSeconds": medians, "infoOverheadFraction": delta, "measurements": measurements})
|
|
save(self.evidence / "results.json", self.results)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--access-file", required=True)
|
|
parser.add_argument("--workdir", required=True)
|
|
parser.add_argument("--evidence-dir", required=True)
|
|
parser.add_argument("--verify-bin", required=True)
|
|
parser.add_argument("--kubeconfig", default=str(Path.home() / ".config/droboticflow/kubeconfigs/poc.config"))
|
|
parser.add_argument("--mode", choices=["stability", "suite", "performance"], required=True)
|
|
parser.add_argument("--duration", type=int, default=7200)
|
|
parser.add_argument("--interval", type=int, default=600)
|
|
parser.add_argument("--case", action="append", choices=["bulk", "reports", "clean", "lifecycle", "crash", "race", "commit", "lost", "connection", "logging"])
|
|
parser.add_argument("--suffix", default="", help="Use a fresh case prefix for a focused rerun")
|
|
parser.add_argument("--plan-barrier", action="store_true", help="Deliver an unchanged real planned response after another client commits; no mocked responses")
|
|
args = parser.parse_args()
|
|
suite = Suite(args)
|
|
if args.mode == "stability":
|
|
suite.case("two-hour-auto-sync", suite.stability)
|
|
elif args.mode == "suite":
|
|
suite.run_suite()
|
|
else:
|
|
suite.performance()
|
|
return 1 if any(v["status"] == "FAIL" for v in suite.results) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|