756 lines
34 KiB
Python
Executable File
756 lines
34 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import http.cookiejar
|
|
import json
|
|
import os
|
|
import secrets
|
|
import shutil
|
|
import socket
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import zipfile
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
DEFAULT_KUBECONFIG = Path("/Users/d-robotics/Downloads/poc.config")
|
|
DEFAULT_PLAYROOM = Path("/Users/d-robotics/Downloads/images")
|
|
NAMESPACE = "real2sim-poc"
|
|
TERMINAL = {"succeeded", "failed", "failed_recovered", "canceled"}
|
|
|
|
|
|
class AcceptanceError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class ApiError(AcceptanceError):
|
|
def __init__(self, status: int, path: str, payload: Any) -> None:
|
|
super().__init__(f"HTTP {status} {path}: {payload}")
|
|
self.status = status
|
|
self.path = path
|
|
self.payload = payload
|
|
|
|
|
|
class ApiClient:
|
|
def __init__(self, base_url: str) -> None:
|
|
self.base_url = base_url.rstrip("/")
|
|
self.cookies = http.cookiejar.CookieJar()
|
|
context = ssl.create_default_context()
|
|
context.check_hostname = False
|
|
context.verify_mode = ssl.CERT_NONE
|
|
self.opener = urllib.request.build_opener(
|
|
urllib.request.ProxyHandler({}),
|
|
urllib.request.HTTPSHandler(context=context),
|
|
urllib.request.HTTPCookieProcessor(self.cookies),
|
|
)
|
|
|
|
def request(
|
|
self,
|
|
method: str,
|
|
path: str,
|
|
payload: Any | None = None,
|
|
*,
|
|
body: bytes | None = None,
|
|
content_type: str = "application/json",
|
|
timeout: int = 180,
|
|
) -> Any:
|
|
data = body
|
|
if payload is not None:
|
|
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
headers = {"Accept": "application/json"}
|
|
if data is not None:
|
|
headers["Content-Type"] = content_type
|
|
request = urllib.request.Request(self.base_url + path, data=data, headers=headers, method=method)
|
|
try:
|
|
with self.opener.open(request, timeout=timeout) as response:
|
|
raw = response.read()
|
|
response_type = response.headers.get("content-type", "")
|
|
if "application/json" in response_type:
|
|
return json.loads(raw.decode("utf-8"))
|
|
return raw.decode("utf-8", errors="replace")
|
|
except urllib.error.HTTPError as exc:
|
|
raw = exc.read()
|
|
try:
|
|
payload = json.loads(raw.decode("utf-8"))
|
|
except Exception:
|
|
payload = raw.decode("utf-8", errors="replace")
|
|
raise ApiError(exc.code, path, payload) from exc
|
|
|
|
def get(self, path: str) -> Any:
|
|
return self.request("GET", path)
|
|
|
|
def post(self, path: str, payload: Any | None = None) -> Any:
|
|
return self.request("POST", path, payload)
|
|
|
|
|
|
def utc_now() -> str:
|
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
|
|
|
|
def kubectl(kubeconfig: Path, *args: str, timeout: int = 3600, check: bool = True) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
["kubectl", "--kubeconfig", str(kubeconfig), *args],
|
|
text=True,
|
|
capture_output=True,
|
|
timeout=timeout,
|
|
check=check,
|
|
)
|
|
|
|
|
|
def free_port() -> int:
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
return int(sock.getsockname()[1])
|
|
|
|
|
|
def start_port_forward(kubeconfig: Path, port: int, out_dir: Path) -> subprocess.Popen[str]:
|
|
log = (out_dir / "port-forward.log").open("w", encoding="utf-8")
|
|
process = subprocess.Popen(
|
|
[
|
|
"kubectl",
|
|
"--kubeconfig",
|
|
str(kubeconfig),
|
|
"-n",
|
|
NAMESPACE,
|
|
"port-forward",
|
|
"svc/real2sim-frontend",
|
|
f"{port}:443",
|
|
"--address",
|
|
"127.0.0.1",
|
|
],
|
|
text=True,
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
)
|
|
deadline = time.time() + 90
|
|
while time.time() < deadline:
|
|
if process.poll() is not None:
|
|
log.close()
|
|
raise AcceptanceError((out_dir / "port-forward.log").read_text(encoding="utf-8", errors="replace"))
|
|
try:
|
|
with socket.create_connection(("127.0.0.1", port), timeout=1):
|
|
return process
|
|
except OSError:
|
|
time.sleep(1)
|
|
process.terminate()
|
|
log.close()
|
|
raise AcceptanceError("timed out waiting for the HTTPS port-forward")
|
|
|
|
|
|
class AcceptanceRun:
|
|
def __init__(self, *, client: ApiClient, out_dir: Path, kubeconfig: Path, build_id: str, timeout_seconds: int) -> None:
|
|
self.client = client
|
|
self.out_dir = out_dir
|
|
self.kubeconfig = kubeconfig
|
|
self.build_id = build_id
|
|
self.timeout_seconds = timeout_seconds
|
|
self.report: dict[str, Any] = {
|
|
"schema": "real2sim.poc_acceptance_report.v1",
|
|
"started_at": utc_now(),
|
|
"finished_at": "",
|
|
"build_id": build_id,
|
|
"namespace": NAMESPACE,
|
|
"version": {},
|
|
"doctor": {},
|
|
"capabilities": {},
|
|
"recipes": {},
|
|
"identities": {},
|
|
"tasks": [],
|
|
"datasets": [],
|
|
"scenes": [],
|
|
"models": [],
|
|
"storage": {},
|
|
"browser": {"status": "pending", "note": "Run browser acceptance after the API gate."},
|
|
"artifacts": [],
|
|
"checks": [],
|
|
"failures": [],
|
|
"warnings": [],
|
|
"pass": False,
|
|
}
|
|
|
|
def save(self) -> None:
|
|
self.out_dir.mkdir(parents=True, exist_ok=True)
|
|
(self.out_dir / "acceptance_report.json").write_text(
|
|
json.dumps(self.report, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
lines = [
|
|
f"# Real2Sim POC Acceptance {self.build_id}",
|
|
"",
|
|
f"- Started: {self.report['started_at']}",
|
|
f"- Finished: {self.report['finished_at'] or 'running'}",
|
|
f"- Pass: **{self.report['pass']}**",
|
|
f"- Tasks: {len(self.report['tasks'])}",
|
|
f"- Failures: {len(self.report['failures'])}",
|
|
"",
|
|
"## Checks",
|
|
*[f"- {'PASS' if item['ok'] else 'FAIL'}: {item['name']}" for item in self.report["checks"]],
|
|
"",
|
|
"## Failures",
|
|
*([f"- {item['stage']}: {item['error']}" for item in self.report["failures"]] or ["- None"]),
|
|
"",
|
|
"## Warnings",
|
|
*([f"- {item}" for item in self.report["warnings"]] or ["- None"]),
|
|
]
|
|
(self.out_dir / "summary.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
def check(self, name: str, ok: bool, detail: Any = None) -> None:
|
|
self.report["checks"].append({"name": name, "ok": bool(ok), "detail": detail})
|
|
self.save()
|
|
if not ok:
|
|
raise AcceptanceError(f"{name}: {detail}")
|
|
|
|
def stage(self, name: str, action: Callable[[], Any]) -> Any | None:
|
|
print(f"\n== {name} ==", flush=True)
|
|
started = time.time()
|
|
try:
|
|
result = action()
|
|
self.report["checks"].append({"name": name, "ok": True, "elapsed_seconds": round(time.time() - started, 2)})
|
|
self.save()
|
|
return result
|
|
except Exception as exc:
|
|
failure = {
|
|
"stage": name,
|
|
"error": f"{type(exc).__name__}: {exc}",
|
|
"elapsed_seconds": round(time.time() - started, 2),
|
|
"next_action": "Inspect the task reports and logs recorded in this acceptance directory, fix the real runtime blocker, then rerun the gate.",
|
|
}
|
|
self.report["checks"].append({"name": name, "ok": False, "elapsed_seconds": failure["elapsed_seconds"]})
|
|
self.report["failures"].append(failure)
|
|
self.save()
|
|
print(f"FAILED: {failure['error']}", file=sys.stderr, flush=True)
|
|
return None
|
|
|
|
def create_task(self, task_type: str, params: dict[str, Any], *, required_artifacts: tuple[str, ...] = ()) -> dict[str, Any]:
|
|
created = self.client.post("/api/tasks", {"type": task_type, "params": params})
|
|
print(f"{task_type}: {created['id']}", flush=True)
|
|
return self.wait_task(created["id"], required_artifacts=required_artifacts)
|
|
|
|
def wait_task(
|
|
self,
|
|
task_id: str,
|
|
*,
|
|
required_artifacts: tuple[str, ...] = (),
|
|
expected_status: str = "succeeded",
|
|
timeout_seconds: int | None = None,
|
|
) -> dict[str, Any]:
|
|
deadline = time.time() + int(timeout_seconds or self.timeout_seconds)
|
|
previous = None
|
|
task: dict[str, Any] = {}
|
|
while time.time() < deadline:
|
|
task = self.client.get(f"/api/tasks/{task_id}")
|
|
marker = (task.get("status"), task.get("progress"), task.get("stage"), task.get("message"))
|
|
if marker != previous:
|
|
print(f" {task.get('status')} {task.get('progress')}% {task.get('stage')} {task.get('message')}", flush=True)
|
|
previous = marker
|
|
if task.get("status") in TERMINAL:
|
|
break
|
|
time.sleep(5)
|
|
else:
|
|
raise AcceptanceError(f"task {task_id} timed out after {timeout_seconds or self.timeout_seconds}s")
|
|
|
|
artifacts = self.client.get(f"/api/tasks/{task_id}/artifacts")
|
|
reports = self.client.get(f"/api/tasks/{task_id}/reports")
|
|
task_record = {
|
|
"id": task_id,
|
|
"type": task.get("type"),
|
|
"status": task.get("status"),
|
|
"progress": task.get("progress"),
|
|
"stage": task.get("stage"),
|
|
"message": task.get("message"),
|
|
"error": task.get("error"),
|
|
"result": task.get("result"),
|
|
"artifact_count": len(artifacts),
|
|
"artifact_names": sorted({str(item.get("name") or "") for item in artifacts}),
|
|
"reports": reports,
|
|
}
|
|
self.report["tasks"].append(task_record)
|
|
self.save()
|
|
if task.get("status") != expected_status:
|
|
log = self.client.get(f"/api/tasks/{task_id}/logs")
|
|
failure_dir = self.out_dir / "task_failures" / task_id
|
|
failure_dir.mkdir(parents=True, exist_ok=True)
|
|
(failure_dir / "task.json").write_text(json.dumps(task, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
(failure_dir / "reports.json").write_text(json.dumps(reports, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
(failure_dir / "artifacts.json").write_text(json.dumps(artifacts, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
(failure_dir / "task.log").write_text(str(log), encoding="utf-8")
|
|
raise AcceptanceError(f"task {task_id} ended as {task.get('status')}: {reports.get('readable_error') or task.get('error')}")
|
|
if expected_status == "succeeded" and not artifacts:
|
|
raise AcceptanceError(f"succeeded task {task_id} has no indexed artifacts")
|
|
names = {str(item.get("name") or "") for item in artifacts}
|
|
missing = sorted(set(required_artifacts) - names)
|
|
if missing:
|
|
raise AcceptanceError(f"task {task_id} is missing required artifacts: {missing}")
|
|
result = task.get("result") or {}
|
|
if expected_status == "succeeded" and not str(result.get("run_storage_uri") or "").startswith("bos://testxyz/real2sim-poc/runs/"):
|
|
raise AcceptanceError(f"task {task_id} has no owned BOS run URI")
|
|
return task
|
|
|
|
|
|
def credential_candidates(root: Path, out_dir: Path) -> list[str]:
|
|
candidates: list[str] = []
|
|
explicit = os.environ.get("REAL2SIM_POC_ADMIN_PASSWORD", "").strip()
|
|
if explicit:
|
|
candidates.append(explicit)
|
|
for path in [out_dir / "acceptance-admin-password.txt", out_dir / "bootstrap-admin-password.txt"]:
|
|
if path.is_file():
|
|
candidates.append(path.read_text(encoding="utf-8").strip())
|
|
prior = sorted((root / "acceptance_runs" / "poc").glob("*/acceptance-admin-password.txt"), key=lambda path: path.stat().st_mtime, reverse=True)
|
|
prior += sorted((root / "acceptance_runs" / "poc").glob("*/bootstrap-admin-password.txt"), key=lambda path: path.stat().st_mtime, reverse=True)
|
|
for path in prior:
|
|
candidates.append(path.read_text(encoding="utf-8").strip())
|
|
return list(dict.fromkeys(value for value in candidates if value))
|
|
|
|
|
|
def authenticate_admin(run: AcceptanceRun) -> str:
|
|
login = None
|
|
current_password = ""
|
|
errors: list[str] = []
|
|
for candidate in credential_candidates(ROOT, run.out_dir):
|
|
try:
|
|
login = run.client.post("/api/auth/login", {"username": "admin", "password": candidate})
|
|
current_password = candidate
|
|
break
|
|
except ApiError as exc:
|
|
errors.append(str(exc.status))
|
|
if login is None:
|
|
raise AcceptanceError(f"admin login failed for all credential candidates: {errors}")
|
|
if login["user"].get("must_change_password"):
|
|
next_password = "Real2Sim!" + secrets.token_urlsafe(18) + "Aa1"
|
|
run.client.post(
|
|
"/api/auth/change-password",
|
|
{"current_password": current_password, "new_password": next_password},
|
|
)
|
|
run.client.post("/api/auth/login", {"username": "admin", "password": next_password})
|
|
current_password = next_password
|
|
password_path = run.out_dir / "acceptance-admin-password.txt"
|
|
password_path.write_text(current_password + "\n", encoding="utf-8")
|
|
password_path.chmod(0o600)
|
|
me = run.client.get("/api/auth/me")["user"]
|
|
run.check("管理员登录且已完成首次改密", me.get("role") == "admin" and not me.get("must_change_password"), me)
|
|
return current_password
|
|
|
|
|
|
def verify_rbac(run: AcceptanceRun) -> None:
|
|
suffix = secrets.token_hex(4)
|
|
records: dict[str, Any] = {}
|
|
for role in ("engineer", "viewer"):
|
|
username = f"acceptance.{role}.{suffix}"
|
|
password = f"{role.title()}!Password123"
|
|
user = run.client.post("/api/users", {"username": username, "password": password, "role": role})
|
|
records[role] = {"id": user["id"], "username": username}
|
|
role_client = ApiClient(run.client.base_url)
|
|
login = role_client.post("/api/auth/login", {"username": username, "password": password})
|
|
if login["user"].get("must_change_password"):
|
|
changed = f"{role.title()}!Changed456"
|
|
role_client.post("/api/auth/change-password", {"current_password": password, "new_password": changed})
|
|
role_client.post("/api/auth/login", {"username": username, "password": changed})
|
|
role_client.get("/api/datasets")
|
|
try:
|
|
payload = {"type": "motrixsim_asset_import" if role == "engineer" else "renderer_benchmark", "params": {}}
|
|
role_client.post("/api/tasks", payload)
|
|
except ApiError as exc:
|
|
if exc.status != 403:
|
|
raise
|
|
else:
|
|
raise AcceptanceError(f"{role} unexpectedly created a forbidden task")
|
|
run.report["identities"] = records
|
|
for record in records.values():
|
|
run.client.request("DELETE", f"/api/users/{record['id']}")
|
|
|
|
|
|
def build_playroom_archive(source: Path, build_id: str) -> tuple[Path, int, str]:
|
|
images = sorted(path for path in source.rglob("*") if path.is_file() and path.suffix.lower() in {".jpg", ".jpeg", ".png", ".webp"})
|
|
if len(images) < 225:
|
|
raise AcceptanceError(f"Playroom source has {len(images)} images; at least 225 are required")
|
|
target = Path(tempfile.gettempdir()) / f"real2sim-playroom-{build_id}.zip"
|
|
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_STORED, allowZip64=True) as archive:
|
|
for image in images:
|
|
archive.write(image, Path("playroom") / image.relative_to(source))
|
|
digest = hashlib.sha256()
|
|
with target.open("rb") as handle:
|
|
while chunk := handle.read(8 * 1024 * 1024):
|
|
digest.update(chunk)
|
|
return target, len(images), digest.hexdigest()
|
|
|
|
|
|
def upload_with_resume(run: AcceptanceRun, archive: Path, checksum: str) -> dict[str, Any]:
|
|
chunk_size = 8 * 1024 * 1024
|
|
created = run.client.post(
|
|
"/api/uploads",
|
|
{
|
|
"filename": archive.name,
|
|
"content_type": "application/zip",
|
|
"total_size": archive.stat().st_size,
|
|
"chunk_size": chunk_size,
|
|
"checksum": checksum,
|
|
},
|
|
)
|
|
upload_id = created["id"]
|
|
total_parts = int(created["total_parts"])
|
|
pause_after = max(1, total_parts // 2)
|
|
with archive.open("rb") as handle:
|
|
for part in range(1, total_parts + 1):
|
|
body = handle.read(chunk_size)
|
|
run.client.request(
|
|
"PUT",
|
|
f"/api/uploads/{upload_id}/parts/{part}",
|
|
body=body,
|
|
content_type="application/octet-stream",
|
|
timeout=600,
|
|
)
|
|
print(f" uploaded part {part}/{total_parts}", flush=True)
|
|
if part == pause_after:
|
|
resumed = run.client.get(f"/api/uploads/{upload_id}")
|
|
expected = list(range(1, pause_after + 1))
|
|
run.check("BOS 上传分片状态可恢复", resumed.get("received_parts") == expected, resumed.get("received_parts"))
|
|
completed = run.client.post(f"/api/uploads/{upload_id}/complete")
|
|
dataset = completed["dataset"]
|
|
run.check("上传数据写入受控 BOS 前缀", str(dataset.get("storage_uri") or "").startswith("bos://testxyz/real2sim-poc/uploads/"), dataset.get("storage_uri"))
|
|
run.report["storage"] = {
|
|
"upload_id": upload_id,
|
|
"storage_uri": dataset.get("storage_uri"),
|
|
"checksum": checksum,
|
|
"size_bytes": archive.stat().st_size,
|
|
"total_parts": total_parts,
|
|
"resume_checkpoint_part": pause_after,
|
|
}
|
|
run.report["datasets"].append({"id": dataset["id"], "role": "playroom_upload", "storage_uri": dataset.get("storage_uri")})
|
|
return dataset
|
|
|
|
|
|
def validate_infrastructure(run: AcceptanceRun) -> None:
|
|
run.report["version"] = run.client.get("/api/version")
|
|
run.report["doctor"] = run.client.get("/api/doctor")
|
|
run.report["capabilities"] = run.client.get("/api/capabilities")
|
|
run.report["recipes"] = run.client.get("/api/recipes")
|
|
run.check("部署 build_id 与验收目标一致", run.report["version"].get("build_id") == run.build_id, run.report["version"].get("build_id"))
|
|
run.check("生产 API doctor 通过", bool(run.report["doctor"].get("ok")), run.report["doctor"])
|
|
run.check("A800 worker 心跳就绪", bool((run.report["doctor"].get("worker") or {}).get("ready")), run.report["doctor"].get("worker"))
|
|
runtimes = run.report["capabilities"].get("runtimes") or {}
|
|
for name in ("anysplat", "mujoco", "motrixsim", "discoverse_act", "rm65_controller"):
|
|
run.check(f"真实运行时就绪: {name}", bool((runtimes.get(name) or {}).get("ready")), runtimes.get(name))
|
|
run.check("BOS 前缀隔离", run.report["capabilities"].get("storage", {}).get("prefix") == "real2sim-poc", run.report["capabilities"].get("storage"))
|
|
|
|
|
|
def exercise_task_operations(run: AcceptanceRun, scene_id: str | None) -> None:
|
|
benchmark = run.create_task("renderer_benchmark", {})
|
|
retry = run.client.post(f"/api/tasks/{benchmark['id']}/retry")
|
|
retried = run.wait_task(retry["id"])
|
|
bulk = run.client.post("/api/tasks/bulk", {"action": "retry", "ids": [benchmark["id"], retried["id"]], "params": {}})
|
|
run.check("任务批量重试逐项返回", len(bulk.get("results") or []) == 2 and all(item.get("ok") for item in bulk["results"]), bulk)
|
|
for item in bulk["results"]:
|
|
run.wait_task(item["result"]["id"], timeout_seconds=300)
|
|
|
|
if not scene_id:
|
|
run.report["warnings"].append("RM65 scene unavailable; cancellation and restart-recovery probes were skipped")
|
|
return
|
|
cancel_probe = run.client.post(
|
|
"/api/tasks",
|
|
{
|
|
"type": "expert_collection",
|
|
"params": {
|
|
"scene_id": scene_id,
|
|
"policy_preset": "rm65_pick_place_6d_ik",
|
|
"episodes": 50,
|
|
"max_attempts": 150,
|
|
"steps": 240,
|
|
"name": "acceptance-cancel-probe",
|
|
},
|
|
},
|
|
)
|
|
wait_until_status(run.client, cancel_probe["id"], "running", 300)
|
|
run.client.post(f"/api/tasks/{cancel_probe['id']}/cancel")
|
|
run.wait_task(cancel_probe["id"], expected_status="canceled", timeout_seconds=300)
|
|
|
|
recovery_probe = run.client.post(
|
|
"/api/tasks",
|
|
{
|
|
"type": "expert_collection",
|
|
"params": {
|
|
"scene_id": scene_id,
|
|
"policy_preset": "rm65_pick_place_6d_ik",
|
|
"episodes": 50,
|
|
"max_attempts": 150,
|
|
"steps": 240,
|
|
"name": "acceptance-recovery-probe",
|
|
},
|
|
},
|
|
)
|
|
wait_until_status(run.client, recovery_probe["id"], "running", 300)
|
|
kubectl(run.kubeconfig, "-n", NAMESPACE, "rollout", "restart", "deployment/real2sim-worker")
|
|
kubectl(run.kubeconfig, "-n", NAMESPACE, "rollout", "status", "deployment/real2sim-worker", "--timeout=3600s", timeout=3700)
|
|
run.wait_task(recovery_probe["id"], expected_status="failed_recovered", timeout_seconds=600)
|
|
run.check("worker 重启后历史任务仍可查询", run.client.get(f"/api/tasks/{benchmark['id']}").get("id") == benchmark["id"])
|
|
|
|
|
|
def wait_until_status(client: ApiClient, task_id: str, status: str, timeout_seconds: int) -> dict[str, Any]:
|
|
deadline = time.time() + timeout_seconds
|
|
while time.time() < deadline:
|
|
task = client.get(f"/api/tasks/{task_id}")
|
|
if task.get("status") == status:
|
|
return task
|
|
if task.get("status") in TERMINAL:
|
|
raise AcceptanceError(f"task {task_id} reached {task.get('status')} before {status}")
|
|
time.sleep(2)
|
|
raise AcceptanceError(f"task {task_id} did not reach {status} within {timeout_seconds}s")
|
|
|
|
|
|
def run_acceptance(args: argparse.Namespace) -> int:
|
|
build_id = args.build_id or kubectl(
|
|
args.kubeconfig,
|
|
"-n",
|
|
NAMESPACE,
|
|
"get",
|
|
"deployment/real2sim-api",
|
|
"-o",
|
|
"jsonpath={.metadata.labels.app\\.kubernetes\\.io/version}",
|
|
).stdout.strip()
|
|
if not build_id:
|
|
raise AcceptanceError("unable to determine deployed build id")
|
|
out_dir = args.out or ROOT / "acceptance_runs" / "poc" / build_id
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
port = args.port or free_port()
|
|
forward = start_port_forward(args.kubeconfig, port, out_dir)
|
|
archive: Path | None = None
|
|
run = AcceptanceRun(
|
|
client=ApiClient(f"https://127.0.0.1:{port}"),
|
|
out_dir=out_dir,
|
|
kubeconfig=args.kubeconfig,
|
|
build_id=build_id,
|
|
timeout_seconds=args.timeout_seconds,
|
|
)
|
|
try:
|
|
if run.stage("管理员认证", lambda: authenticate_admin(run)) is None:
|
|
return finalize(run)
|
|
run.stage("运行时、存储与版本预检", lambda: validate_infrastructure(run))
|
|
run.stage("用户 CRUD 与角色隔离", lambda: verify_rbac(run))
|
|
|
|
uploaded_dataset: dict[str, Any] | None = None
|
|
prepared_dataset_id: str | None = None
|
|
playroom = run.stage("打包真实 Playroom 225 帧", lambda: build_playroom_archive(args.playroom, build_id))
|
|
if playroom:
|
|
archive, image_count, checksum = playroom
|
|
run.report["storage"]["source_image_count"] = image_count
|
|
uploaded_dataset = run.stage("BOS 分片上传与断点续传", lambda: upload_with_resume(run, archive, checksum))
|
|
if uploaded_dataset:
|
|
prepared = run.stage(
|
|
"媒体准备 225 帧",
|
|
lambda: run.create_task(
|
|
"media_prepare",
|
|
{
|
|
"dataset_id": uploaded_dataset["id"],
|
|
"camera_label": "playroom.rgb",
|
|
"target_frame_count": 225,
|
|
"min_total_frames": 80,
|
|
"max_frames_per_video": 225,
|
|
"max_videos": 1,
|
|
"sample_fps": 2,
|
|
"output_name": f"playroom-prepared-{build_id}",
|
|
},
|
|
required_artifacts=("prepared_manifest.json",),
|
|
),
|
|
)
|
|
if prepared:
|
|
prepared_dataset_id = str(prepared["result"]["dataset_id"])
|
|
run.report["datasets"].append({"id": prepared_dataset_id, "role": "playroom_prepared", "frame_count": prepared["result"].get("frame_count")})
|
|
run.stage(
|
|
"Playroom 重建质量预检",
|
|
lambda: run.create_task(
|
|
"reconstruction_preflight",
|
|
{"dataset_id": prepared_dataset_id, "min_frames": 80, "target_frame_count": 225, "allow_low_quality": False, "quality_preset": "capture-high"},
|
|
required_artifacts=("capture_preflight_report.json",),
|
|
),
|
|
)
|
|
|
|
reconstruction_scenes: list[str] = []
|
|
if prepared_dataset_id:
|
|
for index in range(1, 3):
|
|
reconstructed = run.stage(
|
|
f"Playroom AnySplat 真实重建 #{index}",
|
|
lambda index=index: run.create_task(
|
|
"scene_reconstruction",
|
|
{
|
|
"dataset_id": prepared_dataset_id,
|
|
"scene_name": f"Playroom A800 Reconstruction {index}",
|
|
"renderer": "spark",
|
|
"simulators": ["mujoco", "motrixsim"],
|
|
"dry_run": False,
|
|
"preflight_required": True,
|
|
"min_frames": 80,
|
|
"allow_low_quality": False,
|
|
"quality_preset": "capture-high",
|
|
"pose_init_train": True,
|
|
"sfm_matcher": "exhaustive",
|
|
"sfm_guided_matching": True,
|
|
"sfm_frames": "selected",
|
|
"anysplat_keyframe_strategy": "sfm-registered",
|
|
"auto_cover": True,
|
|
},
|
|
required_artifacts=(
|
|
"environment.json",
|
|
"reconstruction_command.json",
|
|
"capture_preflight_report.json",
|
|
"reconstruction_report.json",
|
|
"scene_manifest.json",
|
|
),
|
|
),
|
|
)
|
|
if reconstructed:
|
|
scene_id = str(reconstructed["result"]["scene_id"])
|
|
reconstruction_scenes.append(scene_id)
|
|
run.report["scenes"].append({"id": scene_id, "role": f"playroom_reconstruction_{index}"})
|
|
run.stage(
|
|
f"Playroom 重建仿真双运行时校验 #{index}",
|
|
lambda scene_id=scene_id: run.create_task(
|
|
"simulator_validation",
|
|
{"scene_id": scene_id, "simulators": ["mujoco", "motrixsim"], "level": "runtime_required"},
|
|
required_artifacts=("validation_report.json",),
|
|
),
|
|
)
|
|
if len(reconstruction_scenes) == 2:
|
|
run.stage(
|
|
"两次重建产生独立场景包",
|
|
lambda: run.check("重建 scene_id 独立", len(set(reconstruction_scenes)) == 2, reconstruction_scenes),
|
|
)
|
|
|
|
motrix_task = run.stage(
|
|
"MotrixSim 官方客厅与 RM65 资产导入",
|
|
lambda: run.create_task(
|
|
"motrixsim_asset_import",
|
|
{"source": "/opt/motrixsim-docs", "validation_steps": 120, "max_parlour_meshes": 240},
|
|
required_artifacts=("motrixsim_asset_import_report.json", "rm65_table_pick_place_validation.json", "motrixsim_parlour_rm65_preview.glb"),
|
|
),
|
|
)
|
|
rm65_scene_id = str(motrix_task["result"]["scene_id"]) if motrix_task else None
|
|
if rm65_scene_id:
|
|
run.report["scenes"].append({"id": rm65_scene_id, "role": "rm65_pick_place"})
|
|
run.stage(
|
|
"RM65 场景 runtime_required 校验",
|
|
lambda: run.create_task(
|
|
"simulator_validation",
|
|
{"scene_id": rm65_scene_id, "simulators": ["mujoco", "motrixsim"], "level": "runtime_required"},
|
|
required_artifacts=("validation_report.json",),
|
|
),
|
|
)
|
|
|
|
collection = None
|
|
if rm65_scene_id:
|
|
collection = run.stage(
|
|
"RM65 真实物理专家采集 50 条成功 episode",
|
|
lambda: run.create_task(
|
|
"expert_collection",
|
|
{
|
|
"scene_id": rm65_scene_id,
|
|
"policy_preset": "rm65_pick_place_6d_ik",
|
|
"episodes": 50,
|
|
"max_attempts": 150,
|
|
"steps": 240,
|
|
"workers": 1,
|
|
"seed": 7,
|
|
"image_width": 320,
|
|
"image_height": 240,
|
|
"name": f"RM65 Physical ACT Dataset {build_id}",
|
|
},
|
|
required_artifacts=("manifest.json", "episode_0.hdf5", "episode_0.mp4"),
|
|
),
|
|
)
|
|
collection_dataset_id = str(collection["result"]["dataset_id"]) if collection else None
|
|
if collection_dataset_id:
|
|
run.report["datasets"].append({"id": collection_dataset_id, "role": "rm65_physical_collection", "samples": collection["result"].get("samples")})
|
|
run.stage("采集成功数门槛", lambda: run.check("成功 episode 至少 50", int(collection["result"].get("samples") or 0) >= 50, collection["result"]))
|
|
|
|
trained = None
|
|
if collection_dataset_id:
|
|
trained = run.stage(
|
|
"DISCOVERSE ACT 真实训练",
|
|
lambda: run.create_task(
|
|
"training",
|
|
{
|
|
"dataset_id": collection_dataset_id,
|
|
"recipe": "discoverse_act",
|
|
"epochs": args.act_epochs,
|
|
"batch_size": 8,
|
|
"chunk_size": 32,
|
|
"seed": 1,
|
|
"name": f"RM65 ACT {build_id}",
|
|
},
|
|
required_artifacts=("policy_best.ckpt", "dataset_stats.pkl", "model_card.json", "task_config.py"),
|
|
),
|
|
)
|
|
model_id = str(trained["result"]["model_id"]) if trained else None
|
|
if model_id:
|
|
run.report["models"].append({"id": model_id, "role": "discoverse_act", "metrics": trained["result"].get("metrics")})
|
|
|
|
if model_id and rm65_scene_id:
|
|
inference = run.stage(
|
|
"DISCOVERSE ACT 在线推理 10 回合",
|
|
lambda: run.create_task(
|
|
"inference",
|
|
{
|
|
"model_id": model_id,
|
|
"scene_id": rm65_scene_id,
|
|
"recipe": "discoverse_act",
|
|
"episodes": 10,
|
|
"steps": 240,
|
|
"workers": 1,
|
|
"seed": 11,
|
|
"minimum_success_rate": 0.5,
|
|
},
|
|
required_artifacts=("inference_report.json", "episode_000.mp4", "episode_000_trajectory.json"),
|
|
),
|
|
)
|
|
if inference:
|
|
run.stage("ACT 推理成功率门槛", lambda: run.check("推理成功率至少 50%", float(inference["result"].get("success_rate") or 0) >= 0.5, inference["result"]))
|
|
|
|
run.stage("任务取消、重试、批量操作与重启恢复", lambda: exercise_task_operations(run, rm65_scene_id))
|
|
return finalize(run)
|
|
finally:
|
|
if archive and archive.exists():
|
|
archive.unlink()
|
|
forward.terminate()
|
|
try:
|
|
forward.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
forward.kill()
|
|
|
|
|
|
def finalize(run: AcceptanceRun) -> int:
|
|
run.report["finished_at"] = utc_now()
|
|
run.report["pass"] = not run.report["failures"] and all(item.get("ok") for item in run.report["checks"])
|
|
run.save()
|
|
print(f"\nacceptance_report={run.out_dir / 'acceptance_report.json'}", flush=True)
|
|
print(f"pass={run.report['pass']}", flush=True)
|
|
return 0 if run.report["pass"] else 1
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Run the real Real2Sim production acceptance against the isolated POC namespace.")
|
|
parser.add_argument("--kubeconfig", type=Path, default=Path(os.environ.get("REAL2SIM_POC_KUBECONFIG", DEFAULT_KUBECONFIG)))
|
|
parser.add_argument("--playroom", type=Path, default=Path(os.environ.get("REAL2SIM_POC_PLAYROOM", DEFAULT_PLAYROOM)))
|
|
parser.add_argument("--build-id", default=os.environ.get("REAL2SIM_POC_BUILD_ID", ""))
|
|
parser.add_argument("--out", type=Path, default=Path(os.environ["REAL2SIM_POC_ACCEPTANCE_OUT"]) if os.environ.get("REAL2SIM_POC_ACCEPTANCE_OUT") else None)
|
|
parser.add_argument("--port", type=int, default=0)
|
|
parser.add_argument("--timeout-seconds", type=int, default=int(os.environ.get("REAL2SIM_POC_ACCEPTANCE_TIMEOUT_SECONDS", "28800")))
|
|
parser.add_argument("--act-epochs", type=int, default=int(os.environ.get("REAL2SIM_POC_ACT_EPOCHS", "100")))
|
|
return parser.parse_args()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
raise SystemExit(run_acceptance(parse_args()))
|
|
except KeyboardInterrupt:
|
|
raise SystemExit(130)
|
|
except Exception as exc:
|
|
print(f"acceptance bootstrap failed: {type(exc).__name__}: {exc}", file=sys.stderr)
|
|
raise SystemExit(2)
|