droboticflow/scripts/run_fanyu_full_human_in_the_loop.py

1461 lines
74 KiB
Python
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Repeatable Fanyu v5.8 acceptance harness.
The harness uses ProjectMember roles, an isolated project and an isolated BOS
prefix. Credentials are read only from environment variables. It never embeds
passwords, mutates the production project, or deletes source Capture objects.
"""
from __future__ import annotations
import argparse
import http.client
import json
import os
import secrets
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
from droboticflow_acceptance import DroboticflowClient # noqa: E402
class AcceptanceError(RuntimeError):
pass
def env(name: str, default: str = "", required: bool = False) -> str:
value = os.getenv(name, default).strip()
if required and not value:
raise AcceptanceError(f"{name} is required")
return value
def item_list(payload: Any) -> list[dict[str, Any]]:
if isinstance(payload, list):
return payload
if isinstance(payload, dict) and isinstance(payload.get("items"), list):
return payload["items"]
return []
def upsert_hyperparam(node: dict[str, Any], key: str, value: Any, value_type: str | None = None) -> None:
normalized = str(value)
data = node.setdefault("data", {})
params = data.setdefault("hyperparams", [])
for param in params:
if str(param.get("key", "")).strip() == key:
param["value"] = normalized
if value_type:
param["type"] = value_type
return
params.append({"key": key, "value": normalized, "type": value_type or "number"})
def is_annotation_gate_node(node: dict[str, Any]) -> bool:
data = node.get("data") if isinstance(node.get("data"), dict) else {}
return data.get("operatorId") == "operator-builtin-annotation-gate" or "真实切分" in str(data.get("label", ""))
def clone_workflow_nodes_for_supplier(source_nodes: Any, supplier_id: str) -> list[dict[str, Any]]:
del supplier_id
nodes = json.loads(json.dumps(source_nodes if isinstance(source_nodes, list) else []))
matched = 0
for node in nodes:
if not isinstance(node, dict) or not is_annotation_gate_node(node):
continue
matched += 1
upsert_hyperparam(node, "MAX_REWORK_ROUNDS", 1, "number")
upsert_hyperparam(node, "ANNOTATION_SUPPLIER_ID", "", "string")
if matched != 1:
raise AcceptanceError(f"expected exactly one annotation gate node, found {matched}")
return nodes
def annotation_supplier_ids(nodes: Any) -> set[str]:
values: set[str] = set()
for node in nodes if isinstance(nodes, list) else []:
if not isinstance(node, dict) or not is_annotation_gate_node(node):
continue
data = node.get("data") if isinstance(node.get("data"), dict) else {}
for param in data.get("hyperparams", []) if isinstance(data.get("hyperparams"), list) else []:
if str(param.get("key", "")).strip() == "ANNOTATION_SUPPLIER_ID":
value = str(param.get("value", "")).strip()
if value:
values.add(value)
return values
def human_item_supplier_id(items: list[dict[str, Any]], fallback: str = "") -> str:
suppliers = {str(item.get("supplierId", "")).strip() for item in items if str(item.get("supplierId", "")).strip()}
if len(suppliers) > 1:
raise AcceptanceError(f"human items span multiple suppliers: {sorted(suppliers)}")
if suppliers:
return next(iter(suppliers))
fallback = str(fallback).strip()
if fallback:
return fallback
raise AcceptanceError("human items do not carry a supplierId")
def retry_runs_settled(runs: list[dict[str, Any]], minimum_attempts: int = 3) -> bool:
if len([item for item in runs if item.get("status") == "failed"]) < minimum_attempts:
return False
active = {"queued", "pending", "running", "stopping", "waiting", "waiting_for_audit", "waiting_for_human"}
return all(str(item.get("status", "")) not in active for item in runs)
def annotation_draft_matches(
draft: Any,
slices: list[dict[str, Any]],
segments: list[dict[str, Any]],
) -> bool:
if not isinstance(draft, dict):
return False
expected_slice_ids = [str(item.get("id", "")) for item in slices]
expected_event_ids = [str(item.get("id", "")) for item in segments]
expected_statuses = {str(item.get("id", "")): str(item.get("annotationStatus", "")) for item in slices}
actual_slices = draft.get("sliceSegments") if isinstance(draft.get("sliceSegments"), list) else []
actual_segments = draft.get("segments") if isinstance(draft.get("segments"), list) else []
actual_statuses = draft.get("statuses") if isinstance(draft.get("statuses"), dict) else {}
return (
[str(item.get("id", "")) for item in actual_slices if isinstance(item, dict)] == expected_slice_ids
and [str(item.get("id", "")) for item in actual_segments if isinstance(item, dict)] == expected_event_ids
and {str(key): str(value) for key, value in actual_statuses.items()} == expected_statuses
)
def completed_materialization(value: Any) -> bool:
return (
isinstance(value, dict)
and value.get("status") == "completed"
and bool(str(value.get("artifactId", "")).strip())
and bool(str(value.get("runId", "")).strip())
)
def recoverable_submission_transport_error(error: BaseException) -> bool:
current: BaseException | None = error
while current is not None:
if isinstance(current, (TimeoutError, urllib.error.URLError, http.client.RemoteDisconnected, ConnectionResetError)):
return True
current = current.__cause__
text = str(error).lower()
return any(
marker in text
for marker in ("timed out", "timeout", "remote disconnected", "remote end closed", "connection reset", "urlopen error")
)
class Harness:
def __init__(self, args: argparse.Namespace) -> None:
self.args = args
self.api_url = args.api_url.rstrip("/")
self.state_path = Path(args.state_path)
self.report_path = Path(args.report_path)
self.state: dict[str, Any] = self.load_state()
self.report: dict[str, Any] = {
"schemaVersion": "droboticflow.fanyu.acceptance/v1",
"acceptanceId": args.acceptance_id,
"apiURL": self.api_url,
"startedAt": datetime.now(timezone.utc).isoformat(),
"checks": [],
"resources": {},
"status": "running",
}
self.admin = DroboticflowClient(self.api_url, error_factory=AcceptanceError)
self.annotator = DroboticflowClient(self.api_url, error_factory=AcceptanceError)
self.quality = DroboticflowClient(self.api_url, error_factory=AcceptanceError)
self.acceptance = DroboticflowClient(self.api_url, error_factory=AcceptanceError)
self.supplier_manager = DroboticflowClient(self.api_url, error_factory=AcceptanceError)
self.session_user_ids: dict[str, str] = {}
self.probe_password = self.state.get("probePassword") or secrets.token_urlsafe(24)
self.manager_password = self.state.get("managerPassword") or secrets.token_urlsafe(24)
def load_state(self) -> dict[str, Any]:
if self.state_path.exists():
return json.loads(self.state_path.read_text(encoding="utf-8"))
return {"acceptanceId": self.args.acceptance_id}
def save_state(self) -> None:
self.state_path.parent.mkdir(parents=True, exist_ok=True)
self.state_path.write_text(json.dumps(self.state, ensure_ascii=False, indent=2), encoding="utf-8")
self.state_path.chmod(0o600)
def save_report(self) -> None:
self.report_path.parent.mkdir(parents=True, exist_ok=True)
self.report_path.write_text(json.dumps(self.report, ensure_ascii=False, indent=2), encoding="utf-8")
def check(self, name: str, passed: bool, **evidence: Any) -> None:
self.report["checks"].append({"name": name, "passed": passed, "evidence": evidence})
if not passed:
raise AcceptanceError(f"check failed: {name}: {evidence}")
def remember_resource(self, name: str, value: Any) -> None:
self.state.setdefault("evidence", {})[name] = value
self.report["resources"][name] = value
self.save_state()
def remember_human_item(
self,
item: dict[str, Any],
*,
task: dict[str, Any] | None = None,
slice_ids: list[str] | None = None,
event_ids: list[str] | None = None,
) -> None:
evidence = self.state.setdefault("evidence", {}).setdefault("humanItemsById", {})
item_id = str(item.get("id", "")).strip()
if not item_id:
return
locator = dict(evidence.get(item_id) or {})
locator.update(
{
"humanItemId": item_id,
"taskId": str(item.get("legacyTaskId") or locator.get("taskId") or "").strip(),
"runId": str(item.get("runId") or locator.get("runId") or "").strip(),
"supplierId": str(item.get("supplierId") or locator.get("supplierId") or "").strip(),
"stageId": str(item.get("currentStageId") or locator.get("stageId") or "").strip(),
}
)
if task:
locator["taskId"] = str(task.get("id") or locator.get("taskId") or "").strip()
locator["runId"] = str(task.get("runId") or locator.get("runId") or "").strip()
locator["dataTaskId"] = str(task.get("dataTaskId", "")).strip()
capture_by_run = {
str(entry.get("runId", "")): str(entry.get("captureId", ""))
for entry in self.state.get("evidence", {}).get("captures", [])
if isinstance(entry, dict) and entry.get("runId")
}
if locator.get("runId") in capture_by_run:
locator["captureId"] = capture_by_run[locator["runId"]]
if slice_ids is not None:
locator["sliceIds"] = slice_ids
if event_ids is not None:
locator["eventIds"] = event_ids
evidence[item_id] = locator
self.report["resources"]["humanItems"] = list(evidence.values())
self.save_state()
def capture_directory_for_item(self, item: dict[str, Any]) -> str:
display = item.get("display") if isinstance(item.get("display"), dict) else {}
directories = display.get("captureDirectories") if isinstance(display.get("captureDirectories"), list) else []
for directory in directories:
if str(directory).strip():
return str(directory).strip()
run_id = str(item.get("runId", "")).strip()
for entry in self.state.get("evidence", {}).get("captures", []):
if isinstance(entry, dict) and str(entry.get("runId", "")).strip() == run_id:
capture_id = str(entry.get("captureId", "")).strip()
if capture_id:
return capture_id
raise AcceptanceError(f"cannot resolve Capture directory for human item {item.get('id')}")
def record_browser_fixture(
self,
annotation_item: dict[str, Any],
quality_item: dict[str, Any],
acceptance_item: dict[str, Any],
*,
member_user_id: str,
member_email: str,
supplier_id: str,
) -> None:
expected_stages = (
("annotation", annotation_item, "annotation_combined", {"ready", "rework"}),
("quality", quality_item, "quality_review", {"ready", "submitted", "rework"}),
("acceptance", acceptance_item, "prepackage_acceptance", {"ready", "rework"}),
)
for name, item, stage_id, statuses in expected_stages:
if (
str(item.get("currentStageId", "")) != stage_id
or str(item.get("status", "")) not in statuses
or str(item.get("assignedUserId", "")).strip()
):
raise AcceptanceError(f"browser {name} fixture is already consumed or in the wrong stage: {item}")
quality_payload = quality_item.get("subject", {}).get("payload", {}) if isinstance(quality_item.get("subject"), dict) else {}
if not completed_materialization(quality_payload.get("materialization")):
raise AcceptanceError(f"browser quality fixture has no completed materialization: {quality_item}")
acceptance_payload = acceptance_item.get("subject", {}).get("payload", {}) if isinstance(acceptance_item.get("subject"), dict) else {}
acceptance_evidence = acceptance_payload.get("items") if isinstance(acceptance_payload.get("items"), list) else []
if not acceptance_evidence or not any(
completed_materialization(entry.get("materialization"))
for entry in acceptance_evidence
if isinstance(entry, dict)
):
raise AcceptanceError(f"browser acceptance fixture has no materialized evidence item: {acceptance_item}")
fixture = {
"annotation": {
"captureDirectory": self.capture_directory_for_item(annotation_item),
"taskId": str(annotation_item.get("legacyTaskId", "")),
"humanItemId": str(annotation_item.get("id", "")),
"runId": str(annotation_item.get("runId", "")),
"pool": "available",
},
"quality": {
"captureDirectory": self.capture_directory_for_item(quality_item),
"taskId": str(quality_item.get("legacyTaskId", "")),
"humanItemId": str(quality_item.get("id", "")),
"runId": str(quality_item.get("runId", "")),
"pool": "available",
},
"acceptance": {
"captureDirectory": self.capture_directory_for_item(acceptance_item),
"humanItemId": str(acceptance_item.get("id", "")),
"runId": str(acceptance_item.get("runId", "")),
"pool": "available",
},
"member": {
"userId": member_user_id,
"email": member_email,
"supplierId": supplier_id,
"initialRoles": ["annotator"],
"bulkRoles": ["annotator", "quality_reviewer"],
"disposable": True,
},
}
required = (
fixture["annotation"]["taskId"],
fixture["annotation"]["humanItemId"],
fixture["quality"]["taskId"],
fixture["quality"]["humanItemId"],
fixture["acceptance"]["humanItemId"],
fixture["member"]["userId"],
)
if not all(required):
raise AcceptanceError(f"browser fixture is incomplete: {fixture}")
self.state["browserFixture"] = fixture
self.report["resources"]["browserFixture"] = fixture
self.save_state()
def raw_status(self, client: DroboticflowClient, method: str, path: str, payload: Any | None = None) -> tuple[int, Any]:
body = None if payload is None else json.dumps(payload, ensure_ascii=False).encode("utf-8")
headers = client.request_headers(method, {"Content-Type": "application/json"})
request = urllib.request.Request(
f"{self.api_url}{path}", data=body, headers=headers, method=method
)
try:
with client.opener.open(request, timeout=60) as response:
raw = response.read().decode("utf-8", errors="replace")
return response.status, json.loads(raw) if raw else None
except urllib.error.HTTPError as error:
raw = error.read().decode("utf-8", errors="replace")
try:
return error.code, json.loads(raw)
except json.JSONDecodeError:
return error.code, raw
def login(self) -> None:
accounts = [
("admin", self.admin, "DROBOTICFLOW_ACCEPTANCE_EMAIL", "DROBOTICFLOW_ACCEPTANCE_PASSWORD", "organization admin"),
("annotator", self.annotator, "FANYU_ANNOTATOR_EMAIL", "FANYU_ANNOTATOR_PASSWORD", "annotator"),
("quality", self.quality, "FANYU_QUALITY_EMAIL", "FANYU_QUALITY_PASSWORD", "quality reviewer"),
("acceptance", self.acceptance, "FANYU_ACCEPTANCE_EMAIL", "FANYU_ACCEPTANCE_PASSWORD", "acceptance reviewer"),
]
for key, client, email_name, password_name, label in accounts:
client.login_if_configured(env(email_name, required=True), env(password_name, required=True), label=label)
session = client.request_json("GET", "/api/auth/me")
user_id = str(session.get("user", {}).get("id", "")).strip()
if not user_id:
raise AcceptanceError(f"{label} session has no user id")
self.session_user_ids[key] = user_id
self.check("admin_login", bool(self.session_user_ids.get("admin")), userId=self.session_user_ids.get("admin"))
self.remember_resource("sessionUserIds", self.session_user_ids)
def source_context(self) -> tuple[dict[str, Any], dict[str, Any]]:
source_project_id = env("FANYU_ACCEPTANCE_SOURCE_PROJECT_ID", self.args.source_project_id, required=True)
source_workflow_id = env("FANYU_ACCEPTANCE_SOURCE_WORKFLOW_ID", self.args.source_workflow_id, required=True)
source_project = self.admin.request_json("GET", f"/api/projects/{source_project_id}")
source_workflow = self.admin.request_json("GET", f"/api/workflows/{source_workflow_id}")
self.state.update({"sourceProjectId": source_project_id, "sourceWorkflowId": source_workflow_id})
return source_project, source_workflow
def ensure_project(self) -> None:
source_project, _ = self.source_context()
project_name = f"泛宇 100% 隔离验收 {self.args.acceptance_id}"
projects = item_list(self.admin.request_json("GET", "/api/projects"))
project = next((item for item in projects if item.get("name") == project_name), None)
if project is None:
project = self.admin.request_json(
"POST",
"/api/projects",
{
"name": project_name,
"description": "Capture、三角色人工流程、异常状态和成员管理全分支验收;30 天后清理。",
"storageConnectionId": source_project.get("storageConnectionId"),
"storageBucket": source_project.get("storageBucket"),
"storagePrefix": f"acceptance-100/{self.args.acceptance_id}/artifacts",
},
)
self.state["projectId"] = project["id"]
self.remember_resource("projectId", project["id"])
labels = item_list(self.admin.request_json("GET", "/api/annotation-labels"))
active_label_ids = [str(item.get("id", "")) for item in labels if item.get("status") == "active" and item.get("id")]
if not active_label_ids:
raise AcceptanceError("organization annotation label dictionary is empty")
selection = self.admin.request_json(
"PUT",
f"/api/projects/{project['id']}/annotation-labels",
{"labelIds": active_label_ids},
)
self.check(
"project_annotation_labels",
set(selection.get("labelIds") or []) == set(active_label_ids),
labelCount=len(active_label_ids),
)
def ensure_workflow(self) -> None:
source_project, source_workflow = self.source_context()
project_id = str(self.state.get("projectId", "")).strip()
supplier_id = str(self.state.get("supplierId", "")).strip()
if not project_id or not supplier_id:
raise AcceptanceError("project and supplier members must exist before the workflow is cloned")
workflow_name = f"Fanyu v5.8 100% acceptance {self.args.acceptance_id}"
workflows = item_list(self.admin.request_json("GET", f"/api/projects/{project_id}/workflows"))
workflow = next((item for item in workflows if item.get("name") == workflow_name), None)
if workflow is None:
nodes = clone_workflow_nodes_for_supplier(source_workflow.get("nodes", []), supplier_id)
workflow = self.admin.request_json(
"POST",
f"/api/projects/{project_id}/workflows",
{
"name": workflow_name,
"description": "Isolated Fanyu 100% acceptance workflow",
"type": source_workflow.get("type", "batch"),
"kind": source_workflow.get("kind", "standard"),
"executionScope": "production",
"maxConcurrentRuns": 3,
"nodeTimeoutSeconds": source_workflow.get("nodeTimeoutSeconds", 0),
"runTimeoutSeconds": source_workflow.get("runTimeoutSeconds", 0),
"maxRetryAttempts": 2,
"retryBackoffSeconds": 60,
"workflowInputs": source_workflow.get("workflowInputs", []),
"artifactStorage": source_workflow.get("artifactStorage"),
"nodes": nodes,
"edges": source_workflow.get("edges", []),
},
)
self.admin.request_json("POST", f"/api/workflows/{workflow['id']}/release", {})
workflow = self.admin.request_json("GET", f"/api/workflows/{workflow['id']}")
bound_suppliers = annotation_supplier_ids(workflow.get("nodes", []))
if bound_suppliers:
raise AcceptanceError(
f"v5.8 acceptance workflow must use claim-time supplier binding, found {sorted(bound_suppliers)}; "
"use a new acceptance-id or recreate the isolated workflow"
)
self.state["workflowId"] = workflow["id"]
source_automation = source_workflow.get("captureAutomation") or {}
automation = self.admin.request_json(
"PUT",
f"/api/workflows/{workflow['id']}/capture-automation",
{
"enabled": True,
"connectionId": source_automation.get("connectionId") or source_project.get("storageConnectionId"),
"bucket": source_automation.get("bucket") or source_project.get("storageBucket"),
"prefix": self.args.target_root,
"schema": "orbbec_ego_stereo_v1",
"completionMode": "manifest_and_files",
"manifestName": "source-manifest.json",
"workflowInputId": source_automation.get("workflowInputId", "sourceDirectory"),
"pollIntervalSeconds": 10,
"stabilityWindowSeconds": 30,
"requiredPatterns": source_automation.get("requiredPatterns", []),
"maxRetryAttempts": 2,
"retryBackoffSeconds": 60,
},
)
self.check("capture_automation_config", automation.get("config", {}).get("enabled") is True, config=automation.get("config"))
policy = self.admin.request_json(
"PATCH",
f"/api/workflows/{workflow['id']}/runtime-policy",
{"maxConcurrentCaptureRuns": 2, "acceptanceSampleProbability": 1.0},
)
self.check(
"capture_runtime_policy",
policy.get("maxConcurrentCaptureRuns") == 2 and policy.get("acceptanceSampleProbability") == 1.0,
policy=policy,
)
self.remember_resource(
"workflow",
{"workflowId": workflow["id"], "supplierId": supplier_id, "targetRoot": self.args.target_root},
)
def role_account_specs(self) -> list[tuple[str, list[str]]]:
return [
(env("FANYU_ANNOTATOR_EMAIL", required=True).lower(), ["annotator"]),
(env("FANYU_QUALITY_EMAIL", required=True).lower(), ["quality_reviewer"]),
(env("FANYU_ACCEPTANCE_EMAIL", required=True).lower(), ["acceptance_reviewer"]),
]
def users_by_email(self) -> dict[str, dict[str, Any]]:
users = item_list(self.admin.request_json("GET", "/api/users?includeDisabled=true"))
return {str(item.get("email", "")).lower(): item for item in users}
def ensure_project_member(self, user_id: str, supplier_id: str, roles: list[str]) -> dict[str, Any]:
project_id = str(self.state.get("projectId", "")).strip()
if not project_id:
raise AcceptanceError("projectId is required before project members")
expected_roles = set(roles)
status, payload = self.raw_status(
self.admin,
"POST",
f"/api/projects/{project_id}/members",
{"userId": user_id, "supplierId": supplier_id, "roles": roles},
)
if status == 409 and isinstance(payload, dict):
member = payload.get("member")
elif status == 201:
member = payload
else:
raise AcceptanceError(f"create project member {user_id}: {status} {payload}")
if not isinstance(member, dict) or not member.get("id"):
raise AcceptanceError(f"project member response is incomplete for {user_id}: {payload}")
if str(member.get("supplierId", "")).strip() != supplier_id or set(member.get("roles") or []) != expected_roles:
member = self.admin.request_json(
"PATCH",
f"/api/projects/{project_id}/members/{member['id']}",
{"supplierId": supplier_id, "roles": roles},
)
if str(member.get("supplierId", "")).strip() != supplier_id or set(member.get("roles") or []) != expected_roles:
raise AcceptanceError(
f"project member {member.get('id')} does not match supplier {supplier_id} and roles {sorted(expected_roles)}"
)
return member
def ensure_main_role_members(self, supplier_id: str) -> tuple[dict[str, str], dict[str, dict[str, Any]]]:
by_email = self.users_by_email()
member_ids: dict[str, str] = {}
for email, roles in self.role_account_specs():
user = by_email.get(email)
if user is None:
raise AcceptanceError(f"configured test user not found: {email}")
member = self.ensure_project_member(str(user["id"]), supplier_id, roles)
member_ids[email] = str(member["id"])
self.state["memberIds"] = member_ids
return member_ids, by_email
def ensure_probe_member(self, supplier_id: str) -> tuple[dict[str, Any], dict[str, Any]]:
project_id = str(self.state.get("projectId", "")).strip()
probe_email = f"fanyu-acceptance-probe+{self.args.acceptance_id.lower()}@droboticflow.local"
by_email = self.users_by_email()
probe = by_email.get(probe_email)
if probe is None:
probe = self.admin.request_json(
"POST",
"/api/users",
{
"name": f"Fanyu acceptance probe {self.args.acceptance_id}",
"email": probe_email,
"role": "member",
"supplierId": supplier_id,
"password": self.probe_password,
},
)
else:
probe = self.admin.request_json(
"PUT",
f"/api/users/{probe['id']}",
{"role": "member", "supplierId": supplier_id, "status": "active", "password": self.probe_password},
)
probe_member = self.ensure_project_member(str(probe["id"]), supplier_id, ["annotator"])
self.state.update(
{
"probeUserId": probe["id"],
"probePassword": self.probe_password,
"probeMemberId": probe_member["id"],
}
)
self.remember_resource(
"probeMember",
{"projectId": project_id, "userId": probe["id"], "memberId": probe_member["id"], "supplierId": supplier_id, "roles": ["annotator"]},
)
return probe, probe_member
def ensure_supplier_and_members(self) -> None:
project_id = str(self.state.get("projectId", "")).strip()
if not project_id:
raise AcceptanceError("project must be created before supplier members")
supplier_name = f"Fanyu acceptance supplier {self.args.acceptance_id}"
suppliers = item_list(self.admin.request_json("GET", "/api/suppliers"))
supplier = next((item for item in suppliers if item.get("name") == supplier_name), None)
if supplier is None:
supplier = self.admin.request_json("POST", "/api/suppliers", {"name": supplier_name})
self.state["supplierId"] = supplier["id"]
member_ids, by_email = self.ensure_main_role_members(supplier["id"])
role_accounts = self.role_account_specs()
duplicate_status, _ = self.raw_status(
self.admin,
"POST",
f"/api/projects/{project_id}/members",
{"userId": by_email[role_accounts[0][0]]["id"], "supplierId": supplier["id"], "roles": ["annotator"]},
)
self.check("duplicate_member_409", duplicate_status == 409, status=duplicate_status)
annotator_member_id = member_ids[role_accounts[0][0]]
edited = self.admin.request_json(
"PATCH",
f"/api/projects/{project_id}/members/{annotator_member_id}",
{"supplierId": supplier["id"], "roles": ["annotator", "quality_reviewer"]},
)
self.check("member_multi_role_edit", set(edited.get("roles", [])) == {"annotator", "quality_reviewer"}, roles=edited.get("roles"))
self.admin.request_json(
"PATCH",
f"/api/projects/{project_id}/members/{annotator_member_id}",
{"supplierId": supplier["id"], "roles": ["annotator"]},
)
forbidden_status, _ = self.raw_status(
self.annotator,
"PATCH",
f"/api/projects/{project_id}/members/{annotator_member_id}",
{"supplierId": supplier["id"], "roles": ["acceptance_reviewer"]},
)
self.check("member_management_403", forbidden_status == 403, status=forbidden_status)
manager_email = f"fanyu-acceptance-manager+{self.args.acceptance_id.lower()}@droboticflow.local"
probe, probe_member = self.ensure_probe_member(supplier["id"])
by_email = self.users_by_email()
manager = by_email.get(manager_email)
if manager is None:
manager = self.admin.request_json(
"POST",
"/api/users",
{
"name": f"Fanyu acceptance supplier manager {self.args.acceptance_id}",
"email": manager_email,
"role": "supplier_manager",
"supplierId": supplier["id"],
"password": self.manager_password,
},
)
else:
manager = self.admin.request_json(
"PUT",
f"/api/users/{manager['id']}",
{"role": "supplier_manager", "supplierId": supplier["id"], "status": "active", "password": self.manager_password},
)
self.state.update(
{
"probeUserId": probe["id"],
"probePassword": self.probe_password,
"managerUserId": manager["id"],
"managerPassword": self.manager_password,
}
)
self.state["probeMemberId"] = probe_member["id"]
bulk = self.admin.request_json(
"POST",
f"/api/projects/{project_id}/members/bulk",
{"action": "set_roles", "memberIds": [probe_member["id"]], "roles": ["annotator", "quality_reviewer"]},
)
self.check("member_bulk_set_roles", bulk.get("updated") == 1, response=bulk)
self.admin.request_json(
"PATCH",
f"/api/projects/{project_id}/members/{probe_member['id']}",
{"supplierId": supplier["id"], "roles": ["annotator"]},
)
self.supplier_manager.login_if_configured(manager_email, self.manager_password, label="temporary supplier manager")
visible = item_list(self.supplier_manager.request_json("GET", f"/api/projects/{project_id}/members"))
self.check("supplier_manager_member_scope", all(item.get("supplierId") == supplier["id"] for item in visible), count=len(visible))
manager_status, manager_payload = self.raw_status(
self.supplier_manager,
"POST",
f"/api/projects/{project_id}/members/bulk",
{"action": "set_roles", "memberIds": [probe_member["id"]], "roles": ["acceptance_reviewer"]},
)
self.check(
"supplier_manager_own_scope_management",
manager_status == 200 and isinstance(manager_payload, dict) and manager_payload.get("updated") == 1,
status=manager_status,
response=manager_payload,
)
restored_probe = self.supplier_manager.request_json(
"PATCH",
f"/api/projects/{project_id}/members/{probe_member['id']}",
{"supplierId": supplier["id"], "roles": ["annotator"]},
)
self.check(
"supplier_manager_restores_member_role",
restored_probe.get("supplierId") == supplier["id"] and restored_probe.get("roles") == ["annotator"],
member=restored_probe,
)
self.remember_resource(
"supplierMembers",
{"supplierId": supplier["id"], "memberIds": member_ids, "probeMemberId": probe_member["id"]},
)
def ensure_human_phase_members(self, items: list[dict[str, Any]]) -> str:
supplier_id = human_item_supplier_id(items, fallback=str(self.state.get("supplierId", "")))
suppliers = item_list(self.admin.request_json("GET", "/api/suppliers"))
if not any(str(item.get("id", "")).strip() == supplier_id for item in suppliers):
raise AcceptanceError(f"human item supplier does not exist: {supplier_id}")
self.state["supplierId"] = supplier_id
member_ids, _ = self.ensure_main_role_members(supplier_id)
_, probe_member = self.ensure_probe_member(supplier_id)
self.remember_resource(
"humanPhaseMembers",
{
"supplierId": supplier_id,
"memberIds": member_ids,
"probeMemberId": probe_member["id"],
"humanItemIds": [str(item.get("id", "")) for item in items],
},
)
return supplier_id
def wait_automation(self, predicate, timeout: int = 3600) -> dict[str, Any]:
deadline = time.time() + timeout
while time.time() < deadline:
payload = self.admin.request_json("GET", f"/api/workflows/{self.state['workflowId']}/capture-automation")
if predicate(payload):
return payload
time.sleep(5)
raise AcceptanceError("capture automation wait timed out")
def verify_auto_retry(self) -> None:
project_id = self.state["projectId"]
workflow_name = f"Fanyu deterministic retry {self.args.acceptance_id}"
workflows = item_list(self.admin.request_json("GET", f"/api/projects/{project_id}/workflows"))
workflow = next((item for item in workflows if item.get("name") == workflow_name), None)
if workflow is None:
operators = item_list(self.admin.request_json("GET", "/api/operators"))
operator = next((item for item in operators if item.get("id") == "operator-builtin-shell-command"), None)
if operator is None:
raise AcceptanceError("builtin shell-command operator is unavailable for deterministic retry acceptance")
data = {
"label": "Deterministic retry failure",
"category": operator.get("category", "UTIL"),
"description": "Always fails for retry/backoff acceptance",
"inputs": [],
"outputs": operator.get("outputs", []),
"operatorId": operator["id"],
"operatorImageId": operator.get("imageId", ""),
"image": operator.get("image", ""),
"containerCommand": operator.get("containerCommand", []),
"runtimeContract": operator.get("runtimeContract", {}),
"hyperparams": [{"key": "COMMAND", "value": "exit 17", "type": "string"}],
}
workflow = self.admin.request_json(
"POST",
f"/api/projects/{project_id}/workflows",
{
"name": workflow_name,
"description": "Isolated deterministic retry acceptance",
"type": "batch",
"kind": "standard",
"executionScope": "production",
"maxConcurrentRuns": 1,
"maxRetryAttempts": 2,
"retryBackoffSeconds": 60,
"nodes": [{"id": "retry-failure", "type": "operator", "data": data, "position": {"x": 0, "y": 0}}],
"edges": [],
},
)
workflow = self.admin.request_json("POST", f"/api/workflows/{workflow['id']}/release", {})
self.state["retryWorkflowId"] = workflow["id"]
runs = item_list(self.admin.request_json("GET", f"/api/runs?workflowId={workflow['id']}"))
if not runs:
self.admin.request_json("POST", f"/api/workflows/{workflow['id']}/run", {"trigger": "manual"})
deadline = time.time() + 600
while time.time() < deadline:
runs = item_list(self.admin.request_json("GET", f"/api/runs?workflowId={workflow['id']}"))
terminal = [item for item in runs if item.get("status") == "failed"]
if len(terminal) >= 3:
break
time.sleep(5)
else:
raise AcceptanceError(f"automatic retry did not produce three failed attempts: {runs}")
# The final Run may become visible as failed just before its lineage and
# terminal timestamps are durably reflected by the list endpoint. Give
# that last transition one short, bounded settle window instead of
# treating the read-after-write race as a product failure.
grace_deadline = min(deadline, time.time() + 15)
while time.time() < grace_deadline:
time.sleep(2)
refreshed = item_list(self.admin.request_json("GET", f"/api/runs?workflowId={workflow['id']}"))
runs = refreshed or runs
if retry_runs_settled(runs):
break
if not retry_runs_settled(runs):
raise AcceptanceError(f"automatic retry attempts did not settle within the grace window: {runs}")
details = [self.admin.request_json("GET", f"/api/runs/{item['id']}") for item in runs]
details.sort(key=lambda item: int(item.get("attempt") or 1))
attempts = [int(item.get("attempt") or 1) for item in details]
parents = [item.get("parentRunId", "") for item in details]
self.check("automatic_retry_attempts", attempts == [1, 2, 3], attempts=attempts, runIds=[item["id"] for item in details])
self.check(
"automatic_retry_lineage",
parents[0] == "" and parents[1] == details[0]["id"] and parents[2] == details[1]["id"],
parents=parents,
)
for previous, current in zip(details, details[1:]):
previous_end = datetime.fromisoformat(str(previous["endedAt"]).replace("Z", "+00:00"))
current_available = datetime.fromisoformat(str(current["availableAt"]).replace("Z", "+00:00"))
if (current_available - previous_end).total_seconds() < 59:
raise AcceptanceError(f"retry backoff shorter than 60 seconds: {previous['id']} -> {current['id']}")
self.check("automatic_retry_backoff", True, seconds=60)
retry_evidence = [
{
"runId": item.get("id"),
"parentRunId": item.get("parentRunId", ""),
"attempt": int(item.get("attempt") or 1),
"status": item.get("status"),
}
for item in details
]
self.state["retryWorkflowId"] = workflow["id"]
self.remember_resource("retryRuns", retry_evidence)
def verify_capture_matrix(self) -> None:
expected_missing = {
"missing-left-video",
"missing-right-video",
"missing-left-pts",
"missing-right-pts",
"missing-imu",
"missing-calibration-camera",
"missing-calibration-imu",
"missing-manifest",
"discard-me",
}
expected_complete = {"complete-a", "complete-b", "complete-c", "complete-d", "complete-e", "complete-f", "cancel-me"}
payload = self.wait_automation(
lambda current: expected_missing | expected_complete
<= {item.get("captureId") for item in current.get("records", [])},
timeout=self.args.capture_timeout,
)
by_capture = {item["captureId"]: item for item in payload.get("records", [])}
self.check(
"missing_types_stay_incomplete",
all(by_capture[name]["status"] == "incomplete" and not by_capture[name].get("runId") for name in expected_missing),
statuses={name: by_capture[name]["status"] for name in expected_missing},
)
discard = by_capture["discard-me"]
discarded = self.admin.request_json(
"POST",
f"/api/workflows/{self.state['workflowId']}/capture-automation/records/{discard['id']}/discard",
{"reason": "100% 验收:验证永久放弃"},
)
discarded_record = next(item for item in discarded["records"] if item["id"] == discard["id"])
self.check("capture_discard", discarded_record["status"] == "discarded", record=discarded_record)
restored = self.admin.request_json(
"POST",
f"/api/workflows/{self.state['workflowId']}/capture-automation/records/{discard['id']}/restore",
{"reason": "100% 验收:验证恢复"},
)
restored_record = next(item for item in restored["records"] if item["id"] == discard["id"])
self.check("capture_restore", restored_record["status"] == "incomplete", record=restored_record)
payload = self.wait_automation(
lambda current: any(
item.get("captureId") == "cancel-me" and item.get("status") in {"queued", "running", "waiting_for_human"}
for item in current.get("records", [])
),
timeout=self.args.capture_timeout,
)
cancel_record = next(item for item in payload["records"] if item["captureId"] == "cancel-me")
cancelled = self.admin.request_json(
"POST",
f"/api/workflows/{self.state['workflowId']}/capture-automation/records/{cancel_record['id']}/cancel",
{"reason": "100% 验收:验证用户取消不自动重试", "mode": "graceful"},
)
cancel_record = next(item for item in cancelled["records"] if item["id"] == cancel_record["id"])
self.check("capture_cancel", cancel_record["status"] == "cancelled", record=cancel_record)
time.sleep(70)
still_cancelled = next(
item
for item in self.admin.request_json("GET", f"/api/workflows/{self.state['workflowId']}/capture-automation")["records"]
if item["id"] == cancel_record["id"]
)
self.check("cancel_does_not_auto_retry", still_cancelled["status"] == "cancelled", record=still_cancelled)
reprocessed = self.admin.request_json(
"POST",
f"/api/workflows/{self.state['workflowId']}/capture-automation/records/{cancel_record['id']}/reprocess",
{"reason": "100% 验收:取消后手动重新处理"},
)
retry_record = next(item for item in reprocessed["records"] if item["id"] == cancel_record["id"])
self.check(
"capture_manual_reprocess_lineage",
retry_record["status"] == "queued" and retry_record.get("parentRunId") and retry_record.get("attempt", 0) >= 2,
record=retry_record,
)
stable = self.wait_automation(
lambda current: all(
any(item.get("captureId") == name and item.get("runId") for item in current.get("records", []))
for name in expected_complete
),
timeout=self.args.capture_timeout,
)
ids = [item["id"] for item in stable["records"] if item.get("captureId") in expected_complete]
self.check("capture_idempotence", len(ids) == len(set(ids)) == len(expected_complete), recordIds=ids)
capture_evidence = [
{
"captureId": item.get("captureId"),
"captureRecordId": item.get("id"),
"runId": item.get("runId", ""),
"parentRunId": item.get("parentRunId", ""),
"status": item.get("status"),
"attempt": item.get("attempt", 0),
}
for item in stable.get("records", [])
if item.get("captureId") in expected_missing | expected_complete
]
capture_evidence.sort(key=lambda item: str(item.get("captureId", "")))
self.state["captureRecordIds"] = ids
self.remember_resource("captures", capture_evidence)
def wait_human_items(self, minimum: int = 1, timeout: int = 3600) -> list[dict[str, Any]]:
deadline = time.time() + timeout
query = f"pluginId=annotation-qa&projectId={self.state['projectId']}&workflowId={self.state['workflowId']}&pool=all"
while time.time() < deadline:
items = item_list(self.admin.request_json("GET", f"/api/human-items?{query}"))
candidates = [
item
for item in items
if item.get("legacyTaskId")
and item.get("currentStageId") != "prepackage_acceptance"
and item.get("status") not in {"cancelled", "rejected"}
]
if len(candidates) >= minimum:
return candidates
time.sleep(10)
raise AcceptanceError("human items did not become ready")
def submit_combined(self, item: dict[str, Any], *, draft_first: bool) -> dict[str, Any]:
detail = self.annotator.request_json("GET", f"/api/plugins/annotation-qa/tasks/{item['legacyTaskId']}")
task = detail["task"]
start = float(task.get("coverageStartSecond") or task.get("clipStart") or 0)
end = float(task.get("coverageEndSecond") or task.get("clipEnd") or 0)
if end <= start:
raise AcceptanceError(f"invalid task coverage: {task}")
one = start + (end - start) / 3
two = start + (end - start) * 2 / 3
slices = [
{"id": "acceptance-slice-001", "start": start, "end": one, "annotationStatus": "labeled"},
{"id": "acceptance-slice-002", "start": one, "end": two, "annotationStatus": "empty_confirmed"},
{"id": "acceptance-slice-003", "start": two, "end": end, "annotationStatus": "labeled"},
]
segments = [
{"id": "acceptance-event-001", "labelId": "01", "label": "鲜食处理", "description": "鲜食处理", "start": start, "end": one, "source": "manual", "confidence": 1},
{"id": "acceptance-event-003", "labelId": "15", "label": "顾客交付与内部协作", "description": "顾客交付与内部协作", "start": two, "end": end, "source": "manual", "confidence": 1},
]
self.remember_human_item(
item,
task=task,
slice_ids=[entry["id"] for entry in slices],
event_ids=[entry["id"] for entry in segments],
)
if draft_first:
quality_status, _ = self.raw_status(self.quality, "POST", f"/api/human-items/{item['id']}/claim", {"version": item["version"]})
acceptance_status, _ = self.raw_status(self.acceptance, "POST", f"/api/human-items/{item['id']}/claim", {"version": item["version"]})
self.check("annotation_stage_cross_role_403", quality_status == 403 and acceptance_status == 403, quality=quality_status, acceptance=acceptance_status)
claimed_item = self.claim_human_item(self.annotator, item, "annotator")
if draft_first:
saved = self.annotator.request_json(
"PUT",
f"/api/plugins/annotation-qa/tasks/{item['legacyTaskId']}/drafts",
{
"version": claimed_item["version"],
"sliceSegments": slices,
"operations": [],
"segments": segments,
"statuses": {entry["id"]: entry["annotationStatus"] for entry in slices},
},
)
claimed_item = saved["item"]
restored = self.annotator.request_json("GET", f"/api/plugins/annotation-qa/tasks/{item['legacyTaskId']}")
annotation_draft = restored.get("humanItem", {}).get("subject", {}).get("payload", {}).get("annotationDraft")
self.check(
"server_draft_restore",
annotation_draft_matches(annotation_draft, slices, segments),
taskId=item["legacyTaskId"],
sliceIds=[entry["id"] for entry in slices],
eventIds=[entry["id"] for entry in segments],
statuses={entry["id"]: entry["annotationStatus"] for entry in slices},
)
submission_version = int(claimed_item["version"])
submission_id = f"fanyu-v5-combined-{item['id']}-v{submission_version}"
submission_payload = {
"version": submission_version,
"sliceSegments": slices,
"sliceOperations": [],
"segments": segments,
"requireSliceCompletion": True,
"changeReason": "Fanyu 100% acceptance combined submission",
}
try:
self.annotator.request_json(
"POST",
f"/api/plugins/annotation-qa/tasks/{item['legacyTaskId']}/combined-submissions",
submission_payload,
timeout=min(max(180, int(self.args.human_timeout)), 900),
retries=1,
)
except AcceptanceError as error:
# Physical stereo/PTS/IMU slicing can outlive an HTTP client or
# reverse-proxy timeout. The server reserves an idempotent token
# before materialization, so recover by polling that durable state
# instead of issuing a duplicate POST.
recoverable_transport_markers = ("timed out", "remotedisconnected", "failed 502", "failed 504")
if not any(marker in str(error).lower() for marker in recoverable_transport_markers):
raise
updated = self.wait_combined_submission(item["id"], submission_id)
self.remember_human_item(
updated,
task=task,
slice_ids=[entry["id"] for entry in slices],
event_ids=[entry["id"] for entry in segments],
)
return updated
def quality_decision(self, item: dict[str, Any], decision: str) -> dict[str, Any]:
annotator_status, _ = self.raw_status(self.annotator, "POST", f"/api/human-items/{item['id']}/claim", {"version": item["version"]})
acceptance_status, _ = self.raw_status(self.acceptance, "POST", f"/api/human-items/{item['id']}/claim", {"version": item["version"]})
self.check("quality_stage_cross_role_403", annotator_status == 403 and acceptance_status == 403, annotator=annotator_status, acceptance=acceptance_status)
claimed_item = self.claim_human_item(self.quality, item, "quality")
problem_slice_ids: list[str] = []
if decision == "rejected":
detail = self.quality.request_json("GET", f"/api/plugins/annotation-qa/tasks/{item['legacyTaskId']}")
slice_revisions = detail.get("sliceRevisions") if isinstance(detail.get("sliceRevisions"), list) else []
latest_slices = slice_revisions[-1].get("segments", []) if slice_revisions else []
problem_slice_ids = [str(latest_slices[0].get("id", ""))] if latest_slices else []
if not problem_slice_ids or not problem_slice_ids[0]:
raise AcceptanceError(f"quality rejection has no materialized problem slice: {item}")
self.quality.request_json(
"POST",
f"/api/plugins/annotation-qa/tasks/{item['legacyTaskId']}/reviews",
{
"stage": "quality",
"decision": decision,
"comment": "Fanyu 100% acceptance quality decision",
"reasonCodes": ["acceptance_quality_reject"] if decision == "rejected" else [],
"problemSliceIds": problem_slice_ids,
"videoQuality": "qualified",
"annotationAccuracy": 100,
"annotationCompleteness": 100,
"overallGrade": "A",
},
)
updated = self.admin.request_json("GET", f"/api/human-items/{claimed_item['id']}")["item"]
self.remember_human_item(updated)
return updated
def wait_acceptance_items(self, minimum: int, timeout: int = 3600) -> list[dict[str, Any]]:
deadline = time.time() + timeout
query = f"pluginId=annotation-qa&projectId={self.state['projectId']}&workflowId={self.state['workflowId']}&pool=all"
while time.time() < deadline:
items = item_list(self.admin.request_json("GET", f"/api/human-items?{query}"))
candidates = [
item
for item in items
if item.get("currentStageId") == "prepackage_acceptance" and item.get("status") in {"ready", "rework"}
]
if len(candidates) >= minimum:
return candidates
time.sleep(10)
raise AcceptanceError("prepackage acceptance items did not become ready")
def wait_acceptance_item_for_task(self, task_id: str, timeout: int = 3600) -> dict[str, Any]:
deadline = time.time() + timeout
query = f"pluginId=annotation-qa&projectId={self.state['projectId']}&workflowId={self.state['workflowId']}&pool=all"
while time.time() < deadline:
items = item_list(self.admin.request_json("GET", f"/api/human-items?{query}"))
for item in items:
if item.get("currentStageId") != "prepackage_acceptance" or item.get("status") not in {"ready", "rework"}:
continue
payload = item.get("subject", {}).get("payload", {}) if isinstance(item.get("subject"), dict) else {}
if task_id and task_id in json.dumps(payload, ensure_ascii=False):
return item
time.sleep(2)
raise AcceptanceError(f"prepackage acceptance item for task {task_id} did not become ready")
def approve_acceptance(self, item: dict[str, Any], headpose: str, handpose: str) -> dict[str, Any]:
annotator_status, _ = self.raw_status(self.annotator, "POST", f"/api/human-items/{item['id']}/claim", {"version": item["version"]})
quality_status, _ = self.raw_status(self.quality, "POST", f"/api/human-items/{item['id']}/claim", {"version": item["version"]})
self.check("acceptance_stage_cross_role_403", annotator_status == 403 and quality_status == 403, annotator=annotator_status, quality=quality_status)
claimed_item = self.claim_human_item(self.acceptance, item, "acceptance")
excluded = headpose == "exclude" or handpose == "exclude"
decided = self.acceptance.request_json(
"POST",
f"/api/human-items/{item['id']}/decisions",
{
"version": claimed_item["version"],
"decision": "approved",
"reviewedRevision": claimed_item.get("currentRevision", 0),
"comment": "pose excluded by acceptance matrix" if excluded else "all pose evidence included",
"reasonCodes": ["pose_excluded"] if excluded else [],
"poseDelivery": {
"headpose": headpose,
"handpose": handpose,
"reasonCodes": ["pose_excluded"] if excluded else [],
},
},
)
updated = decided.get("item", decided)
if isinstance(updated, dict):
self.remember_human_item(updated)
return updated
def wait_delivery_packages(self, minimum: int, timeout: int = 3600) -> list[dict[str, Any]]:
deadline = time.time() + timeout
query = f"projectId={self.state['projectId']}&workflowId={self.state['workflowId']}"
while time.time() < deadline:
packages = item_list(self.admin.request_json("GET", f"/api/delivery-packages?{query}"))
accepted = [item for item in packages if item.get("status") == "accepted"]
if len(accepted) >= minimum:
return accepted
time.sleep(10)
raise AcceptanceError("accepted delivery packages did not become ready")
def human_item_for_client(self, client: DroboticflowClient, item_id: str) -> dict[str, Any]:
payload = client.request_json("GET", f"/api/human-items/{item_id}")
item = payload.get("item", payload) if isinstance(payload, dict) else None
if not isinstance(item, dict) or not item.get("id"):
raise AcceptanceError(f"human item not found: {item_id}")
return item
def human_item(self, item_id: str) -> dict[str, Any]:
return self.human_item_for_client(self.admin, item_id)
def claim_human_item(
self,
client: DroboticflowClient,
item: dict[str, Any],
session_key: str,
) -> dict[str, Any]:
item_id = str(item.get("id", "")).strip()
actor_user_id = str(self.session_user_ids.get(session_key, "")).strip()
if not item_id or not actor_user_id:
raise AcceptanceError(f"cannot claim human item without id/session: item={item_id} session={session_key}")
latest = self.human_item_for_client(client, item_id)
def reuse_existing(candidate: dict[str, Any]) -> dict[str, Any] | None:
status = str(candidate.get("status", "")).strip()
if status not in {"claimed", "in_progress"}:
return None
assignee = str(candidate.get("assignedUserId", "")).strip()
if assignee != actor_user_id:
raise AcceptanceError(
f"human item {item_id} is already claimed by {assignee or 'another user'}, expected {actor_user_id}"
)
return candidate
reused = reuse_existing(latest)
if reused is not None:
self.remember_human_item(reused)
return reused
status = str(latest.get("status", "")).strip()
if status not in {"ready", "rework", "submitted"}:
raise AcceptanceError(f"human item {item_id} cannot be claimed from status {status}")
response_status, payload = self.raw_status(
client,
"POST",
f"/api/human-items/{item_id}/claim",
{"version": latest.get("version")},
)
if response_status == 200 and isinstance(payload, dict):
claimed = payload.get("item", payload)
if isinstance(claimed, dict):
self.remember_human_item(claimed)
return claimed
if response_status == 409:
# A retry can race with the successful response from the previous
# process. Re-read once and reuse only our own durable claim.
refreshed = self.human_item_for_client(client, item_id)
reused = reuse_existing(refreshed)
if reused is not None:
self.remember_human_item(reused)
return reused
raise AcceptanceError(f"claim human item {item_id} failed: HTTP {response_status} {payload}")
def wait_combined_submission(
self,
item_id: str,
expected_submission_id: str,
*,
timeout: int | None = None,
poll_seconds: float = 3.0,
) -> dict[str, Any]:
if timeout is None:
timeout = min(max(180, int(getattr(self.args, "human_timeout", 180))), 900)
deadline = time.time() + max(1, timeout)
while time.time() < deadline:
current = self.human_item(item_id)
stage = str(current.get("currentStageId", "")).strip()
pending_id = str(current.get("pendingCombinedSubmissionId", "")).strip()
subject = current.get("subject") if isinstance(current.get("subject"), dict) else {}
payload = subject.get("payload") if isinstance(subject.get("payload"), dict) else {}
if stage == "quality_review" and completed_materialization(payload.get("materialization")):
self.remember_human_item(current)
submissions = self.state.setdefault("evidence", {}).setdefault("combinedSubmissions", {})
submissions[item_id] = {
"submissionId": expected_submission_id,
"humanItemId": item_id,
"materialization": payload["materialization"],
}
self.report["resources"]["combinedSubmissions"] = list(submissions.values())
self.save_state()
return current
if pending_id:
if pending_id != expected_submission_id:
raise AcceptanceError(
f"human item {item_id} is processing a different combined submission: {pending_id}"
)
time.sleep(max(0, poll_seconds))
continue
if stage == "annotation_combined":
raise AcceptanceError(
f"combined submission {expected_submission_id} cleared without materialization; item remained in annotation_combined"
)
raise AcceptanceError(
f"combined submission {expected_submission_id} ended in unexpected stage {stage}: {current}"
)
raise AcceptanceError(f"combined submission {expected_submission_id} did not finish within {timeout} seconds")
def restore_browser_fixture_if_present(self) -> bool:
fixture = self.state.get("browserFixture")
if not isinstance(fixture, dict):
return False
try:
annotation_item = self.human_item(str(fixture["annotation"]["humanItemId"]))
quality_item = self.human_item(str(fixture["quality"]["humanItemId"]))
acceptance_item = self.human_item(str(fixture["acceptance"]["humanItemId"]))
except (KeyError, TypeError) as error:
raise AcceptanceError(f"stored browserFixture is incomplete: {fixture}") from error
supplier_id = self.ensure_human_phase_members([annotation_item, quality_item, acceptance_item])
member_email = f"fanyu-acceptance-probe+{self.args.acceptance_id.lower()}@droboticflow.local"
self.record_browser_fixture(
annotation_item,
quality_item,
acceptance_item,
member_user_id=str(self.state.get("probeUserId", "")),
member_email=member_email,
supplier_id=supplier_id,
)
self.check(
"browser_fixture_resume",
True,
annotationHumanItemId=annotation_item["id"],
qualityHumanItemId=quality_item["id"],
acceptanceHumanItemId=acceptance_item["id"],
)
return True
def verify_human_flow(self) -> None:
if self.restore_browser_fixture_if_present():
return
items = self.wait_human_items(minimum=7, timeout=self.args.human_timeout)
# Interrupted acceptance runs can leave one valid rework item beside
# fresh pool items. Consume fresh items for the deterministic positive
# and escalation matrices first, and keep the prior rework item for a
# browser-stage fixture where another rejection is not required.
items.sort(key=lambda item: (int(item.get("reviewRound") or 0), str(item.get("createdAt") or ""), str(item.get("id") or "")))
supplier_id = self.ensure_human_phase_members(items)
self.check(
"human_items_supplier_alignment",
all(str(item.get("supplierId", "")).strip() in {"", supplier_id} for item in items),
supplierId=supplier_id,
humanItemIds=[item.get("id") for item in items],
)
for item in items:
self.remember_human_item(item)
approved_items: list[dict[str, Any]] = []
for index, source in enumerate(items[:4]):
current = self.human_item(str(source["id"]))
if current.get("status") == "accepted":
approved_items.append(current)
continue
if current.get("currentStageId") == "quality_review" and completed_materialization(
current.get("subject", {}).get("payload", {}).get("materialization")
):
submitted = current
elif current.get("currentStageId") == "annotation_combined":
submitted = self.submit_combined(current, draft_first=index == 0 and int(current.get("reviewRound") or 0) == 0)
else:
raise AcceptanceError(f"positive human item cannot resume from {current.get('currentStageId')}:{current.get('status')}")
if index == 0 and int(submitted.get("reviewRound") or 0) == 0:
rejected = self.quality_decision(submitted, "rejected")
self.check("quality_reject_returns_annotator", rejected["currentStageId"] == "annotation_combined" and rejected["status"] == "rework", item=rejected)
submitted = self.submit_combined(rejected, draft_first=False)
approved = self.quality_decision(submitted, "approved")
self.check(f"quality_approve_{index}", approved["status"] in {"accepted", "ready", "submitted"}, item=approved)
approved_items.append(approved)
acceptance_items = self.wait_acceptance_items(minimum=4, timeout=self.args.human_timeout)
for item in acceptance_items:
self.remember_human_item(item)
combinations = [("include", "include"), ("include", "exclude"), ("exclude", "include"), ("exclude", "exclude")]
acceptance_results = []
for item, (headpose, handpose) in zip(acceptance_items[:4], combinations):
result = self.approve_acceptance(item, headpose, handpose)
self.check(f"pose_delivery_{headpose}_{handpose}", result.get("status") == "accepted", item=result)
acceptance_results.append(result)
packages = self.wait_delivery_packages(minimum=4, timeout=self.args.human_timeout)
package_evidence = []
for package in packages[:4]:
contents = self.admin.request_json("GET", f"/api/delivery-packages/{package['id']}/contents")
names = [artifact.get("name") for artifact in contents.get("artifacts", [])]
package_evidence.append({"id": package["id"], "manifestDigest": package.get("manifestDigest"), "artifacts": names})
self.check(
"delivery_packages_readable",
all("packages-manifest.json" in item["artifacts"] and item["manifestDigest"] for item in package_evidence),
packages=package_evidence,
)
self.state["deliveryPackageIds"] = [item["id"] for item in package_evidence]
self.remember_resource("deliveryPackages", package_evidence)
# Preserve three real, unassigned browser fixtures. The annotation item
# also covers the escalation-reopen branch; the quality item is fully
# materialized but intentionally not reviewed; the acceptance item has
# passed quality but is intentionally not accepted/package-released.
escalation_source = self.human_item(str(items[4]["id"]))
if escalation_source.get("status") == "escalated":
escalated = escalation_source
else:
if escalation_source.get("currentStageId") == "annotation_combined":
escalation_source = self.submit_combined(escalation_source, draft_first=False)
if escalation_source.get("currentStageId") != "quality_review":
raise AcceptanceError(f"escalation fixture cannot resume from {escalation_source}")
escalated = self.quality_decision(escalation_source, "rejected")
if escalated.get("status") == "rework":
escalated = self.submit_combined(escalated, draft_first=False)
escalated = self.quality_decision(escalated, "rejected")
self.check("max_rework_escalates_reopen", escalated["status"] == "escalated", item=escalated)
reopened = self.admin.request_json(
"POST",
f"/api/human-items/{escalated['id']}/escalation",
{"version": escalated["version"], "action": "reopen", "targetStageId": "annotation_combined", "comment": "100% acceptance browser annotation fixture"},
)
annotation_fixture = self.human_item(str(reopened.get("id") or escalated["id"]))
self.check(
"escalation_reopen",
annotation_fixture.get("currentStageId") == "annotation_combined" and annotation_fixture.get("status") == "rework",
item=annotation_fixture,
)
quality_fixture = self.human_item(str(items[5]["id"]))
if quality_fixture.get("currentStageId") == "annotation_combined":
quality_fixture = self.submit_combined(quality_fixture, draft_first=False)
if quality_fixture.get("currentStageId") != "quality_review" or not completed_materialization(
quality_fixture.get("subject", {}).get("payload", {}).get("materialization")
):
raise AcceptanceError(f"quality browser fixture is not materialized: {quality_fixture}")
acceptance_source = self.human_item(str(items[6]["id"]))
if acceptance_source.get("currentStageId") == "annotation_combined":
acceptance_source = self.submit_combined(acceptance_source, draft_first=False)
if acceptance_source.get("status") != "accepted":
if acceptance_source.get("currentStageId") != "quality_review":
raise AcceptanceError(f"acceptance browser source cannot resume from {acceptance_source}")
acceptance_source = self.quality_decision(acceptance_source, "approved")
acceptance_fixture = self.wait_acceptance_item_for_task(
str(acceptance_source.get("legacyTaskId") or items[6].get("legacyTaskId") or ""),
timeout=self.args.human_timeout,
)
annotation_fixture = self.human_item(str(annotation_fixture["id"]))
quality_fixture = self.human_item(str(quality_fixture["id"]))
acceptance_fixture = self.human_item(str(acceptance_fixture["id"]))
for item in (annotation_fixture, quality_fixture, acceptance_fixture):
self.remember_human_item(item)
member_email = f"fanyu-acceptance-probe+{self.args.acceptance_id.lower()}@droboticflow.local"
self.record_browser_fixture(
annotation_fixture,
quality_fixture,
acceptance_fixture,
member_user_id=str(self.state.get("probeUserId", "")),
member_email=member_email,
supplier_id=supplier_id,
)
self.check(
"browser_stage_fixtures_preserved",
True,
annotationHumanItemId=annotation_fixture["id"],
qualityHumanItemId=quality_fixture["id"],
acceptanceHumanItemId=acceptance_fixture["id"],
)
def finalize(self) -> None:
if self.args.finalize_users:
user_keys = ("managerUserId",) if self.state.get("browserFixture") else ("probeUserId", "managerUserId")
for key in user_keys:
user_id = self.state.get(key)
if user_id:
self.admin.request_json("PUT", f"/api/users/{user_id}", {"status": "disabled"})
if not self.state.get("browserFixture"):
self.state.pop("probePassword", None)
self.state.pop("managerPassword", None)
self.report["status"] = "passed"
self.report["finishedAt"] = datetime.now(timezone.utc).isoformat()
self.save_report()
self.save_state()
def run(self) -> None:
self.login()
phases = self.args.phase or ["all"]
if "all" in phases or "setup" in phases:
self.ensure_project()
self.ensure_supplier_and_members()
self.ensure_workflow()
if "all" in phases or "capture" in phases:
self.verify_capture_matrix()
if "all" in phases or "retry" in phases:
self.verify_auto_retry()
if "all" in phases or "human" in phases:
self.verify_human_flow()
self.finalize()
def parse_args() -> argparse.Namespace:
now = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
parser = argparse.ArgumentParser()
parser.add_argument("--api-url", default=env("FANYU_API_URL", "http://120.48.180.80"))
parser.add_argument("--acceptance-id", default=env("FANYU_ACCEPTANCE_ID", now))
parser.add_argument("--source-project-id", default=env("FANYU_ACCEPTANCE_SOURCE_PROJECT_ID"))
parser.add_argument("--source-workflow-id", default=env("FANYU_ACCEPTANCE_SOURCE_WORKFLOW_ID"))
parser.add_argument("--target-root", default=env("FANYU_ACCEPTANCE_TARGET_ROOT", f"acceptance-100/{now}"))
parser.add_argument("--state-path", default=env("FANYU_ACCEPTANCE_STATE_PATH", f"/tmp/fanyu-acceptance-{now}-state.json"))
parser.add_argument("--report-path", default=env("FANYU_ACCEPTANCE_REPORT_PATH", f"/tmp/fanyu-acceptance-{now}-report.json"))
parser.add_argument("--phase", action="append", choices=["all", "setup", "capture", "retry", "human"])
parser.add_argument("--capture-timeout", type=int, default=3600)
parser.add_argument("--human-timeout", type=int, default=7200)
parser.add_argument("--finalize-users", action="store_true")
return parser.parse_args()
def main() -> int:
args = parse_args()
harness = Harness(args)
try:
harness.run()
except Exception as error:
import traceback
traceback.print_exc()
harness.report["status"] = "failed"
harness.report["error"] = str(error)
harness.report["finishedAt"] = datetime.now(timezone.utc).isoformat()
harness.save_report()
print(json.dumps({"status": "failed", "error": str(error), "report": str(harness.report_path)}, ensure_ascii=False), file=sys.stderr)
return 1
print(json.dumps({"status": "passed", "report": str(harness.report_path), "state": str(harness.state_path)}, ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())