droboticflow-new/scripts/holobrain-full-training-workflow-acceptance.sh

467 lines
20 KiB
Bash
Executable File

#!/usr/bin/env bash
: "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:?Set DROBOTICFLOW_ACCEPTANCE_PASSWORD before running acceptance}"
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
# shellcheck source=scripts/lib/service-endpoints.sh
source "$ROOT_DIR/scripts/lib/service-endpoints.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
API_TOKEN_SECRET_NAME="${HOLOBRAIN_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
API_TOKEN_SECRET_KEY="${HOLOBRAIN_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}"
HOLOBRAIN_TRAINING_MODE="${HOLOBRAIN_TRAINING_MODE:-smoke}"
if [[ "$HOLOBRAIN_TRAINING_MODE" != "smoke" && "$HOLOBRAIN_TRAINING_MODE" != "full" ]]; then
echo "ERROR: HOLOBRAIN_TRAINING_MODE must be smoke or full." >&2
exit 2
fi
if [[ -z "${HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS:-}" ]]; then
if [[ "$HOLOBRAIN_TRAINING_MODE" == "smoke" ]]; then
HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS=7200
else
HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS=259200
fi
fi
export HOLOBRAIN_TRAINING_MODE HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS
if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then
public_host="$(
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c '
import ipaddress
import json
import sys
payload = json.load(sys.stdin)
hosts = [item.get("ip") or item.get("hostname") for item in payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])]
hosts = [item for item in hosts if item]
if not hosts:
raise SystemExit("no LoadBalancer ingress host is available")
for host in hosts:
try:
if not ipaddress.ip_address(host).is_private:
print(host)
raise SystemExit(0)
except ValueError:
print(host)
raise SystemExit(0)
print(hosts[0])
'
)"
export API_BASE_URL="http://${public_host}"
fi
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)"
DROBOTICFLOW_API_TOKEN="$(
printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c '
import base64
import json
import os
import sys
try:
payload = json.load(sys.stdin)
except json.JSONDecodeError:
raise SystemExit(0)
raw = payload.get("data", {}).get(os.environ["API_TOKEN_SECRET_KEY"])
if isinstance(raw, str) and raw:
print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="")
'
)"
export DROBOTICFLOW_API_TOKEN
fi
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required." >&2
exit 1
fi
json_get() {
local expression="$1"
python3 -c "import json,sys; print(${expression})"
}
COOKIE_JAR="$(mktemp)"
cleanup() {
rm -f "$COOKIE_JAR"
}
trap cleanup EXIT
api_curl() {
curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \
-b "$COOKIE_JAR" -c "$COOKIE_JAR" -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@"
}
csrf_token() {
awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR"
}
api_curl_write() {
local token
token="$(csrf_token)"
if [[ -n "$token" ]]; then
api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@"
return
fi
api_curl "$@"
}
api_health="$(api_curl "$API_BASE_URL/api/health")"
printf 'api health: %s\n' "$api_health"
printf '%s' "$api_health" | grep -q '"status":"ok"'
login_payload="$(
DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" \
DROBOTICFLOW_ACCEPTANCE_PASSWORD="$DROBOTICFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY'
import json
import os
print(json.dumps({"email": os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["DROBOTICFLOW_ACCEPTANCE_PASSWORD"]}))
PY
)"
api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/login" -d "$login_payload" >/dev/null
project_payload="$(
python3 - <<'PY'
import json
import os
mode = os.environ["HOLOBRAIN_TRAINING_MODE"]
print(json.dumps({"name": f"HoloBrain {mode.title()} Training Acceptance", "description": f"RoboOrchardLab HoloBrain {mode} data/train/export/eval/inference verification", "status": "active"}, ensure_ascii=False))
PY
)"
project_list="$(api_curl "$API_BASE_URL/api/projects")"
project_id="$(printf '%s' "$project_list" | HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 -c '
import json, os, sys
mode = os.environ["HOLOBRAIN_TRAINING_MODE"].title()
name = f"HoloBrain {mode} Training Acceptance"
payload = json.load(sys.stdin)
items = payload.get("items", payload) if isinstance(payload, dict) else payload
for item in items:
if str(item.get("name", "")).strip() == name:
print(item.get("id", ""))
break
')"
if [[ -z "$project_id" ]]; then
project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")"
project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')"
printf 'created project: %s\n' "$project_id"
else
printf 'reusing project: %s\n' "$project_id"
fi
acceptance_suffix="${HOLOBRAIN_ACCEPTANCE_SUFFIX:-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)-$$}"
workflow_payload="$(
ROOT_DIR="$ROOT_DIR" HOLOBRAIN_ACCEPTANCE_SUFFIX="$acceptance_suffix" python3 - <<'PY'
import json
import os
import pathlib
root = pathlib.Path(os.environ["ROOT_DIR"])
catalog = json.loads((root / "internal/api/builtin_operators_catalog.json").read_text(encoding="utf-8"))
operators = {item.get("id"): item for item in catalog}
ids = [
"operator-builtin-holobrain-data-prepare",
"operator-builtin-holobrain-train",
"operator-builtin-holobrain-export",
"operator-builtin-holobrain-robotwin-eval",
"operator-builtin-holobrain-inference-test",
]
missing = [item for item in ids if item not in operators]
if missing:
raise SystemExit(f"missing HoloBrain operators in catalog: {missing}")
def env(name, default=""):
return os.environ.get(name, default)
def override_hyperparams(operator, values):
result = []
seen = set()
for item in operator.get("hyperparams") or []:
next_item = dict(item)
key = str(next_item.get("key", ""))
if key in values:
next_item["value"] = values[key]
seen.add(key)
result.append(next_item)
for key, value in values.items():
if key not in seen:
result.append({"key": key, "value": value, "type": "string", "description": "Acceptance override"})
return result
def operator_image(operator):
slug = str(operator["id"]).removeprefix("operator-builtin-")
specific_key = "HOLOBRAIN_OPERATOR_IMAGE_TAG_" + slug.upper().replace("-", "_")
tag = env(specific_key) or env("HOLOBRAIN_OPERATOR_IMAGE_TAG")
if not tag:
return operator.get("imageId") or operator["image"]
registry = env("HOLOBRAIN_OPERATOR_IMAGE_REGISTRY", "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc")
return f"{registry}/droboticflow-operator-{slug}:{tag}"
def node_data(operator, hyperparams):
resources = operator.get("resources") or {}
image = operator_image(operator)
data = {
"label": operator["name"],
"category": operator["category"],
"description": operator.get("description", ""),
"operatorId": operator["id"],
"operatorImageId": image,
"image": image,
"containerCommand": operator.get("containerCommand", []),
"containerArgs": operator.get("containerArgs", []),
"runtimeContract": operator.get("runtimeContract", {}),
"skillDefinition": operator.get("skillDefinition", ""),
"inputs": operator.get("inputs") or [],
"outputs": operator.get("outputs") or [],
"hyperparams": hyperparams,
}
if resources.get("cpuLimit"):
data["cpu"] = resources["cpuLimit"]
if resources.get("memoryLimit"):
data["memory"] = resources["memoryLimit"]
if resources.get("gpuLimit"):
data["gpu"] = resources["gpuLimit"]
return data
def with_resource_overrides(data, prefix):
cpu = env(prefix + "_CPU")
memory = env(prefix + "_MEMORY")
gpu = env(prefix + "_GPU")
if cpu:
data["cpu"] = cpu
if memory:
data["memory"] = memory
if gpu:
data["gpu"] = gpu
return data
data_values = {
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
"DATA_SOURCE": env("HOLOBRAIN_DATA_SOURCE", "local"),
"ROBOTWIN_SOURCE_PATH": env("HOLOBRAIN_ROBOTWIN_SOURCE_PATH", env("HOLOBRAIN_WORKFLOW_SOURCE_ROOT", "")),
"EXPECTED_LMDB_DIRS_JSON": env("HOLOBRAIN_EXPECTED_LMDB_DIRS_JSON", ""),
"ROBOTWIN_PACK_TASK_NAMES": env("HOLOBRAIN_ROBOTWIN_PACK_TASK_NAMES", ""),
}
data_values = {key: value for key, value in data_values.items() if value}
train_values = {
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
"ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"),
"CONFIG_PATH": "projects/holobrain/configs/config_holobrain_qwen_common.py",
"LAUNCH_MODE": env("HOLOBRAIN_LAUNCH_MODE", "single"),
"GPU_COUNT": env("HOLOBRAIN_GPU_COUNT", "1"),
"CONFIG_KWARGS_JSON": env("HOLOBRAIN_CONFIG_KWARGS_JSON", "{}"),
"SMOKE_MAX_STEP": env("HOLOBRAIN_SMOKE_MAX_STEP", "2"),
"SMOKE_BATCH_SIZE": env("HOLOBRAIN_SMOKE_BATCH_SIZE", "1"),
"SMOKE_NUM_WORKERS": env("HOLOBRAIN_SMOKE_NUM_WORKERS", "0"),
"RESUME_POLICY": env("HOLOBRAIN_RESUME_POLICY", "auto"),
"RESUME_FROM": env("HOLOBRAIN_RESUME_FROM", ""),
"CHECKPOINT_ARTIFACT": env("HOLOBRAIN_CHECKPOINT_ARTIFACT", "latest_valid"),
"OUTPUT_CHECKPOINT_PATH": env("HOLOBRAIN_OUTPUT_CHECKPOINT_PATH", ""),
"PROGRESS_INTERVAL_SECONDS": env("HOLOBRAIN_PROGRESS_INTERVAL_SECONDS", "15"),
"MODEL_DTYPE": env("HOLOBRAIN_MODEL_DTYPE", "bfloat16"),
"INPUT_DTYPE": env("HOLOBRAIN_INPUT_DTYPE", "float32"),
}
export_values = {
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
"ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"),
"CONFIG_PATH": "projects/holobrain/configs/config_holobrain_qwen_common.py",
"CONFIG_KWARGS_JSON": env("HOLOBRAIN_EXPORT_CONFIG_KWARGS_JSON", env("HOLOBRAIN_CONFIG_KWARGS_JSON", "{}")),
"CHECKPOINT_SELECTION": env("HOLOBRAIN_CHECKPOINT_SELECTION", "latest_valid"),
}
eval_values = {
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
"ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"),
"ROBOTWIN_ROOT": env("HOLOBRAIN_ROBOTWIN_ROOT", "/opt/RoboTwin"),
"ROBOTWIN_ASSET_ROOT": env("HOLOBRAIN_ROBOTWIN_ASSET_ROOT", ""),
"INFERENCE_PREFIX": env("HOLOBRAIN_EVAL_INFERENCE_PREFIX", ""),
"UR5_WSG_ARM_DISTANCE": env("HOLOBRAIN_UR5_WSG_ARM_DISTANCE", "0.8"),
"ROBOTWIN_CAMERA_SHADER": env("HOLOBRAIN_ROBOTWIN_CAMERA_SHADER", "default"),
"VULKAN_ICD_FILENAMES": env("HOLOBRAIN_VULKAN_ICD_FILENAMES", ""),
"EVAL_MODE": env("HOLOBRAIN_EVAL_MODE", "ray"),
"TASK_NAMES_JSON": env("HOLOBRAIN_EVAL_TASK_NAMES_JSON", "[]"),
"EPISODE_NUM": env("HOLOBRAIN_EVAL_EPISODE_NUM", "100"),
"GPU_IDS_JSON": env("HOLOBRAIN_EVAL_GPU_IDS_JSON", "[0]"),
"WORKERS_PER_GPU": env("HOLOBRAIN_EVAL_WORKERS_PER_GPU", "1"),
"DEVICE": env("HOLOBRAIN_EVAL_DEVICE", "cuda"),
"SAVE_VIDEO": env("HOLOBRAIN_EVAL_SAVE_VIDEO", "false"),
"SMOKE_TASK_NAME": env("HOLOBRAIN_SMOKE_TASK_NAME", "beat_block_hammer"),
"SMOKE_EVAL_EPISODE_NUM": env("HOLOBRAIN_SMOKE_EVAL_EPISODE_NUM", "1"),
"SMOKE_EVAL_MODE": env("HOLOBRAIN_SMOKE_EVAL_MODE", "local"),
"SMOKE_DEVICE": env("HOLOBRAIN_SMOKE_DEVICE", "cuda"),
}
test_values = {
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
"ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"),
"ROBOTWIN_ROOT": env("HOLOBRAIN_ROBOTWIN_ROOT", "/opt/RoboTwin"),
"ROBOTWIN_ASSET_ROOT": env("HOLOBRAIN_ROBOTWIN_ASSET_ROOT", ""),
"INFERENCE_PREFIX": env("HOLOBRAIN_INFERENCE_PREFIX", ""),
"UR5_WSG_ARM_DISTANCE": env("HOLOBRAIN_UR5_WSG_ARM_DISTANCE", "0.8"),
"ROBOTWIN_CAMERA_SHADER": env("HOLOBRAIN_ROBOTWIN_CAMERA_SHADER", "default"),
"VULKAN_ICD_FILENAMES": env("HOLOBRAIN_VULKAN_ICD_FILENAMES", ""),
"TASK_NAME": env("HOLOBRAIN_INFERENCE_TASK_NAME", env("HOLOBRAIN_SMOKE_TASK_NAME", "beat_block_hammer")),
"EPISODE_NUM": env("HOLOBRAIN_INFERENCE_EPISODE_NUM", "1"),
"INFERENCE_MODE": env("HOLOBRAIN_INFERENCE_MODE", "local"),
"DEVICE": env("HOLOBRAIN_INFERENCE_DEVICE", "cuda"),
}
export_values = {key: value for key, value in export_values.items() if value}
eval_values = {key: value for key, value in eval_values.items() if value}
test_values = {key: value for key, value in test_values.items() if value}
nodes = [
{"id": "data-1", "type": "operator", "data": with_resource_overrides(node_data(operators[ids[0]], override_hyperparams(operators[ids[0]], data_values)), "HOLOBRAIN_DATA_PREPARE"), "position": {"x": 0, "y": 0}},
{"id": "train-1", "type": "operator", "data": node_data(operators[ids[1]], override_hyperparams(operators[ids[1]], train_values)), "position": {"x": 340, "y": 0}},
{"id": "export-1", "type": "operator", "data": node_data(operators[ids[2]], override_hyperparams(operators[ids[2]], export_values)), "position": {"x": 680, "y": 0}},
{"id": "eval-1", "type": "operator", "data": node_data(operators[ids[3]], override_hyperparams(operators[ids[3]], eval_values)), "position": {"x": 1020, "y": 0}},
{"id": "test-1", "type": "operator", "data": node_data(operators[ids[4]], override_hyperparams(operators[ids[4]], test_values)), "position": {"x": 1360, "y": 0}},
]
edges = [
{"id": "edge-data-train", "source": "data-1", "sourceHandle": "out-1", "target": "train-1", "targetHandle": "in-1"},
{"id": "edge-train-export", "source": "train-1", "sourceHandle": "out-1", "target": "export-1", "targetHandle": "in-1"},
{"id": "edge-export-eval", "source": "export-1", "sourceHandle": "out-1", "target": "eval-1", "targetHandle": "in-1"},
{"id": "edge-export-test", "source": "export-1", "sourceHandle": "out-1", "target": "test-1", "targetHandle": "in-1"},
{"id": "edge-data-test", "source": "data-1", "sourceHandle": "out-1", "target": "test-1", "targetHandle": "in-2"},
]
print(json.dumps({
"name": f"HoloBrain {env('HOLOBRAIN_TRAINING_MODE', 'smoke').title()} Training Acceptance",
"description": "Both smoke and full modes require the complete real data -> train -> export -> eval/inference chain.",
"type": "batch",
"nodeTimeoutSeconds": 0,
"runTimeoutSeconds": 0,
"maxRetryAttempts": 0,
"nodes": nodes,
"edges": edges,
}, ensure_ascii=False))
PY
)"
workflow_list="$(api_curl "$API_BASE_URL/api/projects/$project_id/workflows")"
workflow_id="$(printf '%s' "$workflow_list" | HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 -c '
import json, os, sys
mode = os.environ["HOLOBRAIN_TRAINING_MODE"].title()
name = f"HoloBrain {mode} Training Acceptance"
payload = json.load(sys.stdin)
items = payload.get("items", payload) if isinstance(payload, dict) else payload
for item in items:
if str(item.get("name", "")).strip() == name:
print(item.get("id", ""))
break
')"
if [[ -z "$workflow_id" ]]; then
workflow_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")"
workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')"
printf 'created workflow: %s\n' "$workflow_id"
else
workflow_response="$(api_curl_write -X PUT "$API_BASE_URL/api/workflows/$workflow_id" -H 'Content-Type: application/json' -d "$workflow_payload")"
printf 'reused workflow: %s\n' "$workflow_id"
fi
existing_runs="$(api_curl "$API_BASE_URL/api/runs?projectId=$project_id&limit=100")"
active_run_id="$(printf '%s' "$existing_runs" | python3 -c '
import json,sys
payload=json.load(sys.stdin)
items=payload.get("items", payload) if isinstance(payload,dict) else payload
for item in items:
if item.get("status") in {"queued","running","stopping"}:
print(item.get("id", ""))
break
')"
if [[ -n "$active_run_id" ]]; then
echo "ERROR: HoloBrain acceptance already has an active run: $active_run_id" >&2
exit 3
fi
queued_run="$(api_curl_write -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')"
run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')"
printf 'queued run: %s\n' "$run_id"
deadline=$((SECONDS + HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS))
final_run=""
while (( SECONDS < deadline )); do
run_json="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
status="$(printf '%s' "$run_json" | json_get 'json.load(sys.stdin)["status"]')"
node_statuses="$(
printf '%s' "$run_json" | python3 -c 'import json,sys; r=json.load(sys.stdin); print({n.get("nodeId"): n.get("status") for n in r.get("nodeExecutions") or []})'
)"
printf 'run status: %s nodes: %s\n' "$status" "$node_statuses"
if RUN_JSON="$run_json" HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 - <<'PY'
import json
import os
import sys
run = json.loads(os.environ["RUN_JSON"])
statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []}
mode = os.environ["HOLOBRAIN_TRAINING_MODE"]
expected = {"data-1", "train-1", "export-1", "eval-1", "test-1"}
if expected <= statuses.keys() and all(statuses[node_id] == "completed" for node_id in expected):
raise SystemExit(0)
if run.get("status") in {"failed", "cancelled"}:
raise SystemExit(2)
raise SystemExit(1)
PY
then
final_run="$run_json"
break
else
code=$?
if [[ "$code" -eq 2 ]]; then
final_run="$run_json"
break
fi
fi
sleep 10
done
if [[ -z "$final_run" ]]; then
final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
fi
FINAL_RUN="$final_run" HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 - <<'PY'
import json
import os
run = json.loads(os.environ["FINAL_RUN"])
statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []}
artifact_items = [
item
for node in run.get("nodeExecutions") or []
for item in node.get("artifacts") or []
]
artifacts = [item.get("name", "") for item in artifact_items]
artifact_data = {item.get("name", ""): item.get("data") for item in artifact_items if isinstance(item.get("data"), dict)}
mode = os.environ["HOLOBRAIN_TRAINING_MODE"]
expected_nodes = {"data-1", "train-1", "export-1", "eval-1", "test-1"}
expected_artifacts = {
"holobrain-data-manifest.json",
"holobrain-training-report.json",
"holobrain-export-manifest.json",
"holobrain-robotwin-eval-report.json",
"holobrain-inference-result.json",
}
training = artifact_data.get("holobrain-training-report.json") or {}
inference = artifact_data.get("holobrain-inference-result.json") or {}
evaluation = artifact_data.get("holobrain-robotwin-eval-report.json") or {}
accepted = (
run.get("status") == "completed"
and expected_nodes <= statuses.keys()
and all(statuses[node_id] == "completed" for node_id in expected_nodes)
and expected_artifacts <= set(artifacts)
and training.get("trainingMode") == mode
and inference.get("execution", {}).get("actualModelExecution") is True
and inference.get("execution", {}).get("syntheticResult") is False
and evaluation.get("execution", {}).get("backend") == "official-robotwin-evaluator"
and evaluation.get("execution", {}).get("syntheticResult") is False
)
summary = {
"trainingMode": mode,
"projectId": run.get("projectId"),
"workflowId": run.get("workflowId"),
"runId": run.get("id"),
"status": run.get("status"),
"nodeStatuses": statuses,
"artifacts": sorted(artifacts),
"accepted": accepted,
}
print("holobrain workflow acceptance:", json.dumps(summary, ensure_ascii=False))
if not accepted:
raise SystemExit(
f"HoloBrain {mode} acceptance did not complete the real data/train/export/eval/inference contract"
)
PY