forked from longtao.wu/droboticflow
467 lines
20 KiB
Bash
Executable File
467 lines
20 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
: "${DROBOTICFLOW_ACCEPTANCE_PASSWORD:?Set DROBOTICFLOW_ACCEPTANCE_PASSWORD before running acceptance}"
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
export DEPLOY_ENV
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
# shellcheck source=scripts/lib/service-endpoints.sh
|
|
source "$ROOT_DIR/scripts/lib/service-endpoints.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
|
|
API_TOKEN_SECRET_NAME="${HOLOBRAIN_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow-auth-secret}"
|
|
API_TOKEN_SECRET_KEY="${HOLOBRAIN_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}"
|
|
DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}"
|
|
HOLOBRAIN_TRAINING_MODE="${HOLOBRAIN_TRAINING_MODE:-smoke}"
|
|
if [[ "$HOLOBRAIN_TRAINING_MODE" != "smoke" && "$HOLOBRAIN_TRAINING_MODE" != "full" ]]; then
|
|
echo "ERROR: HOLOBRAIN_TRAINING_MODE must be smoke or full." >&2
|
|
exit 2
|
|
fi
|
|
if [[ -z "${HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS:-}" ]]; then
|
|
if [[ "$HOLOBRAIN_TRAINING_MODE" == "smoke" ]]; then
|
|
HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS=7200
|
|
else
|
|
HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS=259200
|
|
fi
|
|
fi
|
|
export HOLOBRAIN_TRAINING_MODE HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS
|
|
|
|
if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then
|
|
public_host="$(
|
|
kubectl -n "$K8S_NAMESPACE" get svc "$API_PUBLIC_SERVICE_NAME" -o json | python3 -c '
|
|
import ipaddress
|
|
import json
|
|
import sys
|
|
payload = json.load(sys.stdin)
|
|
hosts = [item.get("ip") or item.get("hostname") for item in payload.get("status", {}).get("loadBalancer", {}).get("ingress", [])]
|
|
hosts = [item for item in hosts if item]
|
|
if not hosts:
|
|
raise SystemExit("no LoadBalancer ingress host is available")
|
|
for host in hosts:
|
|
try:
|
|
if not ipaddress.ip_address(host).is_private:
|
|
print(host)
|
|
raise SystemExit(0)
|
|
except ValueError:
|
|
print(host)
|
|
raise SystemExit(0)
|
|
print(hosts[0])
|
|
'
|
|
)"
|
|
export API_BASE_URL="http://${public_host}"
|
|
fi
|
|
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
|
|
|
|
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
api_token_secret_json="$(kubectl -n "$K8S_NAMESPACE" get secret "$API_TOKEN_SECRET_NAME" -o json 2>/dev/null || true)"
|
|
DROBOTICFLOW_API_TOKEN="$(
|
|
printf "%s" "$api_token_secret_json" | API_TOKEN_SECRET_KEY="$API_TOKEN_SECRET_KEY" python3 -c '
|
|
import base64
|
|
import json
|
|
import os
|
|
import sys
|
|
try:
|
|
payload = json.load(sys.stdin)
|
|
except json.JSONDecodeError:
|
|
raise SystemExit(0)
|
|
raw = payload.get("data", {}).get(os.environ["API_TOKEN_SECRET_KEY"])
|
|
if isinstance(raw, str) and raw:
|
|
print(base64.b64decode(raw, validate=True).decode("utf-8", errors="replace").strip(), end="")
|
|
'
|
|
)"
|
|
export DROBOTICFLOW_API_TOKEN
|
|
fi
|
|
if [[ -z "${DROBOTICFLOW_API_TOKEN:-}" ]]; then
|
|
echo "ERROR: DROBOTICFLOW_API_TOKEN or ${K8S_NAMESPACE}/${API_TOKEN_SECRET_NAME}.${API_TOKEN_SECRET_KEY} is required." >&2
|
|
exit 1
|
|
fi
|
|
|
|
json_get() {
|
|
local expression="$1"
|
|
python3 -c "import json,sys; print(${expression})"
|
|
}
|
|
|
|
COOKIE_JAR="$(mktemp)"
|
|
cleanup() {
|
|
rm -f "$COOKIE_JAR"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
api_curl() {
|
|
curl --retry 6 --retry-delay 2 --retry-connrefused --retry-all-errors -fsS \
|
|
-b "$COOKIE_JAR" -c "$COOKIE_JAR" -H "Authorization: Bearer ${DROBOTICFLOW_API_TOKEN}" "$@"
|
|
}
|
|
|
|
csrf_token() {
|
|
awk '$6 == "droboticflow_csrf" { value=$7 } END { print value }' "$COOKIE_JAR"
|
|
}
|
|
|
|
api_curl_write() {
|
|
local token
|
|
token="$(csrf_token)"
|
|
if [[ -n "$token" ]]; then
|
|
api_curl -H "X-Droboticflow-CSRF-Token: $token" "$@"
|
|
return
|
|
fi
|
|
api_curl "$@"
|
|
}
|
|
|
|
api_health="$(api_curl "$API_BASE_URL/api/health")"
|
|
printf 'api health: %s\n' "$api_health"
|
|
printf '%s' "$api_health" | grep -q '"status":"ok"'
|
|
|
|
login_payload="$(
|
|
DROBOTICFLOW_ACCEPTANCE_EMAIL="$DROBOTICFLOW_ACCEPTANCE_EMAIL" \
|
|
DROBOTICFLOW_ACCEPTANCE_PASSWORD="$DROBOTICFLOW_ACCEPTANCE_PASSWORD" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
print(json.dumps({"email": os.environ["DROBOTICFLOW_ACCEPTANCE_EMAIL"], "password": os.environ["DROBOTICFLOW_ACCEPTANCE_PASSWORD"]}))
|
|
PY
|
|
)"
|
|
api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/login" -d "$login_payload" >/dev/null
|
|
|
|
project_payload="$(
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
mode = os.environ["HOLOBRAIN_TRAINING_MODE"]
|
|
print(json.dumps({"name": f"HoloBrain {mode.title()} Training Acceptance", "description": f"RoboOrchardLab HoloBrain {mode} data/train/export/eval/inference verification", "status": "active"}, ensure_ascii=False))
|
|
PY
|
|
)"
|
|
project_list="$(api_curl "$API_BASE_URL/api/projects")"
|
|
project_id="$(printf '%s' "$project_list" | HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 -c '
|
|
import json, os, sys
|
|
mode = os.environ["HOLOBRAIN_TRAINING_MODE"].title()
|
|
name = f"HoloBrain {mode} Training Acceptance"
|
|
payload = json.load(sys.stdin)
|
|
items = payload.get("items", payload) if isinstance(payload, dict) else payload
|
|
for item in items:
|
|
if str(item.get("name", "")).strip() == name:
|
|
print(item.get("id", ""))
|
|
break
|
|
')"
|
|
if [[ -z "$project_id" ]]; then
|
|
project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")"
|
|
project_id="$(printf '%s' "$project_response" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'created project: %s\n' "$project_id"
|
|
else
|
|
printf 'reusing project: %s\n' "$project_id"
|
|
fi
|
|
|
|
acceptance_suffix="${HOLOBRAIN_ACCEPTANCE_SUFFIX:-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)-$$}"
|
|
workflow_payload="$(
|
|
ROOT_DIR="$ROOT_DIR" HOLOBRAIN_ACCEPTANCE_SUFFIX="$acceptance_suffix" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import pathlib
|
|
|
|
root = pathlib.Path(os.environ["ROOT_DIR"])
|
|
catalog = json.loads((root / "internal/api/builtin_operators_catalog.json").read_text(encoding="utf-8"))
|
|
operators = {item.get("id"): item for item in catalog}
|
|
ids = [
|
|
"operator-builtin-holobrain-data-prepare",
|
|
"operator-builtin-holobrain-train",
|
|
"operator-builtin-holobrain-export",
|
|
"operator-builtin-holobrain-robotwin-eval",
|
|
"operator-builtin-holobrain-inference-test",
|
|
]
|
|
missing = [item for item in ids if item not in operators]
|
|
if missing:
|
|
raise SystemExit(f"missing HoloBrain operators in catalog: {missing}")
|
|
|
|
def env(name, default=""):
|
|
return os.environ.get(name, default)
|
|
|
|
def override_hyperparams(operator, values):
|
|
result = []
|
|
seen = set()
|
|
for item in operator.get("hyperparams") or []:
|
|
next_item = dict(item)
|
|
key = str(next_item.get("key", ""))
|
|
if key in values:
|
|
next_item["value"] = values[key]
|
|
seen.add(key)
|
|
result.append(next_item)
|
|
for key, value in values.items():
|
|
if key not in seen:
|
|
result.append({"key": key, "value": value, "type": "string", "description": "Acceptance override"})
|
|
return result
|
|
|
|
def operator_image(operator):
|
|
slug = str(operator["id"]).removeprefix("operator-builtin-")
|
|
specific_key = "HOLOBRAIN_OPERATOR_IMAGE_TAG_" + slug.upper().replace("-", "_")
|
|
tag = env(specific_key) or env("HOLOBRAIN_OPERATOR_IMAGE_TAG")
|
|
if not tag:
|
|
return operator.get("imageId") or operator["image"]
|
|
registry = env("HOLOBRAIN_OPERATOR_IMAGE_REGISTRY", "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc")
|
|
return f"{registry}/droboticflow-operator-{slug}:{tag}"
|
|
|
|
def node_data(operator, hyperparams):
|
|
resources = operator.get("resources") or {}
|
|
image = operator_image(operator)
|
|
data = {
|
|
"label": operator["name"],
|
|
"category": operator["category"],
|
|
"description": operator.get("description", ""),
|
|
"operatorId": operator["id"],
|
|
"operatorImageId": image,
|
|
"image": image,
|
|
"containerCommand": operator.get("containerCommand", []),
|
|
"containerArgs": operator.get("containerArgs", []),
|
|
"runtimeContract": operator.get("runtimeContract", {}),
|
|
"skillDefinition": operator.get("skillDefinition", ""),
|
|
"inputs": operator.get("inputs") or [],
|
|
"outputs": operator.get("outputs") or [],
|
|
"hyperparams": hyperparams,
|
|
}
|
|
if resources.get("cpuLimit"):
|
|
data["cpu"] = resources["cpuLimit"]
|
|
if resources.get("memoryLimit"):
|
|
data["memory"] = resources["memoryLimit"]
|
|
if resources.get("gpuLimit"):
|
|
data["gpu"] = resources["gpuLimit"]
|
|
return data
|
|
|
|
def with_resource_overrides(data, prefix):
|
|
cpu = env(prefix + "_CPU")
|
|
memory = env(prefix + "_MEMORY")
|
|
gpu = env(prefix + "_GPU")
|
|
if cpu:
|
|
data["cpu"] = cpu
|
|
if memory:
|
|
data["memory"] = memory
|
|
if gpu:
|
|
data["gpu"] = gpu
|
|
return data
|
|
|
|
data_values = {
|
|
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
|
|
"DATA_SOURCE": env("HOLOBRAIN_DATA_SOURCE", "local"),
|
|
"ROBOTWIN_SOURCE_PATH": env("HOLOBRAIN_ROBOTWIN_SOURCE_PATH", env("HOLOBRAIN_WORKFLOW_SOURCE_ROOT", "")),
|
|
"EXPECTED_LMDB_DIRS_JSON": env("HOLOBRAIN_EXPECTED_LMDB_DIRS_JSON", ""),
|
|
"ROBOTWIN_PACK_TASK_NAMES": env("HOLOBRAIN_ROBOTWIN_PACK_TASK_NAMES", ""),
|
|
}
|
|
data_values = {key: value for key, value in data_values.items() if value}
|
|
train_values = {
|
|
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
|
|
"ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"),
|
|
"CONFIG_PATH": "projects/holobrain/configs/config_holobrain_qwen_common.py",
|
|
"LAUNCH_MODE": env("HOLOBRAIN_LAUNCH_MODE", "single"),
|
|
"GPU_COUNT": env("HOLOBRAIN_GPU_COUNT", "1"),
|
|
"CONFIG_KWARGS_JSON": env("HOLOBRAIN_CONFIG_KWARGS_JSON", "{}"),
|
|
"SMOKE_MAX_STEP": env("HOLOBRAIN_SMOKE_MAX_STEP", "2"),
|
|
"SMOKE_BATCH_SIZE": env("HOLOBRAIN_SMOKE_BATCH_SIZE", "1"),
|
|
"SMOKE_NUM_WORKERS": env("HOLOBRAIN_SMOKE_NUM_WORKERS", "0"),
|
|
"RESUME_POLICY": env("HOLOBRAIN_RESUME_POLICY", "auto"),
|
|
"RESUME_FROM": env("HOLOBRAIN_RESUME_FROM", ""),
|
|
"CHECKPOINT_ARTIFACT": env("HOLOBRAIN_CHECKPOINT_ARTIFACT", "latest_valid"),
|
|
"OUTPUT_CHECKPOINT_PATH": env("HOLOBRAIN_OUTPUT_CHECKPOINT_PATH", ""),
|
|
"PROGRESS_INTERVAL_SECONDS": env("HOLOBRAIN_PROGRESS_INTERVAL_SECONDS", "15"),
|
|
"MODEL_DTYPE": env("HOLOBRAIN_MODEL_DTYPE", "bfloat16"),
|
|
"INPUT_DTYPE": env("HOLOBRAIN_INPUT_DTYPE", "float32"),
|
|
}
|
|
export_values = {
|
|
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
|
|
"ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"),
|
|
"CONFIG_PATH": "projects/holobrain/configs/config_holobrain_qwen_common.py",
|
|
"CONFIG_KWARGS_JSON": env("HOLOBRAIN_EXPORT_CONFIG_KWARGS_JSON", env("HOLOBRAIN_CONFIG_KWARGS_JSON", "{}")),
|
|
"CHECKPOINT_SELECTION": env("HOLOBRAIN_CHECKPOINT_SELECTION", "latest_valid"),
|
|
}
|
|
eval_values = {
|
|
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
|
|
"ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"),
|
|
"ROBOTWIN_ROOT": env("HOLOBRAIN_ROBOTWIN_ROOT", "/opt/RoboTwin"),
|
|
"ROBOTWIN_ASSET_ROOT": env("HOLOBRAIN_ROBOTWIN_ASSET_ROOT", ""),
|
|
"INFERENCE_PREFIX": env("HOLOBRAIN_EVAL_INFERENCE_PREFIX", ""),
|
|
"UR5_WSG_ARM_DISTANCE": env("HOLOBRAIN_UR5_WSG_ARM_DISTANCE", "0.8"),
|
|
"ROBOTWIN_CAMERA_SHADER": env("HOLOBRAIN_ROBOTWIN_CAMERA_SHADER", "default"),
|
|
"VULKAN_ICD_FILENAMES": env("HOLOBRAIN_VULKAN_ICD_FILENAMES", ""),
|
|
"EVAL_MODE": env("HOLOBRAIN_EVAL_MODE", "ray"),
|
|
"TASK_NAMES_JSON": env("HOLOBRAIN_EVAL_TASK_NAMES_JSON", "[]"),
|
|
"EPISODE_NUM": env("HOLOBRAIN_EVAL_EPISODE_NUM", "100"),
|
|
"GPU_IDS_JSON": env("HOLOBRAIN_EVAL_GPU_IDS_JSON", "[0]"),
|
|
"WORKERS_PER_GPU": env("HOLOBRAIN_EVAL_WORKERS_PER_GPU", "1"),
|
|
"DEVICE": env("HOLOBRAIN_EVAL_DEVICE", "cuda"),
|
|
"SAVE_VIDEO": env("HOLOBRAIN_EVAL_SAVE_VIDEO", "false"),
|
|
"SMOKE_TASK_NAME": env("HOLOBRAIN_SMOKE_TASK_NAME", "beat_block_hammer"),
|
|
"SMOKE_EVAL_EPISODE_NUM": env("HOLOBRAIN_SMOKE_EVAL_EPISODE_NUM", "1"),
|
|
"SMOKE_EVAL_MODE": env("HOLOBRAIN_SMOKE_EVAL_MODE", "local"),
|
|
"SMOKE_DEVICE": env("HOLOBRAIN_SMOKE_DEVICE", "cuda"),
|
|
}
|
|
test_values = {
|
|
"TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"),
|
|
"ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"),
|
|
"ROBOTWIN_ROOT": env("HOLOBRAIN_ROBOTWIN_ROOT", "/opt/RoboTwin"),
|
|
"ROBOTWIN_ASSET_ROOT": env("HOLOBRAIN_ROBOTWIN_ASSET_ROOT", ""),
|
|
"INFERENCE_PREFIX": env("HOLOBRAIN_INFERENCE_PREFIX", ""),
|
|
"UR5_WSG_ARM_DISTANCE": env("HOLOBRAIN_UR5_WSG_ARM_DISTANCE", "0.8"),
|
|
"ROBOTWIN_CAMERA_SHADER": env("HOLOBRAIN_ROBOTWIN_CAMERA_SHADER", "default"),
|
|
"VULKAN_ICD_FILENAMES": env("HOLOBRAIN_VULKAN_ICD_FILENAMES", ""),
|
|
"TASK_NAME": env("HOLOBRAIN_INFERENCE_TASK_NAME", env("HOLOBRAIN_SMOKE_TASK_NAME", "beat_block_hammer")),
|
|
"EPISODE_NUM": env("HOLOBRAIN_INFERENCE_EPISODE_NUM", "1"),
|
|
"INFERENCE_MODE": env("HOLOBRAIN_INFERENCE_MODE", "local"),
|
|
"DEVICE": env("HOLOBRAIN_INFERENCE_DEVICE", "cuda"),
|
|
}
|
|
export_values = {key: value for key, value in export_values.items() if value}
|
|
eval_values = {key: value for key, value in eval_values.items() if value}
|
|
test_values = {key: value for key, value in test_values.items() if value}
|
|
nodes = [
|
|
{"id": "data-1", "type": "operator", "data": with_resource_overrides(node_data(operators[ids[0]], override_hyperparams(operators[ids[0]], data_values)), "HOLOBRAIN_DATA_PREPARE"), "position": {"x": 0, "y": 0}},
|
|
{"id": "train-1", "type": "operator", "data": node_data(operators[ids[1]], override_hyperparams(operators[ids[1]], train_values)), "position": {"x": 340, "y": 0}},
|
|
{"id": "export-1", "type": "operator", "data": node_data(operators[ids[2]], override_hyperparams(operators[ids[2]], export_values)), "position": {"x": 680, "y": 0}},
|
|
{"id": "eval-1", "type": "operator", "data": node_data(operators[ids[3]], override_hyperparams(operators[ids[3]], eval_values)), "position": {"x": 1020, "y": 0}},
|
|
{"id": "test-1", "type": "operator", "data": node_data(operators[ids[4]], override_hyperparams(operators[ids[4]], test_values)), "position": {"x": 1360, "y": 0}},
|
|
]
|
|
edges = [
|
|
{"id": "edge-data-train", "source": "data-1", "sourceHandle": "out-1", "target": "train-1", "targetHandle": "in-1"},
|
|
{"id": "edge-train-export", "source": "train-1", "sourceHandle": "out-1", "target": "export-1", "targetHandle": "in-1"},
|
|
{"id": "edge-export-eval", "source": "export-1", "sourceHandle": "out-1", "target": "eval-1", "targetHandle": "in-1"},
|
|
{"id": "edge-export-test", "source": "export-1", "sourceHandle": "out-1", "target": "test-1", "targetHandle": "in-1"},
|
|
{"id": "edge-data-test", "source": "data-1", "sourceHandle": "out-1", "target": "test-1", "targetHandle": "in-2"},
|
|
]
|
|
print(json.dumps({
|
|
"name": f"HoloBrain {env('HOLOBRAIN_TRAINING_MODE', 'smoke').title()} Training Acceptance",
|
|
"description": "Both smoke and full modes require the complete real data -> train -> export -> eval/inference chain.",
|
|
"type": "batch",
|
|
"nodeTimeoutSeconds": 0,
|
|
"runTimeoutSeconds": 0,
|
|
"maxRetryAttempts": 0,
|
|
"nodes": nodes,
|
|
"edges": edges,
|
|
}, ensure_ascii=False))
|
|
PY
|
|
)"
|
|
workflow_list="$(api_curl "$API_BASE_URL/api/projects/$project_id/workflows")"
|
|
workflow_id="$(printf '%s' "$workflow_list" | HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 -c '
|
|
import json, os, sys
|
|
mode = os.environ["HOLOBRAIN_TRAINING_MODE"].title()
|
|
name = f"HoloBrain {mode} Training Acceptance"
|
|
payload = json.load(sys.stdin)
|
|
items = payload.get("items", payload) if isinstance(payload, dict) else payload
|
|
for item in items:
|
|
if str(item.get("name", "")).strip() == name:
|
|
print(item.get("id", ""))
|
|
break
|
|
')"
|
|
if [[ -z "$workflow_id" ]]; then
|
|
workflow_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects/$project_id/workflows" -H 'Content-Type: application/json' -d "$workflow_payload")"
|
|
workflow_id="$(printf '%s' "$workflow_response" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'created workflow: %s\n' "$workflow_id"
|
|
else
|
|
workflow_response="$(api_curl_write -X PUT "$API_BASE_URL/api/workflows/$workflow_id" -H 'Content-Type: application/json' -d "$workflow_payload")"
|
|
printf 'reused workflow: %s\n' "$workflow_id"
|
|
fi
|
|
|
|
existing_runs="$(api_curl "$API_BASE_URL/api/runs?projectId=$project_id&limit=100")"
|
|
active_run_id="$(printf '%s' "$existing_runs" | python3 -c '
|
|
import json,sys
|
|
payload=json.load(sys.stdin)
|
|
items=payload.get("items", payload) if isinstance(payload,dict) else payload
|
|
for item in items:
|
|
if item.get("status") in {"queued","running","stopping"}:
|
|
print(item.get("id", ""))
|
|
break
|
|
')"
|
|
if [[ -n "$active_run_id" ]]; then
|
|
echo "ERROR: HoloBrain acceptance already has an active run: $active_run_id" >&2
|
|
exit 3
|
|
fi
|
|
|
|
queued_run="$(api_curl_write -X POST "$API_BASE_URL/api/workflows/$workflow_id/run" -H 'Content-Type: application/json' -d '{"trigger":"manual"}')"
|
|
run_id="$(printf '%s' "$queued_run" | json_get 'json.load(sys.stdin)["id"]')"
|
|
printf 'queued run: %s\n' "$run_id"
|
|
|
|
deadline=$((SECONDS + HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS))
|
|
final_run=""
|
|
while (( SECONDS < deadline )); do
|
|
run_json="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
|
|
status="$(printf '%s' "$run_json" | json_get 'json.load(sys.stdin)["status"]')"
|
|
node_statuses="$(
|
|
printf '%s' "$run_json" | python3 -c 'import json,sys; r=json.load(sys.stdin); print({n.get("nodeId"): n.get("status") for n in r.get("nodeExecutions") or []})'
|
|
)"
|
|
printf 'run status: %s nodes: %s\n' "$status" "$node_statuses"
|
|
if RUN_JSON="$run_json" HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
run = json.loads(os.environ["RUN_JSON"])
|
|
statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []}
|
|
mode = os.environ["HOLOBRAIN_TRAINING_MODE"]
|
|
expected = {"data-1", "train-1", "export-1", "eval-1", "test-1"}
|
|
if expected <= statuses.keys() and all(statuses[node_id] == "completed" for node_id in expected):
|
|
raise SystemExit(0)
|
|
if run.get("status") in {"failed", "cancelled"}:
|
|
raise SystemExit(2)
|
|
raise SystemExit(1)
|
|
PY
|
|
then
|
|
final_run="$run_json"
|
|
break
|
|
else
|
|
code=$?
|
|
if [[ "$code" -eq 2 ]]; then
|
|
final_run="$run_json"
|
|
break
|
|
fi
|
|
fi
|
|
sleep 10
|
|
done
|
|
if [[ -z "$final_run" ]]; then
|
|
final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")"
|
|
fi
|
|
|
|
FINAL_RUN="$final_run" HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
run = json.loads(os.environ["FINAL_RUN"])
|
|
statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []}
|
|
artifact_items = [
|
|
item
|
|
for node in run.get("nodeExecutions") or []
|
|
for item in node.get("artifacts") or []
|
|
]
|
|
artifacts = [item.get("name", "") for item in artifact_items]
|
|
artifact_data = {item.get("name", ""): item.get("data") for item in artifact_items if isinstance(item.get("data"), dict)}
|
|
mode = os.environ["HOLOBRAIN_TRAINING_MODE"]
|
|
expected_nodes = {"data-1", "train-1", "export-1", "eval-1", "test-1"}
|
|
expected_artifacts = {
|
|
"holobrain-data-manifest.json",
|
|
"holobrain-training-report.json",
|
|
"holobrain-export-manifest.json",
|
|
"holobrain-robotwin-eval-report.json",
|
|
"holobrain-inference-result.json",
|
|
}
|
|
training = artifact_data.get("holobrain-training-report.json") or {}
|
|
inference = artifact_data.get("holobrain-inference-result.json") or {}
|
|
evaluation = artifact_data.get("holobrain-robotwin-eval-report.json") or {}
|
|
accepted = (
|
|
run.get("status") == "completed"
|
|
and expected_nodes <= statuses.keys()
|
|
and all(statuses[node_id] == "completed" for node_id in expected_nodes)
|
|
and expected_artifacts <= set(artifacts)
|
|
and training.get("trainingMode") == mode
|
|
and inference.get("execution", {}).get("actualModelExecution") is True
|
|
and inference.get("execution", {}).get("syntheticResult") is False
|
|
and evaluation.get("execution", {}).get("backend") == "official-robotwin-evaluator"
|
|
and evaluation.get("execution", {}).get("syntheticResult") is False
|
|
)
|
|
summary = {
|
|
"trainingMode": mode,
|
|
"projectId": run.get("projectId"),
|
|
"workflowId": run.get("workflowId"),
|
|
"runId": run.get("id"),
|
|
"status": run.get("status"),
|
|
"nodeStatuses": statuses,
|
|
"artifacts": sorted(artifacts),
|
|
"accepted": accepted,
|
|
}
|
|
print("holobrain workflow acceptance:", json.dumps(summary, ensure_ascii=False))
|
|
if not accepted:
|
|
raise SystemExit(
|
|
f"HoloBrain {mode} acceptance did not complete the real data/train/export/eval/inference contract"
|
|
)
|
|
PY
|