diff --git a/backend/operator-runner/test_main.py b/backend/operator-runner/test_main.py index 1a9a8e81..a0c1a543 100644 --- a/backend/operator-runner/test_main.py +++ b/backend/operator-runner/test_main.py @@ -3416,20 +3416,23 @@ class ShellCommandOperatorTests(unittest.TestCase): eval_report = MODULE.load_artifact_payload(next(item for item in eval_artifacts if item["name"] == "holobrain-robotwin-eval-report.json")) self.assertEqual(eval_report["datasetType"], "holobrain-robotwin-evaluation") - sample = root / "sample.npz" - sample.write_bytes(b"sample") inference_artifacts = MODULE.builtin_holobrain_inference_test( "run-1", "test", root / "test-out", { "label": "HoloBrain 模型测试", - "hyperparams": [{"key": "SAMPLE_INPUT_PATH", "value": str(sample)}], + "hyperparams": [ + {"key": "ROBOORCHARDLAB_ROOT", "value": str(repo)}, + {"key": "ENV_JSON", "value": "{\"HOLOBRAIN_OPERATOR_DRY_RUN\":\"1\"}"}, + ], }, [export_manifest], ) inference_report = MODULE.load_artifact_payload(inference_artifacts[0]) self.assertEqual(inference_report["datasetType"], "holobrain-inference-test") + self.assertEqual(inference_report["status"], "dry-run") + self.assertFalse(inference_report["execution"]["actualModelExecution"]) def test_builtin_shell_command_requires_script_id(self): with tempfile.TemporaryDirectory() as temp_dir: diff --git a/backend/operators/builtin/act_policy_train/repository.json b/backend/operators/builtin/act_policy_train/repository.json index d0fc9b88..8093cffa 100644 --- a/backend/operators/builtin/act_policy_train/repository.json +++ b/backend/operators/builtin/act_policy_train/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "act_policy_train", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, @@ -58,15 +59,15 @@ "requiresGpu": true }, "runtimeDependencies": [ - "real-act-training-command", - "trajectory-demonstration-files", + "huggingface-lerobot-act", + "lerobot-dataset-or-local-root", "gpu-a800" ], "schemaVersion": "droboticflow.operator.runtime/v1" }, "runtimeDependencies": [ - "real-act-training-command", - "trajectory-demonstration-files", + "huggingface-lerobot-act", + "lerobot-dataset-or-local-root", "gpu-a800" ], "runtimeHandler": "builtin_act_policy_train", diff --git a/backend/operators/builtin/action_state_quality_validate/repository.json b/backend/operators/builtin/action_state_quality_validate/repository.json index 08f1975e..b1a53e18 100644 --- a/backend/operators/builtin/action_state_quality_validate/repository.json +++ b/backend/operators/builtin/action_state_quality_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "action_state_quality_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/agibot_world2026_validate/repository.json b/backend/operators/builtin/agibot_world2026_validate/repository.json index d00df9f0..6327cb84 100644 --- a/backend/operators/builtin/agibot_world2026_validate/repository.json +++ b/backend/operators/builtin/agibot_world2026_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "agibot_world2026_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/aloha_hdf5_validate/repository.json b/backend/operators/builtin/aloha_hdf5_validate/repository.json index 778347cb..3daa2075 100644 --- a/backend/operators/builtin/aloha_hdf5_validate/repository.json +++ b/backend/operators/builtin/aloha_hdf5_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "aloha_hdf5_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/audit_sample/repository.json b/backend/operators/builtin/audit_sample/repository.json index 321694ad..6488d840 100644 --- a/backend/operators/builtin/audit_sample/repository.json +++ b/backend/operators/builtin/audit_sample/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "audit_sample", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/bop_dataset_validate/repository.json b/backend/operators/builtin/bop_dataset_validate/repository.json index 15fb57e1..909c7556 100644 --- a/backend/operators/builtin/bop_dataset_validate/repository.json +++ b/backend/operators/builtin/bop_dataset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "bop_dataset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/bos_incremental_scan/repository.json b/backend/operators/builtin/bos_incremental_scan/repository.json index 3cd7144e..f064098a 100644 --- a/backend/operators/builtin/bos_incremental_scan/repository.json +++ b/backend/operators/builtin/bos_incremental_scan/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "bos_incremental_scan", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/bridgedata_tfds_manifest_validate/repository.json b/backend/operators/builtin/bridgedata_tfds_manifest_validate/repository.json index 800133a1..f87c8baf 100644 --- a/backend/operators/builtin/bridgedata_tfds_manifest_validate/repository.json +++ b/backend/operators/builtin/bridgedata_tfds_manifest_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "bridgedata_tfds_manifest_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/calvin_episode_validate/repository.json b/backend/operators/builtin/calvin_episode_validate/repository.json index 1e26104e..cbde8750 100644 --- a/backend/operators/builtin/calvin_episode_validate/repository.json +++ b/backend/operators/builtin/calvin_episode_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "calvin_episode_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/camera_calibration_validate/repository.json b/backend/operators/builtin/camera_calibration_validate/repository.json index 743866d7..3d88daee 100644 --- a/backend/operators/builtin/camera_calibration_validate/repository.json +++ b/backend/operators/builtin/camera_calibration_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "camera_calibration_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/coco_annotation_validate/repository.json b/backend/operators/builtin/coco_annotation_validate/repository.json index 1129cc6b..2b5dd8ce 100644 --- a/backend/operators/builtin/coco_annotation_validate/repository.json +++ b/backend/operators/builtin/coco_annotation_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "coco_annotation_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/conditional_route/repository.json b/backend/operators/builtin/conditional_route/repository.json index 12468546..522c7c84 100644 --- a/backend/operators/builtin/conditional_route/repository.json +++ b/backend/operators/builtin/conditional_route/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "conditional_route", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/csv_to_json/repository.json b/backend/operators/builtin/csv_to_json/repository.json index b2f14b7d..74f1b8ba 100644 --- a/backend/operators/builtin/csv_to_json/repository.json +++ b/backend/operators/builtin/csv_to_json/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "csv_to_json", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/dataset_provenance_audit/repository.json b/backend/operators/builtin/dataset_provenance_audit/repository.json index 3fb59a86..72b8331b 100644 --- a/backend/operators/builtin/dataset_provenance_audit/repository.json +++ b/backend/operators/builtin/dataset_provenance_audit/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "dataset_provenance_audit", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/dataset_schema_contract_validate/repository.json b/backend/operators/builtin/dataset_schema_contract_validate/repository.json index a1f2e4a2..6551f60a 100644 --- a/backend/operators/builtin/dataset_schema_contract_validate/repository.json +++ b/backend/operators/builtin/dataset_schema_contract_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "dataset_schema_contract_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/dataset_split_leakage_validate/repository.json b/backend/operators/builtin/dataset_split_leakage_validate/repository.json index 1df60335..4588770a 100644 --- a/backend/operators/builtin/dataset_split_leakage_validate/repository.json +++ b/backend/operators/builtin/dataset_split_leakage_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "dataset_split_leakage_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/dataset_split_manifest/repository.json b/backend/operators/builtin/dataset_split_manifest/repository.json index d96b8be8..561d28a7 100644 --- a/backend/operators/builtin/dataset_split_manifest/repository.json +++ b/backend/operators/builtin/dataset_split_manifest/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "dataset_split_manifest", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/dexycb_dataset_validate/repository.json b/backend/operators/builtin/dexycb_dataset_validate/repository.json index 84e6d3f4..76e3fa3c 100644 --- a/backend/operators/builtin/dexycb_dataset_validate/repository.json +++ b/backend/operators/builtin/dexycb_dataset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "dexycb_dataset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/diffusion_policy_train/repository.json b/backend/operators/builtin/diffusion_policy_train/repository.json index cacacf7d..17c7a314 100644 --- a/backend/operators/builtin/diffusion_policy_train/repository.json +++ b/backend/operators/builtin/diffusion_policy_train/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "diffusion_policy_train", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, @@ -58,14 +59,14 @@ "requiresGpu": true }, "runtimeDependencies": [ - "diffusion-policy-training-command", + "real-stanford-diffusion-policy", "zarr-or-lerobot-trajectory-files", "gpu-a800" ], "schemaVersion": "droboticflow.operator.runtime/v1" }, "runtimeDependencies": [ - "diffusion-policy-training-command", + "real-stanford-diffusion-policy", "zarr-or-lerobot-trajectory-files", "gpu-a800" ], diff --git a/backend/operators/builtin/directory_scan/repository.json b/backend/operators/builtin/directory_scan/repository.json index 1e85f19b..e76bb08a 100644 --- a/backend/operators/builtin/directory_scan/repository.json +++ b/backend/operators/builtin/directory_scan/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "directory_scan", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/droid_raw_episode_validate/repository.json b/backend/operators/builtin/droid_raw_episode_validate/repository.json index 7ef3d6fc..23580969 100644 --- a/backend/operators/builtin/droid_raw_episode_validate/repository.json +++ b/backend/operators/builtin/droid_raw_episode_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "droid_raw_episode_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/embodiment_metadata_validate/repository.json b/backend/operators/builtin/embodiment_metadata_validate/repository.json index 4575fbb6..6510abd3 100644 --- a/backend/operators/builtin/embodiment_metadata_validate/repository.json +++ b/backend/operators/builtin/embodiment_metadata_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "embodiment_metadata_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/episode_boundary_validate/repository.json b/backend/operators/builtin/episode_boundary_validate/repository.json index ce7c6848..dc76a217 100644 --- a/backend/operators/builtin/episode_boundary_validate/repository.json +++ b/backend/operators/builtin/episode_boundary_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "episode_boundary_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/force_torque_sensor_validate/repository.json b/backend/operators/builtin/force_torque_sensor_validate/repository.json index 9986e6ab..e5160326 100644 --- a/backend/operators/builtin/force_torque_sensor_validate/repository.json +++ b/backend/operators/builtin/force_torque_sensor_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "force_torque_sensor_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/furniturebench_pkl_demonstration_validate/repository.json b/backend/operators/builtin/furniturebench_pkl_demonstration_validate/repository.json index 1c7f6670..19bb3b32 100644 --- a/backend/operators/builtin/furniturebench_pkl_demonstration_validate/repository.json +++ b/backend/operators/builtin/furniturebench_pkl_demonstration_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "furniturebench_pkl_demonstration_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/graspnet_dataset_validate/repository.json b/backend/operators/builtin/graspnet_dataset_validate/repository.json index db16ae01..48b50675 100644 --- a/backend/operators/builtin/graspnet_dataset_validate/repository.json +++ b/backend/operators/builtin/graspnet_dataset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "graspnet_dataset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/hand_object_pose_validate/repository.json b/backend/operators/builtin/hand_object_pose_validate/repository.json index 345aa7be..f0da1d61 100644 --- a/backend/operators/builtin/hand_object_pose_validate/repository.json +++ b/backend/operators/builtin/hand_object_pose_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "hand_object_pose_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/hdf5_demonstration_summary/repository.json b/backend/operators/builtin/hdf5_demonstration_summary/repository.json index c1477b78..58a15c3f 100644 --- a/backend/operators/builtin/hdf5_demonstration_summary/repository.json +++ b/backend/operators/builtin/hdf5_demonstration_summary/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "hdf5_demonstration_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/hf_dataset_probe/repository.json b/backend/operators/builtin/hf_dataset_probe/repository.json index 0c48e43b..f826497f 100644 --- a/backend/operators/builtin/hf_dataset_probe/repository.json +++ b/backend/operators/builtin/hf_dataset_probe/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "hf_dataset_probe", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/ho3d_dataset_validate/repository.json b/backend/operators/builtin/ho3d_dataset_validate/repository.json index 4fb0b787..aba5dcb4 100644 --- a/backend/operators/builtin/ho3d_dataset_validate/repository.json +++ b/backend/operators/builtin/ho3d_dataset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "ho3d_dataset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/holobrain_data_prepare/repository.json b/backend/operators/builtin/holobrain_data_prepare/repository.json new file mode 100644 index 00000000..a5b2bd43 --- /dev/null +++ b/backend/operators/builtin/holobrain_data_prepare/repository.json @@ -0,0 +1,80 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", + "requiresPersistentWorkspaceEvidence": true, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "DATASET", + "containerArgs": [], + "containerCommand": [ + "python", + "/app/backend/operator-runner/main.py" + ], + "containerEntrypoint": "backend/operator-runner/main.py", + "entrypoint": "operator.py", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-data-prepare:5289510-holobrain-data-guard2-20260708050323", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-data-prepare@sha256:0df15a87262c4099bf5b6c9dbbf17ea54fcac40657dd93ca6ff49e6af69c3b43", + "metadata": "operator.json", + "name": "HoloBrain 数据准备", + "operatorId": "operator-builtin-holobrain-data-prepare", + "operatorSlug": "holobrain_data_prepare", + "preferredSlug": "holobrain_data_prepare", + "pythonImport": "backend.operators.builtin.holobrain_data_prepare.operator:run", + "repositoryId": "builtin/holobrain_data_prepare", + "runtimeContract": { + "acceptance": { + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": false, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_holobrain_data_prepare", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "longRunning": true, + "operatorId": "operator-builtin-holobrain-data-prepare", + "operatorSlug": "holobrain_data_prepare", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "minGpuCount": 0, + "recommendedGpuType": "", + "requiresGpu": false + }, + "runtimeDependencies": [ + "official-robotwin2-dataset", + "huggingface-hub", + "real-robotwin-files" + ], + "schemaVersion": "droboticflow.operator.runtime/v1" + }, + "runtimeDependencies": [ + "official-robotwin2-dataset", + "huggingface-hub", + "real-robotwin-files" + ], + "runtimeHandler": "builtin_holobrain_data_prepare", + "runtimeSource": "backend/operators/builtin/holobrain_data_prepare/operator.py", + "schemaVersion": "droboticflow.operator.repository/v1", + "sourceRoot": "backend/operators/builtin/holobrain_data_prepare", + "strictPocAcceptanceExternals": [], + "type": "builtin" +} diff --git a/backend/operators/builtin/holobrain_export/repository.json b/backend/operators/builtin/holobrain_export/repository.json new file mode 100644 index 00000000..daf9c499 --- /dev/null +++ b/backend/operators/builtin/holobrain_export/repository.json @@ -0,0 +1,79 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", + "requiresPersistentWorkspaceEvidence": true, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "containerArgs": [], + "containerCommand": [ + "python", + "/app/backend/operator-runner/main.py" + ], + "containerEntrypoint": "backend/operator-runner/main.py", + "entrypoint": "operator.py", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-export:a34570f-holobrain-exportrepair5-20260709014631", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-export@sha256:90e90351b00fac0405740b87281608ce51e2171e0a78ed81295a317a5f93e7f6", + "metadata": "operator.json", + "name": "HoloBrain 模型导出", + "operatorId": "operator-builtin-holobrain-export", + "operatorSlug": "holobrain_export", + "preferredSlug": "holobrain_export", + "pythonImport": "backend.operators.builtin.holobrain_export.operator:run", + "repositoryId": "builtin/holobrain_export", + "runtimeContract": { + "acceptance": { + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": true, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_holobrain_export", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "operatorId": "operator-builtin-holobrain-export", + "operatorSlug": "holobrain_export", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "minGpuCount": 1, + "recommendedGpuType": "NVIDIA A800 80GB", + "requiresGpu": true + }, + "runtimeDependencies": [ + "roboorchardlab-holobrain-export", + "holobrain-checkpoint", + "gpu-a800" + ], + "schemaVersion": "droboticflow.operator.runtime/v1" + }, + "runtimeDependencies": [ + "roboorchardlab-holobrain-export", + "holobrain-checkpoint", + "gpu-a800" + ], + "runtimeHandler": "builtin_holobrain_export", + "runtimeSource": "backend/operators/builtin/holobrain_export/operator.py", + "schemaVersion": "droboticflow.operator.repository/v1", + "sourceRoot": "backend/operators/builtin/holobrain_export", + "strictPocAcceptanceExternals": [], + "type": "builtin" +} diff --git a/backend/operators/builtin/holobrain_inference_test/repository.json b/backend/operators/builtin/holobrain_inference_test/repository.json new file mode 100644 index 00000000..360f1a30 --- /dev/null +++ b/backend/operators/builtin/holobrain_inference_test/repository.json @@ -0,0 +1,79 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", + "requiresPersistentWorkspaceEvidence": true, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "containerArgs": [], + "containerCommand": [ + "python", + "/app/backend/operator-runner/main.py" + ], + "containerEntrypoint": "backend/operator-runner/main.py", + "entrypoint": "operator.py", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-inference-test:5289510-holobrain-downstream2-202607082304", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-inference-test@sha256:3a79fd20722ce23a8b64eab3cd31e1b0434fef2564b3b6f55c02933d667abf33", + "metadata": "operator.json", + "name": "HoloBrain 模型测试", + "operatorId": "operator-builtin-holobrain-inference-test", + "operatorSlug": "holobrain_inference_test", + "preferredSlug": "holobrain_inference_test", + "pythonImport": "backend.operators.builtin.holobrain_inference_test.operator:run", + "repositoryId": "builtin/holobrain_inference_test", + "runtimeContract": { + "acceptance": { + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": true, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_holobrain_inference_test", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "operatorId": "operator-builtin-holobrain-inference-test", + "operatorSlug": "holobrain_inference_test", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "minGpuCount": 1, + "recommendedGpuType": "NVIDIA A800 80GB", + "requiresGpu": true + }, + "runtimeDependencies": [ + "roboorchardlab-holobrain-pipeline", + "real-sample-path", + "gpu-a800" + ], + "schemaVersion": "droboticflow.operator.runtime/v1" + }, + "runtimeDependencies": [ + "roboorchardlab-holobrain-pipeline", + "real-sample-path", + "gpu-a800" + ], + "runtimeHandler": "builtin_holobrain_inference_test", + "runtimeSource": "backend/operators/builtin/holobrain_inference_test/operator.py", + "schemaVersion": "droboticflow.operator.repository/v1", + "sourceRoot": "backend/operators/builtin/holobrain_inference_test", + "strictPocAcceptanceExternals": [], + "type": "builtin" +} diff --git a/backend/operators/builtin/holobrain_operator.py b/backend/operators/builtin/holobrain_operator.py index 885d1fe4..b7d58d1a 100644 --- a/backend/operators/builtin/holobrain_operator.py +++ b/backend/operators/builtin/holobrain_operator.py @@ -42,6 +42,19 @@ DEFAULT_ARROW_CAMERA_MAP = { } DEFAULT_ARROW_JOINT_INDEXES_16_TO_14 = [0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 15] TRUE_VALUES = {"1", "true", "yes", "on"} +TRAINING_MODES = {"smoke", "full"} +PROGRESS_MARKER = "DROBOTICFLOW_PROGRESS " +MODEL_DTYPE_ALIASES = { + "bf16": "bfloat16", + "bfloat16": "bfloat16", + "torch.bfloat16": "bfloat16", +} +INPUT_DTYPE_ALIASES = { + "fp32": "float32", + "float": "float32", + "float32": "float32", + "torch.float32": "float32", +} def _params(helpers: HelperMap, node_data: dict[str, Any]) -> dict[str, str]: @@ -74,6 +87,24 @@ def _bool(params: dict[str, str], key: str, default: bool = False) -> bool: raise RuntimeError(f"{key} must be a boolean") +def _training_mode(params: dict[str, str], default: str = "full") -> str: + mode = _text(params, "TRAINING_MODE", default).lower() + if mode not in TRAINING_MODES: + raise RuntimeError("TRAINING_MODE must be smoke or full") + return mode + + +def _utc_now() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def _write_json_atomic(path: pathlib.Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp") + temporary.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + os.replace(temporary, path) + + def _json_value(params: dict[str, str], key: str, default: Any) -> Any: value = _text(params, key) if not value: @@ -143,16 +174,35 @@ def _first_payload(inputs: list[dict[str, Any]], helpers: HelperMap, dataset_typ def _safe_link_or_copy(source: pathlib.Path, target: pathlib.Path) -> None: - if target.exists() or target.is_symlink(): - if target.resolve() == source.resolve(): + source = source.resolve() + if not source.exists(): + raise RuntimeError(f"source path does not exist: {source}") + + def remove_target() -> None: + if not os.path.lexists(target): + return + if target.is_symlink() and target.resolve() == source: return if target.is_dir() and not target.is_symlink(): shutil.rmtree(target) else: target.unlink() + + if os.path.lexists(target): + if target.resolve() == source: + return + remove_target() target.parent.mkdir(parents=True, exist_ok=True) try: target.symlink_to(source, target_is_directory=source.is_dir()) + except FileExistsError: + # A retried node or another initializer may have created the exact + # link between the existence check and symlink_to(). Treat that as a + # successful idempotent initialization instead of failing the Run. + if target.is_symlink() and target.resolve() == source: + return + remove_target() + target.symlink_to(source, target_is_directory=source.is_dir()) except OSError as exc: if source.is_dir(): raise RuntimeError(f"Unable to create directory symlink from {target} to {source}") from exc @@ -738,6 +788,7 @@ def _run_command( output_path: pathlib.Path, env: dict[str, str], metadata: dict[str, Any], + progress_callback: Callable[[str | None], None] | None = None, ) -> tuple[pathlib.Path, pathlib.Path, pathlib.Path]: output_path.mkdir(parents=True, exist_ok=True) command_path = _write_command_file(output_path, command, cwd, metadata) @@ -774,9 +825,13 @@ def _run_command( log_file.flush() console.write(line) console.flush() + if progress_callback is not None: + progress_callback(line) continue selector.unregister(key.fileobj) key.fileobj.close() + if progress_callback is not None: + progress_callback(None) returncode = process.wait() if returncode != 0: tail = _tail_text(stderr_path) or _tail_text(stdout_path) @@ -812,17 +867,281 @@ def _parse_training_progress(*paths: pathlib.Path) -> dict[str, Any]: } +def _checkpoint_mtime(path: pathlib.Path) -> float: + try: + if path.is_dir(): + return max((item.stat().st_mtime for item in path.rglob("*") if item.is_file()), default=path.stat().st_mtime) + return path.stat().st_mtime + except OSError: + return 0.0 + + +def _valid_model_checkpoint(path: pathlib.Path) -> bool: + if not path.is_file() or path.stat().st_size <= 0: + return False + if path.suffix == ".safetensors": + return _valid_safetensors_header(path) + return path.suffix.lower() in {".bin", ".pt", ".pth"} + + +def _checkpoint_model_file(path: pathlib.Path) -> pathlib.Path | None: + if path.is_file(): + return path if _valid_model_checkpoint(path) else None + if not path.is_dir(): + return None + preferred = [ + path / "model.safetensors", + path / "pytorch_model.bin", + ] + candidates = ( + preferred + + sorted(path.glob("model*.safetensors")) + + sorted(path.glob("pytorch_model*.bin")) + + sorted(path.glob("model*.pt")) + + sorted(path.glob("model*.pth")) + ) + selected = next((item for item in candidates if _valid_model_checkpoint(item)), None) + if selected is not None: + return selected + interrupted_safetensors = path / "model.safetensors" + if interrupted_safetensors.exists(): + try: + repaired, _ = _resolve_safetensors_checkpoint(interrupted_safetensors) + except RuntimeError: + return None + if _valid_model_checkpoint(repaired): + return repaired + return None + + +def _valid_resume_checkpoint(path: pathlib.Path) -> bool: + if not path.is_dir() or _checkpoint_model_file(path) is None: + return False + state_patterns = ( + "optimizer*", + "scheduler*", + "random_states*", + "custom_checkpoint*", + "trainer_progress*", + ) + return any(item.is_file() and item.stat().st_size > 0 for pattern in state_patterns for item in path.glob(pattern)) + + +def _latest_valid_resume_checkpoint(workspace: pathlib.Path) -> pathlib.Path | None: + checkpoint_dir = workspace / "checkpoints" + if not checkpoint_dir.is_dir(): + return None + candidates = [item for item in checkpoint_dir.iterdir() if _valid_resume_checkpoint(item)] + return max(candidates, key=_checkpoint_mtime) if candidates else None + + def _latest_checkpoint(workspace: pathlib.Path) -> pathlib.Path | None: candidates: list[pathlib.Path] = [] checkpoint_dir = workspace / "checkpoints" - if checkpoint_dir.exists(): - candidates.extend(item for item in checkpoint_dir.iterdir() if item.is_dir() or item.is_file()) - candidates.extend(item for item in workspace.rglob("*.safetensors") if item.is_file()) - candidates.extend(item for item in workspace.rglob("*.pt") if item.is_file()) - candidates.extend(item for item in workspace.rglob("*.pth") if item.is_file()) - if not candidates: + if checkpoint_dir.is_dir(): + for item in checkpoint_dir.iterdir(): + model_file = _checkpoint_model_file(item) + if model_file is not None: + candidates.append(model_file) + for pattern in ("*.safetensors", "*.bin", "*.pt", "*.pth"): + candidates.extend(item for item in workspace.glob(pattern) if _valid_model_checkpoint(item)) + return max(candidates, key=_checkpoint_mtime) if candidates else None + + +def _resolve_resume_checkpoint(params: dict[str, str], kwargs: dict[str, Any], workspace: pathlib.Path) -> tuple[pathlib.Path | None, str]: + policy = _text(params, "RESUME_POLICY", "auto").lower() + if policy not in {"auto", "explicit", "never"}: + raise RuntimeError("RESUME_POLICY must be auto, explicit, or never") + configured = _text(params, "RESUME_FROM") or str(kwargs.get("resume_from") or "").strip() + if policy == "never": + if configured: + raise RuntimeError("RESUME_FROM/config resume_from cannot be set when RESUME_POLICY=never") + kwargs.pop("resume_from", None) + return None, policy + if configured: + selected = _path(configured) + if not _valid_resume_checkpoint(selected): + raise RuntimeError(f"RESUME_FROM is not a valid Accelerate training-state checkpoint: {selected}") + kwargs["resume_from"] = str(selected) + return selected, "explicit" + if policy == "explicit": + raise RuntimeError("RESUME_FROM is required when RESUME_POLICY=explicit") + selected = _latest_valid_resume_checkpoint(workspace) + if selected is not None: + kwargs["resume_from"] = str(selected) + return selected, policy + + +def _select_output_checkpoint(params: dict[str, str], workspace: pathlib.Path) -> tuple[pathlib.Path | None, str]: + selection = _text(params, "CHECKPOINT_ARTIFACT", "latest_valid").lower() + if selection == "none": + return None, selection + if selection == "explicit": + configured = _text(params, "OUTPUT_CHECKPOINT_PATH") + if not configured: + raise RuntimeError("OUTPUT_CHECKPOINT_PATH is required when CHECKPOINT_ARTIFACT=explicit") + selected = _path(configured) + if not _valid_model_checkpoint(selected): + raise RuntimeError(f"OUTPUT_CHECKPOINT_PATH is not a valid model checkpoint: {selected}") + return selected, selection + if selection != "latest_valid": + raise RuntimeError("CHECKPOINT_ARTIFACT must be latest_valid, explicit, or none") + return _latest_checkpoint(workspace), selection + + +def _normalize_dtype(value: Any, aliases: dict[str, str]) -> str: + normalized = str(value or "").strip().lower() + return aliases.get(normalized, normalized) + + +def _dtype_preflight( + params: dict[str, str], + kwargs: dict[str, Any], + config_path: pathlib.Path, + output_path: pathlib.Path, +) -> tuple[dict[str, Any], pathlib.Path]: + requested_model = _normalize_dtype(_text(params, "MODEL_DTYPE", "bfloat16"), MODEL_DTYPE_ALIASES) + requested_input = _normalize_dtype(_text(params, "INPUT_DTYPE", "float32"), INPUT_DTYPE_ALIASES) + configured_model = _normalize_dtype( + kwargs.get("model_dtype") or kwargs.get("torch_dtype") or requested_model, + MODEL_DTYPE_ALIASES, + ) + configured_input = _normalize_dtype(kwargs.get("input_dtype") or requested_input, INPUT_DTYPE_ALIASES) + errors: list[str] = [] + if requested_model != "bfloat16" or configured_model != "bfloat16": + errors.append("official HoloBrain Qwen model weights require bfloat16") + if requested_input != "float32" or configured_input != "float32": + errors.append("official RoboTwin processors require float32 image/depth tensors before model-side casting") + payload = { + "schemaVersion": "droboticflow.holobrain.dtype-preflight/v1", + "valid": not errors, + "configPath": str(config_path), + "modelDtype": configured_model, + "inputDtype": configured_input, + "compatiblePair": {"model": "bfloat16", "input": "float32"}, + "errors": errors, + "checkedAt": _utc_now(), + } + path = output_path / "holobrain-dtype-preflight.json" + _write_json_atomic(path, payload) + if errors: + raise RuntimeError("HoloBrain dtype preflight failed: " + "; ".join(errors)) + return payload, path + + +def _apply_smoke_training_defaults(params: dict[str, str], kwargs: dict[str, Any]) -> dict[str, Any]: + next_kwargs = dict(kwargs) + max_step = max(1, _int(params, "SMOKE_MAX_STEP", 2)) + next_kwargs.setdefault("max_step", max_step) + next_kwargs.setdefault("save_step_freq", max(1, min(max_step, _int(params, "SMOKE_SAVE_STEP_FREQ", 1)))) + next_kwargs.setdefault("step_log_freq", max(1, min(max_step, _int(params, "SMOKE_LOG_STEP_FREQ", 1)))) + next_kwargs.setdefault("batch_size", max(1, _int(params, "SMOKE_BATCH_SIZE", 1))) + next_kwargs.setdefault("num_workers", max(0, _int(params, "SMOKE_NUM_WORKERS", 0))) + next_kwargs.setdefault("training_datasets", ["robotwin2_0"]) + next_kwargs.setdefault("deploy_datasets", ["robotwin2_0"]) + return next_kwargs + + +def _parse_duration_seconds(value: str) -> int | None: + value = value.strip().rstrip(".") + if not value or value.upper() == "N/A": return None - return max(candidates, key=lambda item: item.stat().st_mtime) + parts = value.split(":") + try: + numbers = [int(item) for item in parts] + except ValueError: + return None + if len(numbers) == 3: + return numbers[0] * 3600 + numbers[1] * 60 + numbers[2] + if len(numbers) == 2: + return numbers[0] * 60 + numbers[1] + return numbers[0] if len(numbers) == 1 else None + + +class _TrainingProgressReporter: + def __init__( + self, + *, + run_id: str, + node_id: str, + output_path: pathlib.Path, + workspace: pathlib.Path, + max_step: int, + training_mode: str, + interval_seconds: int, + ) -> None: + self.workspace = workspace + self.output_path = output_path + self.workspace_path = workspace / "droboticflow-progress.json" + self.artifact_path = output_path / "holobrain-progress.json" + self.interval_seconds = max(1, interval_seconds) + self.last_emit = 0.0 + self.payload: dict[str, Any] = { + "schemaVersion": "droboticflow.execution-progress/v1", + "runId": run_id, + "nodeId": node_id, + "stage": "train", + "status": "running", + "trainingMode": training_mode, + "epoch": None, + "step": 0, + "maxStep": max_step, + "samplesPerSecond": None, + "estimatedRemainingSeconds": None, + "checkpoint": "", + "updatedAt": _utc_now(), + } + + def _refresh_checkpoint(self) -> bool: + latest = _latest_checkpoint(self.workspace) + value = str(latest) if latest else "" + if value == self.payload.get("checkpoint"): + return False + self.payload["checkpoint"] = value + return True + + def _parse_line(self, line: str) -> bool: + changed = False + match = re.search(r"Epoch\[(\d+)\].*?GlobalStep\[(\d+)\]", line) + if match: + epoch = int(match.group(1)) + step = int(match.group(2)) + changed = epoch != self.payload.get("epoch") or step != self.payload.get("step") + self.payload["epoch"] = epoch + self.payload["step"] = step + speed = re.search(r"(?:Average )?Training Speed:\s*([0-9]+(?:\.[0-9]+)?)\s+samples/sec", line, re.IGNORECASE) + if speed: + value = float(speed.group(1)) + changed = changed or value != self.payload.get("samplesPerSecond") + self.payload["samplesPerSecond"] = value + remaining = re.search(r"Estimated Remaining Time:\s*([^\t\n]+)", line, re.IGNORECASE) + if remaining: + value = _parse_duration_seconds(remaining.group(1)) + changed = changed or value != self.payload.get("estimatedRemainingSeconds") + self.payload["estimatedRemainingSeconds"] = value + return changed + + def emit(self, *, force: bool = False) -> None: + now = time.monotonic() + checkpoint_changed = self._refresh_checkpoint() + if not force and not checkpoint_changed and now - self.last_emit < self.interval_seconds: + return + self.last_emit = now + self.payload["updatedAt"] = _utc_now() + _write_json_atomic(self.workspace_path, self.payload) + _write_json_atomic(self.artifact_path, self.payload) + sys.stdout.write(PROGRESS_MARKER + json.dumps(self.payload, ensure_ascii=False, separators=(",", ":")) + "\n") + sys.stdout.flush() + + def observe(self, line: str | None) -> None: + changed = self._parse_line(line) if line else False + self.emit(force=changed) + + def finish(self, status: str) -> None: + self.payload["status"] = status + if status == "completed": + self.payload["estimatedRemainingSeconds"] = 0 + self.emit(force=True) def _valid_safetensors_header(path: pathlib.Path) -> bool: @@ -1054,13 +1373,24 @@ def run_data_prepare( helpers: HelperMap, ) -> list[dict[str, Any]]: params = _params(helpers, node_data) + training_mode = _training_mode(params) + effective_params = dict(params) + if training_mode == "smoke" and not _text(effective_params, "ARROW_MAX_EPISODES"): + effective_params["ARROW_MAX_EPISODES"] = str(max(1, _int(params, "SMOKE_MAX_EPISODES", 1))) output_path = pathlib.Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) - expected = tuple(str(item).strip() for item in _json_list(params, "EXPECTED_LMDB_DIRS_JSON", list(DEFAULT_EXPECTED_LMDB_DIRS)) if str(item).strip()) - source_root_text = _text(params, "ROBOTWIN_SOURCE_PATH") - data_source = _text(params, "DATA_SOURCE", "local").lower() + default_expected = [DEFAULT_EXPECTED_LMDB_DIRS[0]] if training_mode == "smoke" else list(DEFAULT_EXPECTED_LMDB_DIRS) + expected = tuple( + str(item).strip() + for item in _json_list(effective_params, "EXPECTED_LMDB_DIRS_JSON", default_expected) + if str(item).strip() + ) + if not expected: + expected = tuple(default_expected) + source_root_text = _text(effective_params, "ROBOTWIN_SOURCE_PATH") + data_source = _text(effective_params, "DATA_SOURCE", "local").lower() local_root = _workspace_path_for_bos_uri(source_root_text) if data_source == "bos" else (_path(source_root_text) if source_root_text else output_path / "robotwin-source") - hf_repo = _text(params, "ROBOTWIN_HF_REPO", ROBOTWIN_DATASET_REPO) + hf_repo = _text(effective_params, "ROBOTWIN_HF_REPO", ROBOTWIN_DATASET_REPO) if data_source in {"local", "local_lmdb", "local_raw", "bos"}: if not source_root_text: @@ -1075,9 +1405,9 @@ def run_data_prepare( discovered = _discover_dataset_dirs(local_root, expected) conversions: list[dict[str, Any]] = [] if len(discovered) < len(expected) and data_source in {"local", "local_raw", "bos"}: - converted, conversions = _pack_robotwin_arrow_to_lmdb(local_root, output_path, expected, params) + converted, conversions = _pack_robotwin_arrow_to_lmdb(local_root, output_path, expected, effective_params) if not converted: - converted, conversions = _pack_robotwin_raw_to_lmdb(local_root, output_path, expected, params) + converted, conversions = _pack_robotwin_raw_to_lmdb(local_root, output_path, expected, effective_params) if converted: discovered_by_name = {str(item.get("name")): item for item in discovered} for item in converted: @@ -1115,6 +1445,7 @@ def run_data_prepare( payload = { "datasetType": "holobrain-data-manifest", "valid": True, + "trainingMode": training_mode, "dataSource": data_source, "source": { "kind": data_source, @@ -1132,6 +1463,17 @@ def run_data_prepare( "fileCount": file_count, "sampledSizeBytes": sampled_size_bytes, "sampleCandidates": sample_candidates, + "dtypeContract": { + "storedRgb": "encoded-bytes", + "storedDepth": "sensor-native", + "trainingTensor": "float32", + "model": "bfloat16", + }, + "smokePolicy": { + "realDataRequired": True, + "maxEpisodes": _int(effective_params, "ARROW_MAX_EPISODES", 0) if training_mode == "smoke" else 0, + "syntheticDataAllowed": False, + }, "requiresRealData": True, } return _framework_artifacts(helpers, run_id, node_id, output_path, "holobrain-data-manifest.json", payload, node_data=node_data) @@ -1139,6 +1481,7 @@ def run_data_prepare( def _materialize_data_for_repo(repo_root: pathlib.Path, params: dict[str, str], inputs: list[dict[str, Any]], helpers: HelperMap) -> tuple[pathlib.Path, dict[str, Any]]: payload = _first_payload(inputs, helpers, "holobrain-data-manifest") + training_mode = _training_mode(params, str(payload.get("trainingMode") or "full")) data_root = _text(params, "ROBOTWIN_DATA_ROOT") or str(payload.get("robotwinDataRoot") or payload.get("holoBrainDataRoot") or "") if not data_root: raise RuntimeError("HoloBrain training requires ROBOTWIN_DATA_ROOT or an upstream holobrain-data-manifest") @@ -1146,7 +1489,28 @@ def _materialize_data_for_repo(repo_root: pathlib.Path, params: dict[str, str], if not source.exists(): raise RuntimeError(f"RoboTwin data root does not exist: {source}") target = repo_root / "projects" / "holobrain" / "data" / "robotwin2.0" - _safe_link_or_copy(source, target) + if training_mode == "smoke": + # The official robotwin2_0 config names both clean and randomized + # directories. A real smoke subset may intentionally contain only a + # clean LMDB. Build an idempotent overlay and map the second config + # path to the same real subset instead of mutating the source mount. + if target.is_symlink() or (target.exists() and not target.is_dir()): + target.unlink() + target.mkdir(parents=True, exist_ok=True) + for child in sorted(source.iterdir()): + if child.is_dir(): + _safe_link_or_copy(child, target / child.name) + clean = target / "aloha_agilex_demo_clean" + randomized = target / "aloha_agilex_demo_randomized" + if clean.exists() and not randomized.exists(): + _safe_link_or_copy(clean, randomized) + if not clean.exists() or not randomized.exists(): + raise RuntimeError( + "Smoke training requires a real aloha_agilex_demo_clean LMDB; " + "the idempotent RoboTwin config overlay could not be initialized" + ) + else: + _safe_link_or_copy(source, target) return target, payload @@ -1159,6 +1523,7 @@ def run_train( helpers: HelperMap, ) -> list[dict[str, Any]]: params = _params(helpers, node_data) + training_mode = _training_mode(params) output_path = pathlib.Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) repo_root = _ensure_roboorchard_repo(params, output_path) @@ -1169,12 +1534,24 @@ def run_train( data_root, data_payload = _materialize_data_for_repo(repo_root, params, inputs, helpers) workspace = _path(_text(params, "WORKSPACE", str(output_path / "holobrain-workspace"))) logging_dir = _path(_text(params, "LOGGING_DIR", str(workspace / "logs"))) + workspace.mkdir(parents=True, exist_ok=True) + logging_dir.mkdir(parents=True, exist_ok=True) kwargs = _json_dict(params, "CONFIG_KWARGS_JSON", {}) - kwargs_path = None - if kwargs: - kwargs_path = output_path / "config-kwargs.json" - kwargs_path.write_text(json.dumps(kwargs, ensure_ascii=False, indent=2), encoding="utf-8") + if training_mode == "smoke": + kwargs = _apply_smoke_training_defaults(params, kwargs) + resume_checkpoint, resume_policy = _resolve_resume_checkpoint(params, kwargs, workspace) + initial_checkpoint = _text(params, "INITIAL_CHECKPOINT_PATH") + if initial_checkpoint: + initial_checkpoint_path = _path(initial_checkpoint) + if not _valid_model_checkpoint(initial_checkpoint_path): + raise RuntimeError(f"INITIAL_CHECKPOINT_PATH is not a valid model checkpoint: {initial_checkpoint_path}") + kwargs["checkpoint"] = str(initial_checkpoint_path) + dtype_preflight, dtype_preflight_path = _dtype_preflight(params, kwargs, config_path, output_path) + kwargs_path = output_path / "config-kwargs.json" + kwargs_path.write_text(json.dumps(kwargs, ensure_ascii=False, indent=2), encoding="utf-8") launch_mode = _text(params, "LAUNCH_MODE", "single").lower() + if launch_mode not in {"single", "accelerate"}: + raise RuntimeError("LAUNCH_MODE must be single or accelerate") script = project_dir / "scripts" / "train.py" if launch_mode == "accelerate": gpu_count = _int(params, "GPU_COUNT", 1) @@ -1196,8 +1573,7 @@ def run_train( command.extend([str(script), "--workspace", str(workspace), "--logging_dir", str(logging_dir), "--config", str(config_path)]) else: command = [sys.executable, str(script), "--workspace", str(workspace), "--logging_dir", str(logging_dir), "--config", str(config_path)] - if kwargs_path is not None: - command.extend(["--kwargs", str(kwargs_path)]) + command.extend(["--kwargs", str(kwargs_path)]) env = _command_env(params, output_path) hf_runtime = { "hfHome": env.get("HF_HOME", ""), @@ -1206,28 +1582,50 @@ def run_train( and env.get("TRANSFORMERS_OFFLINE", "") == "1" and env.get("HF_DATASETS_OFFLINE", "") == "1", } - command_path, stdout_path, stderr_path = _run_command( - command, - project_dir, - output_path, - env, - { - "framework": "RoboOrchardLab HoloBrain train", - "config": str(config_path), - "workspace": str(workspace), - "dataRoot": str(data_root), - "model": HOLOBRAIN_MODEL_REPO, - "hfRuntime": hf_runtime, - "launchMode": launch_mode, - "fullTrainingDefault": True, - }, + max_step = int(kwargs.get("max_step", _int(params, "MAX_STEP", 100000))) + progress_reporter = _TrainingProgressReporter( + run_id=run_id, + node_id=node_id, + output_path=output_path, + workspace=workspace, + max_step=max_step, + training_mode=training_mode, + interval_seconds=max(1, _int(params, "PROGRESS_INTERVAL_SECONDS", 15)), ) + progress_reporter.emit(force=True) + try: + command_path, stdout_path, stderr_path = _run_command( + command, + project_dir, + output_path, + env, + { + "framework": "RoboOrchardLab HoloBrain train", + "config": str(config_path), + "workspace": str(workspace), + "dataRoot": str(data_root), + "model": HOLOBRAIN_MODEL_REPO, + "hfRuntime": hf_runtime, + "launchMode": launch_mode, + "trainingMode": training_mode, + "resumePolicy": resume_policy, + "resumeFrom": str(resume_checkpoint) if resume_checkpoint else "", + "fullTrainingDefault": training_mode == "full", + }, + progress_callback=progress_reporter.observe, + ) + except Exception: + progress_reporter.finish("failed") + raise + progress_reporter.finish("completed") progress = _parse_training_progress(stdout_path, stderr_path) - latest_checkpoint = _latest_checkpoint(workspace) + latest_checkpoint, checkpoint_artifact_selection = _select_output_checkpoint(params, workspace) + structured_progress = json.loads(progress_reporter.artifact_path.read_text(encoding="utf-8")) report = { "datasetType": "holobrain-training-report", "valid": True, "status": "completed", + "trainingMode": training_mode, "framework": "RoboOrchardLab HoloBrain", "model": { "repo": HOLOBRAIN_MODEL_REPO, @@ -1237,7 +1635,7 @@ def run_train( "config": { "path": str(config_path), "sourceUrl": "https://github.com/HorizonRobotics/RoboOrchardLab/blob/master/projects/holobrain/configs/config_holobrain_qwen_common.py", - "maxStep": int(kwargs.get("max_step", _int(params, "MAX_STEP", 100000))), + "maxStep": max_step, "kwargs": kwargs, }, "data": { @@ -1249,11 +1647,26 @@ def run_train( "stderrPath": str(stderr_path), "trainingWorkspace": str(workspace), "latestCheckpoint": str(latest_checkpoint) if latest_checkpoint else "", + "checkpointArtifactSelection": checkpoint_artifact_selection, + "resume": { + "policy": resume_policy, + "selected": str(resume_checkpoint) if resume_checkpoint else "", + "automaticallySelected": resume_policy == "auto" and resume_checkpoint is not None, + }, + "dtypePreflight": dtype_preflight, + "progress": structured_progress, "currentStep": progress.get("currentStep"), "lossSamples": progress.get("lossSamples", []), "finalLoss": progress.get("finalLoss"), } - extras = [(command_path, "json"), (stdout_path, "text"), (stderr_path, "text")] + extras = [ + (command_path, "json"), + (stdout_path, "text"), + (stderr_path, "text"), + (kwargs_path, "json"), + (dtype_preflight_path, "json"), + (progress_reporter.artifact_path, "json"), + ] if latest_checkpoint and latest_checkpoint.is_file(): extras.append((latest_checkpoint, "model")) return _framework_artifacts(helpers, run_id, node_id, output_path, "holobrain-training-report.json", report, extras, node_data) @@ -1286,19 +1699,51 @@ def run_export( project_dir = repo_root / DEFAULT_PROJECT_DIR config_path = repo_root / _text(params, "CONFIG_PATH", DEFAULT_CONFIG) train_payload = _first_payload(inputs, helpers, "holobrain-training-report") + training_mode = _training_mode(params, str(train_payload.get("trainingMode") or "full")) train_workspace = _path(_text(params, "TRAINING_WORKSPACE") or str(train_payload.get("trainingWorkspace") or "")) - checkpoint = _text(params, "CHECKPOINT_PATH") or str(train_payload.get("latestCheckpoint") or "") - if not checkpoint and train_workspace.exists(): - latest = _latest_checkpoint(train_workspace) - checkpoint = str(latest) if latest else "" + checkpoint_selection = _text(params, "CHECKPOINT_SELECTION", "latest_valid").lower() + if checkpoint_selection not in {"latest_valid", "upstream", "explicit"}: + raise RuntimeError("CHECKPOINT_SELECTION must be latest_valid, upstream, or explicit") + explicit_checkpoint = _text(params, "CHECKPOINT_PATH") + upstream_checkpoint = str(train_payload.get("latestCheckpoint") or "").strip() + checkpoint = "" + if checkpoint_selection == "explicit": + if not explicit_checkpoint: + raise RuntimeError("CHECKPOINT_PATH is required when CHECKPOINT_SELECTION=explicit") + checkpoint = explicit_checkpoint + elif checkpoint_selection == "upstream": + checkpoint = upstream_checkpoint or explicit_checkpoint + else: + if explicit_checkpoint: + checkpoint = explicit_checkpoint + elif train_workspace.exists(): + latest = _latest_checkpoint(train_workspace) + checkpoint = str(latest) if latest else "" + if not checkpoint: + checkpoint = upstream_checkpoint + if training_mode == "smoke" and not checkpoint: + raise RuntimeError("Smoke export requires a valid checkpoint produced by the smoke training node") checkpoint_validation: dict[str, Any] = {"validated": False, "reason": "not-configured"} if checkpoint: checkpoint_path, checkpoint_validation = _resolve_safetensors_checkpoint(_path(checkpoint)) + if checkpoint_path.is_dir(): + checkpoint_model = _checkpoint_model_file(checkpoint_path) + if checkpoint_model is not None: + checkpoint_path = checkpoint_model + checkpoint_validation = { + **checkpoint_validation, + "validated": True, + "resolvedFromTrainingStateDir": checkpoint, + "checkpoint": str(checkpoint_path), + } + if training_mode == "smoke" and not _valid_model_checkpoint(checkpoint_path): + raise RuntimeError(f"Selected HoloBrain export checkpoint is not valid: {checkpoint_path}") checkpoint = str(checkpoint_path) kwargs = _json_dict(params, "CONFIG_KWARGS_JSON", {}) if checkpoint: kwargs["checkpoint"] = checkpoint workspace = _path(_text(params, "EXPORT_WORKSPACE", str(output_path / "holobrain-export"))) + workspace.mkdir(parents=True, exist_ok=True) source_urdf_dir = project_dir / "urdf" if source_urdf_dir.is_dir(): shutil.copytree(source_urdf_dir, workspace / "urdf", dirs_exist_ok=True) @@ -1317,6 +1762,8 @@ def run_export( { "framework": "RoboOrchardLab HoloBrain export", "workspace": str(workspace), + "trainingMode": training_mode, + "checkpointSelection": checkpoint_selection, "checkpoint": checkpoint, "checkpointValidation": checkpoint_validation, }, @@ -1333,10 +1780,12 @@ def run_export( "datasetType": "holobrain-export-manifest", "valid": model_dir.exists(), "status": "completed", + "trainingMode": training_mode, "exportWorkspace": str(workspace), "exportedModelDir": str(model_dir), "modelArchive": str(archive_path) if archive_path.exists() else "", "checkpoint": checkpoint, + "checkpointSelection": checkpoint_selection, "checkpointValidation": checkpoint_validation, "modelSafetensorsValidation": model_validation, "commandPath": str(command_path), @@ -1363,10 +1812,20 @@ def run_robotwin_eval( output_path.mkdir(parents=True, exist_ok=True) repo_root = _ensure_roboorchard_repo(params, output_path) project_dir = repo_root / DEFAULT_PROJECT_DIR + export_payload = _first_payload(inputs, helpers, "holobrain-export-manifest") + training_mode = _training_mode(params, str(export_payload.get("trainingMode") or "full")) model_dir = _model_dir_from_inputs(params, inputs, helpers) output_json = output_path / "robotwin-eval-result.json" - task_names = [str(item) for item in _json_list(params, "TASK_NAMES_JSON", []) if str(item).strip()] + task_names = ( + [_text(params, "SMOKE_TASK_NAME", "beat_block_hammer")] + if training_mode == "smoke" + else [str(item) for item in _json_list(params, "TASK_NAMES_JSON", []) if str(item).strip()] + ) gpu_ids = [str(item) for item in _json_list(params, "GPU_IDS_JSON", [0]) if str(item).strip() != ""] + eval_mode = _text(params, "SMOKE_EVAL_MODE", "local") if training_mode == "smoke" else _text(params, "EVAL_MODE", "ray") + if eval_mode not in {"local", "ray"}: + raise RuntimeError("EVAL_MODE must be local or ray") + episode_num = _int(params, "SMOKE_EVAL_EPISODE_NUM", 1) if training_mode == "smoke" else _int(params, "EPISODE_NUM", 100) command = [ sys.executable, str(project_dir / "scripts" / "robotwin_eval.py"), @@ -1375,9 +1834,9 @@ def run_robotwin_eval( "--inference_prefix", _text(params, "INFERENCE_PREFIX", "robotwin2_0"), "--episode_num", - str(_int(params, "EPISODE_NUM", 100)), + str(episode_num), "--mode", - _text(params, "EVAL_MODE", "ray"), + eval_mode, "--config_type", _text(params, "CONFIG_TYPE", "demo_clean"), "--output_path", @@ -1387,7 +1846,10 @@ def run_robotwin_eval( ] if task_names: command.extend(["--task_names", *task_names]) - if gpu_ids: + device = _text(params, "SMOKE_DEVICE", "cuda") if training_mode == "smoke" else _text(params, "DEVICE") + if device: + command.extend(["--device", device]) + if gpu_ids and eval_mode == "ray": command.extend(["--gpu_ids", *gpu_ids]) if _bool(params, "SAVE_VIDEO", False): command.append("--save_video") @@ -1397,7 +1859,14 @@ def run_robotwin_eval( project_dir, output_path, env, - {"framework": "RoboOrchardLab HoloBrain RoboTwin eval", "modelDir": str(model_dir)}, + { + "framework": "RoboOrchardLab HoloBrain RoboTwin eval", + "modelDir": str(model_dir), + "trainingMode": training_mode, + "taskNames": task_names, + "episodeNum": episode_num, + "evalMode": eval_mode, + }, ) metrics: dict[str, Any] = {} if output_json.exists(): @@ -1406,6 +1875,15 @@ def run_robotwin_eval( "datasetType": "holobrain-robotwin-evaluation", "valid": output_json.exists(), "status": "completed", + "trainingMode": training_mode, + "execution": { + "backend": "official-robotwin-evaluator", + "mode": eval_mode, + "taskNames": task_names, + "episodeNum": episode_num, + "device": device, + "syntheticResult": False, + }, "modelDir": str(model_dir), "metrics": metrics, "commandPath": str(command_path), @@ -1435,59 +1913,87 @@ def run_inference_test( params = _params(helpers, node_data) output_path = pathlib.Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) + repo_root = _ensure_roboorchard_repo(params, output_path) + project_dir = repo_root / DEFAULT_PROJECT_DIR + export_payload = _first_payload(inputs, helpers, "holobrain-export-manifest") + training_mode = _training_mode(params, str(export_payload.get("trainingMode") or "full")) model_dir = _model_dir_from_inputs(params, inputs, helpers) - sample_path = _text(params, "SAMPLE_INPUT_PATH") - if not sample_path: - for payload in _artifact_payloads(inputs, helpers): - candidates = payload.get("sampleCandidates") - if isinstance(candidates, list) and candidates: - sample_path = str(candidates[0]) - break - if not sample_path: - raise RuntimeError("HoloBrain inference test requires SAMPLE_INPUT_PATH or upstream real sampleCandidates") - sample = _path(sample_path) - if not sample.exists(): - raise RuntimeError(f"SAMPLE_INPUT_PATH does not exist: {sample}") - script = output_path / "holobrain_inference_smoke.py" - result_path = output_path / "holobrain-inference-result.json" - script.write_text( - """ -import json -import pathlib -import sys - -model_dir = pathlib.Path(sys.argv[1]) -sample_path = pathlib.Path(sys.argv[2]) -result_path = pathlib.Path(sys.argv[3]) - -config_files = sorted(path.name for path in model_dir.glob("*.config.json")) -if not config_files: - raise SystemExit(f"no inference config files found in {model_dir}") -payload = { - "datasetType": "holobrain-inference-test", - "valid": True, - "modelDir": str(model_dir), - "samplePath": str(sample_path), - "sampleSuffix": sample_path.suffix, - "inferencePrefixes": [name[:-len(".config.json")] for name in config_files], - "note": "Validated exported HoloBrain pipeline files against a real sample path; full action inference requires a task-specific sample decoder.", -} -result_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") -""".strip() - + "\n", - encoding="utf-8", - ) - command = [sys.executable, str(script), str(model_dir), str(sample), str(result_path)] + inference_prefix = _text(params, "INFERENCE_PREFIX", "robotwin2_0") + task_name = _text(params, "TASK_NAME", "beat_block_hammer") + episode_num = max(1, _int(params, "EPISODE_NUM", 1)) + inference_mode = _text(params, "INFERENCE_MODE", "local").lower() + if inference_mode not in {"local", "ray"}: + raise RuntimeError("INFERENCE_MODE must be local or ray") + device = _text(params, "DEVICE", "cuda") + metrics_path = output_path / "holobrain-inference-metrics.json" + command = [ + sys.executable, + str(project_dir / "scripts" / "robotwin_eval.py"), + "--model_dir", + str(model_dir), + "--inference_prefix", + inference_prefix, + "--episode_num", + str(episode_num), + "--mode", + inference_mode, + "--config_type", + _text(params, "CONFIG_TYPE", "demo_clean"), + "--output_path", + str(metrics_path), + "--workers_per_gpu", + str(max(1, _int(params, "WORKERS_PER_GPU", 1))), + "--task_names", + task_name, + "--device", + device, + ] + if inference_mode == "ray": + gpu_ids = [str(item) for item in _json_list(params, "GPU_IDS_JSON", [0]) if str(item).strip() != ""] + if not gpu_ids: + raise RuntimeError("GPU_IDS_JSON must contain at least one GPU id when INFERENCE_MODE=ray") + command.extend(["--gpu_ids", *gpu_ids]) env = _command_env(params, output_path) command_path, stdout_path, stderr_path = _run_command( command, - output_path, + project_dir, output_path, env, - {"framework": "HoloBrain inference smoke", "modelDir": str(model_dir), "samplePath": str(sample)}, + { + "framework": "RoboOrchardLab HoloBrain real inference smoke", + "modelDir": str(model_dir), + "trainingMode": training_mode, + "taskName": task_name, + "episodeNum": episode_num, + "inferenceMode": inference_mode, + }, ) - payload = json.loads(result_path.read_text(encoding="utf-8")) - payload.update({"commandPath": str(command_path), "stdoutPath": str(stdout_path), "stderrPath": str(stderr_path)}) + dry_run = str(env.get("HOLOBRAIN_OPERATOR_DRY_RUN", "")).strip().lower() in TRUE_VALUES + if not metrics_path.exists() and not dry_run: + raise RuntimeError("Official RoboTwin inference smoke completed without writing metrics") + metrics = json.loads(metrics_path.read_text(encoding="utf-8")) if metrics_path.exists() else {} + payload = { + "datasetType": "holobrain-inference-test", + "valid": not dry_run, + "status": "dry-run" if dry_run else "completed", + "trainingMode": training_mode, + "modelDir": str(model_dir), + "metrics": metrics, + "execution": { + "backend": "official-robotwin-evaluator", + "actualModelExecution": not dry_run, + "syntheticResult": False, + "dryRun": dry_run, + "inferencePrefix": inference_prefix, + "taskName": task_name, + "episodeNum": episode_num, + "mode": inference_mode, + "device": device, + }, + "commandPath": str(command_path), + "stdoutPath": str(stdout_path), + "stderrPath": str(stderr_path), + } return _framework_artifacts( helpers, run_id, @@ -1495,6 +2001,6 @@ result_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encodi output_path, "holobrain-inference-result.json", payload, - [(command_path, "json"), (stdout_path, "text"), (stderr_path, "text")], + [(command_path, "json"), (stdout_path, "text"), (stderr_path, "text"), (metrics_path, "json")], node_data, ) diff --git a/backend/operators/builtin/holobrain_robotwin_eval/repository.json b/backend/operators/builtin/holobrain_robotwin_eval/repository.json new file mode 100644 index 00000000..87105ecb --- /dev/null +++ b/backend/operators/builtin/holobrain_robotwin_eval/repository.json @@ -0,0 +1,79 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", + "requiresPersistentWorkspaceEvidence": true, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "containerArgs": [], + "containerCommand": [ + "python", + "/app/backend/operator-runner/main.py" + ], + "containerEntrypoint": "backend/operator-runner/main.py", + "entrypoint": "operator.py", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-robotwin-eval:bd25938-holobrain-evalsapien-20260709032957", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-robotwin-eval@sha256:4f0ca2efcb3d69ad672df06dc28805ce3587c5ad8df4ee790677ea719fc32add", + "metadata": "operator.json", + "name": "HoloBrain RoboTwin 评测", + "operatorId": "operator-builtin-holobrain-robotwin-eval", + "operatorSlug": "holobrain_robotwin_eval", + "preferredSlug": "holobrain_robotwin_eval", + "pythonImport": "backend.operators.builtin.holobrain_robotwin_eval.operator:run", + "repositoryId": "builtin/holobrain_robotwin_eval", + "runtimeContract": { + "acceptance": { + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": true, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_holobrain_robotwin_eval", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "operatorId": "operator-builtin-holobrain-robotwin-eval", + "operatorSlug": "holobrain_robotwin_eval", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "minGpuCount": 1, + "recommendedGpuType": "NVIDIA A800 80GB", + "requiresGpu": true + }, + "runtimeDependencies": [ + "roboorchardlab-robotwin-eval", + "robotwin-benchmark", + "gpu-a800" + ], + "schemaVersion": "droboticflow.operator.runtime/v1" + }, + "runtimeDependencies": [ + "roboorchardlab-robotwin-eval", + "robotwin-benchmark", + "gpu-a800" + ], + "runtimeHandler": "builtin_holobrain_robotwin_eval", + "runtimeSource": "backend/operators/builtin/holobrain_robotwin_eval/operator.py", + "schemaVersion": "droboticflow.operator.repository/v1", + "sourceRoot": "backend/operators/builtin/holobrain_robotwin_eval", + "strictPocAcceptanceExternals": [], + "type": "builtin" +} diff --git a/backend/operators/builtin/holobrain_train/repository.json b/backend/operators/builtin/holobrain_train/repository.json new file mode 100644 index 00000000..4d48def9 --- /dev/null +++ b/backend/operators/builtin/holobrain_train/repository.json @@ -0,0 +1,82 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", + "requiresPersistentWorkspaceEvidence": true, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "containerArgs": [], + "containerCommand": [ + "python", + "/app/backend/operator-runner/main.py" + ], + "containerEntrypoint": "backend/operator-runner/main.py", + "entrypoint": "operator.py", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-train:5289510-holobrain-offline28-20260708044102", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-train@sha256:a21bb31ad50efed8cb12ab18e76ba6a394f648582273cbac47a305129198ed0b", + "metadata": "operator.json", + "name": "HoloBrain 完整训练", + "operatorId": "operator-builtin-holobrain-train", + "operatorSlug": "holobrain_train", + "preferredSlug": "holobrain_train", + "pythonImport": "backend.operators.builtin.holobrain_train.operator:run", + "repositoryId": "builtin/holobrain_train", + "runtimeContract": { + "acceptance": { + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": true, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_holobrain_train", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "longRunning": true, + "operatorId": "operator-builtin-holobrain-train", + "operatorSlug": "holobrain_train", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "minGpuCount": 1, + "recommendedGpuType": "NVIDIA A800 80GB", + "requiresGpu": true + }, + "runtimeDependencies": [ + "roboorchardlab-holobrain-train", + "official-robotwin2-dataset", + "qwen-vl-checkpoint", + "gpu-a800" + ], + "schemaVersion": "droboticflow.operator.runtime/v1" + }, + "runtimeDependencies": [ + "roboorchardlab-holobrain-train", + "official-robotwin2-dataset", + "qwen-vl-checkpoint", + "gpu-a800" + ], + "runtimeHandler": "builtin_holobrain_train", + "runtimeSource": "backend/operators/builtin/holobrain_train/operator.py", + "schemaVersion": "droboticflow.operator.repository/v1", + "sourceRoot": "backend/operators/builtin/holobrain_train", + "strictPocAcceptanceExternals": [], + "type": "builtin" +} diff --git a/backend/operators/builtin/human_review/repository.json b/backend/operators/builtin/human_review/repository.json index a16409d6..b5cef47e 100644 --- a/backend/operators/builtin/human_review/repository.json +++ b/backend/operators/builtin/human_review/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "human_review", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/joint_limit_validate/repository.json b/backend/operators/builtin/joint_limit_validate/repository.json index 2e6b5da5..3bcb56b6 100644 --- a/backend/operators/builtin/joint_limit_validate/repository.json +++ b/backend/operators/builtin/joint_limit_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "joint_limit_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/json_field_extract/repository.json b/backend/operators/builtin/json_field_extract/repository.json index 19ab8b84..a4684514 100644 --- a/backend/operators/builtin/json_field_extract/repository.json +++ b/backend/operators/builtin/json_field_extract/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "json_field_extract", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/json_filter/repository.json b/backend/operators/builtin/json_filter/repository.json index 3e3b662e..b246a8d3 100644 --- a/backend/operators/builtin/json_filter/repository.json +++ b/backend/operators/builtin/json_filter/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "json_filter", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/json_map_rename/repository.json b/backend/operators/builtin/json_map_rename/repository.json index e6277b98..bc95c5cd 100644 --- a/backend/operators/builtin/json_map_rename/repository.json +++ b/backend/operators/builtin/json_map_rename/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "json_map_rename", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/json_to_csv/repository.json b/backend/operators/builtin/json_to_csv/repository.json index ee5a2e0d..d897d330 100644 --- a/backend/operators/builtin/json_to_csv/repository.json +++ b/backend/operators/builtin/json_to_csv/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "json_to_csv", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/language_instruction_validate/repository.json b/backend/operators/builtin/language_instruction_validate/repository.json index 6cbfbd1f..eb77a09c 100644 --- a/backend/operators/builtin/language_instruction_validate/repository.json +++ b/backend/operators/builtin/language_instruction_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "language_instruction_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/lerobot_dataset_validate/repository.json b/backend/operators/builtin/lerobot_dataset_validate/repository.json index ed231c4a..3f31acfa 100644 --- a/backend/operators/builtin/lerobot_dataset_validate/repository.json +++ b/backend/operators/builtin/lerobot_dataset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "lerobot_dataset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/lerobot_metadata_consistency_validate/repository.json b/backend/operators/builtin/lerobot_metadata_consistency_validate/repository.json index f7f0f874..1377b737 100644 --- a/backend/operators/builtin/lerobot_metadata_consistency_validate/repository.json +++ b/backend/operators/builtin/lerobot_metadata_consistency_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "lerobot_metadata_consistency_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/lerobot_summary/repository.json b/backend/operators/builtin/lerobot_summary/repository.json index 8d2b8669..c8b59eef 100644 --- a/backend/operators/builtin/lerobot_summary/repository.json +++ b/backend/operators/builtin/lerobot_summary/repository.json @@ -16,8 +16,8 @@ ], "containerEntrypoint": "backend/operator-runner/main.py", "entrypoint": "operator.py", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-lerobot-summary:builtin", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-lerobot-summary@sha256:c8e1878dced01da3da3a068cd53a395b03061b27f3e52e20cef2faedf4081dbf", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-lerobot-summary:69b0e0e-poc-20260630072404", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-lerobot-summary:69b0e0e-poc-20260630072404@sha256:9cb7288be1a06c0098d0babd83da3e1b821be02644483b8d02ad92145d43dd95", "metadata": "operator.json", "name": "LeRobot 数据集概览", "operatorId": "operator-builtin-lerobot-summary", @@ -49,6 +49,7 @@ "operatorSlug": "lerobot_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/lerobot_v3_episode_index_validate/repository.json b/backend/operators/builtin/lerobot_v3_episode_index_validate/repository.json index c82a971c..e2e5e13c 100644 --- a/backend/operators/builtin/lerobot_v3_episode_index_validate/repository.json +++ b/backend/operators/builtin/lerobot_v3_episode_index_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "lerobot_v3_episode_index_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/lerobot_video_alignment_validate/repository.json b/backend/operators/builtin/lerobot_video_alignment_validate/repository.json index b1105f7c..6ff18140 100644 --- a/backend/operators/builtin/lerobot_video_alignment_validate/repository.json +++ b/backend/operators/builtin/lerobot_video_alignment_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "lerobot_video_alignment_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/libero_hdf5_task_validate/repository.json b/backend/operators/builtin/libero_hdf5_task_validate/repository.json index 0bca0962..e3a039c5 100644 --- a/backend/operators/builtin/libero_hdf5_task_validate/repository.json +++ b/backend/operators/builtin/libero_hdf5_task_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "libero_hdf5_task_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/maniskill_trajectory_validate/repository.json b/backend/operators/builtin/maniskill_trajectory_validate/repository.json index a0bbc453..636551e2 100644 --- a/backend/operators/builtin/maniskill_trajectory_validate/repository.json +++ b/backend/operators/builtin/maniskill_trajectory_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "maniskill_trajectory_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/mcap_topic_summary/repository.json b/backend/operators/builtin/mcap_topic_summary/repository.json index 903a79d6..1415fed4 100644 --- a/backend/operators/builtin/mcap_topic_summary/repository.json +++ b/backend/operators/builtin/mcap_topic_summary/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "mcap_topic_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/media_reference_validate/repository.json b/backend/operators/builtin/media_reference_validate/repository.json index 6e468e07..e725835e 100644 --- a/backend/operators/builtin/media_reference_validate/repository.json +++ b/backend/operators/builtin/media_reference_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "media_reference_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/merge/repository.json b/backend/operators/builtin/merge/repository.json index 4b423be8..4a9b6f0a 100644 --- a/backend/operators/builtin/merge/repository.json +++ b/backend/operators/builtin/merge/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "merge", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/mjcf_actuator_validate/repository.json b/backend/operators/builtin/mjcf_actuator_validate/repository.json index 3da2908b..3ce36c93 100644 --- a/backend/operators/builtin/mjcf_actuator_validate/repository.json +++ b/backend/operators/builtin/mjcf_actuator_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "mjcf_actuator_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/model_inference/repository.json b/backend/operators/builtin/model_inference/repository.json index bed74575..9fbf4384 100644 --- a/backend/operators/builtin/model_inference/repository.json +++ b/backend/operators/builtin/model_inference/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "model_inference", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/mongodb_write/repository.json b/backend/operators/builtin/mongodb_write/repository.json index 18a51071..8d89e7f4 100644 --- a/backend/operators/builtin/mongodb_write/repository.json +++ b/backend/operators/builtin/mongodb_write/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "mongodb_write", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/openvla_finetune/repository.json b/backend/operators/builtin/openvla_finetune/repository.json index f8963bdd..6fc80fa7 100644 --- a/backend/operators/builtin/openvla_finetune/repository.json +++ b/backend/operators/builtin/openvla_finetune/repository.json @@ -16,8 +16,8 @@ ], "containerEntrypoint": "backend/operator-runner/main.py", "entrypoint": "operator.py", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-openvla-finetune:builtin", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-openvla-finetune@sha256:36184d088fb2ca79a69c6c11adfba4099a1afa32eac7ec3da6408946044837d2", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-openvla-finetune:1442552-poc-20260630062516", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-openvla-finetune:1442552-poc-20260630062516@sha256:dda74e65fa48fd5d33afcece7779455e95569ce036ac74a07789718e69e67b79", "metadata": "operator.json", "name": "OpenVLA 微调训练", "operatorId": "operator-builtin-openvla-finetune", @@ -49,6 +49,7 @@ "operatorSlug": "openvla_finetune", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, @@ -58,15 +59,17 @@ "requiresGpu": true }, "runtimeDependencies": [ - "openvla-training-command", - "vla-sample-action-manifest", + "openvla-lora-finetune", + "rlds-vla-dataset", + "peft", "gpu-a800" ], "schemaVersion": "droboticflow.operator.runtime/v1" }, "runtimeDependencies": [ - "openvla-training-command", - "vla-sample-action-manifest", + "openvla-lora-finetune", + "rlds-vla-dataset", + "peft", "gpu-a800" ], "runtimeHandler": "builtin_openvla_finetune", diff --git a/backend/operators/builtin/openvla_inference/repository.json b/backend/operators/builtin/openvla_inference/repository.json index 5350b95c..00ce637f 100644 --- a/backend/operators/builtin/openvla_inference/repository.json +++ b/backend/operators/builtin/openvla_inference/repository.json @@ -16,8 +16,8 @@ ], "containerEntrypoint": "backend/operator-runner/main.py", "entrypoint": "operator.py", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-openvla-inference:builtin", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-openvla-inference@sha256:3336a89556d87b18a268be0234ff95acd6b23405e90c0da6fe9cdcbf7d3c524e", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-openvla-inference:a0c1e77-poc-20260630052949", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-openvla-inference:a0c1e77-poc-20260630052949@sha256:80d5e8e36b2b160f98133c060869e81d78461c6e4feb0166f0984830f3513a3c", "metadata": "operator.json", "name": "OpenVLA 推理", "operatorId": "operator-builtin-openvla-inference", @@ -49,6 +49,7 @@ "operatorSlug": "openvla_inference", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, @@ -58,15 +59,15 @@ "requiresGpu": true }, "runtimeDependencies": [ - "openvla-inference-command", - "openvla-model-weights", + "openvla-transformers-inference", + "openvla-model-or-upstream-adapter", "gpu-a800" ], "schemaVersion": "droboticflow.operator.runtime/v1" }, "runtimeDependencies": [ - "openvla-inference-command", - "openvla-model-weights", + "openvla-transformers-inference", + "openvla-model-or-upstream-adapter", "gpu-a800" ], "runtimeHandler": "builtin_openvla_inference", diff --git a/backend/operators/builtin/package/repository.json b/backend/operators/builtin/package/repository.json index 92ecc627..2becd41b 100644 --- a/backend/operators/builtin/package/repository.json +++ b/backend/operators/builtin/package/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "package", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/parquet_table_summary/repository.json b/backend/operators/builtin/parquet_table_summary/repository.json index b6725386..17fd01b0 100644 --- a/backend/operators/builtin/parquet_table_summary/repository.json +++ b/backend/operators/builtin/parquet_table_summary/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "parquet_table_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/partnet_mobility_asset_validate/repository.json b/backend/operators/builtin/partnet_mobility_asset_validate/repository.json index 13c91162..6d5768a9 100644 --- a/backend/operators/builtin/partnet_mobility_asset_validate/repository.json +++ b/backend/operators/builtin/partnet_mobility_asset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "partnet_mobility_asset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/pointcloud_summary/repository.json b/backend/operators/builtin/pointcloud_summary/repository.json index 97b2fc65..41da604a 100644 --- a/backend/operators/builtin/pointcloud_summary/repository.json +++ b/backend/operators/builtin/pointcloud_summary/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "pointcloud_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/real_source/repository.json b/backend/operators/builtin/real_source/repository.json index 4510cf71..4bea0d1a 100644 --- a/backend/operators/builtin/real_source/repository.json +++ b/backend/operators/builtin/real_source/repository.json @@ -39,7 +39,7 @@ "requiresManifest": true }, "entrypoint": "/app/backend/operator-runner/main.py", - "handler": "builtin_source", + "handler": "source", "imagePolicy": { "allowsSharedBaseLayer": true, "forbidsSharedOperatorRunnerFinalImage": true, @@ -49,6 +49,7 @@ "operatorSlug": "real_source", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/rgbd_frame_pair_validate/repository.json b/backend/operators/builtin/rgbd_frame_pair_validate/repository.json index c7e72f19..df6b7045 100644 --- a/backend/operators/builtin/rgbd_frame_pair_validate/repository.json +++ b/backend/operators/builtin/rgbd_frame_pair_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "rgbd_frame_pair_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/rh20t_scene_validate/repository.json b/backend/operators/builtin/rh20t_scene_validate/repository.json index 1107b4e5..28e92171 100644 --- a/backend/operators/builtin/rh20t_scene_validate/repository.json +++ b/backend/operators/builtin/rh20t_scene_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "rh20t_scene_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/rlbench_dataset_validate/repository.json b/backend/operators/builtin/rlbench_dataset_validate/repository.json index a5e8411e..2e086a14 100644 --- a/backend/operators/builtin/rlbench_dataset_validate/repository.json +++ b/backend/operators/builtin/rlbench_dataset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "rlbench_dataset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/rlds_episode_summary/repository.json b/backend/operators/builtin/rlds_episode_summary/repository.json index 675ff61f..db6f09f8 100644 --- a/backend/operators/builtin/rlds_episode_summary/repository.json +++ b/backend/operators/builtin/rlds_episode_summary/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "rlds_episode_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/robocasa_hdf5_demonstration_validate/repository.json b/backend/operators/builtin/robocasa_hdf5_demonstration_validate/repository.json index d075d789..1f369e25 100644 --- a/backend/operators/builtin/robocasa_hdf5_demonstration_validate/repository.json +++ b/backend/operators/builtin/robocasa_hdf5_demonstration_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "robocasa_hdf5_demonstration_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/robomimic_hdf5_validate/repository.json b/backend/operators/builtin/robomimic_hdf5_validate/repository.json index c450df6e..09d26e25 100644 --- a/backend/operators/builtin/robomimic_hdf5_validate/repository.json +++ b/backend/operators/builtin/robomimic_hdf5_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "robomimic_hdf5_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/robomind_hdf5_quality_validate/repository.json b/backend/operators/builtin/robomind_hdf5_quality_validate/repository.json index 7156e988..1b2dc37d 100644 --- a/backend/operators/builtin/robomind_hdf5_quality_validate/repository.json +++ b/backend/operators/builtin/robomind_hdf5_quality_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "robomind_hdf5_quality_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/roboset_hdf5_validate/repository.json b/backend/operators/builtin/roboset_hdf5_validate/repository.json index f828a1af..bc64595c 100644 --- a/backend/operators/builtin/roboset_hdf5_validate/repository.json +++ b/backend/operators/builtin/roboset_hdf5_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "roboset_hdf5_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/robot_model_asset_validate/repository.json b/backend/operators/builtin/robot_model_asset_validate/repository.json index 57010fd4..f228eb4f 100644 --- a/backend/operators/builtin/robot_model_asset_validate/repository.json +++ b/backend/operators/builtin/robot_model_asset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "robot_model_asset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/roboturk_real_hdf5_validate/repository.json b/backend/operators/builtin/roboturk_real_hdf5_validate/repository.json index a8186800..b9550d7b 100644 --- a/backend/operators/builtin/roboturk_real_hdf5_validate/repository.json +++ b/backend/operators/builtin/roboturk_real_hdf5_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "roboturk_real_hdf5_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/ros_joint_state_validate/repository.json b/backend/operators/builtin/ros_joint_state_validate/repository.json index d6e166dd..757cfa81 100644 --- a/backend/operators/builtin/ros_joint_state_validate/repository.json +++ b/backend/operators/builtin/ros_joint_state_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "ros_joint_state_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/rosbag_summary/repository.json b/backend/operators/builtin/rosbag_summary/repository.json index 41e7e795..f9cf1c51 100644 --- a/backend/operators/builtin/rosbag_summary/repository.json +++ b/backend/operators/builtin/rosbag_summary/repository.json @@ -16,8 +16,8 @@ ], "containerEntrypoint": "backend/operator-runner/main.py", "entrypoint": "operator.py", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-rosbag-summary:builtin", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-rosbag-summary@sha256:9d41ed0073c84068869b7c844cf2a5362601dab551835a0ce0a3b1a9d00de9c5", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-rosbag-summary:69b0e0e-poc-20260630072404", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-rosbag-summary:69b0e0e-poc-20260630072404@sha256:ac7d5fbcd7cf2b80c51e6f5f972bc66d7b9563ca3ceeaac7ceba6b3101c9b1f0", "metadata": "operator.json", "name": "ROSBag 数据概览", "operatorId": "operator-builtin-rosbag-summary", @@ -49,6 +49,7 @@ "operatorSlug": "rosbag_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/rtx_action_schema_validate/repository.json b/backend/operators/builtin/rtx_action_schema_validate/repository.json index b1ea5c94..c5c7a9f0 100644 --- a/backend/operators/builtin/rtx_action_schema_validate/repository.json +++ b/backend/operators/builtin/rtx_action_schema_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "rtx_action_schema_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/segmentation_mask_validate/repository.json b/backend/operators/builtin/segmentation_mask_validate/repository.json index 02ea0476..7a046582 100644 --- a/backend/operators/builtin/segmentation_mask_validate/repository.json +++ b/backend/operators/builtin/segmentation_mask_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "segmentation_mask_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/sensor_timestamp_sync_validate/repository.json b/backend/operators/builtin/sensor_timestamp_sync_validate/repository.json index ee42917e..1817a126 100644 --- a/backend/operators/builtin/sensor_timestamp_sync_validate/repository.json +++ b/backend/operators/builtin/sensor_timestamp_sync_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "sensor_timestamp_sync_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/shell_command/repository.json b/backend/operators/builtin/shell_command/repository.json index 60daa1dc..783d1872 100644 --- a/backend/operators/builtin/shell_command/repository.json +++ b/backend/operators/builtin/shell_command/repository.json @@ -48,7 +48,8 @@ "operatorId": "operator-builtin-shell-command", "operatorSlug": "shell_command", "parameters": { - "commandWrapperAllowed": true, + "commandWrapperAllowed": false, + "controlledScriptAllowed": true, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/sim3d_asset_generate/repository.json b/backend/operators/builtin/sim3d_asset_generate/repository.json index 2cd7dfe4..bd2d2b6a 100644 --- a/backend/operators/builtin/sim3d_asset_generate/repository.json +++ b/backend/operators/builtin/sim3d_asset_generate/repository.json @@ -39,7 +39,7 @@ "requiresManifest": true }, "entrypoint": "/app/backend/operator-runner/main.py", - "handler": "builtin_sim3d_asset_generate", + "handler": "sim3d_asset_generate", "imagePolicy": { "allowsSharedBaseLayer": true, "forbidsSharedOperatorRunnerFinalImage": true, @@ -49,6 +49,7 @@ "operatorSlug": "sim3d_asset_generate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/template_render/repository.json b/backend/operators/builtin/template_render/repository.json index 331bc5b9..dd1964ba 100644 --- a/backend/operators/builtin/template_render/repository.json +++ b/backend/operators/builtin/template_render/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "template_render", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/tf_transform_tree_validate/repository.json b/backend/operators/builtin/tf_transform_tree_validate/repository.json index 8b7aaf34..9f83c7a3 100644 --- a/backend/operators/builtin/tf_transform_tree_validate/repository.json +++ b/backend/operators/builtin/tf_transform_tree_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "tf_transform_tree_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/tfrecord_structural_validate/repository.json b/backend/operators/builtin/tfrecord_structural_validate/repository.json index 7a5e8d2f..2ac15ea1 100644 --- a/backend/operators/builtin/tfrecord_structural_validate/repository.json +++ b/backend/operators/builtin/tfrecord_structural_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "tfrecord_structural_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/trajectory_integrity_validate/repository.json b/backend/operators/builtin/trajectory_integrity_validate/repository.json index 79f8cfae..60e0dc36 100644 --- a/backend/operators/builtin/trajectory_integrity_validate/repository.json +++ b/backend/operators/builtin/trajectory_integrity_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "trajectory_integrity_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/umi_zarr_dataset_validate/repository.json b/backend/operators/builtin/umi_zarr_dataset_validate/repository.json index 8361d798..f8f068f8 100644 --- a/backend/operators/builtin/umi_zarr_dataset_validate/repository.json +++ b/backend/operators/builtin/umi_zarr_dataset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "umi_zarr_dataset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/upload/repository.json b/backend/operators/builtin/upload/repository.json index b73b70de..0c4e133b 100644 --- a/backend/operators/builtin/upload/repository.json +++ b/backend/operators/builtin/upload/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "upload", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/video_annotate/repository.json b/backend/operators/builtin/video_annotate/repository.json index 4d313378..d6c27072 100644 --- a/backend/operators/builtin/video_annotate/repository.json +++ b/backend/operators/builtin/video_annotate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "video_annotate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/video_filter/repository.json b/backend/operators/builtin/video_filter/repository.json index acad69f0..283e919b 100644 --- a/backend/operators/builtin/video_filter/repository.json +++ b/backend/operators/builtin/video_filter/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "video_filter", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/video_quality/repository.json b/backend/operators/builtin/video_quality/repository.json index 537c88d0..845d09d2 100644 --- a/backend/operators/builtin/video_quality/repository.json +++ b/backend/operators/builtin/video_quality/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "video_quality", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/video_slice/repository.json b/backend/operators/builtin/video_slice/repository.json index 2f4d2ca5..0916a9cf 100644 --- a/backend/operators/builtin/video_slice/repository.json +++ b/backend/operators/builtin/video_slice/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "video_slice", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/video_target_clip/repository.json b/backend/operators/builtin/video_target_clip/repository.json index c0aa4eef..d7c38eb3 100644 --- a/backend/operators/builtin/video_target_clip/repository.json +++ b/backend/operators/builtin/video_target_clip/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "video_target_clip", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/vla_model_evaluate/repository.json b/backend/operators/builtin/vla_model_evaluate/repository.json index 21a676c8..dda321c5 100644 --- a/backend/operators/builtin/vla_model_evaluate/repository.json +++ b/backend/operators/builtin/vla_model_evaluate/repository.json @@ -39,7 +39,7 @@ "requiresManifest": true }, "entrypoint": "/app/backend/operator-runner/main.py", - "handler": "builtin_vla_model_evaluate", + "handler": "vla_model_evaluate", "imagePolicy": { "allowsSharedBaseLayer": true, "forbidsSharedOperatorRunnerFinalImage": true, @@ -49,6 +49,7 @@ "operatorSlug": "vla_model_evaluate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/vla_model_inference/repository.json b/backend/operators/builtin/vla_model_inference/repository.json index 870c24d9..a20af3a5 100644 --- a/backend/operators/builtin/vla_model_inference/repository.json +++ b/backend/operators/builtin/vla_model_inference/repository.json @@ -39,7 +39,7 @@ "requiresManifest": true }, "entrypoint": "/app/backend/operator-runner/main.py", - "handler": "builtin_vla_model_inference", + "handler": "vla_model_inference", "imagePolicy": { "allowsSharedBaseLayer": true, "forbidsSharedOperatorRunnerFinalImage": true, @@ -49,6 +49,7 @@ "operatorSlug": "vla_model_inference", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/vla_model_train/repository.json b/backend/operators/builtin/vla_model_train/repository.json index 2f76fe70..7d36f4f3 100644 --- a/backend/operators/builtin/vla_model_train/repository.json +++ b/backend/operators/builtin/vla_model_train/repository.json @@ -39,7 +39,7 @@ "requiresManifest": true }, "entrypoint": "/app/backend/operator-runner/main.py", - "handler": "builtin_vla_model_train", + "handler": "vla_model_train", "imagePolicy": { "allowsSharedBaseLayer": true, "forbidsSharedOperatorRunnerFinalImage": true, @@ -49,6 +49,7 @@ "operatorSlug": "vla_model_train", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/vla_sample_manifest_validate/repository.json b/backend/operators/builtin/vla_sample_manifest_validate/repository.json index 88843373..851cb51a 100644 --- a/backend/operators/builtin/vla_sample_manifest_validate/repository.json +++ b/backend/operators/builtin/vla_sample_manifest_validate/repository.json @@ -39,7 +39,7 @@ "requiresManifest": true }, "entrypoint": "/app/backend/operator-runner/main.py", - "handler": "builtin_vla_sample_manifest_validate", + "handler": "vla_sample_manifest_validate", "imagePolicy": { "allowsSharedBaseLayer": true, "forbidsSharedOperatorRunnerFinalImage": true, @@ -49,6 +49,7 @@ "operatorSlug": "vla_sample_manifest_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/webdataset_shard_summary/repository.json b/backend/operators/builtin/webdataset_shard_summary/repository.json index b8f102dd..34ada45d 100644 --- a/backend/operators/builtin/webdataset_shard_summary/repository.json +++ b/backend/operators/builtin/webdataset_shard_summary/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "webdataset_shard_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/wilor_hand_pose/repository.json b/backend/operators/builtin/wilor_hand_pose/repository.json new file mode 100644 index 00000000..68b02b6a --- /dev/null +++ b/backend/operators/builtin/wilor_hand_pose/repository.json @@ -0,0 +1,82 @@ +{ + "acceptance": { + "hardcodedOutputAccepted": false, + "mockOnlyAccepted": false, + "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", + "requiresPersistentWorkspaceEvidence": true, + "requiresRealInputs": true, + "shallowOnlyAccepted": false, + "simulationOnlyAccepted": false + }, + "category": "AI", + "containerArgs": [], + "containerCommand": [ + "python", + "/app/backend/operator-runner/main.py" + ], + "containerEntrypoint": "backend/operator-runner/main.py", + "entrypoint": "operator.py", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-wilor-hand-pose:wilor-hand-pose-io", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-wilor-hand-pose@sha256:454fe90ab54513bb3e022ced3f8fddfe6fd78a3e347780d66488ce0d2c05287c", + "metadata": "operator.json", + "name": "WiLoR 3D手部姿态估计", + "operatorId": "operator-builtin-wilor-hand-pose", + "operatorSlug": "wilor_hand_pose", + "preferredSlug": "wilor_hand_pose", + "pythonImport": "backend.operators.builtin.wilor_hand_pose.operator:run", + "repositoryId": "builtin/wilor_hand_pose", + "runtimeContract": { + "acceptance": { + "requiresArtifactChecksum": true, + "requiresFixtureChecksum": true, + "requiresGpuEvidence": true, + "requiresRealFixture": true, + "requiresWorkflowTransfer": true + }, + "artifacts": { + "requiresChecksum": true, + "requiresLineage": true, + "requiresManifest": true + }, + "entrypoint": "/app/backend/operator-runner/main.py", + "handler": "builtin_wilor_hand_pose", + "imagePolicy": { + "allowsSharedBaseLayer": true, + "forbidsSharedOperatorRunnerFinalImage": true, + "independentFinalImage": true + }, + "operatorId": "operator-builtin-wilor-hand-pose", + "operatorSlug": "wilor_hand_pose", + "parameters": { + "commandWrapperAllowed": false, + "controlledScriptAllowed": false, + "source": "hyperparams", + "structured": true + }, + "resources": { + "externalGpuService": true, + "minGpuCount": 0, + "recommendedGpuType": "NVIDIA A800 80GB", + "requiresGpu": false + }, + "runtimeDependencies": [ + "wilor-inference-service", + "image-video-files", + "http-api", + "gpu-a800-service" + ], + "schemaVersion": "droboticflow.operator.runtime/v1" + }, + "runtimeDependencies": [ + "wilor-inference-service", + "image-video-files", + "http-api", + "gpu-a800-service" + ], + "runtimeHandler": "builtin_wilor_hand_pose", + "runtimeSource": "backend/operators/builtin/wilor_hand_pose/operator.py", + "schemaVersion": "droboticflow.operator.repository/v1", + "sourceRoot": "backend/operators/builtin/wilor_hand_pose", + "strictPocAcceptanceExternals": [], + "type": "builtin" +} diff --git a/backend/operators/builtin/xplatform_dataset_index/repository.json b/backend/operators/builtin/xplatform_dataset_index/repository.json index 0bed5b66..e1020934 100644 --- a/backend/operators/builtin/xplatform_dataset_index/repository.json +++ b/backend/operators/builtin/xplatform_dataset_index/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "xplatform_dataset_index", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/xplatform_dataset_validate/repository.json b/backend/operators/builtin/xplatform_dataset_validate/repository.json index 56c4f555..f630bb5f 100644 --- a/backend/operators/builtin/xplatform_dataset_validate/repository.json +++ b/backend/operators/builtin/xplatform_dataset_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "xplatform_dataset_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/yolo_detection_validate/repository.json b/backend/operators/builtin/yolo_detection_validate/repository.json index d6735cf5..c928a16f 100644 --- a/backend/operators/builtin/yolo_detection_validate/repository.json +++ b/backend/operators/builtin/yolo_detection_validate/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "yolo_detection_validate", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/yolo_embodied_detection_train/repository.json b/backend/operators/builtin/yolo_embodied_detection_train/repository.json index eaf12bfb..4b2026fb 100644 --- a/backend/operators/builtin/yolo_embodied_detection_train/repository.json +++ b/backend/operators/builtin/yolo_embodied_detection_train/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "yolo_embodied_detection_train", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/yolo_video_target_clip/repository.json b/backend/operators/builtin/yolo_video_target_clip/repository.json index 8620d7de..23329b3a 100644 --- a/backend/operators/builtin/yolo_video_target_clip/repository.json +++ b/backend/operators/builtin/yolo_video_target_clip/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "yolo_video_target_clip", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/backend/operators/builtin/zarr_trajectory_summary/repository.json b/backend/operators/builtin/zarr_trajectory_summary/repository.json index ddfb4c1c..7618919e 100644 --- a/backend/operators/builtin/zarr_trajectory_summary/repository.json +++ b/backend/operators/builtin/zarr_trajectory_summary/repository.json @@ -49,6 +49,7 @@ "operatorSlug": "zarr_trajectory_summary", "parameters": { "commandWrapperAllowed": false, + "controlledScriptAllowed": false, "source": "hyperparams", "structured": true }, diff --git a/cmd/worker/main.go b/cmd/worker/main.go index f58bdee7..36372198 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -84,21 +84,13 @@ func main() { fmt.Printf("[worker] failed to initialize runtime: %v\n", err) os.Exit(1) } + runtime.SetBuildInfo(worker.BuildInfo{ + BuildCommit: cfg.BuildCommit, + BuildTime: cfg.BuildTime, + Image: cfg.Image, + }) - go func() { - ticker := time.NewTicker(time.Duration(cfg.PollIntervalMS) * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if err := runtime.ProcessAvailable(ctx); err != nil { - fmt.Printf("[worker] processAvailable failed: %v\n", err) - } - } - } - }() + go runtime.RunScheduler(ctx, time.Duration(cfg.PollIntervalMS)*time.Millisecond) server := &http.Server{ Addr: fmt.Sprintf("0.0.0.0:%d", cfg.WorkerPort), diff --git a/docs/operators/HoloBrain-RoboTwin-评测.md b/docs/operators/HoloBrain-RoboTwin-评测.md index db8d241a..29a4316e 100644 --- a/docs/operators/HoloBrain-RoboTwin-评测.md +++ b/docs/operators/HoloBrain-RoboTwin-评测.md @@ -6,7 +6,7 @@ ## 作用 -消费导出的 HoloBrain 模型,调用官方 robotwin_eval.py 生成 RoboTwin 评测指标。 +调用官方 robotwin_eval.py 执行真实 HoloBrain RoboTwin 评测;smoke 为单任务单 episode。 ## 定位与职责边界 @@ -15,7 +15,7 @@ ## 平台语义 -HoloBrain RoboTwin 评测算子;消费导出的 HoloBrain 模型,调用官方 robotwin_eval.py 输出评测指标。 +HoloBrain RoboTwin 评测算子;full 保留官方多任务评测默认,smoke 固定执行 1 个真实 RoboTwin episode,必须由官方 evaluator 产生指标。 ## 运行时契约 @@ -47,6 +47,7 @@ flowchart LR ## 关键超参数 +- `TRAINING_MODE`:类型 `string`,默认值 `full`。full 或 smoke;smoke 使用专用单任务单 episode 参数 可直接使用默认值,也可在节点级覆盖。 - `ROBOORCHARDLAB_ROOT`:类型 `string`,默认值 `/opt/RoboOrchardLab`。RoboOrchardLab 仓库路径 可直接使用默认值,也可在节点级覆盖。 - `MODEL_DIR`:类型 `string`,默认值 空字符串或由工作流显式填写。导出模型目录;为空时消费上游导出 manifest 通常建议在工作流中显式配置。 - `INFERENCE_PREFIX`:类型 `string`,默认值 `robotwin2_0`。导出 pipeline config prefix 可直接使用默认值,也可在节点级覆盖。 @@ -56,6 +57,10 @@ flowchart LR - `GPU_IDS_JSON`:类型 `json`,默认值 `[0]`。Ray 评测 GPU id 数组 可直接使用默认值,也可在节点级覆盖。 - `WORKERS_PER_GPU`:类型 `number`,默认值 `1`。每张 GPU 的并发 worker 数 可直接使用默认值,也可在节点级覆盖。 - `SAVE_VIDEO`:类型 `boolean`,默认值 `false`。是否保存评测视频 可直接使用默认值,也可在节点级覆盖。 +- `SMOKE_TASK_NAME`:类型 `string`,默认值 `beat_block_hammer`。smoke 实际执行的 RoboTwin 任务 可直接使用默认值,也可在节点级覆盖。 +- `SMOKE_EVAL_EPISODE_NUM`:类型 `number`,默认值 `1`。smoke episode 数 可直接使用默认值,也可在节点级覆盖。 +- `SMOKE_EVAL_MODE`:类型 `string`,默认值 `local`。smoke evaluator 模式 可直接使用默认值,也可在节点级覆盖。 +- `SMOKE_DEVICE`:类型 `string`,默认值 `cuda`。smoke 模型执行设备 可直接使用默认值,也可在节点级覆盖。 - `RECOMMENDED_GPU`:类型 `string`,默认值 `1x NVIDIA A800 80GB`。推荐 GPU 配置 可直接使用默认值,也可在节点级覆盖。 ## 执行前准备 @@ -82,6 +87,7 @@ flowchart LR "type": "builtin", "category": "AI", "hyperparams": { + "TRAINING_MODE": "full", "ROBOORCHARDLAB_ROOT": "/opt/RoboOrchardLab", "MODEL_DIR": "<按节点填写>", "INFERENCE_PREFIX": "robotwin2_0", @@ -91,6 +97,10 @@ flowchart LR "GPU_IDS_JSON": "[0]", "WORKERS_PER_GPU": "1", "SAVE_VIDEO": "false", + "SMOKE_TASK_NAME": "beat_block_hammer", + "SMOKE_EVAL_EPISODE_NUM": "1", + "SMOKE_EVAL_MODE": "local", + "SMOKE_DEVICE": "cuda", "RECOMMENDED_GPU": "1x NVIDIA A800 80GB" } } diff --git a/docs/operators/HoloBrain-完整训练.md b/docs/operators/HoloBrain-完整训练.md index 07e8ce96..3315dba9 100644 --- a/docs/operators/HoloBrain-完整训练.md +++ b/docs/operators/HoloBrain-完整训练.md @@ -6,7 +6,7 @@ ## 作用 -使用 RoboOrchardLab 官方 HoloBrain Qwen 配置执行 100000 step 全量训练,并输出训练报告。 +执行 HoloBrain 训练;full 保留官方长训默认,smoke 用真实小数据短训并输出可恢复 checkpoint 与结构化进度。 ## 定位与职责边界 @@ -15,7 +15,7 @@ ## 平台语义 -HoloBrain 完整训练算子;使用 RoboOrchardLab 官方 projects/holobrain/scripts/train.py 和 config_holobrain_qwen_common.py,默认执行 100000 step 全量训练并输出训练报告。 +HoloBrain 训练算子;TRAINING_MODE=full 保留官方 100000 step 默认,smoke 默认在真实 RoboTwin 小数据上执行 2 step 并生成 checkpoint。失败重试默认恢复最新有效 Accelerate checkpoint;进度写入 workspace 并以 DROBOTICFLOW_PROGRESS {json} 输出;dtype 在启动前检查。 ## 运行时契约 @@ -47,17 +47,26 @@ flowchart LR ## 关键超参数 -- `ROBOORCHARDLAB_ROOT`:类型 `string`,默认值 `/opt/RoboOrchardLab`。RoboOrchardLab 仓库路径;不存在时可克隆 可直接使用默认值,也可在节点级覆盖。 -- `ROBOORCHARDLAB_REF`:类型 `string`,默认值 `master`。克隆 RoboOrchardLab 时使用的分支或 tag 可直接使用默认值,也可在节点级覆盖。 -- `CLONE_ROBOORCHARDLAB`:类型 `boolean`,默认值 `true`。容器内缺少仓库时是否自动 clone 可直接使用默认值,也可在节点级覆盖。 +- `TRAINING_MODE`:类型 `string`,默认值 `full`。full 或 smoke;full 不覆盖官方长训默认,smoke 使用短步数真实训练 可直接使用默认值,也可在节点级覆盖。 +- `ROBOORCHARDLAB_ROOT`:类型 `string`,默认值 `/opt/RoboOrchardLab`。镜像内预置的 RoboOrchardLab 仓库路径;缺失时直接失败 可直接使用默认值,也可在节点级覆盖。 - `CONFIG_PATH`:类型 `string`,默认值 `projects/holobrain/configs/config_holobrain_qwen_common.py`。HoloBrain Qwen 训练配置 可直接使用默认值,也可在节点级覆盖。 - `ROBOTWIN_DATA_ROOT`:类型 `string`,默认值 空字符串或由工作流显式填写。RoboTwin 数据根目录;为空时消费上游数据准备 manifest 通常建议在工作流中显式配置。 - `WORKSPACE`:类型 `string`,默认值 空字符串或由工作流显式填写。训练 workspace;为空时使用节点输出目录下 holobrain-workspace 通常建议在工作流中显式配置。 - `LAUNCH_MODE`:类型 `string`,默认值 `single`。single 或 accelerate 可直接使用默认值,也可在节点级覆盖。 - `GPU_COUNT`:类型 `number`,默认值 `1`。accelerate 模式使用的进程/GPU 数 可直接使用默认值,也可在节点级覆盖。 - `CONFIG_KWARGS_JSON`:类型 `json`,默认值 `{}`。传给 train.py --kwargs 的 JSON;默认不覆盖 max_step=100000 可直接使用默认值,也可在节点级覆盖。 +- `SMOKE_MAX_STEP`:类型 `number`,默认值 `2`。smoke 真实训练最大 step;默认 2 可直接使用默认值,也可在节点级覆盖。 +- `SMOKE_BATCH_SIZE`:类型 `number`,默认值 `1`。smoke batch size 可直接使用默认值,也可在节点级覆盖。 +- `SMOKE_NUM_WORKERS`:类型 `number`,默认值 `0`。smoke 数据加载进程数 可直接使用默认值,也可在节点级覆盖。 +- `RESUME_POLICY`:类型 `string`,默认值 `auto`。auto 自动选择 workspace/checkpoints 下最新有效训练状态;explicit 要求 RESUME_FROM;never 禁用恢复 可直接使用默认值,也可在节点级覆盖。 +- `RESUME_FROM`:类型 `string`,默认值 空字符串或由工作流显式填写。显式 Accelerate 训练状态 checkpoint 目录 通常建议在工作流中显式配置。 +- `INITIAL_CHECKPOINT_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。可选初始模型权重文件;与训练状态恢复分开 通常建议在工作流中显式配置。 +- `CHECKPOINT_ARTIFACT`:类型 `string`,默认值 `latest_valid`。latest_valid、explicit 或 none;控制训练报告暴露的模型产物 可直接使用默认值,也可在节点级覆盖。 +- `OUTPUT_CHECKPOINT_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。CHECKPOINT_ARTIFACT=explicit 时使用的模型 checkpoint 通常建议在工作流中显式配置。 +- `PROGRESS_INTERVAL_SECONDS`:类型 `number`,默认值 `15`。结构化进度写入 droboticflow-progress.json、holobrain-progress.json 并输出 DROBOTICFLOW_PROGRESS {json};该值为最长刷新间隔 可直接使用默认值,也可在节点级覆盖。 +- `MODEL_DTYPE`:类型 `string`,默认值 `bfloat16`。官方 HoloBrain Qwen 模型 dtype;不兼容值在启动前失败 可直接使用默认值,也可在节点级覆盖。 +- `INPUT_DTYPE`:类型 `string`,默认值 `float32`。官方 RoboTwin processor 输入 tensor dtype;不兼容值在启动前失败 可直接使用默认值,也可在节点级覆盖。 - `WANDB_MODE`:类型 `string`,默认值 `offline`。训练日志模式 可直接使用默认值,也可在节点级覆盖。 -- `HF_TOKEN`:类型 `string`,默认值 空字符串或由工作流显式填写。可选 Hugging Face token,用于模型/权重下载 通常建议在工作流中显式配置。 - `RECOMMENDED_GPU`:类型 `string`,默认值 `1x NVIDIA A800 80GB`。推荐 GPU 配置 可直接使用默认值,也可在节点级覆盖。 ## 执行前准备 @@ -84,17 +93,26 @@ flowchart LR "type": "builtin", "category": "AI", "hyperparams": { + "TRAINING_MODE": "full", "ROBOORCHARDLAB_ROOT": "/opt/RoboOrchardLab", - "ROBOORCHARDLAB_REF": "master", - "CLONE_ROBOORCHARDLAB": "true", "CONFIG_PATH": "projects/holobrain/configs/config_holobrain_qwen_common.py", "ROBOTWIN_DATA_ROOT": "<按节点填写>", "WORKSPACE": "<按节点填写>", "LAUNCH_MODE": "single", "GPU_COUNT": "1", "CONFIG_KWARGS_JSON": "{}", + "SMOKE_MAX_STEP": "2", + "SMOKE_BATCH_SIZE": "1", + "SMOKE_NUM_WORKERS": "0", + "RESUME_POLICY": "auto", + "RESUME_FROM": "<按节点填写>", + "INITIAL_CHECKPOINT_PATH": "<按节点填写>", + "CHECKPOINT_ARTIFACT": "latest_valid", + "OUTPUT_CHECKPOINT_PATH": "<按节点填写>", + "PROGRESS_INTERVAL_SECONDS": "15", + "MODEL_DTYPE": "bfloat16", + "INPUT_DTYPE": "float32", "WANDB_MODE": "offline", - "HF_TOKEN": "<按节点填写>", "RECOMMENDED_GPU": "1x NVIDIA A800 80GB" } } diff --git a/docs/operators/HoloBrain-数据准备.md b/docs/operators/HoloBrain-数据准备.md index 975fcddc..6c7a661e 100644 --- a/docs/operators/HoloBrain-数据准备.md +++ b/docs/operators/HoloBrain-数据准备.md @@ -6,7 +6,7 @@ ## 作用 -下载或校验官方 RoboTwin 2.0 真实数据,并生成 HoloBrain 训练可消费的数据 manifest。 +校验真实 RoboTwin 2.0 数据并生成 HoloBrain manifest;支持 smoke 真实小数据和 full 完整数据模式。 ## 定位与职责边界 @@ -15,7 +15,7 @@ ## 平台语义 -HoloBrain 数据准备算子;下载或校验官方 RoboTwin 2.0 真实数据,将其映射为 RoboOrchardLab HoloBrain 训练期望的数据目录并输出 manifest。 +HoloBrain 数据准备算子;TRAINING_MODE=smoke 使用真实小数据契约(Arrow 默认最多 1 个 episode),full 保持完整数据要求;两种模式都拒绝合成或空输入。 ## 运行时契约 @@ -47,14 +47,12 @@ flowchart LR ## 关键超参数 -- `DATA_SOURCE`:类型 `string`,默认值 `hf_snapshot`。数据来源:hf_snapshot 或 local 可直接使用默认值,也可在节点级覆盖。 -- `ROBOTWIN_HF_REPO`:类型 `string`,默认值 `TianxingChen/RoboTwin2.0`。官方 RoboTwin 2.0 Hugging Face dataset repo 可直接使用默认值,也可在节点级覆盖。 -- `ALLOW_DOWNLOAD`:类型 `boolean`,默认值 `true`。允许从 Hugging Face 下载真实数据;关闭时仅校验本地路径 可直接使用默认值,也可在节点级覆盖。 -- `ROBOTWIN_SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。本地 RoboTwin/HoloBrain 数据根目录;DATA_SOURCE=local 时必填 通常建议在工作流中显式配置。 -- `EXPECTED_DATASETS_JSON`:类型 `json`,默认值 `["aloha_agilex_demo_clean","aloha_agilex_demo_randomized","ur5_wsg_demo_clean"]`。期望发现的数据集目录名 可直接使用默认值,也可在节点级覆盖。 -- `HF_ALLOW_PATTERNS_JSON`:类型 `json`,默认值 `[]`。可选 Hugging Face snapshot allow_patterns,空数组表示不过滤 可直接使用默认值,也可在节点级覆盖。 -- `HF_IGNORE_PATTERNS_JSON`:类型 `json`,默认值 `[]`。可选 Hugging Face snapshot ignore_patterns 可直接使用默认值,也可在节点级覆盖。 -- `HF_TOKEN`:类型 `string`,默认值 空字符串或由工作流显式填写。可选 Hugging Face token 通常建议在工作流中显式配置。 +- `TRAINING_MODE`:类型 `string`,默认值 `full`。full 保持完整训练数据行为;smoke 使用真实小数据完成端到端验证 可直接使用默认值,也可在节点级覆盖。 +- `DATA_SOURCE`:类型 `string`,默认值 `local`。数据来源;仅支持 local,不支持运行时下载 可直接使用默认值,也可在节点级覆盖。 +- `ROBOTWIN_SOURCE_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。已预置的 RoboTwin/HoloBrain 真实数据根目录;必填 通常建议在工作流中显式配置。 +- `EXPECTED_LMDB_DIRS_JSON`:类型 `json`,默认值 空字符串或由工作流显式填写。可选覆盖 LMDB 目录;空值时 smoke 只要求 aloha_agilex_demo_clean,full 要求三个官方目录 通常建议在工作流中显式配置。 +- `SMOKE_MAX_EPISODES`:类型 `number`,默认值 `1`。smoke 从真实 Arrow 数据转换的最大 episode 数 可直接使用默认值,也可在节点级覆盖。 +- `ARROW_MAX_EPISODES`:类型 `number`,默认值 空字符串或由工作流显式填写。显式 Arrow 转换 episode 上限;空值时由模式决定 通常建议在工作流中显式配置。 ## 执行前准备 @@ -80,14 +78,12 @@ flowchart LR "type": "builtin", "category": "DATASET", "hyperparams": { - "DATA_SOURCE": "hf_snapshot", - "ROBOTWIN_HF_REPO": "TianxingChen/RoboTwin2.0", - "ALLOW_DOWNLOAD": "true", + "TRAINING_MODE": "full", + "DATA_SOURCE": "local", "ROBOTWIN_SOURCE_PATH": "<按节点填写>", - "EXPECTED_DATASETS_JSON": "[\"aloha_agilex_demo_clean\",\"aloha_agilex_demo_randomized\",\"ur5_wsg_demo_clean\"]", - "HF_ALLOW_PATTERNS_JSON": "[]", - "HF_IGNORE_PATTERNS_JSON": "[]", - "HF_TOKEN": "<按节点填写>" + "EXPECTED_LMDB_DIRS_JSON": "<按节点填写>", + "SMOKE_MAX_EPISODES": "1", + "ARROW_MAX_EPISODES": "<按节点填写>" } } ``` diff --git a/docs/operators/HoloBrain-模型导出.md b/docs/operators/HoloBrain-模型导出.md index 6e1694bf..c879a6b5 100644 --- a/docs/operators/HoloBrain-模型导出.md +++ b/docs/operators/HoloBrain-模型导出.md @@ -6,7 +6,7 @@ ## 作用 -消费 HoloBrain 训练 checkpoint,调用官方 export.py 导出可部署模型目录。 +从显式、上游或训练 workspace 的最新有效 checkpoint 导出可部署 HoloBrain 模型。 ## 定位与职责边界 @@ -15,7 +15,7 @@ ## 平台语义 -HoloBrain 模型导出算子;消费训练 workspace/checkpoint,调用 RoboOrchardLab 官方 export.py 输出可部署模型目录和处理器配置。 +HoloBrain 模型导出算子;按 CHECKPOINT_SELECTION 消费显式、上游或训练 workspace 中最新有效 checkpoint。smoke 模式没有真实训练 checkpoint 时直接失败。 ## 运行时契约 @@ -47,10 +47,12 @@ flowchart LR ## 关键超参数 +- `TRAINING_MODE`:类型 `string`,默认值 `full`。full 或 smoke;smoke 强制要求上游真实 checkpoint 可直接使用默认值,也可在节点级覆盖。 - `ROBOORCHARDLAB_ROOT`:类型 `string`,默认值 `/opt/RoboOrchardLab`。RoboOrchardLab 仓库路径 可直接使用默认值,也可在节点级覆盖。 - `CONFIG_PATH`:类型 `string`,默认值 `projects/holobrain/configs/config_holobrain_qwen_common.py`。HoloBrain Qwen 导出配置 可直接使用默认值,也可在节点级覆盖。 - `TRAINING_WORKSPACE`:类型 `string`,默认值 空字符串或由工作流显式填写。训练 workspace;为空时消费上游训练报告 通常建议在工作流中显式配置。 - `CHECKPOINT_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。显式 checkpoint 路径;为空时使用最新训练 checkpoint 通常建议在工作流中显式配置。 +- `CHECKPOINT_SELECTION`:类型 `string`,默认值 `latest_valid`。latest_valid、upstream 或 explicit 可直接使用默认值,也可在节点级覆盖。 - `EXPORT_WORKSPACE`:类型 `string`,默认值 空字符串或由工作流显式填写。导出目录;为空时使用节点输出目录下 holobrain-export 通常建议在工作流中显式配置。 - `CONFIG_KWARGS_JSON`:类型 `json`,默认值 `{}`。额外 export.py --kwargs JSON 可直接使用默认值,也可在节点级覆盖。 - `RECOMMENDED_GPU`:类型 `string`,默认值 `1x NVIDIA A800 80GB`。export reload-test 使用 CUDA 可直接使用默认值,也可在节点级覆盖。 @@ -79,10 +81,12 @@ flowchart LR "type": "builtin", "category": "AI", "hyperparams": { + "TRAINING_MODE": "full", "ROBOORCHARDLAB_ROOT": "/opt/RoboOrchardLab", "CONFIG_PATH": "projects/holobrain/configs/config_holobrain_qwen_common.py", "TRAINING_WORKSPACE": "<按节点填写>", "CHECKPOINT_PATH": "<按节点填写>", + "CHECKPOINT_SELECTION": "latest_valid", "EXPORT_WORKSPACE": "<按节点填写>", "CONFIG_KWARGS_JSON": "{}", "RECOMMENDED_GPU": "1x NVIDIA A800 80GB" diff --git a/docs/operators/HoloBrain-模型测试.md b/docs/operators/HoloBrain-模型测试.md index d82bfe1f..37f3aea6 100644 --- a/docs/operators/HoloBrain-模型测试.md +++ b/docs/operators/HoloBrain-模型测试.md @@ -6,7 +6,7 @@ ## 作用 -消费导出模型和真实样本路径,执行 HoloBrain 训练后模型测试入口校验。 +加载导出模型并通过官方 RoboTwin evaluator 执行真实模型推理,不接受文件存在性或硬编码结果冒充成功。 ## 定位与职责边界 @@ -15,7 +15,7 @@ ## 平台语义 -HoloBrain 模型测试算子;消费导出模型和真实样本路径,校验导出 pipeline 文件和样本绑定,作为训练后推理烟测入口。 +HoloBrain 模型测试算子;加载导出的真实模型并通过官方 RoboTwin evaluator 执行至少 1 个 episode,不再用文件存在性或硬编码结果冒充推理。 ## 运行时契约 @@ -47,8 +47,16 @@ flowchart LR ## 关键超参数 +- `TRAINING_MODE`:类型 `string`,默认值 `full`。记录上游 full/smoke 模式 可直接使用默认值,也可在节点级覆盖。 +- `ROBOORCHARDLAB_ROOT`:类型 `string`,默认值 `/opt/RoboOrchardLab`。RoboOrchardLab 仓库路径 可直接使用默认值,也可在节点级覆盖。 - `MODEL_DIR`:类型 `string`,默认值 空字符串或由工作流显式填写。导出模型目录;为空时消费上游导出 manifest 通常建议在工作流中显式配置。 -- `SAMPLE_INPUT_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。真实样本文件路径;为空时消费上游数据 manifest 的 sampleCandidates 通常建议在工作流中显式配置。 +- `INFERENCE_PREFIX`:类型 `string`,默认值 `robotwin2_0`。导出 pipeline config prefix 可直接使用默认值,也可在节点级覆盖。 +- `TASK_NAME`:类型 `string`,默认值 `beat_block_hammer`。实际执行的 RoboTwin 推理任务 可直接使用默认值,也可在节点级覆盖。 +- `EPISODE_NUM`:类型 `number`,默认值 `1`。真实推理 episode 数,至少为 1 可直接使用默认值,也可在节点级覆盖。 +- `INFERENCE_MODE`:类型 `string`,默认值 `local`。local 或 ray 可直接使用默认值,也可在节点级覆盖。 +- `DEVICE`:类型 `string`,默认值 `cuda`。模型推理设备 可直接使用默认值,也可在节点级覆盖。 +- `GPU_IDS_JSON`:类型 `json`,默认值 `[0]`。ray 模式 GPU id 数组 可直接使用默认值,也可在节点级覆盖。 +- `WORKERS_PER_GPU`:类型 `number`,默认值 `1`。ray 模式每卡 worker 数 可直接使用默认值,也可在节点级覆盖。 - `RECOMMENDED_GPU`:类型 `string`,默认值 `1x NVIDIA A800 80GB`。完整推理测试推荐 GPU 配置 可直接使用默认值,也可在节点级覆盖。 ## 执行前准备 @@ -75,8 +83,16 @@ flowchart LR "type": "builtin", "category": "AI", "hyperparams": { + "TRAINING_MODE": "full", + "ROBOORCHARDLAB_ROOT": "/opt/RoboOrchardLab", "MODEL_DIR": "<按节点填写>", - "SAMPLE_INPUT_PATH": "<按节点填写>", + "INFERENCE_PREFIX": "robotwin2_0", + "TASK_NAME": "beat_block_hammer", + "EPISODE_NUM": "1", + "INFERENCE_MODE": "local", + "DEVICE": "cuda", + "GPU_IDS_JSON": "[0]", + "WORKERS_PER_GPU": "1", "RECOMMENDED_GPU": "1x NVIDIA A800 80GB" } } diff --git a/docs/operators/README.md b/docs/operators/README.md index f8bc1424..32199940 100644 --- a/docs/operators/README.md +++ b/docs/operators/README.md @@ -2,7 +2,7 @@ 当前目录按“每个内建算子一份 Markdown”组织,内容基于当前平台真实算子定义按算子名称去重生成。 -- 当前收录数量:`101` +- 当前收录数量:`102` - 文档定位:单算子详细手册,重点覆盖输入输出契约、运行时路径、配置示例、排障与验收。 - 配套总览:[`../builtin-operators-manual.zh-CN.md`](../builtin-operators-manual.zh-CN.md) @@ -23,6 +23,7 @@ - [HoloBrain 模型测试](HoloBrain-模型测试.md) - [OpenVLA 微调训练](OpenVLA-微调训练.md) - [OpenVLA 推理](OpenVLA-推理.md) +- [WiLoR 3D手部姿态估计](WiLoR-3D手部姿态估计.md) - [YOLO 具身目标检测训练](YOLO-具身目标检测训练.md) - [模型推理](模型推理.md) - [轻量策略评测](轻量策略评测.md) diff --git a/docs/operators/WiLoR-3D手部姿态估计.md b/docs/operators/WiLoR-3D手部姿态估计.md new file mode 100644 index 00000000..fd78c12b --- /dev/null +++ b/docs/operators/WiLoR-3D手部姿态估计.md @@ -0,0 +1,131 @@ +# WiLoR 3D手部姿态估计 + +- 算子 ID:`operator-builtin-wilor-hand-pose` +- 类别:`AI` +- 类型:`builtin` + +## 作用 + +调用 WiLoR GPU 推理服务,对图片或视频执行 3D 手部姿态估计,并输出标注图、标注视频、关键点 JSON 和运行证据。 + +## 定位与职责边界 + +- 该算子属于 `AI` 类,当前登记类型是 `builtin`。 +- 平台会在 `/api/operators` 中自动提供该算子元数据,运行时由 operator-runner 委派到独立算子入口。 + +## 平台语义 + +WiLoR 3D 手部姿态估计算子;读取图片或视频 URL/上游媒体 artifact,调用已部署的 WiLoR GPU 服务,输出标注图或标注视频、关键点 JSON、mesh OBJ 和服务端 CUDA/A800 运行证据。 + +## 运行时契约 + +- Worker 会先把上游 artifact 汇总到 `INPUT_ARTIFACTS_PATH`,再把节点配置通过 `NODE_DATA_JSON` 注入运行容器。 +- 节点执行结果会写入 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 对应的文件,再由 worker 汇总进运行记录。 +- 大文件和大 JSON 不应依赖 API 内联传输,应该通过 `artifact.path`、`dataRef.payloadPath`、共享 PVC 或对象存储引用传递。 +- 当前独立实现入口是 `operator-repos/builtin/wilor_hand_pose/src/operator.py` 中的 `run(...)`,平台 runtimeHandler 为 `builtin_wilor_hand_pose`。 + +## 设计边界 + +- 该算子应该只承担单一职责,不应在一个节点里同时混合索引、筛选、上传、审核等多个阶段性行为。 +- 如果一个节点要消费多个语义不同的结果,优先通过多节点拆分,而不是把所有逻辑塞进一个超大脚本。 + +## 数据流示意 + +```mermaid +flowchart LR + A[上游输入或运行时注入] --> B[WiLoR 3D手部姿态估计] + B --> C[artifact / stage manifest / 外部副作用] +``` + +## 输入端口详解 + +- `in-1` / **media** / `json` / `bus`:图片/视频 artifact,或包含 url/path 字段的媒体清单 作为输入端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 输出端口详解 + +- `out-1` / **wilorResult** / `json` / `bus`:WiLoR 结果 manifest、标注媒体和关键点 JSON 作为输出端口时,适合结构化清单、规则结果、摘要对象等可直接序列化的数据。 通过统一 artifact 契约传递;大 payload 会外置到 stage manifest / payloadPath。 + +## 关键超参数 + +- `WILOR_API_BASE_URL`:类型 `string`,默认值 `http://120.48.65.232`。WiLoR 推理服务地址 可直接使用默认值,也可在节点级覆盖。 +- `INPUT_URL`:类型 `string`,默认值 `http://120.48.65.232/examples/sample-hands.png`。输入图片或视频 URL;为空时读取上游媒体 artifact 可直接使用默认值,也可在节点级覆盖。 +- `INPUT_PATH`:类型 `string`,默认值 空字符串或由工作流显式填写。输入媒体在 runner 容器内的本地路径;优先级低于 INPUT_URL 通常建议在工作流中显式配置。 +- `MEDIA_KIND`:类型 `string`,默认值 `auto`。媒体类型,auto/image/video 可直接使用默认值,也可在节点级覆盖。 +- `HAND_CONFIDENCE`:类型 `number`,默认值 `0.30`。手部检测置信度阈值,范围 0.05-0.95 可直接使用默认值,也可在节点级覆盖。 +- `RESCALE_FACTOR`:类型 `number`,默认值 `2.5`。WiLoR crop 重缩放因子,范围 1.0-4.0 可直接使用默认值,也可在节点级覆盖。 +- `INCLUDE_MESH`:类型 `boolean`,默认值 `true`。图片推理是否下载第一只手的 OBJ mesh 可直接使用默认值,也可在节点级覆盖。 +- `FRAME_STRIDE`:类型 `integer`,默认值 `3`。视频抽帧间隔,范围 1-30 可直接使用默认值,也可在节点级覆盖。 +- `MAX_FRAMES`:类型 `integer`,默认值 `120`。视频最多采样帧数 可直接使用默认值,也可在节点级覆盖。 +- `DOWNLOAD_OUTPUTS`:类型 `boolean`,默认值 `true`。是否下载 WiLoR 返回的标注媒体和 JSON 文件 可直接使用默认值,也可在节点级覆盖。 +- `REQUEST_TIMEOUT_SECONDS`:类型 `integer`,默认值 `300`。单次 API 请求超时时间,单位秒 可直接使用默认值,也可在节点级覆盖。 +- `WILOR_SERVICE_IMAGE`:类型 `string`,默认值 `ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/custom-operators/wilor-inference:f3a4957-gpu-20260703172135`。WiLoR GPU 服务镜像 tag,仅用于运行证据记录 可直接使用默认值,也可在节点级覆盖。 +- `WILOR_SERVICE_IMAGE_ID`:类型 `string`,默认值 `ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/custom-operators/wilor-inference@sha256:be8369126716ad63ea62a93c8bc3e56425a5370cd77144e32b74254f1ff33055`。WiLoR GPU 服务 digest-pinned imageId,仅用于运行证据记录 可直接使用默认值,也可在节点级覆盖。 + +## 执行前准备 + +- 先确认上下游端口类型兼容,特别是 `json`、`file`、`video` 等端口不要混接。 +- 为每个关键超参数补齐明确值,避免依赖隐式默认值进入生产工作流。 +- 对需要跨节点长期保留的产物,提前规划对象存储前缀或外部落库集合,而不是只写临时目录。 + +## 数据通信与存储路径 + +- 标准输入来自 `INPUT_ARTIFACTS_PATH` 指向的 artifact 列表文件,artifact 可同时包含 `data`、`path`、`metadata` 与 `dataRef`。 +- 标准输出写入 `OUTPUT_DIR`,并通过 `stage-manifest.json` 对外暴露产物路径、payload 引用和摘要元数据。 + +## 典型上下游编排 + +- 该算子通常位于同类数据处理节点之间,承担结构化中间层职责。 + +## 节点配置示例 + +```json +{ + "label": "WiLoR 3D手部姿态估计", + "operatorId": "operator-builtin-wilor-hand-pose", + "type": "builtin", + "category": "AI", + "hyperparams": { + "WILOR_API_BASE_URL": "http://120.48.65.232", + "INPUT_URL": "http://120.48.65.232/examples/sample-hands.png", + "INPUT_PATH": "<按节点填写>", + "MEDIA_KIND": "auto", + "HAND_CONFIDENCE": "0.30", + "RESCALE_FACTOR": "2.5", + "INCLUDE_MESH": "true", + "FRAME_STRIDE": "3", + "MAX_FRAMES": "120", + "DOWNLOAD_OUTPUTS": "true", + "REQUEST_TIMEOUT_SECONDS": "300", + "WILOR_SERVICE_IMAGE": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/custom-operators/wilor-inference:f3a4957-gpu-20260703172135", + "WILOR_SERVICE_IMAGE_ID": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/custom-operators/wilor-inference@sha256:be8369126716ad63ea62a93c8bc3e56425a5370cd77144e32b74254f1ff33055" + } +} +``` + +## 输入输出示例 + +- 典型输入:worker 会把上游 artifact 序列化为 `INPUT_ARTIFACTS_PATH` 文件供节点读取。 +- 典型输出:节点会在 `OUTPUT_DIR` 下写业务文件,并在 `stage-manifest.json` 里登记对外可消费的结果。 + +## 成功判定 + +- 节点 execution 状态为 `completed`,且日志中没有隐藏的回退错误或重试未果记录。 +- `stage-manifest.json` 中能看到期望的 artifact 条目,且下游节点可以正常消费这些结果。 + +## 常见失败与排查 + +- 如果看不到预期结果,先检查 `OUTPUT_DIR`、`RESULT_OUTPUT_PATH` 和 `STAGE_MANIFEST_PATH` 是否真实写出。 +- 如果下游收不到数据,先检查当前节点输出 artifact 是否设置了正确的 `portId`、`portName` 和 `type`。 +- 如果日志中只有摘要没有大 payload,优先排查 `dataRef.payloadPath` 或外置文件是否存在,而不是误以为结果丢失。 + +## 验收检查清单 + +- 至少在一条真实工作流中跑通该算子,而不是只验证单元测试或空输入。 +- 确认该节点输出被下游节点实际消费,避免出现“本节点成功但链路无效”的假阳性。 +- 保留一次成功运行的示例日志、stage manifest 和关键产物路径,便于后续回归对照。 + +## 相关文档 + +- [AI 类索引](README.md) +- [算子开发文档](../operator-authoring.md) +- [内建算子总手册](../builtin-operators-manual.zh-CN.md) diff --git a/internal/api/acceptance_import.go b/internal/api/acceptance_import.go index 69f6be9c..00c0bd42 100644 --- a/internal/api/acceptance_import.go +++ b/internal/api/acceptance_import.go @@ -30,11 +30,15 @@ var requiredPocAcceptanceFixtureSourceNames = []string{"csv", "json", "lerobot", type pocAcceptanceImportRequest struct { Result map[string]any `json:"result"` Source struct { - Environment string `json:"environment,omitempty"` - Namespace string `json:"namespace,omitempty"` - JobName string `json:"jobName,omitempty"` - Image string `json:"image,omitempty"` - ImageID string `json:"imageId,omitempty"` + Environment string `json:"environment,omitempty"` + Namespace string `json:"namespace,omitempty"` + JobName string `json:"jobName,omitempty"` + Image string `json:"image,omitempty"` + ImageID string `json:"imageId,omitempty"` + SourceCommit string `json:"sourceCommit,omitempty"` + CatalogHash string `json:"catalogHash,omitempty"` + CatalogCount int `json:"catalogCount,omitempty"` + CompletedAt string `json:"completedAt,omitempty"` } `json:"source,omitempty"` } @@ -448,7 +452,7 @@ func acceptanceResultError(payload map[string]any) string { return "" } -func acceptanceResultArtifactData(payload map[string]any, source pocAcceptanceImportRequest) map[string]any { +func acceptanceResultArtifactData(payload map[string]any, source pocAcceptanceImportRequest, completedAt string) map[string]any { data := make(map[string]any, len(payload)+1) for key, value := range payload { data[key] = value @@ -469,9 +473,77 @@ func acceptanceResultArtifactData(payload map[string]any, source pocAcceptanceIm if source.Source.ImageID != "" { sourceData["imageId"] = source.Source.ImageID } + if source.Source.SourceCommit != "" { + sourceData["sourceCommit"] = strings.TrimSpace(source.Source.SourceCommit) + } + if source.Source.CatalogHash != "" { + sourceData["catalogHash"] = strings.TrimSpace(source.Source.CatalogHash) + } + if source.Source.CatalogCount > 0 { + sourceData["catalogCount"] = source.Source.CatalogCount + } + if source.Source.CompletedAt != "" { + sourceData["completedAt"] = strings.TrimSpace(source.Source.CompletedAt) + } if len(sourceData) > 0 { data["importSource"] = sourceData } + staticPayload, _ := payload["static"].(map[string]any) + resultSource, _ := payload["source"].(map[string]any) + if stringField(data, "sourceCommit") == "" { + value := strings.TrimSpace(source.Source.SourceCommit) + if value == "" { + value = stringField(resultSource, "sourceCommit") + } + if value == "" { + value = stringField(staticPayload, "sourceCommit") + } + if value != "" { + data["sourceCommit"] = value + } + } + if stringField(data, "catalogHash") == "" { + value := strings.TrimSpace(source.Source.CatalogHash) + if value == "" { + value = stringField(resultSource, "catalogHash") + } + if value == "" { + value = stringField(staticPayload, "catalogHash") + } + if value != "" { + data["catalogHash"] = value + } + } + if numberField(data, "catalogCount") == 0 { + count := source.Source.CatalogCount + if count <= 0 { + count = numberField(resultSource, "catalogCount") + } + if count <= 0 { + count = numberField(staticPayload, "catalogCount") + } + if count <= 0 { + count = numberField(staticPayload, "operatorFileCount") + } + if count <= 0 { + count = numberField(payload, "operatorCount") + } + if count > 0 { + data["catalogCount"] = count + } + } + if stringField(data, "completedAt") == "" { + value := strings.TrimSpace(source.Source.CompletedAt) + if value == "" { + value = stringField(resultSource, "completedAt") + } + if value == "" { + value = strings.TrimSpace(completedAt) + } + if value != "" { + data["completedAt"] = value + } + } return data } @@ -564,10 +636,6 @@ func (server *server) handlePocAcceptanceImport(writer http.ResponseWriter, requ } runStatus := acceptanceResultStatus(payload.Result) - artifactData := acceptanceResultArtifactData(payload.Result, payload) - rawArtifact, _ := json.Marshal(artifactData) - hash := sha256.Sum256(rawArtifact) - runID := acceptanceRunID(payload.Result) startedAt := stringField(payload.Result, "startedAt") if startedAt == "" { startedAt = now @@ -576,6 +644,10 @@ func (server *server) handlePocAcceptanceImport(writer http.ResponseWriter, requ if endedAt == "" { endedAt = now } + artifactData := acceptanceResultArtifactData(payload.Result, payload, endedAt) + rawArtifact, _ := json.Marshal(artifactData) + hash := sha256.Sum256(rawArtifact) + runID := acceptanceRunID(payload.Result) artifact := domain.ExecutionArtifact{ ID: "artifact-" + runID, RunID: runID, diff --git a/internal/api/api_token.go b/internal/api/api_token.go index 97a77f3c..e50f59d7 100644 --- a/internal/api/api_token.go +++ b/internal/api/api_token.go @@ -10,7 +10,7 @@ import ( type apiTokenContextKey struct{} func isPublicEndpoint(request *http.Request) bool { - if request.URL.Path == "/api/health" || isWorkerAPIHealthEndpoint(request) { + if request.URL.Path == "/api/health" || request.URL.Path == "/api/ready" || isWorkerAPIHealthEndpoint(request) { return true } if request.URL.Path == "/api/public/settings" { diff --git a/internal/api/auth.go b/internal/api/auth.go index e8868912..87288204 100644 --- a/internal/api/auth.go +++ b/internal/api/auth.go @@ -117,12 +117,13 @@ func isExternalBusinessWriteEndpoint(path string) bool { } func isWorkerAPIHealthEndpoint(request *http.Request) bool { - return request.Method == http.MethodGet && request.URL.Path == "/worker-api/health" + return request.Method == http.MethodGet && (request.URL.Path == "/worker-api/health" || request.URL.Path == "/worker-api/ready") } func isSessionPublicEndpoint(request *http.Request) bool { path := request.URL.Path return path == "/api/health" || + path == "/api/ready" || path == "/api/public/settings" || isWorkerAPIHealthEndpoint(request) || isExternalBusinessWriteEndpoint(path) || diff --git a/internal/api/builtin_operators.go b/internal/api/builtin_operators.go index 3092ea5b..98286b22 100644 --- a/internal/api/builtin_operators.go +++ b/internal/api/builtin_operators.go @@ -2,6 +2,8 @@ package api import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "strings" @@ -30,6 +32,23 @@ var reservedMongoCollections = map[string]struct{}{ //go:embed builtin_operators_catalog.json var builtinOperatorCatalogJSON []byte +func builtinOperatorCatalogFingerprint() (string, int, error) { + var catalog any + if err := json.Unmarshal(builtinOperatorCatalogJSON, &catalog); err != nil { + return "", 0, fmt.Errorf("parse builtin operator catalog for fingerprint: %w", err) + } + records, ok := catalog.([]any) + if !ok || len(records) == 0 { + return "", 0, fmt.Errorf("builtin operator catalog is empty") + } + canonical, err := json.Marshal(catalog) + if err != nil { + return "", 0, fmt.Errorf("canonicalize builtin operator catalog: %w", err) + } + digest := sha256.Sum256(canonical) + return hex.EncodeToString(digest[:]), len(records), nil +} + func builtinOperatorRecords(now string, runnerImage string) ([]domain.OperatorRecord, error) { var records []domain.OperatorRecord if err := json.Unmarshal(builtinOperatorCatalogJSON, &records); err != nil { diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index 38e2ee52..cf914a1f 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -10569,8 +10569,14 @@ "python", "/app/backend/operator-runner/main.py" ], - "description": "校验已预置的官方 RoboTwin 2.0 真实数据,并生成 HoloBrain 训练可消费的数据 manifest。运行时不下载数据。", + "description": "校验真实 RoboTwin 2.0 数据并生成 HoloBrain manifest;支持 smoke 真实小数据和 full 完整数据模式。", "hyperparams": [ + { + "description": "full 保持完整训练数据行为;smoke 使用真实小数据完成端到端验证", + "key": "TRAINING_MODE", + "type": "string", + "value": "full" + }, { "description": "数据来源;仅支持 local,不支持运行时下载", "key": "DATA_SOURCE", @@ -10584,10 +10590,22 @@ "value": "" }, { - "description": "期望发现的数据集目录名", - "key": "EXPECTED_DATASETS_JSON", + "description": "可选覆盖 LMDB 目录;空值时 smoke 只要求 aloha_agilex_demo_clean,full 要求三个官方目录", + "key": "EXPECTED_LMDB_DIRS_JSON", "type": "json", - "value": "[\"robotwin2_0\",\"robotwin2_0_ur5_wsg\"]" + "value": "" + }, + { + "description": "smoke 从真实 Arrow 数据转换的最大 episode 数", + "key": "SMOKE_MAX_EPISODES", + "type": "number", + "value": "1" + }, + { + "description": "显式 Arrow 转换 episode 上限;空值时由模式决定", + "key": "ARROW_MAX_EPISODES", + "type": "number", + "value": "" } ], "id": "operator-builtin-holobrain-data-prepare", @@ -10648,7 +10666,7 @@ }, "runtimeHandler": "builtin_holobrain_data_prepare", "runtimeSource": "operator-repos/builtin/holobrain_data_prepare/src/operator.py", - "skillDefinition": "HoloBrain 数据准备算子;校验已预置的官方 RoboTwin 2.0 真实数据,将其映射为 RoboOrchardLab HoloBrain 训练期望的数据目录并输出 manifest。运行时不下载数据,缺少真实数据会直接失败。", + "skillDefinition": "HoloBrain 数据准备算子;TRAINING_MODE=smoke 使用真实小数据契约(Arrow 默认最多 1 个 episode),full 保持完整数据要求;两种模式都拒绝合成或空输入。", "type": "builtin" }, { @@ -10665,8 +10683,14 @@ "python", "/app/backend/operator-runner/main.py" ], - "description": "使用 RoboOrchardLab 官方 HoloBrain Qwen 配置执行 100000 step 全量训练,并输出训练报告。", + "description": "执行 HoloBrain 训练;full 保留官方长训默认,smoke 用真实小数据短训并输出可恢复 checkpoint 与结构化进度。", "hyperparams": [ + { + "description": "full 或 smoke;full 不覆盖官方长训默认,smoke 使用短步数真实训练", + "key": "TRAINING_MODE", + "type": "string", + "value": "full" + }, { "description": "镜像内预置的 RoboOrchardLab 仓库路径;缺失时直接失败", "key": "ROBOORCHARDLAB_ROOT", @@ -10709,6 +10733,72 @@ "type": "json", "value": "{}" }, + { + "description": "smoke 真实训练最大 step;默认 2", + "key": "SMOKE_MAX_STEP", + "type": "number", + "value": "2" + }, + { + "description": "smoke batch size", + "key": "SMOKE_BATCH_SIZE", + "type": "number", + "value": "1" + }, + { + "description": "smoke 数据加载进程数", + "key": "SMOKE_NUM_WORKERS", + "type": "number", + "value": "0" + }, + { + "description": "auto 自动选择 workspace/checkpoints 下最新有效训练状态;explicit 要求 RESUME_FROM;never 禁用恢复", + "key": "RESUME_POLICY", + "type": "string", + "value": "auto" + }, + { + "description": "显式 Accelerate 训练状态 checkpoint 目录", + "key": "RESUME_FROM", + "type": "string", + "value": "" + }, + { + "description": "可选初始模型权重文件;与训练状态恢复分开", + "key": "INITIAL_CHECKPOINT_PATH", + "type": "string", + "value": "" + }, + { + "description": "latest_valid、explicit 或 none;控制训练报告暴露的模型产物", + "key": "CHECKPOINT_ARTIFACT", + "type": "string", + "value": "latest_valid" + }, + { + "description": "CHECKPOINT_ARTIFACT=explicit 时使用的模型 checkpoint", + "key": "OUTPUT_CHECKPOINT_PATH", + "type": "string", + "value": "" + }, + { + "description": "结构化进度写入 droboticflow-progress.json、holobrain-progress.json 并输出 DROBOTICFLOW_PROGRESS {json};该值为最长刷新间隔", + "key": "PROGRESS_INTERVAL_SECONDS", + "type": "number", + "value": "15" + }, + { + "description": "官方 HoloBrain Qwen 模型 dtype;不兼容值在启动前失败", + "key": "MODEL_DTYPE", + "type": "string", + "value": "bfloat16" + }, + { + "description": "官方 RoboTwin processor 输入 tensor dtype;不兼容值在启动前失败", + "key": "INPUT_DTYPE", + "type": "string", + "value": "float32" + }, { "description": "训练日志模式", "key": "WANDB_MODE", @@ -10795,7 +10885,7 @@ }, "runtimeHandler": "builtin_holobrain_train", "runtimeSource": "operator-repos/builtin/holobrain_train/src/operator.py", - "skillDefinition": "HoloBrain 完整训练算子;使用 RoboOrchardLab 官方 projects/holobrain/scripts/train.py 和 config_holobrain_qwen_common.py,默认执行 100000 step 全量训练并输出训练报告。", + "skillDefinition": "HoloBrain 训练算子;TRAINING_MODE=full 保留官方 100000 step 默认,smoke 默认在真实 RoboTwin 小数据上执行 2 step 并生成 checkpoint。失败重试默认恢复最新有效 Accelerate checkpoint;进度写入 workspace 并以 DROBOTICFLOW_PROGRESS {json} 输出;dtype 在启动前检查。", "type": "builtin" }, { @@ -10812,8 +10902,14 @@ "python", "/app/backend/operator-runner/main.py" ], - "description": "消费 HoloBrain 训练 checkpoint,调用官方 export.py 导出可部署模型目录。", + "description": "从显式、上游或训练 workspace 的最新有效 checkpoint 导出可部署 HoloBrain 模型。", "hyperparams": [ + { + "description": "full 或 smoke;smoke 强制要求上游真实 checkpoint", + "key": "TRAINING_MODE", + "type": "string", + "value": "full" + }, { "description": "RoboOrchardLab 仓库路径", "key": "ROBOORCHARDLAB_ROOT", @@ -10838,6 +10934,12 @@ "type": "string", "value": "" }, + { + "description": "latest_valid、upstream 或 explicit", + "key": "CHECKPOINT_SELECTION", + "type": "string", + "value": "latest_valid" + }, { "description": "导出目录;为空时使用节点输出目录下 holobrain-export", "key": "EXPORT_WORKSPACE", @@ -10928,7 +11030,7 @@ }, "runtimeHandler": "builtin_holobrain_export", "runtimeSource": "operator-repos/builtin/holobrain_export/src/operator.py", - "skillDefinition": "HoloBrain 模型导出算子;消费训练 workspace/checkpoint,调用 RoboOrchardLab 官方 export.py 输出可部署模型目录和处理器配置。", + "skillDefinition": "HoloBrain 模型导出算子;按 CHECKPOINT_SELECTION 消费显式、上游或训练 workspace 中最新有效 checkpoint。smoke 模式没有真实训练 checkpoint 时直接失败。", "type": "builtin" }, { @@ -10945,8 +11047,14 @@ "python", "/app/backend/operator-runner/main.py" ], - "description": "消费导出的 HoloBrain 模型,调用官方 robotwin_eval.py 生成 RoboTwin 评测指标。", + "description": "调用官方 robotwin_eval.py 执行真实 HoloBrain RoboTwin 评测;smoke 为单任务单 episode。", "hyperparams": [ + { + "description": "full 或 smoke;smoke 使用专用单任务单 episode 参数", + "key": "TRAINING_MODE", + "type": "string", + "value": "full" + }, { "description": "RoboOrchardLab 仓库路径", "key": "ROBOORCHARDLAB_ROOT", @@ -11001,6 +11109,30 @@ "type": "boolean", "value": "false" }, + { + "description": "smoke 实际执行的 RoboTwin 任务", + "key": "SMOKE_TASK_NAME", + "type": "string", + "value": "beat_block_hammer" + }, + { + "description": "smoke episode 数", + "key": "SMOKE_EVAL_EPISODE_NUM", + "type": "number", + "value": "1" + }, + { + "description": "smoke evaluator 模式", + "key": "SMOKE_EVAL_MODE", + "type": "string", + "value": "local" + }, + { + "description": "smoke 模型执行设备", + "key": "SMOKE_DEVICE", + "type": "string", + "value": "cuda" + }, { "description": "推荐 GPU 配置", "key": "RECOMMENDED_GPU", @@ -11079,7 +11211,7 @@ }, "runtimeHandler": "builtin_holobrain_robotwin_eval", "runtimeSource": "operator-repos/builtin/holobrain_robotwin_eval/src/operator.py", - "skillDefinition": "HoloBrain RoboTwin 评测算子;消费导出的 HoloBrain 模型,调用官方 robotwin_eval.py 输出评测指标。", + "skillDefinition": "HoloBrain RoboTwin 评测算子;full 保留官方多任务评测默认,smoke 固定执行 1 个真实 RoboTwin episode,必须由官方 evaluator 产生指标。", "type": "builtin" }, { @@ -11096,8 +11228,20 @@ "python", "/app/backend/operator-runner/main.py" ], - "description": "消费导出模型和真实样本路径,执行 HoloBrain 训练后模型测试入口校验。", + "description": "加载导出模型并通过官方 RoboTwin evaluator 执行真实模型推理,不接受文件存在性或硬编码结果冒充成功。", "hyperparams": [ + { + "description": "记录上游 full/smoke 模式", + "key": "TRAINING_MODE", + "type": "string", + "value": "full" + }, + { + "description": "RoboOrchardLab 仓库路径", + "key": "ROBOORCHARDLAB_ROOT", + "type": "string", + "value": "/opt/RoboOrchardLab" + }, { "description": "导出模型目录;为空时消费上游导出 manifest", "key": "MODEL_DIR", @@ -11105,10 +11249,46 @@ "value": "" }, { - "description": "真实样本文件路径;为空时消费上游数据 manifest 的 sampleCandidates", - "key": "SAMPLE_INPUT_PATH", + "description": "导出 pipeline config prefix", + "key": "INFERENCE_PREFIX", "type": "string", - "value": "" + "value": "robotwin2_0" + }, + { + "description": "实际执行的 RoboTwin 推理任务", + "key": "TASK_NAME", + "type": "string", + "value": "beat_block_hammer" + }, + { + "description": "真实推理 episode 数,至少为 1", + "key": "EPISODE_NUM", + "type": "number", + "value": "1" + }, + { + "description": "local 或 ray", + "key": "INFERENCE_MODE", + "type": "string", + "value": "local" + }, + { + "description": "模型推理设备", + "key": "DEVICE", + "type": "string", + "value": "cuda" + }, + { + "description": "ray 模式 GPU id 数组", + "key": "GPU_IDS_JSON", + "type": "json", + "value": "[0]" + }, + { + "description": "ray 模式每卡 worker 数", + "key": "WORKERS_PER_GPU", + "type": "number", + "value": "1" }, { "description": "完整推理测试推荐 GPU 配置", @@ -11188,7 +11368,7 @@ }, "runtimeHandler": "builtin_holobrain_inference_test", "runtimeSource": "operator-repos/builtin/holobrain_inference_test/src/operator.py", - "skillDefinition": "HoloBrain 模型测试算子;消费导出模型和真实样本路径,校验导出 pipeline 文件和样本绑定,作为训练后推理烟测入口。", + "skillDefinition": "HoloBrain 模型测试算子;加载导出的真实模型并通过官方 RoboTwin evaluator 执行至少 1 个 episode,不再用文件存在性或硬编码结果冒充推理。", "type": "builtin" } ] diff --git a/internal/api/dashboard_operator_inventory_test.go b/internal/api/dashboard_operator_inventory_test.go new file mode 100644 index 00000000..558cd705 --- /dev/null +++ b/internal/api/dashboard_operator_inventory_test.go @@ -0,0 +1,19 @@ +package api + +import ( + "testing" + + "droboticflow/internal/domain" +) + +func TestEnrichDashboardOperatorInventorySeparatesCurrentCustomAndHistorical(t *testing.T) { + var summary domain.DashboardSummary + enrichDashboardOperatorInventory(&summary, []domain.OperatorRecord{ + {ID: "operator-builtin-upload", Type: "builtin"}, + {ID: "operator-custom-demo", Type: "custom"}, + {ID: "operator-builtin-retired", Type: "builtin"}, + }) + if summary.Operators.Total != 3 || summary.Operators.Builtin != 1 || summary.Operators.Custom != 1 || summary.Operators.Historical != 1 { + t.Fatalf("operator inventory = %#v", summary.Operators) + } +} diff --git a/internal/api/object_storage_browser.go b/internal/api/object_storage_browser.go index aff4d2a1..7d4b8cb0 100644 --- a/internal/api/object_storage_browser.go +++ b/internal/api/object_storage_browser.go @@ -45,6 +45,16 @@ type objectStorageBrowser interface { BuildPreviewURL(context.Context, string, string) (string, error) } +type objectStoragePage struct { + Items []domain.ObjectStorageObjectSummary + NextMarker string + HasMore bool +} + +type pagedObjectStorageBrowser interface { + ListObjectsPage(context.Context, string, string, string, int) (objectStoragePage, error) +} + type bosObjectStorageBrowser struct { connection *domain.StorageConnectionRecord client *bos.Client @@ -185,55 +195,90 @@ func (browser *bosObjectStorageBrowser) ListBuckets(ctx context.Context) ([]doma } func (browser *bosObjectStorageBrowser) ListObjects(ctx context.Context, bucket string, prefix string) ([]domain.ObjectStorageObjectSummary, error) { - normalizedBucket := strings.TrimSpace(bucket) - requestPrefix := storageObjectRequestPrefix(prefix) items := make([]domain.ObjectStorageObjectSummary, 0) seen := make(map[string]struct{}) marker := "" for { - result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ - Prefix: requestPrefix, - Delimiter: "/", - MaxKeys: 1000, - Marker: marker, - }) + page, err := browser.ListObjectsPage(ctx, bucket, prefix, marker, 1000) if err != nil { return nil, err } - for _, item := range result.CommonPrefixes { - prefixValue := strings.Trim(strings.TrimSpace(item.Prefix), "/") - if prefixValue == "" || prefixValue == strings.Trim(strings.TrimSpace(prefix), "/") { + for _, item := range page.Items { + key := item.Kind + "|" + item.Key + if _, exists := seen[key]; exists { continue } - if _, exists := seen["dir|"+prefixValue]; exists { - continue - } - seen["dir|"+prefixValue] = struct{}{} - items = append(items, objectStorageSummaryFromPrefix(normalizedBucket, prefixValue)) + seen[key] = struct{}{} + items = append(items, item) } - for _, item := range result.Contents { - key := strings.Trim(strings.TrimSpace(item.Key), "/") - if key == "" || key == strings.Trim(strings.TrimSpace(prefix), "/") { - continue - } - if _, exists := seen["file|"+key]; exists { - continue - } - seen["file|"+key] = struct{}{} - items = append(items, objectStorageSummaryFromListObject(normalizedBucket, item)) - } - if result == nil || !result.IsTruncated { + if !page.HasMore || page.NextMarker == "" || page.NextMarker == marker { break } - nextMarker := strings.TrimSpace(result.NextMarker) - if nextMarker == "" || nextMarker == marker { - break - } - marker = nextMarker + marker = page.NextMarker } return items, nil } +func (browser *bosObjectStorageBrowser) ListObjectsPage( + ctx context.Context, + bucket string, + prefix string, + marker string, + limit int, +) (objectStoragePage, error) { + normalizedBucket := strings.TrimSpace(bucket) + normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/") + if limit <= 0 { + limit = 100 + } + if limit > 1000 { + limit = 1000 + } + result, err := browser.client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{ + Prefix: storageObjectRequestPrefix(prefix), + Delimiter: "/", + MaxKeys: limit, + Marker: strings.TrimSpace(marker), + }) + if err != nil { + return objectStoragePage{}, err + } + if result == nil { + return objectStoragePage{Items: []domain.ObjectStorageObjectSummary{}}, nil + } + items := make([]domain.ObjectStorageObjectSummary, 0, len(result.CommonPrefixes)+len(result.Contents)) + seen := make(map[string]struct{}) + for _, item := range result.CommonPrefixes { + prefixValue := strings.Trim(strings.TrimSpace(item.Prefix), "/") + if prefixValue == "" || prefixValue == normalizedPrefix { + continue + } + key := "dir|" + prefixValue + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + items = append(items, objectStorageSummaryFromPrefix(normalizedBucket, prefixValue)) + } + for _, item := range result.Contents { + keyValue := strings.Trim(strings.TrimSpace(item.Key), "/") + if keyValue == "" || keyValue == normalizedPrefix { + continue + } + key := "file|" + keyValue + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + items = append(items, objectStorageSummaryFromListObject(normalizedBucket, item)) + } + return objectStoragePage{ + Items: items, + NextMarker: strings.TrimSpace(result.NextMarker), + HasMore: result.IsTruncated, + }, nil +} + func (browser *bosObjectStorageBrowser) CreateBucket(_ context.Context, bucket string) (domain.ObjectStorageBucketSummary, error) { if _, err := browser.client.PutBucket(strings.TrimSpace(bucket)); err != nil { return domain.ObjectStorageBucketSummary{}, err @@ -879,6 +924,7 @@ func (server *server) handleObjectStorageOverview(writer http.ResponseWriter, re } func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, request *http.Request) { + query := request.URL.Query() connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId")) browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID) if err != nil { @@ -893,7 +939,18 @@ func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, req renderValidationError(writer, validationIssue{Field: "bucket", Message: err.Error()}) return } - prefix := strings.Trim(strings.TrimSpace(request.URL.Query().Get("prefix")), "/") + prefix := strings.Trim(strings.TrimSpace(query.Get("prefix")), "/") + marker := strings.TrimSpace(query.Get("marker")) + paginationRequested := query.Has("limit") || query.Has("marker") + limit := 100 + if rawLimit := strings.TrimSpace(query.Get("limit")); rawLimit != "" { + parsedLimit, parseErr := strconv.Atoi(rawLimit) + if parseErr != nil || parsedLimit < 1 || parsedLimit > 1000 { + renderValidationError(writer, validationIssue{Field: "limit", Message: "limit must be an integer between 1 and 1000"}) + return + } + limit = parsedLimit + } if browser == nil || connection == nil { renderJSON(writer, http.StatusOK, domain.ObjectStorageBrowseResult{ ConnectionID: connectionID, @@ -902,10 +959,22 @@ func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, req Bucket: bucket, Prefix: prefix, Items: []domain.ObjectStorageObjectSummary{}, + Limit: limit, }) return } - items, err := browser.ListObjects(request.Context(), bucket, prefix) + page := objectStoragePage{} + if paginationRequested { + if pagedBrowser, ok := browser.(pagedObjectStorageBrowser); ok { + page, err = pagedBrowser.ListObjectsPage(request.Context(), bucket, prefix, marker, limit) + } else { + var items []domain.ObjectStorageObjectSummary + items, err = browser.ListObjects(request.Context(), bucket, prefix) + page = paginateObjectStorageItems(items, marker, limit) + } + } else { + page.Items, err = browser.ListObjects(request.Context(), bucket, prefix) + } if err != nil { renderError(writer, http.StatusInternalServerError, err) return @@ -918,10 +987,37 @@ func (server *server) handleObjectStorageObjects(writer http.ResponseWriter, req Configured: true, Bucket: bucket, Prefix: prefix, - Items: items, + Items: page.Items, + NextMarker: page.NextMarker, + HasMore: page.HasMore, + Limit: limit, }) } +func paginateObjectStorageItems(items []domain.ObjectStorageObjectSummary, marker string, limit int) objectStoragePage { + start := 0 + marker = strings.TrimSpace(marker) + if marker != "" { + for index, item := range items { + if item.Key == marker { + start = index + 1 + break + } + } + } + if start > len(items) { + start = len(items) + } + end := min(start+limit, len(items)) + pageItems := append([]domain.ObjectStorageObjectSummary(nil), items[start:end]...) + hasMore := end < len(items) + nextMarker := "" + if hasMore && len(pageItems) > 0 { + nextMarker = pageItems[len(pageItems)-1].Key + } + return objectStoragePage{Items: pageItems, NextMarker: nextMarker, HasMore: hasMore} +} + func (server *server) handleObjectStorageBucketCreate(writer http.ResponseWriter, request *http.Request) { renderJSON(writer, http.StatusMethodNotAllowed, map[string]string{"error": "bucket management is disabled in object storage browser"}) } diff --git a/internal/api/object_storage_pagination_test.go b/internal/api/object_storage_pagination_test.go new file mode 100644 index 00000000..98a41e99 --- /dev/null +++ b/internal/api/object_storage_pagination_test.go @@ -0,0 +1,23 @@ +package api + +import ( + "testing" + + "droboticflow/internal/domain" +) + +func TestPaginateObjectStorageItemsUsesLastKeyAsMarker(t *testing.T) { + items := []domain.ObjectStorageObjectSummary{ + {Key: "a"}, + {Key: "b"}, + {Key: "c"}, + } + first := paginateObjectStorageItems(items, "", 2) + if len(first.Items) != 2 || first.NextMarker != "b" || !first.HasMore { + t.Fatalf("first page = %#v", first) + } + second := paginateObjectStorageItems(items, first.NextMarker, 2) + if len(second.Items) != 1 || second.Items[0].Key != "c" || second.NextMarker != "" || second.HasMore { + t.Fatalf("second page = %#v", second) + } +} diff --git a/internal/api/server.go b/internal/api/server.go index a5dbd828..c9893d09 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -2,6 +2,7 @@ package api import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -62,6 +63,9 @@ type server struct { customOperatorImageImporterImage string customOperatorImageImportTimeoutSeconds int customOperatorImageArchiveMaxMiB int + buildCommit string + buildTime string + image string } type runJobTerminator interface { @@ -399,6 +403,7 @@ func workflowNodeByID(workflow *domain.WorkflowRecord, nodeID string) *domain.Wo func runSummaryFromRecord(run domain.RunRecord) domain.RunSummary { return domain.RunSummary{ ID: run.ID, + OrganizationID: run.OrganizationID, WorkflowID: run.WorkflowID, WorkflowName: run.WorkflowName, ProjectID: run.ProjectID, @@ -415,6 +420,8 @@ func runSummaryFromRecord(run domain.RunRecord) domain.RunSummary { EndedAt: run.EndedAt, DurationMS: run.DurationMS, CurrentNodeID: run.CurrentNodeID, + Failure: run.Failure, + Progress: run.Progress, Error: run.Error, CreatedAt: run.CreatedAt, UpdatedAt: run.UpdatedAt, @@ -1141,6 +1148,9 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig customOperatorImageImporterImage: appConfig.CustomOperatorImageImporterImage, customOperatorImageImportTimeoutSeconds: appConfig.CustomOperatorImageImportTimeoutSeconds, customOperatorImageArchiveMaxMiB: appConfig.CustomOperatorImageArchiveMaxMiB, + buildCommit: appConfig.BuildCommit, + buildTime: appConfig.BuildTime, + image: appConfig.Image, } router := chi.NewRouter() router.Use(instance.apiTokenAuth) @@ -1148,6 +1158,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Use(instance.csrfProtection) router.Get("/api/health", instance.handleHealth) + router.Get("/api/ready", instance.handleReady) router.Get("/api/auth/bootstrap-status", instance.handleAuthBootstrapStatus) router.Post("/api/auth/bootstrap-super-admin", instance.handleAuthBootstrapSuperAdmin) router.Post("/api/auth/login", instance.handleAuthLogin) @@ -1167,6 +1178,7 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig router.Post("/users/{id}/reset-password", instance.handleAdminUserResetPassword) }) router.Get("/worker-api/health", instance.handleWorkerAPIHealth) + router.Get("/worker-api/ready", instance.handleWorkerAPIReady) router.Get("/api/public/settings", instance.handlePublicSettingsGet) router.Group(func(router chi.Router) { router.Use(instance.requireWorkerAPIRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleOperator)) @@ -1381,8 +1393,40 @@ func invalidConfigurationHandler(err error) http.Handler { }) } +func (server *server) serviceMetadata() map[string]any { + return map[string]any{ + "service": "droboticflow-api", + "buildCommit": server.buildCommit, + "buildTime": server.buildTime, + "image": server.image, + } +} + func (server *server) handleHealth(writer http.ResponseWriter, _ *http.Request) { - renderJSON(writer, http.StatusOK, map[string]string{"status": "ok", "service": "droboticflow-api"}) + response := server.serviceMetadata() + response["status"] = "ok" + renderJSON(writer, http.StatusOK, response) +} + +func (server *server) handleReady(writer http.ResponseWriter, request *http.Request) { + ctx, cancel := context.WithTimeout(request.Context(), 2*time.Second) + defer cancel() + response := server.serviceMetadata() + if err := server.store.Ready(ctx); err != nil { + response["status"] = "not_ready" + response["ready"] = false + response["checks"] = map[string]any{ + "store": map[string]string{"status": "failed", "error": err.Error()}, + } + renderJSON(writer, http.StatusServiceUnavailable, response) + return + } + response["status"] = "ready" + response["ready"] = true + response["checks"] = map[string]any{ + "store": map[string]string{"status": "ok"}, + } + renderJSON(writer, http.StatusOK, response) } func (server *server) handleAIStatus(writer http.ResponseWriter, request *http.Request) { @@ -1494,7 +1538,13 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http. renderError(writer, http.StatusInternalServerError, err) return } - summary := store.BuildDashboardSummary(projects, operators, runSummaries, auditTasks) + settings, err := server.store.GetSettings(request.Context()) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + summary := store.BuildDashboardSummaryAt(projects, operators, runSummaries, auditTasks, settings.Timezone, time.Now()) + enrichDashboardOperatorInventory(&summary, operators) server.enrichDashboardPOC(request.Context(), request, &summary) renderJSON(writer, http.StatusOK, summary) return @@ -1504,10 +1554,49 @@ func (server *server) handleDashboard(writer http.ResponseWriter, request *http. renderError(writer, http.StatusInternalServerError, err) return } + operators, err := server.listOperatorsForRequest(request.Context(), request) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + enrichDashboardOperatorInventory(&summary, operators) server.enrichDashboardPOC(request.Context(), request, &summary) renderJSON(writer, http.StatusOK, summary) } +func enrichDashboardOperatorInventory(summary *domain.DashboardSummary, operators []domain.OperatorRecord) { + if summary == nil { + return + } + var catalog []struct { + ID string `json:"id"` + } + if err := json.Unmarshal(builtinOperatorCatalogJSON, &catalog); err != nil { + return + } + currentBuiltinIDs := make(map[string]struct{}, len(catalog)) + for _, item := range catalog { + if id := strings.TrimSpace(item.ID); id != "" { + currentBuiltinIDs[id] = struct{}{} + } + } + summary.Operators.Total = len(operators) + summary.Operators.Builtin = 0 + summary.Operators.Custom = 0 + summary.Operators.Historical = 0 + for _, operator := range operators { + if _, current := currentBuiltinIDs[strings.TrimSpace(operator.ID)]; current { + summary.Operators.Builtin++ + continue + } + if strings.EqualFold(strings.TrimSpace(operator.Type), "builtin") { + summary.Operators.Historical++ + continue + } + summary.Operators.Custom++ + } +} + func (server *server) handlePocAcceptanceReadiness(writer http.ResponseWriter, request *http.Request) { renderJSON(writer, http.StatusOK, server.dashboardPOCReadiness(request.Context(), request)) } @@ -1633,6 +1722,13 @@ func (server *server) ensurePocGPUWorkflowAcceptanceWorkflow(ctx context.Context if strings.TrimSpace(workflow.Name) != pocGPUWorkflowAcceptanceWorkflowName { continue } + active, err := server.store.HasActiveRun(ctx, workflow.ID) + if err != nil { + return nil, err + } + if active { + return nil, store.ErrWorkflowRunActive + } workflow.OrganizationID = normalizedTenantID(project.OrganizationID) workflow.ProjectID = project.ID workflow.Description = "Queues a real GPU-requesting workflow run for POC GPU acceptance." @@ -1687,11 +1783,25 @@ func (server *server) handlePocGPUWorkflowAcceptance(writer http.ResponseWriter, } workflow, err := server.ensurePocGPUWorkflowAcceptanceWorkflow(request.Context(), project, sourcePath) if err != nil { + if errors.Is(err, store.ErrWorkflowRunActive) { + renderJSON(writer, http.StatusConflict, map[string]string{ + "error": "验收工作流已有进行中的运行", + "code": "workflow_run_active", + }) + return + } renderError(writer, http.StatusInternalServerError, err) return } run, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerManual) if err != nil { + if errors.Is(err, store.ErrWorkflowRunActive) { + renderJSON(writer, http.StatusConflict, map[string]string{ + "error": "验收工作流已有进行中的运行", + "code": "workflow_run_active", + }) + return + } renderError(writer, http.StatusInternalServerError, err) return } @@ -1846,6 +1956,144 @@ func dashboardLatestAcceptancePayload(runs []domain.RunRecord) (dashboardAccepta return latestPayload, latestRun, latestPayload != nil } +const dashboardAcceptanceFreshnessWindow = 24 * time.Hour + +type dashboardAcceptanceFreshness struct { + Status string + StaleReason string + UnknownReason string + SourceCommit string + CatalogHash string + CatalogCount int + CompletedAt string + CurrentSourceCommit string + CurrentCatalogHash string + CurrentCatalogCount int +} + +func dashboardFirstString(values ...any) string { + for _, value := range values { + if text := dashboardString(value); text != "" { + return text + } + } + return "" +} + +func dashboardFirstInt(values ...any) int { + for _, value := range values { + if number := dashboardInt(value); number > 0 { + return number + } + } + return 0 +} + +func normalizedCatalogHash(value string) string { + value = strings.ToLower(strings.TrimSpace(value)) + return strings.TrimPrefix(value, "sha256:") +} + +func sourceCommitsMatch(left string, right string) bool { + left = strings.ToLower(strings.TrimSpace(left)) + right = strings.ToLower(strings.TrimSpace(right)) + if left == "" || right == "" { + return false + } + if left == right { + return true + } + if len(left) >= 7 && len(right) >= 7 { + return strings.HasPrefix(left, right) || strings.HasPrefix(right, left) + } + return false +} + +func evaluateDashboardAcceptanceFreshness( + payload dashboardAcceptancePayload, + run domain.RunRecord, + currentSourceCommit string, + currentCatalogHash string, + currentCatalogCount int, + now time.Time, +) dashboardAcceptanceFreshness { + importSource := dashboardMap(payload["importSource"]) + sourcePayload := dashboardMap(payload["source"]) + staticPayload := dashboardMap(payload["static"]) + result := dashboardAcceptanceFreshness{ + Status: "unknown", + SourceCommit: dashboardFirstString(payload["sourceCommit"], importSource["sourceCommit"], sourcePayload["sourceCommit"], staticPayload["sourceCommit"]), + CatalogHash: dashboardFirstString(payload["catalogHash"], importSource["catalogHash"], sourcePayload["catalogHash"], staticPayload["catalogHash"]), + CatalogCount: dashboardFirstInt(payload["catalogCount"], importSource["catalogCount"], sourcePayload["catalogCount"], staticPayload["catalogCount"], staticPayload["operatorFileCount"], payload["operatorCount"]), + CompletedAt: dashboardFirstString(payload["completedAt"], importSource["completedAt"], sourcePayload["completedAt"], payload["endedAt"], run.EndedAt, run.UpdatedAt), + CurrentSourceCommit: strings.TrimSpace(currentSourceCommit), + CurrentCatalogHash: strings.TrimSpace(currentCatalogHash), + CurrentCatalogCount: currentCatalogCount, + } + + staleReasons := make([]string, 0, 4) + unknownReasons := make([]string, 0, 4) + if result.CompletedAt == "" { + unknownReasons = append(unknownReasons, "completed_at_missing") + } else if completedAt, err := time.Parse(time.RFC3339Nano, result.CompletedAt); err != nil { + unknownReasons = append(unknownReasons, "completed_at_invalid") + } else if completedAt.After(now.Add(5 * time.Minute)) { + unknownReasons = append(unknownReasons, "completed_at_in_future") + } else if now.Sub(completedAt) > dashboardAcceptanceFreshnessWindow { + staleReasons = append(staleReasons, "acceptance_older_than_24h") + } + + if result.SourceCommit == "" { + unknownReasons = append(unknownReasons, "source_commit_missing") + } else if result.CurrentSourceCommit == "" { + unknownReasons = append(unknownReasons, "current_source_commit_unknown") + } else if !sourceCommitsMatch(result.SourceCommit, result.CurrentSourceCommit) { + staleReasons = append(staleReasons, "source_commit_mismatch") + } + + if normalizedCatalogHash(result.CatalogHash) == "" { + unknownReasons = append(unknownReasons, "catalog_hash_missing") + } else if normalizedCatalogHash(result.CurrentCatalogHash) == "" { + unknownReasons = append(unknownReasons, "current_catalog_hash_unknown") + } else if normalizedCatalogHash(result.CatalogHash) != normalizedCatalogHash(result.CurrentCatalogHash) { + staleReasons = append(staleReasons, "catalog_hash_mismatch") + } + if result.CatalogCount <= 0 { + unknownReasons = append(unknownReasons, "catalog_count_missing") + } else if result.CurrentCatalogCount <= 0 { + unknownReasons = append(unknownReasons, "current_catalog_count_unknown") + } else if result.CatalogCount != result.CurrentCatalogCount { + staleReasons = append(staleReasons, "catalog_count_mismatch") + } + + if len(staleReasons) > 0 { + result.Status = "stale" + result.StaleReason = strings.Join(staleReasons, ",") + return result + } + if len(unknownReasons) > 0 { + result.UnknownReason = strings.Join(unknownReasons, ",") + return result + } + result.Status = "fresh" + return result +} + +func dashboardFreshnessEvidence(freshness dashboardAcceptanceFreshness) map[string]any { + return map[string]any{ + "freshness": freshness.Status, + "staleReason": freshness.StaleReason, + "unknownReason": freshness.UnknownReason, + "sourceCommit": freshness.SourceCommit, + "catalogHash": freshness.CatalogHash, + "catalogCount": freshness.CatalogCount, + "completedAt": freshness.CompletedAt, + "currentSourceCommit": freshness.CurrentSourceCommit, + "currentCatalogHash": freshness.CurrentCatalogHash, + "currentCatalogCount": freshness.CurrentCatalogCount, + } +} + func dashboardStrictAcceptancePassed(payload dashboardAcceptancePayload) bool { requiredExternals := dashboardStringList(payload["requiredExternals"]) externalPassed := dashboardStringList(payload["externalVerificationPassed"]) @@ -2041,7 +2289,7 @@ func dashboardGPUWorkflowPreflightGate(runs []domain.RunRecord) domain.Dashboard ) } -func dashboardStrictAcceptanceGate(payload dashboardAcceptancePayload, found bool) domain.DashboardGate { +func dashboardStrictAcceptanceGate(payload dashboardAcceptancePayload, found bool, freshness dashboardAcceptanceFreshness) domain.DashboardGate { if !found { return dashboardGate("unknown", "未导入严格验收", "还没有找到 POC strict acceptance 结果 artifact。", nil) } @@ -2053,6 +2301,15 @@ func dashboardStrictAcceptanceGate(payload dashboardAcceptancePayload, found boo "externalMissing": dashboardStringList(payload["externalVerificationMissing"]), "blockingMissingCoverage": dashboardStringList(payload["blockingMissingCoverage"]), } + for key, value := range dashboardFreshnessEvidence(freshness) { + evidence[key] = value + } + if freshness.Status == "stale" { + return dashboardGate("unknown", "严格验收证据已过期", "最新 POC acceptance artifact 已过期或与当前代码/算子目录不一致,不能作为当前阻塞或通过依据。", evidence) + } + if freshness.Status != "fresh" { + return dashboardGate("unknown", "严格验收新鲜度未知", "最新 POC acceptance artifact 缺少完整的提交、算子目录或完成时间证据。", evidence) + } if dashboardStrictAcceptancePassed(payload) { return dashboardGate("passed", "严格验收通过", "完整外部依赖、真实 fixture、算子仓库策略和算子覆盖均已通过。", evidence) } @@ -2062,6 +2319,71 @@ func dashboardStrictAcceptanceGate(payload dashboardAcceptancePayload, found boo return dashboardGate("blocked", "严格验收未完成", "最新 POC acceptance 结果还不能证明完整外部验收通过。", evidence) } +func dashboardBusinessAcceptancePassed(payload dashboardAcceptancePayload) bool { + requiredExternals := dashboardStringList(payload["requiredExternals"]) + externalPassed := dashboardStringList(payload["externalVerificationPassed"]) + return dashboardString(payload["status"]) == "passed" && + dashboardBool(payload["fullExternalAcceptance"]) && + dashboardString(payload["acceptanceScope"]) == "full" && + dashboardStringListContains(requiredExternals, "bos") && + dashboardStringListContains(requiredExternals, "mongo") && + len(dashboardStringList(payload["externalVerificationMissing"])) == 0 && + dashboardStringListContains(externalPassed, "bos") && + dashboardStringListContains(externalPassed, "mongo") && + dashboardBool(payload["allFixturesReal"]) && + dashboardInt(payload["generatedFixtureCount"]) == 0 && + dashboardInt(payload["realFixtureCount"]) > 0 +} + +func dashboardBusinessAcceptanceGate(payload dashboardAcceptancePayload, found bool, freshness dashboardAcceptanceFreshness) domain.DashboardGate { + if !found { + return dashboardGate("unknown", "当前业务验收未知", "还没有当前版本的完整业务验收 artifact。", nil) + } + evidence := dashboardFreshnessEvidence(freshness) + evidence["status"] = dashboardString(payload["status"]) + evidence["externalMissing"] = dashboardStringList(payload["externalVerificationMissing"]) + if freshness.Status != "fresh" { + return dashboardGate("unknown", "当前业务验收待刷新", "验收 artifact 不是当前代码和算子目录的新鲜证据。", evidence) + } + if dashboardBusinessAcceptancePassed(payload) { + return dashboardGate("passed", "当前业务验收通过", "当前版本的外部依赖和真实 fixture 验收均已通过。", evidence) + } + return dashboardGate("blocked", "当前业务验收未通过", "当前版本仍有外部依赖、fixture 或预检项未通过。", evidence) +} + +func dashboardOperatorCoveragePassed(payload dashboardAcceptancePayload, currentCatalogCount int) bool { + operatorCount := dashboardInt(payload["operatorCount"]) + return currentCatalogCount > 0 && + operatorCount == currentCatalogCount && + dashboardInt(payload["operatorPassedCount"]) == operatorCount && + dashboardInt(payload["coveredOperatorCount"]) == operatorCount && + dashboardInt(payload["failedCount"]) == 0 && + dashboardInt(payload["requiredSkippedCount"]) == 0 && + dashboardBool(payload["allOperatorCoverageComplete"]) && + dashboardBool(payload["blockingOperatorCoverageComplete"]) && + len(dashboardStringList(payload["missingCoverage"])) == 0 && + len(dashboardStringList(payload["blockingMissingCoverage"])) == 0 && + dashboardBool(payload["operatorRepositoryCoverageComplete"]) && + payload["operatorRepositoryPolicyClean"] != false +} + +func dashboardOperatorCoverageGate(payload dashboardAcceptancePayload, found bool, freshness dashboardAcceptanceFreshness) domain.DashboardGate { + if !found { + return dashboardGate("unknown", "算子覆盖未知", "还没有当前算子目录的覆盖验收 artifact。", nil) + } + evidence := dashboardFreshnessEvidence(freshness) + evidence["operatorCount"] = dashboardInt(payload["operatorCount"]) + evidence["coveredOperatorCount"] = dashboardInt(payload["coveredOperatorCount"]) + evidence["missingCoverage"] = dashboardStringList(payload["missingCoverage"]) + if freshness.Status != "fresh" { + return dashboardGate("unknown", "算子覆盖待刷新", "验收 artifact 与当前算子目录不一致或新鲜度未知。", evidence) + } + if dashboardOperatorCoveragePassed(payload, freshness.CurrentCatalogCount) { + return dashboardGate("passed", "算子覆盖完整", "当前内置算子目录已完整执行并通过仓库策略检查。", evidence) + } + return dashboardGate("blocked", "算子覆盖不完整", "当前内置算子目录仍有未覆盖、失败或仓库策略问题。", evidence) +} + func appendDashboardAction(actions []string, action string) []string { action = strings.TrimSpace(action) if action == "" { @@ -2076,11 +2398,17 @@ func appendDashboardAction(actions []string, action string) []string { } func dashboardOverallPOCStatus(poc domain.DashboardPOCReadiness) (string, string, string) { - gates := []domain.DashboardGate{poc.GPU, poc.GPUWorkflowPreflight, poc.StrictAcceptance} + gates := []domain.DashboardGate{ + poc.ServiceAvailability, + poc.GPUCapacity, + poc.GPUWorkflowPreflight, + poc.BusinessAcceptance, + poc.OperatorCoverage, + } unknown := false for _, gate := range gates { if gate.Status == "blocked" { - return "blocked", "POC 验收阻塞", "GPU 或 strict acceptance 仍有未完成 gate。" + return "blocked", "POC 验收阻塞", "服务、GPU、当前业务验收或算子覆盖仍有未完成 gate。" } if gate.Status != "passed" { unknown = true @@ -2089,7 +2417,19 @@ func dashboardOverallPOCStatus(poc domain.DashboardPOCReadiness) (string, string if unknown { return "unknown", "POC 验收待确认", "当前缺少完整的 POC gate 证据。" } - return "passed", "POC 验收通过", "GPU 和 strict acceptance gate 均已通过。" + return "passed", "POC 验收通过", "服务、GPU、当前业务验收和算子覆盖 gate 均已通过。" +} + +func (server *server) dashboardServiceAvailabilityGate(status string, message string) domain.DashboardGate { + evidence := map[string]any{ + "buildCommit": server.buildCommit, + "buildTime": server.buildTime, + "image": server.image, + } + if status == "passed" { + return dashboardGate("passed", "服务可用", message, evidence) + } + return dashboardGate("blocked", "服务不可用", message, evidence) } func normalizeDashboardPOCReadinessLists(poc *domain.DashboardPOCReadiness) { @@ -2119,11 +2459,17 @@ func normalizeDashboardPOCReadinessLists(poc *domain.DashboardPOCReadiness) { func (server *server) dashboardPOCReadiness(ctx context.Context, request *http.Request) domain.DashboardPOCReadiness { runs, err := server.listRunsForRequest(ctx, request, "") if err != nil { + gpu := server.dashboardGPUCapacityGate(ctx) poc := domain.DashboardPOCReadiness{ - Status: "unknown", - Label: "POC 验收待确认", - Message: "无法读取 runs 中的 POC acceptance artifact:" + err.Error(), - GPU: server.dashboardGPUCapacityGate(ctx), + Status: "unknown", + Label: "POC 验收待确认", + Message: "无法读取 runs 中的 POC acceptance artifact:" + err.Error(), + Freshness: "unknown", + ServiceAvailability: server.dashboardServiceAvailabilityGate("blocked", "API 无法读取运行存储:"+err.Error()), + GPUCapacity: gpu, + BusinessAcceptance: dashboardGate("unknown", "当前业务验收未知", "无法读取 POC acceptance artifact。", nil), + OperatorCoverage: dashboardGate("unknown", "算子覆盖未知", "无法读取 POC acceptance artifact。", nil), + GPU: gpu, GPUWorkflowPreflight: dashboardGate( "unknown", "GPU workflow 预检未知", @@ -2137,12 +2483,45 @@ func (server *server) dashboardPOCReadiness(ctx context.Context, request *http.R return poc } payload, latestRun, found := dashboardLatestAcceptancePayload(runs) - poc := domain.DashboardPOCReadiness{ - GPU: server.dashboardGPUCapacityGate(ctx), - GPUWorkflowPreflight: dashboardGPUWorkflowPreflightGate(runs), - StrictAcceptance: dashboardStrictAcceptanceGate(payload, found), + currentCatalogHash, currentCatalogCount, catalogErr := builtinOperatorCatalogFingerprint() + if catalogErr != nil { + currentCatalogHash = "" + currentCatalogCount = 0 + } + freshness := dashboardAcceptanceFreshness{ + Status: "unknown", + UnknownReason: "acceptance_artifact_missing", + CurrentSourceCommit: server.buildCommit, + CurrentCatalogHash: currentCatalogHash, + CurrentCatalogCount: currentCatalogCount, } if found { + freshness = evaluateDashboardAcceptanceFreshness(payload, latestRun, server.buildCommit, currentCatalogHash, currentCatalogCount, time.Now()) + if catalogErr != nil && freshness.Status != "stale" { + freshness.Status = "unknown" + freshness.UnknownReason = "current_catalog_unavailable" + } + } + gpu := server.dashboardGPUCapacityGate(ctx) + poc := domain.DashboardPOCReadiness{ + Freshness: freshness.Status, + StaleReason: freshness.StaleReason, + CurrentSourceCommit: freshness.CurrentSourceCommit, + CurrentCatalogHash: freshness.CurrentCatalogHash, + CurrentCatalogCount: freshness.CurrentCatalogCount, + ServiceAvailability: server.dashboardServiceAvailabilityGate("passed", "API 已成功读取当前运行、配置和验收证据。"), + GPUCapacity: gpu, + BusinessAcceptance: dashboardBusinessAcceptanceGate(payload, found, freshness), + OperatorCoverage: dashboardOperatorCoverageGate(payload, found, freshness), + GPU: gpu, + GPUWorkflowPreflight: dashboardGPUWorkflowPreflightGate(runs), + StrictAcceptance: dashboardStrictAcceptanceGate(payload, found, freshness), + } + if found { + poc.SourceCommit = freshness.SourceCommit + poc.CatalogHash = freshness.CatalogHash + poc.CatalogCount = freshness.CatalogCount + poc.CompletedAt = freshness.CompletedAt poc.LatestAcceptanceRunID = latestRun.ID poc.LatestAcceptanceUpdatedAt = latestRun.UpdatedAt poc.LatestAcceptanceStatus = dashboardString(payload["status"]) @@ -3151,6 +3530,18 @@ func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Workflow") { return } + active, err := server.store.HasActiveRun(request.Context(), item.ID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if active { + renderJSON(writer, http.StatusConflict, map[string]string{ + "error": "工作流存在进行中的运行,请等待任务结束或停止任务后再编辑", + "code": "workflow_run_active", + }) + return + } var payload workflowPatch if !decodeJSON(writer, request, &payload) { @@ -3396,6 +3787,18 @@ func (server *server) handleWorkflowRun(writer http.ResponseWriter, request *htt if !server.ensureTenantAccess(writer, request, workflow.OrganizationID, "Workflow") { return } + active, err := server.store.HasActiveRun(request.Context(), workflow.ID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if active { + renderJSON(writer, http.StatusConflict, map[string]string{ + "error": "工作流已有进行中的运行", + "code": "workflow_run_active", + }) + return + } if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil { requestPayload := scanConfigRequestFromWorkflow(workflow.Name, workflow.Description, workflow.Schedule, workflow.ScanConfig) if issues := validateScanConfigDraftPayload(requestPayload, workflow.ScanConfig); len(issues) > 0 { @@ -3428,8 +3831,11 @@ func (server *server) handleWorkflowRun(writer http.ResponseWriter, request *htt renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()}) return } - if errors.Is(err, store.ErrWorkflowAlreadyCompleted) { - renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流已成功执行,不支持再次手动运行"}) + if errors.Is(err, store.ErrWorkflowRunActive) { + renderJSON(writer, http.StatusConflict, map[string]string{ + "error": "工作流已有进行中的运行", + "code": "workflow_run_active", + }) return } renderError(writer, http.StatusInternalServerError, err) @@ -3487,6 +3893,18 @@ func (server *server) handleRunsRerunSubtree(writer http.ResponseWriter, request renderJSON(writer, http.StatusConflict, map[string]string{"error": "只有失败的运行才支持从指定节点重新执行下游"}) return } + active, err := server.store.HasActiveRun(request.Context(), sourceRun.WorkflowID) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + if active { + renderJSON(writer, http.StatusConflict, map[string]string{ + "error": "工作流已有进行中的运行", + "code": "workflow_run_active", + }) + return + } var payload rerunSubtreeRequest if !decodeJSON(writer, request, &payload) { return @@ -5363,7 +5781,119 @@ func (server *server) handleOperatorLightweightFileDelete(writer http.ResponseWr renderJSON(writer, http.StatusOK, response) } +func runStatusIsValid(status domain.RunStatus) bool { + switch status { + case domain.RunStatusQueued, + domain.RunStatusRunning, + domain.RunStatusStopping, + domain.RunStatusWaitingForAudit, + domain.RunStatusCompleted, + domain.RunStatusFailed, + domain.RunStatusCancelled: + return true + default: + return false + } +} + +func runSummaryPaginationRequested(request *http.Request) bool { + query := request.URL.Query() + for _, key := range []string{"limit", "cursor", "q", "projectId", "workflowId"} { + if _, ok := query[key]; ok { + return true + } + } + return false +} + +func decodeRunSummaryCursor(raw string) (*domain.RunSummaryCursor, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + payload, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil { + return nil, fmt.Errorf("cursor must be a valid opaque run cursor") + } + var cursor domain.RunSummaryCursor + if err := json.Unmarshal(payload, &cursor); err != nil { + return nil, fmt.Errorf("cursor must be a valid opaque run cursor") + } + if strings.TrimSpace(cursor.CreatedAt) == "" || strings.TrimSpace(cursor.ID) == "" { + return nil, fmt.Errorf("cursor must include createdAt and id") + } + return &cursor, nil +} + +func encodeRunSummaryCursor(cursor *domain.RunSummaryCursor) string { + if cursor == nil { + return "" + } + payload, err := json.Marshal(cursor) + if err != nil { + return "" + } + return base64.RawURLEncoding.EncodeToString(payload) +} + +func parseRunSummaryQuery(request *http.Request) (domain.RunSummaryQuery, []validationIssue) { + values := request.URL.Query() + query := domain.RunSummaryQuery{ + Limit: 50, + Status: domain.RunStatus(strings.TrimSpace(values.Get("status"))), + Search: strings.TrimSpace(values.Get("q")), + ProjectID: strings.TrimSpace(values.Get("projectId")), + WorkflowID: strings.TrimSpace(values.Get("workflowId")), + } + issues := make([]validationIssue, 0, 3) + if rawLimit, ok := values["limit"]; ok { + value := "" + if len(rawLimit) > 0 { + value = rawLimit[0] + } + limit, issue := parseDatabaseBrowserPageQuery(value, "limit", 50, 1, 100) + if issue != nil { + issues = append(issues, *issue) + } else { + query.Limit = limit + } + } + if query.Status != "" && !runStatusIsValid(query.Status) { + issues = append(issues, validationIssue{Field: "status", Message: "status must be queued, running, stopping, waiting_for_audit, completed, failed, or cancelled"}) + } + if len(query.Search) > 200 { + issues = append(issues, validationIssue{Field: "q", Message: "q must be at most 200 characters"}) + } + cursor, err := decodeRunSummaryCursor(values.Get("cursor")) + if err != nil { + issues = append(issues, validationIssue{Field: "cursor", Message: err.Error()}) + } else { + query.Cursor = cursor + } + return query, issues +} + func (server *server) handleRunsList(writer http.ResponseWriter, request *http.Request) { + if runSummaryPaginationRequested(request) { + query, issues := parseRunSummaryQuery(request) + if len(issues) > 0 { + renderValidationError(writer, issues...) + return + } + page, err := server.listRunSummaryPageForRequest(request.Context(), request, query) + if err != nil { + renderError(writer, http.StatusInternalServerError, err) + return + } + renderJSON(writer, http.StatusOK, map[string]any{ + "items": page.Items, + "total": page.Total, + "statusCounts": page.StatusCounts, + "nextCursor": encodeRunSummaryCursor(page.NextCursor), + "hasMore": page.HasMore, + }) + return + } items, err := server.listRunSummariesForRequest(request.Context(), request, request.URL.Query().Get("status")) if err != nil { renderError(writer, http.StatusInternalServerError, err) diff --git a/internal/api/server_test.go b/internal/api/server_test.go index 03a9c01d..de8c2d96 100644 --- a/internal/api/server_test.go +++ b/internal/api/server_test.go @@ -1703,9 +1703,15 @@ func TestDashboardIncludesPOCGatesFromAcceptanceArtifactAndWorkerCapacity(t *tes if body.POC.LatestAcceptanceRunID != "run-poc-operator-acceptance-blocked" { t.Fatalf("latest acceptance run = %q", body.POC.LatestAcceptanceRunID) } - if body.POC.GPU.Status != "blocked" || body.POC.StrictAcceptance.Status != "blocked" { + if body.POC.GPU.Status != "blocked" || body.POC.StrictAcceptance.Status != "unknown" { t.Fatalf("unexpected POC gate statuses: gpu=%s strict=%s", body.POC.GPU.Status, body.POC.StrictAcceptance.Status) } + if body.POC.Freshness != "stale" || body.POC.StaleReason == "" { + t.Fatalf("expected historical acceptance artifact to be stale: %#v", body.POC) + } + if body.POC.ServiceAvailability.Status != "passed" || body.POC.GPUCapacity.Status != body.POC.GPU.Status || body.POC.BusinessAcceptance.Status != "unknown" || body.POC.OperatorCoverage.Status != "unknown" { + t.Fatalf("unexpected split readiness gates: %#v", body.POC) + } if body.POC.GPUWorkflowPreflight.Status != "passed" { t.Fatalf("expected GPU workflow preflight gate to pass, got %#v", body.POC.GPUWorkflowPreflight) } @@ -1880,9 +1886,12 @@ func TestPocAcceptanceReadinessAllowsSharedAPITokenWhenSessionAuthIsEnabled(t *t if readiness.Status != "blocked" || readiness.LatestAcceptanceRunID != "run-poc-operator-acceptance-readiness-blocked" { t.Fatalf("unexpected readiness summary: %#v", readiness) } - if readiness.GPU.Status != "blocked" || readiness.GPUWorkflowPreflight.Status != "unknown" || readiness.StrictAcceptance.Status != "blocked" { + if readiness.GPU.Status != "blocked" || readiness.GPUWorkflowPreflight.Status != "unknown" || readiness.StrictAcceptance.Status != "unknown" { t.Fatalf("unexpected readiness gates: gpu=%s gpuWorkflow=%s strict=%s", readiness.GPU.Status, readiness.GPUWorkflowPreflight.Status, readiness.StrictAcceptance.Status) } + if readiness.Freshness != "stale" || readiness.BusinessAcceptance.Status != "unknown" || readiness.OperatorCoverage.Status != "unknown" { + t.Fatalf("expected historical readiness evidence to be stale, got %#v", readiness) + } if !containsString(readiness.ExternalVerificationMissing, "mongo") || readiness.RealFixtureCount != 6 || readiness.GeneratedFixtureCount != 0 { t.Fatalf("readiness evidence missing: %#v", readiness) } @@ -3741,30 +3750,15 @@ func TestPocGPUWorkflowAcceptanceAllowsSharedAPITokenWhenSessionAuthIsEnabled(t secondRequest := httptest.NewRequest(http.MethodPost, "/api/acceptance/poc-gpu-workflow", bytes.NewReader([]byte(`{"sourcePath":"/workspace/datasets/fixtures/poc-operator-acceptance/xplatform-v2"}`))) secondRequest.Header.Set("Authorization", "Bearer shared-acceptance-token") handler.ServeHTTP(secondRecorder, secondRequest) - if secondRecorder.Code != http.StatusAccepted { - t.Fatalf("second GPU workflow acceptance status: %d body=%s", secondRecorder.Code, secondRecorder.Body.String()) + if secondRecorder.Code != http.StatusConflict { + t.Fatalf("expected active acceptance run conflict, got %d body=%s", secondRecorder.Code, secondRecorder.Body.String()) } - var secondBody struct { - Project domain.ProjectRecord `json:"project"` - Workflow domain.WorkflowRecord `json:"workflow"` - Run domain.RunRecord `json:"run"` + var conflictBody map[string]string + if err := json.Unmarshal(secondRecorder.Body.Bytes(), &conflictBody); err != nil { + t.Fatalf("decode conflict response: %v", err) } - if err := json.Unmarshal(secondRecorder.Body.Bytes(), &secondBody); err != nil { - t.Fatalf("decode second response: %v", err) - } - if secondBody.Project.ID != body.Project.ID || secondBody.Workflow.ID != body.Workflow.ID { - t.Fatalf("expected second acceptance call to reuse project/workflow, got project=%q workflow=%q want project=%q workflow=%q", secondBody.Project.ID, secondBody.Workflow.ID, body.Project.ID, body.Workflow.ID) - } - if secondBody.Run.ID == body.Run.ID { - t.Fatalf("expected second acceptance call to create a fresh run, got %q", secondBody.Run.ID) - } - secondSourceNode := secondBody.Workflow.Nodes[0] - secondParams := map[string]string{} - for _, param := range secondSourceNode.Data.Hyperparams { - secondParams[param.Key] = param.Value - } - if secondParams["SOURCE_PATH"] != "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform-v2" { - t.Fatalf("second source hyperparams = %+v", secondParams) + if conflictBody["code"] != "workflow_run_active" { + t.Fatalf("expected workflow_run_active, got %#v", conflictBody) } projects, err := dataStore.ListProjectsByOrganization(context.Background(), domain.DefaultOrganizationID) if err != nil { @@ -7320,7 +7314,7 @@ func TestOperatorImageArchiveImportAllowsAPITokenWhenSessionAuthRequired(t *test } } -func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { +func TestCompletedWorkflowCanBeRunAgainManually(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -7367,16 +7361,27 @@ func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { bytes.NewReader([]byte(`{"trigger":"manual"}`)), ) handler.ServeHTTP(runRecorder, runRequest) - if runRecorder.Code != http.StatusConflict { - t.Fatalf("expected 409 for completed workflow rerun, got %d body=%s", runRecorder.Code, runRecorder.Body.String()) + if runRecorder.Code != http.StatusAccepted { + t.Fatalf("expected 202 for completed workflow rerun, got %d body=%s", runRecorder.Code, runRecorder.Body.String()) } - var body map[string]string + var body domain.RunRecord if err := json.Unmarshal(runRecorder.Body.Bytes(), &body); err != nil { t.Fatalf("decode rerun response: %v", err) } - if body["error"] == "" { - t.Fatalf("expected rerun error message") + if body.Status != domain.RunStatusQueued || body.WorkflowID != workflow.ID { + t.Fatalf("expected queued rerun for workflow %q, got %#v", workflow.ID, body) + } + + activeRecorder := httptest.NewRecorder() + activeRequest := httptest.NewRequest( + http.MethodPost, + "/api/workflows/"+workflow.ID+"/run", + bytes.NewReader([]byte(`{"trigger":"manual"}`)), + ) + handler.ServeHTTP(activeRecorder, activeRequest) + if activeRecorder.Code != http.StatusConflict || !strings.Contains(activeRecorder.Body.String(), "workflow_run_active") { + t.Fatalf("expected active rerun conflict, got %d body=%s", activeRecorder.Code, activeRecorder.Body.String()) } } @@ -10330,3 +10335,280 @@ func TestSettingsRejectsInvalidTimezone(t *testing.T) { t.Fatalf("expected invalid timezone to fail, got %d body=%s", updateRecorder.Code, updateRecorder.Body.String()) } } + +func TestRunsListCursorPaginationFiltersAndKeepsLegacyResponse(t *testing.T) { + dataStore := store.NewMemoryStore() + ctx := context.Background() + for _, run := range []domain.RunRecord{ + {ID: "run-page-3", WorkflowID: "workflow-a", WorkflowName: "Vision Flow", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusFailed, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"}, + {ID: "run-page-2", WorkflowID: "workflow-a", WorkflowName: "Vision Flow", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusRunning, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"}, + {ID: "run-page-1", WorkflowID: "workflow-a", WorkflowName: "Vision Flow", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T01:00:00Z", UpdatedAt: "2026-07-10T01:00:00Z"}, + } { + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run %s: %v", run.ID, err) + } + } + handler := NewHandlerWithConfig(dataStore, nil, config.AppConfig{}) + + type pageResponse struct { + Items []domain.RunSummary `json:"items"` + Total int64 `json:"total"` + StatusCounts map[domain.RunStatus]int64 `json:"statusCounts"` + NextCursor string `json:"nextCursor"` + HasMore bool `json:"hasMore"` + } + firstRecorder := httptest.NewRecorder() + handler.ServeHTTP(firstRecorder, httptest.NewRequest(http.MethodGet, "/api/runs?limit=2&q=vision&projectId=project-a", nil)) + if firstRecorder.Code != http.StatusOK { + t.Fatalf("first page status = %d body=%s", firstRecorder.Code, firstRecorder.Body.String()) + } + var first pageResponse + if err := json.Unmarshal(firstRecorder.Body.Bytes(), &first); err != nil { + t.Fatalf("decode first page: %v", err) + } + if first.Total != 3 || len(first.Items) != 2 || first.Items[0].ID != "run-page-3" || first.Items[1].ID != "run-page-2" || !first.HasMore || first.NextCursor == "" { + t.Fatalf("unexpected first page: %#v", first) + } + if first.StatusCounts[domain.RunStatusFailed] != 1 || first.StatusCounts[domain.RunStatusRunning] != 1 || first.StatusCounts[domain.RunStatusCompleted] != 1 { + t.Fatalf("unexpected status counts: %#v", first.StatusCounts) + } + + secondRecorder := httptest.NewRecorder() + secondURL := "/api/runs?limit=2&q=vision&projectId=project-a&cursor=" + url.QueryEscape(first.NextCursor) + handler.ServeHTTP(secondRecorder, httptest.NewRequest(http.MethodGet, secondURL, nil)) + var second pageResponse + if err := json.Unmarshal(secondRecorder.Body.Bytes(), &second); err != nil { + t.Fatalf("decode second page: %v", err) + } + if secondRecorder.Code != http.StatusOK || second.Total != 3 || len(second.Items) != 1 || second.Items[0].ID != "run-page-1" || second.HasMore || second.NextCursor != "" { + t.Fatalf("unexpected second page status=%d page=%#v", secondRecorder.Code, second) + } + + filteredRecorder := httptest.NewRecorder() + handler.ServeHTTP(filteredRecorder, httptest.NewRequest(http.MethodGet, "/api/runs?limit=50&status=failed&q=vision", nil)) + var filtered pageResponse + if err := json.Unmarshal(filteredRecorder.Body.Bytes(), &filtered); err != nil { + t.Fatalf("decode filtered page: %v", err) + } + if filtered.Total != 3 || len(filtered.Items) != 1 || filtered.Items[0].ID != "run-page-3" || filtered.StatusCounts[domain.RunStatusRunning] != 1 { + t.Fatalf("unexpected filtered page: %#v", filtered) + } + + legacyRecorder := httptest.NewRecorder() + handler.ServeHTTP(legacyRecorder, httptest.NewRequest(http.MethodGet, "/api/runs?status=failed", nil)) + var legacy map[string]json.RawMessage + if err := json.Unmarshal(legacyRecorder.Body.Bytes(), &legacy); err != nil { + t.Fatalf("decode legacy response: %v", err) + } + if _, ok := legacy["items"]; !ok { + t.Fatalf("legacy response missing items: %s", legacyRecorder.Body.String()) + } + if _, ok := legacy["total"]; ok { + t.Fatalf("legacy status-only response unexpectedly changed shape: %s", legacyRecorder.Body.String()) + } + + for _, path := range []string{ + "/api/runs?limit=0", + "/api/runs?limit=50&status=unknown", + "/api/runs?limit=50&cursor=not-a-cursor", + } { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path, nil)) + if recorder.Code != http.StatusBadRequest { + t.Fatalf("expected validation error for %s, got %d body=%s", path, recorder.Code, recorder.Body.String()) + } + } +} + +type readinessErrorStore struct { + store.Store +} + +func (readinessErrorStore) Ready(context.Context) error { + return errors.New("mongo unavailable") +} + +func TestHealthAndReadinessExposeBuildMetadataAndRemainPublic(t *testing.T) { + dataStore := store.NewMemoryStore() + appConfig := config.AppConfig{ + APIToken: "required-token", + RequireSessionAuth: true, + BuildCommit: "3cda38c", + BuildTime: "2026-07-10T12:00:00Z", + Image: "ccr.example/droboticflow-api:3cda38c", + } + handler := NewHandlerWithConfig(dataStore, nil, appConfig) + + for _, path := range []string{"/api/health", "/api/ready"} { + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path, nil)) + if recorder.Code != http.StatusOK { + t.Fatalf("%s status = %d body=%s", path, recorder.Code, recorder.Body.String()) + } + var body map[string]any + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("decode %s: %v", path, err) + } + if body["buildCommit"] != "3cda38c" || body["buildTime"] != "2026-07-10T12:00:00Z" || body["image"] != "ccr.example/droboticflow-api:3cda38c" { + t.Fatalf("%s build metadata = %#v", path, body) + } + } + + notReadyHandler := NewHandlerWithConfig(readinessErrorStore{Store: dataStore}, nil, appConfig) + recorder := httptest.NewRecorder() + notReadyHandler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/ready", nil)) + if recorder.Code != http.StatusServiceUnavailable || !strings.Contains(recorder.Body.String(), "mongo unavailable") { + t.Fatalf("not-ready response = %d body=%s", recorder.Code, recorder.Body.String()) + } +} + +func TestWorkflowUpdateRejectsActiveRunButAllowsCompletedRun(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "Editable workflow"}) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run := domain.RunRecord{ID: "run-edit-guard", WorkflowID: workflow.ID, Status: domain.RunStatusRunning, CreatedAt: "2026-07-10T01:00:00Z", UpdatedAt: "2026-07-10T01:00:00Z"} + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + handler := NewHandlerWithConfig(dataStore, nil, config.AppConfig{}) + + activeRecorder := httptest.NewRecorder() + handler.ServeHTTP(activeRecorder, httptest.NewRequest(http.MethodPut, "/api/workflows/"+workflow.ID, bytes.NewReader([]byte(`{"description":"blocked"}`)))) + if activeRecorder.Code != http.StatusConflict || !strings.Contains(activeRecorder.Body.String(), "workflow_run_active") { + t.Fatalf("active update response = %d body=%s", activeRecorder.Code, activeRecorder.Body.String()) + } + + run.Status = domain.RunStatusCompleted + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("complete run: %v", err) + } + completedRecorder := httptest.NewRecorder() + handler.ServeHTTP(completedRecorder, httptest.NewRequest(http.MethodPut, "/api/workflows/"+workflow.ID, bytes.NewReader([]byte(`{"description":"allowed"}`)))) + if completedRecorder.Code != http.StatusOK { + t.Fatalf("completed update response = %d body=%s", completedRecorder.Code, completedRecorder.Body.String()) + } +} + +func TestBuiltinOperatorCatalogFingerprintUsesAuthoritativeCatalog(t *testing.T) { + hash, count, err := builtinOperatorCatalogFingerprint() + if err != nil { + t.Fatalf("catalog fingerprint: %v", err) + } + records, err := builtinOperatorRecords("", "") + if err != nil { + t.Fatalf("catalog records: %v", err) + } + if count != len(records) || count == 0 { + t.Fatalf("catalog count = %d, records = %d", count, len(records)) + } + if len(hash) != 64 { + t.Fatalf("catalog hash = %q, want sha256 hex", hash) + } + again, againCount, err := builtinOperatorCatalogFingerprint() + if err != nil || again != hash || againCount != count { + t.Fatalf("catalog fingerprint is not stable: hash=%q/%q count=%d/%d err=%v", hash, again, count, againCount, err) + } +} + +func TestAcceptanceArtifactDataExtractsFreshnessSourceFields(t *testing.T) { + payload := map[string]any{ + "operatorCount": float64(102), + "static": map[string]any{ + "operatorFileCount": float64(102), + }, + } + request := pocAcceptanceImportRequest{} + request.Source.SourceCommit = " 3cda38c " + request.Source.CatalogHash = " abcdef " + request.Source.CatalogCount = 102 + request.Source.CompletedAt = " 2026-07-10T12:00:00Z " + data := acceptanceResultArtifactData(payload, request, "2026-07-10T13:00:00Z") + if data["sourceCommit"] != "3cda38c" || data["catalogHash"] != "abcdef" || data["catalogCount"] != 102 || data["completedAt"] != "2026-07-10T12:00:00Z" { + t.Fatalf("freshness fields not normalized: %#v", data) + } + importSource := dashboardMap(data["importSource"]) + if importSource["sourceCommit"] != "3cda38c" || importSource["catalogCount"] != 102 { + t.Fatalf("import source metadata missing: %#v", importSource) + } +} + +func TestEvaluateDashboardAcceptanceFreshness(t *testing.T) { + now := time.Date(2026, time.July, 10, 12, 0, 0, 0, time.UTC) + base := dashboardAcceptancePayload{ + "sourceCommit": "3cda38c", + "catalogHash": "sha256:abcdef", + "catalogCount": float64(102), + "completedAt": "2026-07-10T11:00:00Z", + } + run := domain.RunRecord{UpdatedAt: "2026-07-10T11:00:00Z"} + + fresh := evaluateDashboardAcceptanceFreshness(base, run, "3cda38c1b0bea3df30e7f6f292cd3216ac14185d", "abcdef", 102, now) + if fresh.Status != "fresh" || fresh.StaleReason != "" || fresh.CompletedAt == "" { + t.Fatalf("expected fresh acceptance evidence, got %#v", fresh) + } + + cases := []struct { + name string + mutate func(dashboardAcceptancePayload) + reasonPart string + }{ + {name: "age", mutate: func(payload dashboardAcceptancePayload) { payload["completedAt"] = "2026-07-09T11:59:59Z" }, reasonPart: "acceptance_older_than_24h"}, + {name: "commit", mutate: func(payload dashboardAcceptancePayload) { payload["sourceCommit"] = "different" }, reasonPart: "source_commit_mismatch"}, + {name: "hash", mutate: func(payload dashboardAcceptancePayload) { payload["catalogHash"] = "different" }, reasonPart: "catalog_hash_mismatch"}, + {name: "count", mutate: func(payload dashboardAcceptancePayload) { payload["catalogCount"] = float64(101) }, reasonPart: "catalog_count_mismatch"}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + payload := dashboardAcceptancePayload{} + for key, value := range base { + payload[key] = value + } + testCase.mutate(payload) + freshness := evaluateDashboardAcceptanceFreshness(payload, run, "3cda38c1b0bea3df30e7f6f292cd3216ac14185d", "abcdef", 102, now) + if freshness.Status != "stale" || !strings.Contains(freshness.StaleReason, testCase.reasonPart) { + t.Fatalf("expected stale reason %q, got %#v", testCase.reasonPart, freshness) + } + gate := dashboardStrictAcceptanceGate(payload, true, freshness) + if gate.Status != "unknown" { + t.Fatalf("stale strict acceptance must be unknown, got %#v", gate) + } + }) + } + + unknown := evaluateDashboardAcceptanceFreshness(dashboardAcceptancePayload{"completedAt": "2026-07-10T11:00:00Z"}, run, "", "abcdef", 102, now) + if unknown.Status != "unknown" || unknown.UnknownReason == "" { + t.Fatalf("expected incomplete metadata to be unknown, got %#v", unknown) + } +} + +func TestFreshAcceptanceSplitsBusinessAndOperatorCoverageGates(t *testing.T) { + payload := dashboardAcceptancePayload(pocAcceptanceFullResult("fresh-split-gates")) + freshness := dashboardAcceptanceFreshness{ + Status: "fresh", + SourceCommit: "3cda38c", + CatalogHash: "abcdef", + CatalogCount: 28, + CompletedAt: "2026-07-10T11:00:00Z", + CurrentSourceCommit: "3cda38c", + CurrentCatalogHash: "abcdef", + CurrentCatalogCount: 28, + } + strict := dashboardStrictAcceptanceGate(payload, true, freshness) + business := dashboardBusinessAcceptanceGate(payload, true, freshness) + coverage := dashboardOperatorCoverageGate(payload, true, freshness) + if strict.Status != "passed" || business.Status != "passed" || coverage.Status != "passed" { + t.Fatalf("expected fresh gates to pass: strict=%#v business=%#v coverage=%#v", strict, business, coverage) + } + + freshness.Status = "stale" + freshness.StaleReason = "catalog_hash_mismatch" + strict = dashboardStrictAcceptanceGate(payload, true, freshness) + business = dashboardBusinessAcceptanceGate(payload, true, freshness) + coverage = dashboardOperatorCoverageGate(payload, true, freshness) + if strict.Status != "unknown" || business.Status != "unknown" || coverage.Status != "unknown" { + t.Fatalf("stale artifact must not remain passed or blocked: strict=%#v business=%#v coverage=%#v", strict, business, coverage) + } +} diff --git a/internal/api/tenant.go b/internal/api/tenant.go index 2b66325a..8d80a86a 100644 --- a/internal/api/tenant.go +++ b/internal/api/tenant.go @@ -187,6 +187,13 @@ func (server *server) listRunSummariesForRequest(ctx context.Context, request *h return server.store.ListRunSummaries(ctx, status) } +func (server *server) listRunSummaryPageForRequest(ctx context.Context, request *http.Request, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) { + if server.tenantEnforced(request) { + return server.store.ListRunSummariesPageByOrganization(ctx, server.requestOrganizationID(request), query) + } + return server.store.ListRunSummariesPage(ctx, query) +} + func (server *server) listAuditTasksForRequest(ctx context.Context, request *http.Request, status string) ([]domain.AuditTaskRecord, error) { if server.tenantEnforced(request) { return server.store.ListAuditTasksByOrganization(ctx, server.requestOrganizationID(request), status) diff --git a/internal/api/worker_proxy.go b/internal/api/worker_proxy.go index fbaedcf1..2b730879 100644 --- a/internal/api/worker_proxy.go +++ b/internal/api/worker_proxy.go @@ -68,6 +68,10 @@ func (server *server) handleWorkerAPIHealth(writer http.ResponseWriter, request server.proxyWorkerAPI(writer, request, "/worker-api/health", workerAPIShortProxyTimeout) } +func (server *server) handleWorkerAPIReady(writer http.ResponseWriter, request *http.Request) { + server.proxyWorkerAPI(writer, request, "/worker-api/ready", workerAPIShortProxyTimeout) +} + func (server *server) handleWorkerAPIStats(writer http.ResponseWriter, request *http.Request) { query := url.Values{} if raw := strings.TrimSpace(request.URL.RawQuery); raw != "" { diff --git a/internal/config/config.go b/internal/config/config.go index 18eb5af5..50fd1fef 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -40,6 +40,9 @@ type AppConfig struct { CustomOperatorImageImporterImage string CustomOperatorImageImportTimeoutSeconds int CustomOperatorImageArchiveMaxMiB int + BuildCommit string + BuildTime string + Image string } func LoadValidated() (AppConfig, error) { @@ -142,6 +145,9 @@ func LoadValidated() (AppConfig, error) { CustomOperatorImageImporterImage: getenvTrimmed("CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE", "quay.io/skopeo/stable:v1.16.1"), CustomOperatorImageImportTimeoutSeconds: customOperatorImageImportTimeoutSeconds, CustomOperatorImageArchiveMaxMiB: customOperatorImageArchiveMaxMiB, + BuildCommit: strings.TrimSpace(os.Getenv("DROBOTICFLOW_BUILD_COMMIT")), + BuildTime: strings.TrimSpace(os.Getenv("DROBOTICFLOW_BUILD_TIME")), + Image: strings.TrimSpace(os.Getenv("DROBOTICFLOW_IMAGE")), }, nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 6551c6fe..04111c5f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -36,6 +36,9 @@ var configEnvKeys = []string{ "CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE", "CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS", "CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB", + "DROBOTICFLOW_BUILD_COMMIT", + "DROBOTICFLOW_BUILD_TIME", + "DROBOTICFLOW_IMAGE", } func clearConfigEnv(t *testing.T) { @@ -143,6 +146,9 @@ func TestLoadValidatedAcceptsExplicitStartupEnv(t *testing.T) { t.Setenv("CUSTOM_OPERATOR_IMAGE_IMPORTER_IMAGE", " ccr.example/tools/skopeo:1 ") t.Setenv("CUSTOM_OPERATOR_IMAGE_IMPORT_TIMEOUT_SECONDS", "900") t.Setenv("CUSTOM_OPERATOR_IMAGE_ARCHIVE_MAX_MIB", "1024") + t.Setenv("DROBOTICFLOW_BUILD_COMMIT", " 3cda38c ") + t.Setenv("DROBOTICFLOW_BUILD_TIME", " 2026-07-10T12:00:00Z ") + t.Setenv("DROBOTICFLOW_IMAGE", " ccr.example/droboticflow-api:3cda38c ") cfg, err := LoadValidated() if err != nil { @@ -187,6 +193,9 @@ func TestLoadValidatedAcceptsExplicitStartupEnv(t *testing.T) { if !cfg.RequireSessionAuth || !cfg.SessionCookieSecure { t.Fatalf("expected explicit boolean envs to enable auth and secure cookie") } + if cfg.BuildCommit != "3cda38c" || cfg.BuildTime != "2026-07-10T12:00:00Z" || cfg.Image != "ccr.example/droboticflow-api:3cda38c" { + t.Fatalf("build metadata = commit %q time %q image %q", cfg.BuildCommit, cfg.BuildTime, cfg.Image) + } } func TestLoadValidatedTreatsWhitespaceStringEnvAsUnset(t *testing.T) { diff --git a/internal/domain/object_storage_browser.go b/internal/domain/object_storage_browser.go index e36cfcb2..dc03fb33 100644 --- a/internal/domain/object_storage_browser.go +++ b/internal/domain/object_storage_browser.go @@ -40,6 +40,9 @@ type ObjectStorageBrowseResult struct { Bucket string `json:"bucket"` Prefix string `json:"prefix,omitempty"` Items []ObjectStorageObjectSummary `json:"items"` + NextMarker string `json:"nextMarker,omitempty"` + HasMore bool `json:"hasMore"` + Limit int `json:"limit,omitempty"` } type ObjectStorageBucketCreateInput struct { diff --git a/internal/domain/types.go b/internal/domain/types.go index a1ae7cbd..4a967362 100644 --- a/internal/domain/types.go +++ b/internal/domain/types.go @@ -523,6 +523,30 @@ type RunMetadata struct { LightweightOperatorCodeManifests map[string]LightweightOperatorCodeManifest `json:"lightweightOperatorCodeManifests,omitempty" bson:"lightweightOperatorCodeManifests,omitempty"` } +// RunFailure is the machine-readable counterpart to RunRecord.Error. Error is +// intentionally retained for backwards compatibility and human-readable logs. +type RunFailure struct { + Code string `json:"code" bson:"code"` + Stage string `json:"stage,omitempty" bson:"stage,omitempty"` + NodeID string `json:"nodeId,omitempty" bson:"nodeId,omitempty"` + Message string `json:"message" bson:"message"` + Retryable bool `json:"retryable" bson:"retryable"` + Suggestion string `json:"suggestion,omitempty" bson:"suggestion,omitempty"` +} + +// RunProgress carries structured progress for long-running operators. It is +// optional so existing runners and persisted Run documents remain compatible. +type RunProgress struct { + Stage string `json:"stage,omitempty" bson:"stage,omitempty"` + Epoch int `json:"epoch,omitempty" bson:"epoch,omitempty"` + Step int `json:"step,omitempty" bson:"step,omitempty"` + MaxStep int `json:"maxStep,omitempty" bson:"maxStep,omitempty"` + SamplesPerSecond float64 `json:"samplesPerSecond,omitempty" bson:"samplesPerSecond,omitempty"` + EstimatedRemainingSeconds int64 `json:"estimatedRemainingSeconds,omitempty" bson:"estimatedRemainingSeconds,omitempty"` + Checkpoint string `json:"checkpoint,omitempty" bson:"checkpoint,omitempty"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + type RunRecord struct { ID string `json:"id" bson:"id"` OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` @@ -552,35 +576,61 @@ type RunRecord struct { Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"` K8sRefs []K8sRef `json:"k8sRefs" bson:"k8sRefs"` Metadata *RunMetadata `json:"metadata,omitempty" bson:"metadata,omitempty"` + Failure *RunFailure `json:"failure,omitempty" bson:"failure,omitempty"` + Progress *RunProgress `json:"progress,omitempty" bson:"progress,omitempty"` Error string `json:"error,omitempty" bson:"error,omitempty"` CreatedAt string `json:"createdAt" bson:"createdAt"` UpdatedAt string `json:"updatedAt" bson:"updatedAt"` } type RunSummary struct { - ID string `json:"id" bson:"id"` - OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` - WorkflowID string `json:"workflowId" bson:"workflowId"` - WorkflowName string `json:"workflowName" bson:"workflowName"` - ProjectID string `json:"projectId" bson:"projectId"` - ProjectName string `json:"projectName" bson:"projectName"` - Trigger RunTrigger `json:"trigger" bson:"trigger"` - Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` - ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` - RestartFromNodeID string `json:"restartFromNodeId,omitempty" bson:"restartFromNodeId,omitempty"` - AvailableAt string `json:"availableAt,omitempty" bson:"availableAt,omitempty"` - ExecutionLeaseOwner string `json:"executionLeaseOwner,omitempty" bson:"executionLeaseOwner,omitempty"` - ExecutionLeaseExpiresAt string `json:"executionLeaseExpiresAt,omitempty" bson:"executionLeaseExpiresAt,omitempty"` - Status RunStatus `json:"status" bson:"status"` - StopMode RunStopMode `json:"stopMode,omitempty" bson:"stopMode,omitempty"` - StopRequestedAt string `json:"stopRequestedAt,omitempty" bson:"stopRequestedAt,omitempty"` - StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` - EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` - DurationMS int64 `json:"durationMs,omitempty" bson:"durationMs,omitempty"` - CurrentNodeID string `json:"currentNodeId,omitempty" bson:"currentNodeId,omitempty"` - Error string `json:"error,omitempty" bson:"error,omitempty"` - CreatedAt string `json:"createdAt" bson:"createdAt"` - UpdatedAt string `json:"updatedAt" bson:"updatedAt"` + ID string `json:"id" bson:"id"` + OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"` + WorkflowID string `json:"workflowId" bson:"workflowId"` + WorkflowName string `json:"workflowName" bson:"workflowName"` + ProjectID string `json:"projectId" bson:"projectId"` + ProjectName string `json:"projectName" bson:"projectName"` + Trigger RunTrigger `json:"trigger" bson:"trigger"` + Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"` + ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"` + RestartFromNodeID string `json:"restartFromNodeId,omitempty" bson:"restartFromNodeId,omitempty"` + AvailableAt string `json:"availableAt,omitempty" bson:"availableAt,omitempty"` + ExecutionLeaseOwner string `json:"executionLeaseOwner,omitempty" bson:"executionLeaseOwner,omitempty"` + ExecutionLeaseExpiresAt string `json:"executionLeaseExpiresAt,omitempty" bson:"executionLeaseExpiresAt,omitempty"` + Status RunStatus `json:"status" bson:"status"` + StopMode RunStopMode `json:"stopMode,omitempty" bson:"stopMode,omitempty"` + StopRequestedAt string `json:"stopRequestedAt,omitempty" bson:"stopRequestedAt,omitempty"` + StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"` + EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"` + DurationMS int64 `json:"durationMs,omitempty" bson:"durationMs,omitempty"` + CurrentNodeID string `json:"currentNodeId,omitempty" bson:"currentNodeId,omitempty"` + Failure *RunFailure `json:"failure,omitempty" bson:"failure,omitempty"` + Progress *RunProgress `json:"progress,omitempty" bson:"progress,omitempty"` + Error string `json:"error,omitempty" bson:"error,omitempty"` + CreatedAt string `json:"createdAt" bson:"createdAt"` + UpdatedAt string `json:"updatedAt" bson:"updatedAt"` +} + +type RunSummaryCursor struct { + CreatedAt string `json:"createdAt"` + ID string `json:"id"` +} + +type RunSummaryQuery struct { + Limit int + Cursor *RunSummaryCursor + Status RunStatus + Search string + ProjectID string + WorkflowID string +} + +type RunSummaryPage struct { + Items []RunSummary + Total int64 + StatusCounts map[RunStatus]int64 + NextCursor *RunSummaryCursor + HasMore bool } type AuditTaskPayload struct { @@ -767,8 +817,10 @@ type DashboardSummary struct { Archived int `json:"archived"` } `json:"projects"` Operators struct { - Total int `json:"total"` - Custom int `json:"custom"` + Total int `json:"total"` + Builtin int `json:"builtin"` + Custom int `json:"custom"` + Historical int `json:"historical"` } `json:"operators"` Jobs struct { Today int `json:"today"` @@ -813,6 +865,19 @@ type DashboardPOCReadiness struct { RealFixtureCount int `json:"realFixtureCount"` GeneratedFixtureCount int `json:"generatedFixtureCount"` AllFixturesReal bool `json:"allFixturesReal"` + SourceCommit string `json:"sourceCommit,omitempty"` + CatalogHash string `json:"catalogHash,omitempty"` + CatalogCount int `json:"catalogCount,omitempty"` + CompletedAt string `json:"completedAt,omitempty"` + Freshness string `json:"freshness"` + StaleReason string `json:"staleReason,omitempty"` + CurrentSourceCommit string `json:"currentSourceCommit,omitempty"` + CurrentCatalogHash string `json:"currentCatalogHash,omitempty"` + CurrentCatalogCount int `json:"currentCatalogCount,omitempty"` + ServiceAvailability DashboardGate `json:"serviceAvailability"` + GPUCapacity DashboardGate `json:"gpuCapacity"` + BusinessAcceptance DashboardGate `json:"businessAcceptance"` + OperatorCoverage DashboardGate `json:"operatorCoverage"` GPU DashboardGate `json:"gpu"` GPUWorkflowPreflight DashboardGate `json:"gpuWorkflowPreflight"` DCloud DashboardGate `json:"dcloud"` diff --git a/internal/k8s/launcher.go b/internal/k8s/launcher.go index c124df19..79f0d54b 100644 --- a/internal/k8s/launcher.go +++ b/internal/k8s/launcher.go @@ -117,9 +117,22 @@ const ( ) type JobInspection struct { - Reference *JobReference - State JobState - Output string + Reference *JobReference + State JobState + Output string + CreatedAt string + BlockingReason string + BlockingMessage string +} + +type OrphanRuntime struct { + Kind string `json:"kind"` + Name string `json:"name"` + Namespace string `json:"namespace"` + RunID string `json:"runId"` + CreatedAt string `json:"createdAt"` + Cleaned bool `json:"cleaned"` + CleanupError string `json:"cleanupError,omitempty"` } type GPUCapacityNode struct { @@ -822,6 +835,26 @@ func (launcher *Launcher) readJobOutput(ctx context.Context, jobName string) (st return string(output), nil } +func (launcher *Launcher) readJobOutputNow(ctx context.Context, jobName string) (string, error) { + pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName}) + if err != nil { + return "", err + } + if len(pods.Items) == 0 { + return "", fmt.Errorf("no pod found for job %s", jobName) + } + stream, err := launcher.clientset.CoreV1().Pods(launcher.namespace).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}).Stream(ctx) + if err != nil { + return "", err + } + defer stream.Close() + output, err := io.ReadAll(stream) + if err != nil { + return "", err + } + return string(output), nil +} + // findJobPodName returns the name of the (first) pod backing jobName, waiting up // to ~30s for it to exist. Returns "" if no pod appears in time. func (launcher *Launcher) findJobPodName(ctx context.Context, jobName string) string { @@ -1158,9 +1191,106 @@ func (launcher *Launcher) TerminateRun(ctx context.Context, namespace string, ru return err } } + pods, err := launcher.clientset.CoreV1().Pods(targetNamespace).List( + ctx, + metav1.ListOptions{LabelSelector: "runId=" + strings.TrimSpace(runID)}, + ) + if err != nil { + return err + } + propagation := metav1.DeletePropagationBackground + for _, pod := range pods.Items { + err := launcher.clientset.CoreV1().Pods(targetNamespace).Delete(ctx, pod.Name, metav1.DeleteOptions{PropagationPolicy: &propagation}) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + } return nil } +func orphanRuntimeCreatedAt(timestamp metav1.Time) string { + if timestamp.IsZero() { + return "" + } + return timestamp.UTC().Format(time.RFC3339Nano) +} + +func (launcher *Launcher) ReconcileOrphanedRuntimes(ctx context.Context, knownRunIDs map[string]struct{}, olderThan time.Time, cleanup bool) ([]OrphanRuntime, error) { + if launcher == nil || launcher.clientset == nil { + return nil, nil + } + result := make([]OrphanRuntime, 0) + jobs, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId"}) + if err != nil { + return nil, err + } + for _, job := range jobs.Items { + runID := strings.TrimSpace(job.Labels["runId"]) + if runID == "" || !job.CreationTimestamp.Time.Before(olderThan) { + continue + } + if _, exists := knownRunIDs[runID]; exists { + continue + } + item := OrphanRuntime{ + Kind: "Job", + Name: job.Name, + Namespace: job.Namespace, + RunID: runID, + CreatedAt: orphanRuntimeCreatedAt(job.CreationTimestamp), + } + if cleanup { + if err := launcher.TerminateJob(ctx, job.Namespace, job.Name); err != nil { + item.CleanupError = err.Error() + } else { + item.Cleaned = true + } + } + result = append(result, item) + } + + pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "runId"}) + if err != nil { + return nil, err + } + for _, pod := range pods.Items { + runID := strings.TrimSpace(pod.Labels["runId"]) + if runID == "" || !pod.CreationTimestamp.Time.Before(olderThan) { + continue + } + if _, exists := knownRunIDs[runID]; exists { + continue + } + item := OrphanRuntime{ + Kind: "Pod", + Name: pod.Name, + Namespace: pod.Namespace, + RunID: runID, + CreatedAt: orphanRuntimeCreatedAt(pod.CreationTimestamp), + } + if cleanup { + propagation := metav1.DeletePropagationBackground + err := launcher.clientset.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{PropagationPolicy: &propagation}) + if err != nil && !apierrors.IsNotFound(err) { + item.CleanupError = err.Error() + } else { + item.Cleaned = true + } + } + result = append(result, item) + } + sort.Slice(result, func(i int, j int) bool { + if result[i].RunID == result[j].RunID { + if result[i].Kind == result[j].Kind { + return result[i].Name < result[j].Name + } + return result[i].Kind < result[j].Kind + } + return result[i].RunID < result[j].RunID + }) + return result, nil +} + func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*LaunchResult, error) { if err := ValidateGPURequest(input.GPULimit); err != nil { return nil, err @@ -1198,9 +1328,30 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc reference := &JobReference{JobName: jobName, Namespace: launcher.namespace} waitParentCtx := ctx + stopCancellationForwarder := func() {} if input.LongRunning { - waitParentCtx = context.WithoutCancel(ctx) + // Long-running training deliberately ignores ordinary workflow/node deadlines, + // but it must still detach promptly when the worker shuts down or loses its + // execution lease. The Kubernetes Job is left in place for the next worker to + // reconcile; only short-lived jobs are deleted on cancellation below. + detachedCtx, cancelDetached := context.WithCancel(context.Background()) + forwarderDone := make(chan struct{}) + go func() { + select { + case <-ctx.Done(): + if errors.Is(ctx.Err(), context.Canceled) { + cancelDetached() + } + case <-forwarderDone: + } + }() + waitParentCtx = detachedCtx + stopCancellationForwarder = func() { + close(forwarderDone) + cancelDetached() + } } + defer stopCancellationForwarder() waitCtx, cancel := context.WithDeadline(waitParentCtx, waitDeadlineForLaunch(waitParentCtx, input, time.Now())) defer cancel() ticker := time.NewTicker(time.Second) @@ -1310,7 +1461,7 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc select { case <-waitCtx.Done(): stopSampler() - if errors.Is(waitCtx.Err(), context.Canceled) { + if errors.Is(waitCtx.Err(), context.Canceled) && !input.LongRunning { terminateCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) _ = launcher.TerminateJob(terminateCtx, launcher.namespace, jobName) cancel() @@ -1324,6 +1475,50 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc } } +var terminalPendingReasons = map[string]struct{}{ + "CreateContainerConfigError": {}, + "CreateContainerError": {}, + "ErrImagePull": {}, + "ImagePullBackOff": {}, + "InvalidImageName": {}, + "Unschedulable": {}, +} + +func terminalPendingReason(reason string) bool { + _, ok := terminalPendingReasons[strings.TrimSpace(reason)] + return ok +} + +func podBlockingState(pod corev1.Pod) (string, string) { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && terminalPendingReason(condition.Reason) { + return condition.Reason, condition.Message + } + } + for _, status := range append(append([]corev1.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) { + if status.State.Waiting != nil && terminalPendingReason(status.State.Waiting.Reason) { + return status.State.Waiting.Reason, status.State.Waiting.Message + } + } + if terminalPendingReason(pod.Status.Reason) { + return pod.Status.Reason, pod.Status.Message + } + return "", "" +} + +func (launcher *Launcher) inspectBlockingState(ctx context.Context, jobName string) (string, string, error) { + pods, err := launcher.clientset.CoreV1().Pods(launcher.namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + jobName}) + if err != nil { + return "", "", err + } + for _, pod := range pods.Items { + if reason, message := podBlockingState(pod); reason != "" { + return reason, message, nil + } + } + return "", "", nil +} + func (launcher *Launcher) Inspect(ctx context.Context, runID string, nodeID string) (*JobInspection, error) { selectorParts := []string{"runId=" + strings.TrimSpace(runID)} if trimmedNodeID := strings.TrimSpace(nodeID); trimmedNodeID != "" { @@ -1343,23 +1538,31 @@ func (launcher *Launcher) Inspect(ctx context.Context, runID string, nodeID stri }) job := jobs.Items[len(jobs.Items)-1] reference := &JobReference{JobName: job.Name, Namespace: launcher.namespace} + createdAt := orphanRuntimeCreatedAt(job.CreationTimestamp) switch { case job.Status.Succeeded > 0: - output, err := launcher.readJobOutput(ctx, job.Name) + output, err := launcher.readJobOutputNow(ctx, job.Name) if err != nil { return nil, err } - return &JobInspection{Reference: reference, State: JobStateSucceeded, Output: output}, nil + return &JobInspection{Reference: reference, State: JobStateSucceeded, Output: output, CreatedAt: createdAt}, nil case job.Status.Failed > 0: - output, outputErr := launcher.readJobOutput(ctx, job.Name) + output, outputErr := launcher.readJobOutputNow(ctx, job.Name) diagnostics := launcher.readJobDiagnostics(ctx, &job, output, outputErr) if strings.TrimSpace(diagnostics) != "" { output = diagnostics } - return &JobInspection{Reference: reference, State: JobStateFailed, Output: output}, nil - case job.Status.Active > 0: - return &JobInspection{Reference: reference, State: JobStateActive}, nil - default: - return &JobInspection{Reference: reference, State: JobStatePending}, nil + return &JobInspection{Reference: reference, State: JobStateFailed, Output: output, CreatedAt: createdAt}, nil } + reason, message, err := launcher.inspectBlockingState(ctx, job.Name) + if err != nil { + return nil, err + } + if reason != "" { + return &JobInspection{Reference: reference, State: JobStatePending, CreatedAt: createdAt, BlockingReason: reason, BlockingMessage: message}, nil + } + if job.Status.Active > 0 { + return &JobInspection{Reference: reference, State: JobStateActive, CreatedAt: createdAt}, nil + } + return &JobInspection{Reference: reference, State: JobStatePending, CreatedAt: createdAt}, nil } diff --git a/internal/k8s/launcher_test.go b/internal/k8s/launcher_test.go index 820038ce..d9639bf2 100644 --- a/internal/k8s/launcher_test.go +++ b/internal/k8s/launcher_test.go @@ -626,6 +626,144 @@ func TestTerminateRunDeletesJobsOneByOne(t *testing.T) { } } +func TestInspectClassifiesImagePullBackOffAsBlockingPending(t *testing.T) { + createdAt := metav1.NewTime(time.Now().UTC().Add(-15 * time.Minute)) + clientset := fake.NewSimpleClientset( + &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "run-1-node-1-a1", Namespace: "droboticflow", Labels: map[string]string{"runId": "run-1", "nodeId": "node-1"}, CreationTimestamp: createdAt}, + Status: batchv1.JobStatus{Active: 1}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "run-1-node-1-a1-pod", Namespace: "droboticflow", Labels: map[string]string{"job-name": "run-1-node-1-a1", "runId": "run-1", "nodeId": "node-1"}, CreationTimestamp: createdAt}, + Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{ + Name: "operator-runner", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{ + Reason: "ImagePullBackOff", + Message: "back-off pulling image", + }}, + }}}, + }, + ) + launcher := &Launcher{namespace: "droboticflow", clientset: clientset} + inspection, err := launcher.Inspect(context.Background(), "run-1", "node-1") + if err != nil { + t.Fatalf("inspect job: %v", err) + } + if inspection.State != JobStatePending || inspection.BlockingReason != "ImagePullBackOff" { + t.Fatalf("expected blocking pending inspection, got %#v", inspection) + } + if !strings.Contains(inspection.BlockingMessage, "pulling image") || inspection.CreatedAt == "" { + t.Fatalf("expected blocker detail and creation time, got %#v", inspection) + } +} + +func TestInspectKeepsHealthyLongRunningJobActive(t *testing.T) { + createdAt := metav1.NewTime(time.Now().UTC().Add(-48 * time.Hour)) + clientset := fake.NewSimpleClientset( + &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "run-long-train-a1", Namespace: "droboticflow", Labels: map[string]string{"runId": "run-long", "nodeId": "train"}, CreationTimestamp: createdAt}, + Status: batchv1.JobStatus{Active: 1}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "run-long-train-pod", Namespace: "droboticflow", Labels: map[string]string{"job-name": "run-long-train-a1", "runId": "run-long", "nodeId": "train"}, CreationTimestamp: createdAt}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + ) + launcher := &Launcher{namespace: "droboticflow", clientset: clientset} + inspection, err := launcher.Inspect(context.Background(), "run-long", "train") + if err != nil { + t.Fatalf("inspect job: %v", err) + } + if inspection.State != JobStateActive || inspection.BlockingReason != "" { + t.Fatalf("expected healthy long-running job to remain active, got %#v", inspection) + } +} + +func TestLaunchCancellationDetachesWithoutDeletingLongRunningJob(t *testing.T) { + clientset := fake.NewSimpleClientset() + launcher := &Launcher{ + namespace: "droboticflow", + workspacePVCName: "workspace", + workspaceMountPath: "/workspace", + clientset: clientset, + } + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 1) + go func() { + _, err := launcher.Launch(ctx, LaunchInput{Image: "runner:latest", RunID: "run-long", WorkflowID: "workflow-1", NodeID: "train", LongRunning: true}) + result <- err + }() + deadline := time.Now().Add(time.Second) + for { + jobs, err := clientset.BatchV1().Jobs("droboticflow").List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("list jobs: %v", err) + } + if len(jobs.Items) == 1 { + break + } + if time.Now().After(deadline) { + t.Fatal("timed out waiting for long-running job creation") + } + time.Sleep(5 * time.Millisecond) + } + cancel() + select { + case err := <-result: + if err == nil || !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("expected detached launch to return cancellation, got %v", err) + } + case <-time.After(time.Second): + t.Fatal("long-running launch did not detach after cancellation") + } + jobs, err := clientset.BatchV1().Jobs("droboticflow").List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("list jobs after cancellation: %v", err) + } + if len(jobs.Items) != 1 || jobs.Items[0].Name != "run-long-train-a1" { + t.Fatalf("expected long-running job preserved for reconciliation, got %#v", jobs.Items) + } +} + +func TestReconcileOrphanedRuntimesReportsAndOptionallyCleansOldUnknownResources(t *testing.T) { + old := metav1.NewTime(time.Now().UTC().Add(-48 * time.Hour)) + recent := metav1.NewTime(time.Now().UTC().Add(-time.Hour)) + clientset := fake.NewSimpleClientset( + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "known-job", Namespace: "droboticflow", Labels: map[string]string{"runId": "run-known"}, CreationTimestamp: old}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "orphan-job", Namespace: "droboticflow", Labels: map[string]string{"runId": "run-orphan"}, CreationTimestamp: old}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "recent-job", Namespace: "droboticflow", Labels: map[string]string{"runId": "run-recent"}, CreationTimestamp: recent}}, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan-pod", Namespace: "droboticflow", Labels: map[string]string{"runId": "run-orphan"}, CreationTimestamp: old}}, + ) + launcher := &Launcher{namespace: "droboticflow", clientset: clientset} + orphans, err := launcher.ReconcileOrphanedRuntimes(context.Background(), map[string]struct{}{"run-known": {}}, time.Now().UTC().Add(-24*time.Hour), false) + if err != nil { + t.Fatalf("inspect orphans: %v", err) + } + if len(orphans) != 2 || orphans[0].RunID != "run-orphan" || orphans[1].RunID != "run-orphan" { + t.Fatalf("expected old unknown job and pod only, got %#v", orphans) + } + for _, item := range orphans { + if item.Cleaned { + t.Fatalf("cleanup must default to disabled, got %#v", item) + } + } + + cleaned, err := launcher.ReconcileOrphanedRuntimes(context.Background(), map[string]struct{}{"run-known": {}}, time.Now().UTC().Add(-24*time.Hour), true) + if err != nil { + t.Fatalf("clean orphans: %v", err) + } + if len(cleaned) != 2 || !cleaned[0].Cleaned || !cleaned[1].Cleaned { + t.Fatalf("expected both orphan resources cleaned, got %#v", cleaned) + } + jobs, err := clientset.BatchV1().Jobs("droboticflow").List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("list jobs: %v", err) + } + if len(jobs.Items) != 2 { + t.Fatalf("expected known and recent jobs to remain, got %#v", jobs.Items) + } +} + func TestReadJobDiagnosticsIncludesPodStatusAndEvents(t *testing.T) { job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{Name: "run-1-node-a", Namespace: "droboticflow"}, diff --git a/internal/store/errors.go b/internal/store/errors.go index 252c2310..7617d664 100644 --- a/internal/store/errors.go +++ b/internal/store/errors.go @@ -2,4 +2,4 @@ package store import "errors" -var ErrWorkflowAlreadyCompleted = errors.New("workflow has completed successfully and cannot be run again manually") +var ErrWorkflowRunActive = errors.New("workflow already has an active run") diff --git a/internal/store/memory.go b/internal/store/memory.go index 7e4569ac..d1fe818a 100644 --- a/internal/store/memory.go +++ b/internal/store/memory.go @@ -53,6 +53,10 @@ func NewMemoryStore() *MemoryStore { } } +func (store *MemoryStore) Ready(_ context.Context) error { + return nil +} + func sortProjects(items []domain.ProjectRecord) { sort.Slice(items, func(left int, right int) bool { return items[left].CreatedAt > items[right].CreatedAt }) } @@ -807,6 +811,7 @@ func (store *MemoryStore) CountRunsByStatus(_ context.Context) (map[domain.RunSt func summarizeRun(run domain.RunRecord) domain.RunSummary { return domain.RunSummary{ + OrganizationID: run.OrganizationID, ID: run.ID, WorkflowID: run.WorkflowID, WorkflowName: run.WorkflowName, @@ -824,6 +829,8 @@ func summarizeRun(run domain.RunRecord) domain.RunSummary { EndedAt: run.EndedAt, DurationMS: run.DurationMS, CurrentNodeID: run.CurrentNodeID, + Failure: run.Failure, + Progress: run.Progress, Error: run.Error, CreatedAt: run.CreatedAt, UpdatedAt: run.UpdatedAt, @@ -868,6 +875,156 @@ func (store *MemoryStore) ListRunSummariesByOrganization(_ context.Context, orga return items, nil } +func runSummaryMatchesQuery(item domain.RunSummary, query domain.RunSummaryQuery) bool { + if query.ProjectID != "" && item.ProjectID != query.ProjectID { + return false + } + if query.WorkflowID != "" && item.WorkflowID != query.WorkflowID { + return false + } + search := strings.ToLower(strings.TrimSpace(query.Search)) + if search == "" { + return true + } + for _, value := range []string{ + item.ID, + item.WorkflowID, + item.WorkflowName, + item.ProjectID, + item.ProjectName, + item.CurrentNodeID, + item.Error, + } { + if strings.Contains(strings.ToLower(value), search) { + return true + } + } + return false +} + +func runSummaryBeforeCursor(item domain.RunSummary, cursor *domain.RunSummaryCursor) bool { + if cursor == nil { + return true + } + if item.CreatedAt == cursor.CreatedAt { + return item.ID < cursor.ID + } + return item.CreatedAt < cursor.CreatedAt +} + +func (store *MemoryStore) listRunSummariesPageLocked(organizationID string, query domain.RunSummaryQuery) domain.RunSummaryPage { + statusCounts := emptyRunStatusCounts() + matching := make([]domain.RunSummary, 0) + var total int64 + for _, run := range store.runs { + if organizationID != "" && !organizationMatches(run.OrganizationID, organizationID) { + continue + } + summary := summarizeRun(run) + if !runSummaryMatchesQuery(summary, query) { + continue + } + statusCounts[summary.Status]++ + total++ + if query.Status != "" && summary.Status != query.Status { + continue + } + matching = append(matching, summary) + } + + sortRunSummaries(matching) + pageItems := make([]domain.RunSummary, 0, normalizedRunSummaryLimit(query.Limit)+1) + for _, item := range matching { + if runSummaryBeforeCursor(item, query.Cursor) { + pageItems = append(pageItems, item) + } + } + limit := normalizedRunSummaryLimit(query.Limit) + hasMore := len(pageItems) > limit + if hasMore { + pageItems = pageItems[:limit] + } + var nextCursor *domain.RunSummaryCursor + if hasMore && len(pageItems) > 0 { + last := pageItems[len(pageItems)-1] + nextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID} + } + return domain.RunSummaryPage{ + Items: pageItems, + Total: total, + StatusCounts: statusCounts, + NextCursor: nextCursor, + HasMore: hasMore, + } +} + +func (store *MemoryStore) ListRunSummariesPage(_ context.Context, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) { + store.mu.Lock() + defer store.mu.Unlock() + return store.listRunSummariesPageLocked("", query), nil +} + +func (store *MemoryStore) ListRunSummariesPageByOrganization(_ context.Context, organizationID string, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) { + store.mu.Lock() + defer store.mu.Unlock() + return store.listRunSummariesPageLocked(organizationID, query), nil +} + +func (store *MemoryStore) hasActiveRunLocked(workflowID string) bool { + for _, run := range store.runs { + if run.WorkflowID == workflowID && isActiveRunStatus(run.Status) { + return true + } + } + return false +} + +func (store *MemoryStore) HasActiveRun(_ context.Context, workflowID string) (bool, error) { + store.mu.Lock() + defer store.mu.Unlock() + return store.hasActiveRunLocked(workflowID), nil +} + +func (store *MemoryStore) UpdateRunProgress(_ context.Context, runID string, progress *domain.RunProgress) error { + store.mu.Lock() + defer store.mu.Unlock() + run, ok := store.runs[runID] + if !ok { + return fmt.Errorf("run %s not found", runID) + } + if progress == nil { + run.Progress = nil + run.UpdatedAt = nowISO() + } else { + value := *progress + if strings.TrimSpace(value.UpdatedAt) == "" { + value.UpdatedAt = nowISO() + } + run.Progress = &value + run.UpdatedAt = value.UpdatedAt + } + store.runs[runID] = run + return nil +} + +func (store *MemoryStore) UpdateRunFailure(_ context.Context, runID string, failure *domain.RunFailure) error { + store.mu.Lock() + defer store.mu.Unlock() + run, ok := store.runs[runID] + if !ok { + return fmt.Errorf("run %s not found", runID) + } + if failure == nil { + run.Failure = nil + } else { + value := *failure + run.Failure = &value + } + run.UpdatedAt = nowISO() + store.runs[runID] = run + return nil +} + func (store *MemoryStore) GetRun(_ context.Context, id string) (*domain.RunRecord, error) { store.mu.Lock() defer store.mu.Unlock() @@ -887,9 +1044,6 @@ func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigge if !ok { return nil, fmt.Errorf("workflow %s not found", workflowID) } - if workflow.Status == domain.WorkflowStatusCompleted && workflow.Kind != domain.WorkflowKindScan && trigger != domain.RunTriggerScheduled { - return nil, ErrWorkflowAlreadyCompleted - } projectName := "" if workflow.ProjectID != "" { project, ok := store.projects[workflow.ProjectID] @@ -1617,8 +1771,12 @@ func (store *MemoryStore) GetDashboardSummary(ctx context.Context) (domain.Dashb if err != nil { return domain.DashboardSummary{}, err } + settings, err := store.GetSettings(ctx) + if err != nil { + return domain.DashboardSummary{}, err + } - return BuildDashboardSummary(projects, operators, runs, auditTasks), nil + return BuildDashboardSummaryAt(projects, operators, runs, auditTasks, settings.Timezone, time.Now()), nil } func BuildDashboardSummary( @@ -1626,6 +1784,17 @@ func BuildDashboardSummary( operators []domain.OperatorRecord, runs []domain.RunSummary, auditTasks []domain.AuditTaskRecord, +) domain.DashboardSummary { + return BuildDashboardSummaryAt(projects, operators, runs, auditTasks, domain.DefaultWorkflowScheduleTimezone, time.Now()) +} + +func BuildDashboardSummaryAt( + projects []domain.ProjectRecord, + operators []domain.OperatorRecord, + runs []domain.RunSummary, + auditTasks []domain.AuditTaskRecord, + timezone string, + now time.Time, ) domain.DashboardSummary { completedRuns := make([]domain.RunSummary, 0) failedRuns := make([]domain.RunSummary, 0) @@ -1672,7 +1841,21 @@ func BuildDashboardSummary( } } totalFinished := len(completedRuns) + len(failedRuns) - summary.Jobs.Today = len(runs) + location, err := time.LoadLocation(strings.TrimSpace(timezone)) + if err != nil { + location, _ = time.LoadLocation(domain.DefaultWorkflowScheduleTimezone) + } + localNow := now.In(location) + for _, run := range runs { + createdAt, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(run.CreatedAt)) + if err != nil { + continue + } + localCreatedAt := createdAt.In(location) + if localCreatedAt.Year() == localNow.Year() && localCreatedAt.YearDay() == localNow.YearDay() { + summary.Jobs.Today++ + } + } if totalFinished == 0 { summary.Jobs.SuccessRate = 100 } else { diff --git a/internal/store/memory_test.go b/internal/store/memory_test.go index ec80dc57..5b5557a0 100644 --- a/internal/store/memory_test.go +++ b/internal/store/memory_test.go @@ -286,14 +286,14 @@ func TestAppendNodeExecutionResourceSampleAppendsInPlace(t *testing.T) { NodeExecutions: []domain.NodeExecutionRecord{ { NodeID: "sam3", - NodeLabel: "SAM3 视频分割", - Status: domain.NodeExecutionStatusRunning, + NodeLabel: "SAM3 视频分割", + Status: domain.NodeExecutionStatusRunning, ResourceSeries: []domain.ResourceSample{{Timestamp: "t0", GPU: 10, GPUMemoryUsedMiB: 1000}}, }, { NodeID: "source", - NodeLabel: "数据源", - Status: domain.NodeExecutionStatusCompleted, + NodeLabel: "数据源", + Status: domain.NodeExecutionStatusCompleted, ResourceSeries: []domain.ResourceSample{{Timestamp: "t1"}}, }, }, @@ -577,7 +577,7 @@ func TestMemoryStoreStorageConnectionCRUD(t *testing.T) { } } -func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { +func TestCompletedWorkflowCanBeRunAgainManuallyWhenInactive(t *testing.T) { ctx := context.Background() dataStore := store.NewMemoryStore() @@ -617,18 +617,56 @@ func TestCompletedWorkflowCannotBeRunAgainManually(t *testing.T) { t.Fatalf("mark workflow completed: %v", err) } - if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err == nil { - t.Fatalf("expected manual rerun to be rejected") - } else if !errors.Is(err, store.ErrWorkflowAlreadyCompleted) { - t.Fatalf("expected ErrWorkflowAlreadyCompleted, got %v", err) + manualRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("expected completed workflow rerun to be allowed, got %v", err) + } + if manualRun == nil || manualRun.Status != domain.RunStatusQueued { + t.Fatalf("expected manual rerun queued, got %#v", manualRun) + } +} + +func TestHasActiveRunRecognizesBlockingStatuses(t *testing.T) { + ctx := context.Background() + for _, status := range []domain.RunStatus{ + domain.RunStatusQueued, + domain.RunStatusRunning, + domain.RunStatusStopping, + domain.RunStatusWaitingForAudit, + } { + t.Run(string(status), func(t *testing.T) { + dataStore := store.NewMemoryStore() + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "active guard"}) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + if err := dataStore.UpsertRun(ctx, domain.RunRecord{ + ID: "run-active-" + string(status), + WorkflowID: workflow.ID, + Status: status, + }); err != nil { + t.Fatalf("upsert active run: %v", err) + } + active, err := dataStore.HasActiveRun(ctx, workflow.ID) + if err != nil { + t.Fatalf("check active run: %v", err) + } + if !active { + t.Fatalf("expected %s to count as active", status) + } + }) } - scheduledRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled) + dataStore := store.NewMemoryStore() + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{Name: "scheduled concurrency"}) if err != nil { - t.Fatalf("expected scheduled run to remain allowed, got %v", err) + t.Fatalf("create scheduled workflow: %v", err) } - if scheduledRun == nil || scheduledRun.Status != domain.RunStatusQueued { - t.Fatalf("expected scheduled run queued, got %#v", scheduledRun) + if err := dataStore.UpsertRun(ctx, domain.RunRecord{ID: "run-active-manual", WorkflowID: workflow.ID, Status: domain.RunStatusRunning}); err != nil { + t.Fatalf("upsert active scheduled run: %v", err) + } + if _, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual); err != nil { + t.Fatalf("internal queue creation remains governed by worker concurrency limits, got %v", err) } } @@ -666,6 +704,10 @@ func TestCreateRunTracksPreviousAndCurrentRunTimestamps(t *testing.T) { if firstRun.CurrentRunAt == "" { t.Fatalf("expected first currentRunAt populated") } + firstRun.Status = domain.RunStatusCompleted + if err := dataStore.UpsertRun(ctx, *firstRun); err != nil { + t.Fatalf("complete first run: %v", err) + } secondRun, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerScheduled) if err != nil { @@ -678,3 +720,125 @@ func TestCreateRunTracksPreviousAndCurrentRunTimestamps(t *testing.T) { t.Fatalf("expected second currentRunAt populated") } } + +func TestListRunSummariesPageFiltersCountsAndAdvancesCursor(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + runs := []domain.RunRecord{ + {ID: "run-5", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusFailed, Error: "GPU failed", CreatedAt: "2026-07-10T05:00:00Z", UpdatedAt: "2026-07-10T05:00:00Z", Failure: &domain.RunFailure{Code: "gpu_failed", Message: "GPU failed"}}, + {ID: "run-4", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusRunning, CreatedAt: "2026-07-10T04:00:00Z", UpdatedAt: "2026-07-10T04:00:00Z", Progress: &domain.RunProgress{Stage: "train", Step: 4, MaxStep: 10, UpdatedAt: "2026-07-10T04:00:00Z"}}, + {ID: "run-3", OrganizationID: "org-a", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T03:00:00Z", UpdatedAt: "2026-07-10T03:00:00Z"}, + {ID: "run-2", OrganizationID: "org-a", WorkflowID: "workflow-audio", WorkflowName: "Audio Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusCancelled, CreatedAt: "2026-07-10T02:00:00Z", UpdatedAt: "2026-07-10T02:00:00Z"}, + {ID: "run-1", OrganizationID: "org-b", WorkflowID: "workflow-vision", WorkflowName: "Vision Train", ProjectID: "project-a", ProjectName: "Project A", Status: domain.RunStatusFailed, CreatedAt: "2026-07-10T01:00:00Z", UpdatedAt: "2026-07-10T01:00:00Z"}, + } + for _, run := range runs { + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run %s: %v", run.ID, err) + } + } + + query := domain.RunSummaryQuery{Limit: 2, Search: "vision", ProjectID: "project-a"} + first, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query) + if err != nil { + t.Fatalf("list first page: %v", err) + } + if first.Total != 3 || len(first.Items) != 2 || !first.HasMore || first.NextCursor == nil { + t.Fatalf("unexpected first page: %#v", first) + } + if first.Items[0].ID != "run-5" || first.Items[1].ID != "run-4" { + t.Fatalf("unexpected first page order: %#v", first.Items) + } + if first.StatusCounts[domain.RunStatusFailed] != 1 || first.StatusCounts[domain.RunStatusRunning] != 1 || first.StatusCounts[domain.RunStatusCompleted] != 1 { + t.Fatalf("unexpected status counts: %#v", first.StatusCounts) + } + if first.Items[0].Failure == nil || first.Items[0].Failure.Code != "gpu_failed" || first.Items[1].Progress == nil || first.Items[1].Progress.Step != 4 { + t.Fatalf("structured failure/progress missing from summaries: %#v", first.Items) + } + + query.Cursor = first.NextCursor + second, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query) + if err != nil { + t.Fatalf("list second page: %v", err) + } + if second.Total != 3 || len(second.Items) != 1 || second.Items[0].ID != "run-3" || second.HasMore || second.NextCursor != nil { + t.Fatalf("unexpected second page: %#v", second) + } + + query.Cursor = nil + query.Status = domain.RunStatusFailed + failed, err := dataStore.ListRunSummariesPageByOrganization(ctx, "org-a", query) + if err != nil { + t.Fatalf("list failed page: %v", err) + } + if failed.Total != 3 || len(failed.Items) != 1 || failed.Items[0].ID != "run-5" { + t.Fatalf("unexpected failed page: %#v", failed) + } + if failed.StatusCounts[domain.RunStatusRunning] != 1 { + t.Fatalf("status counts should remain global to non-status filters: %#v", failed.StatusCounts) + } +} + +func TestUpdateRunProgressAndFailureUseStructuredFields(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + run := domain.RunRecord{ID: "run-structured", Status: domain.RunStatusRunning, CreatedAt: "2026-07-10T01:00:00Z", UpdatedAt: "2026-07-10T01:00:00Z"} + if err := dataStore.UpsertRun(ctx, run); err != nil { + t.Fatalf("upsert run: %v", err) + } + progress := &domain.RunProgress{Stage: "train", Epoch: 2, Step: 40, MaxStep: 100, SamplesPerSecond: 8.5, Checkpoint: "checkpoint-40", UpdatedAt: "2026-07-10T01:05:00Z"} + if err := dataStore.UpdateRunProgress(ctx, run.ID, progress); err != nil { + t.Fatalf("update progress: %v", err) + } + failure := &domain.RunFailure{Code: "dtype_mismatch", Stage: "preflight", NodeID: "train", Message: "dtype mismatch", Retryable: false, Suggestion: "align model and input dtype"} + if err := dataStore.UpdateRunFailure(ctx, run.ID, failure); err != nil { + t.Fatalf("update failure: %v", err) + } + stored, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if stored.Progress == nil || stored.Progress.Step != 40 || stored.Progress.Checkpoint != "checkpoint-40" { + t.Fatalf("unexpected progress: %#v", stored.Progress) + } + if stored.Failure == nil || stored.Failure.Code != "dtype_mismatch" || stored.Failure.Suggestion == "" { + t.Fatalf("unexpected failure: %#v", stored.Failure) + } + if err := dataStore.UpdateRunProgress(ctx, run.ID, nil); err != nil { + t.Fatalf("clear progress: %v", err) + } + if err := dataStore.UpdateRunFailure(ctx, run.ID, nil); err != nil { + t.Fatalf("clear failure: %v", err) + } + cleared, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get cleared run: %v", err) + } + if cleared.Progress != nil || cleared.Failure != nil { + t.Fatalf("expected structured fields cleared, got progress=%#v failure=%#v", cleared.Progress, cleared.Failure) + } +} + +func TestBuildDashboardSummaryCountsTodayInConfiguredTimezone(t *testing.T) { + now := time.Date(2026, time.July, 10, 16, 30, 0, 0, time.UTC) // 2026-07-11 00:30 in Asia/Shanghai. + runs := []domain.RunSummary{ + {ID: "previous-local-day", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T15:59:59Z"}, + {ID: "local-midnight", Status: domain.RunStatusCompleted, CreatedAt: "2026-07-10T16:00:00Z"}, + {ID: "local-late", Status: domain.RunStatusFailed, CreatedAt: "2026-07-11T15:59:59Z"}, + {ID: "next-local-day", Status: domain.RunStatusFailed, CreatedAt: "2026-07-11T16:00:00Z"}, + {ID: "invalid-time", Status: domain.RunStatusRunning, CreatedAt: "not-a-time"}, + } + summary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "Asia/Shanghai", now) + if summary.Jobs.Today != 2 { + t.Fatalf("Asia/Shanghai jobs.today = %d, want 2", summary.Jobs.Today) + } + + utcSummary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "UTC", now) + if utcSummary.Jobs.Today != 2 { + t.Fatalf("UTC jobs.today = %d, want 2", utcSummary.Jobs.Today) + } + + fallbackSummary := store.BuildDashboardSummaryAt(nil, nil, runs, nil, "Mars/Base", now) + if fallbackSummary.Jobs.Today != summary.Jobs.Today { + t.Fatalf("invalid timezone should fall back to %s: got %d want %d", domain.DefaultWorkflowScheduleTimezone, fallbackSummary.Jobs.Today, summary.Jobs.Today) + } +} diff --git a/internal/store/mongo.go b/internal/store/mongo.go index 4acbd2f2..0ca3ea80 100644 --- a/internal/store/mongo.go +++ b/internal/store/mongo.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "regexp" "sort" "strings" "time" @@ -56,6 +57,10 @@ func NewMongoStore(db *mongo.Database) *MongoStore { } } +func (store *MongoStore) Ready(ctx context.Context) error { + return store.db.Client().Ping(ctx, nil) +} + func decodeMany[T any](ctx context.Context, cursor *mongo.Cursor) ([]T, error) { defer cursor.Close(ctx) items := make([]T, 0) @@ -709,6 +714,8 @@ func runSummaryProjection() bson.M { "endedAt": 1, "durationMs": 1, "currentNodeId": 1, + "failure": 1, + "progress": 1, "error": 1, "createdAt": 1, "updatedAt": 1, @@ -751,6 +758,190 @@ func (store *MongoStore) ListRunSummariesByOrganization(ctx context.Context, org return decodeMany[domain.RunSummary](ctx, cursor) } +func combineRunSummaryFilters(filters ...bson.M) bson.M { + items := make(bson.A, 0, len(filters)) + for _, filter := range filters { + if len(filter) > 0 { + items = append(items, filter) + } + } + if len(items) == 0 { + return bson.M{} + } + if len(items) == 1 { + return items[0].(bson.M) + } + return bson.M{"$and": items} +} + +func runSummaryBaseFilter(query domain.RunSummaryQuery) bson.M { + filters := make([]bson.M, 0, 3) + if query.ProjectID != "" { + filters = append(filters, bson.M{"projectId": query.ProjectID}) + } + if query.WorkflowID != "" { + filters = append(filters, bson.M{"workflowId": query.WorkflowID}) + } + if search := strings.TrimSpace(query.Search); search != "" { + pattern := primitive.Regex{Pattern: regexp.QuoteMeta(search), Options: "i"} + filters = append(filters, bson.M{"$or": bson.A{ + bson.M{"id": pattern}, + bson.M{"workflowId": pattern}, + bson.M{"workflowName": pattern}, + bson.M{"projectId": pattern}, + bson.M{"projectName": pattern}, + bson.M{"currentNodeId": pattern}, + bson.M{"error": pattern}, + }}) + } + return combineRunSummaryFilters(filters...) +} + +func runSummaryCursorFilter(cursor *domain.RunSummaryCursor) bson.M { + if cursor == nil { + return bson.M{} + } + return bson.M{"$or": bson.A{ + bson.M{"createdAt": bson.M{"$lt": cursor.CreatedAt}}, + bson.M{"createdAt": cursor.CreatedAt, "id": bson.M{"$lt": cursor.ID}}, + }} +} + +func (store *MongoStore) listRunSummariesPage(ctx context.Context, organizationID string, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) { + baseFilter := runSummaryBaseFilter(query) + if organizationID != "" { + baseFilter = organizationScopedFilter(organizationID, baseFilter) + } + + statusFilter := bson.M{} + if query.Status != "" { + statusFilter["status"] = query.Status + } + matchedFilter := combineRunSummaryFilters(baseFilter, statusFilter) + total, err := store.runs.CountDocuments(ctx, baseFilter) + if err != nil { + return domain.RunSummaryPage{}, err + } + + statusCounts := emptyRunStatusCounts() + pipeline := mongo.Pipeline{} + if len(baseFilter) > 0 { + pipeline = append(pipeline, bson.D{{Key: "$match", Value: baseFilter}}) + } + pipeline = append(pipeline, bson.D{{Key: "$group", Value: bson.M{ + "_id": "$status", + "count": bson.M{"$sum": 1}, + }}}) + countsCursor, err := store.runs.Aggregate(ctx, pipeline) + if err != nil { + return domain.RunSummaryPage{}, err + } + defer countsCursor.Close(ctx) + for countsCursor.Next(ctx) { + var item struct { + Status domain.RunStatus `bson:"_id"` + Count int64 `bson:"count"` + } + if err := countsCursor.Decode(&item); err != nil { + return domain.RunSummaryPage{}, err + } + statusCounts[item.Status] = item.Count + } + if err := countsCursor.Err(); err != nil { + return domain.RunSummaryPage{}, err + } + + limit := normalizedRunSummaryLimit(query.Limit) + pageFilter := combineRunSummaryFilters(matchedFilter, runSummaryCursorFilter(query.Cursor)) + cursor, err := store.runs.Find( + ctx, + pageFilter, + options.Find(). + SetProjection(runSummaryProjection()). + SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}). + SetLimit(int64(limit+1)), + ) + if err != nil { + return domain.RunSummaryPage{}, err + } + items, err := decodeMany[domain.RunSummary](ctx, cursor) + if err != nil { + return domain.RunSummaryPage{}, err + } + hasMore := len(items) > limit + if hasMore { + items = items[:limit] + } + var nextCursor *domain.RunSummaryCursor + if hasMore && len(items) > 0 { + last := items[len(items)-1] + nextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID} + } + return domain.RunSummaryPage{ + Items: items, + Total: total, + StatusCounts: statusCounts, + NextCursor: nextCursor, + HasMore: hasMore, + }, nil +} + +func (store *MongoStore) ListRunSummariesPage(ctx context.Context, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) { + return store.listRunSummariesPage(ctx, "", query) +} + +func (store *MongoStore) ListRunSummariesPageByOrganization(ctx context.Context, organizationID string, query domain.RunSummaryQuery) (domain.RunSummaryPage, error) { + return store.listRunSummariesPage(ctx, organizationID, query) +} + +func (store *MongoStore) HasActiveRun(ctx context.Context, workflowID string) (bool, error) { + count, err := store.runs.CountDocuments(ctx, bson.M{ + "workflowId": workflowID, + "status": bson.M{"$in": activeRunStatuses()}, + }, options.Count().SetLimit(1)) + return count > 0, err +} + +func (store *MongoStore) UpdateRunProgress(ctx context.Context, runID string, progress *domain.RunProgress) error { + updatedAt := nowISO() + update := bson.M{"$set": bson.M{"updatedAt": updatedAt}} + if progress == nil { + update["$unset"] = bson.M{"progress": ""} + } else { + value := *progress + if strings.TrimSpace(value.UpdatedAt) == "" { + value.UpdatedAt = updatedAt + } + updatedAt = value.UpdatedAt + update["$set"] = bson.M{"progress": value, "updatedAt": updatedAt} + } + result, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, update) + if err != nil { + return err + } + if result.MatchedCount == 0 { + return fmt.Errorf("run %s not found", runID) + } + return nil +} + +func (store *MongoStore) UpdateRunFailure(ctx context.Context, runID string, failure *domain.RunFailure) error { + update := bson.M{"$set": bson.M{"updatedAt": nowISO()}} + if failure == nil { + update["$unset"] = bson.M{"failure": ""} + } else { + update["$set"] = bson.M{"failure": *failure, "updatedAt": nowISO()} + } + result, err := store.runs.UpdateOne(ctx, bson.M{"id": runID}, update) + if err != nil { + return err + } + if result.MatchedCount == 0 { + return fmt.Errorf("run %s not found", runID) + } + return nil +} + func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunRecord, error) { var item domain.RunRecord err := store.runs.FindOne(ctx, bson.M{"id": id}).Decode(&item) @@ -772,9 +963,6 @@ func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigg if workflow == nil { return nil, fmt.Errorf("workflow %s not found", workflowID) } - if workflow.Status == domain.WorkflowStatusCompleted && workflow.Kind != domain.WorkflowKindScan && trigger != domain.RunTriggerScheduled { - return nil, ErrWorkflowAlreadyCompleted - } projectName := "" if workflow.ProjectID != "" { project, err := store.GetProject(ctx, workflow.ProjectID) @@ -1507,7 +1695,11 @@ func (store *MongoStore) GetDashboardSummary(ctx context.Context) (domain.Dashbo if err != nil { return domain.DashboardSummary{}, err } - summary := BuildDashboardSummary(projects, operators, runs, auditTasks) + settings, err := store.GetSettings(ctx) + if err != nil { + return domain.DashboardSummary{}, err + } + summary := BuildDashboardSummaryAt(projects, operators, runs, auditTasks, settings.Timezone, time.Now()) summary.Jobs.SuccessRate = float64(int(summary.Jobs.SuccessRate*10)) / 10 return summary, nil } diff --git a/internal/store/mongo_indexes.go b/internal/store/mongo_indexes.go index 6d80892e..a6707f4b 100644 --- a/internal/store/mongo_indexes.go +++ b/internal/store/mongo_indexes.go @@ -70,9 +70,14 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error { store.runs: { uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}), index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), + index("organizationId_createdAt_id", bson.D{{Key: "organizationId", Value: 1}, {Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}), + index("createdAt_id", bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}), + index("status_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}), + index("status_updatedAt", bson.D{{Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}), index("status_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}), index("status_lease_availableAt_createdAt", bson.D{{Key: "status", Value: 1}, {Key: "executionLeaseExpiresAt", Value: 1}, {Key: "availableAt", Value: 1}, {Key: "createdAt", Value: -1}}), index("workflowId_createdAt", bson.D{{Key: "workflowId", Value: 1}, {Key: "createdAt", Value: -1}}), + index("workflowId_status", bson.D{{Key: "workflowId", Value: 1}, {Key: "status", Value: 1}}), index("projectId_createdAt", bson.D{{Key: "projectId", Value: 1}, {Key: "createdAt", Value: -1}}), }, store.sourceStates: { diff --git a/internal/store/run_queries.go b/internal/store/run_queries.go new file mode 100644 index 00000000..8f4eee0a --- /dev/null +++ b/internal/store/run_queries.go @@ -0,0 +1,43 @@ +package store + +import "droboticflow/internal/domain" + +func activeRunStatuses() []domain.RunStatus { + return []domain.RunStatus{ + domain.RunStatusQueued, + domain.RunStatusRunning, + domain.RunStatusStopping, + domain.RunStatusWaitingForAudit, + } +} + +func isActiveRunStatus(status domain.RunStatus) bool { + for _, activeStatus := range activeRunStatuses() { + if status == activeStatus { + return true + } + } + return false +} + +func emptyRunStatusCounts() map[domain.RunStatus]int64 { + return map[domain.RunStatus]int64{ + domain.RunStatusQueued: 0, + domain.RunStatusRunning: 0, + domain.RunStatusStopping: 0, + domain.RunStatusWaitingForAudit: 0, + domain.RunStatusCompleted: 0, + domain.RunStatusFailed: 0, + domain.RunStatusCancelled: 0, + } +} + +func normalizedRunSummaryLimit(limit int) int { + if limit <= 0 { + return 50 + } + if limit > 100 { + return 100 + } + return limit +} diff --git a/internal/store/store.go b/internal/store/store.go index 3d088dfd..a82cbb3e 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -14,6 +14,7 @@ var ( ) type Store interface { + Ready(context.Context) error EnsureDefaultOrganization(context.Context) (*domain.OrganizationRecord, error) ListOrganizations(context.Context) ([]domain.OrganizationRecord, error) GetOrganization(context.Context, string) (*domain.OrganizationRecord, error) @@ -65,6 +66,9 @@ type Store interface { CountRunsByStatus(context.Context) (map[domain.RunStatus]int, error) ListRunSummaries(context.Context, string) ([]domain.RunSummary, error) ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error) + ListRunSummariesPage(context.Context, domain.RunSummaryQuery) (domain.RunSummaryPage, error) + ListRunSummariesPageByOrganization(context.Context, string, domain.RunSummaryQuery) (domain.RunSummaryPage, error) + HasActiveRun(context.Context, string) (bool, error) GetRun(context.Context, string) (*domain.RunRecord, error) CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error) ClaimNextQueuedRun(context.Context, string, time.Duration) (*domain.RunRecord, error) @@ -74,6 +78,8 @@ type Store interface { ReleaseRunLease(context.Context, string, string) error RequestRunStop(context.Context, string, domain.RunStopMode) (*domain.RunRecord, error) UpsertRun(context.Context, domain.RunRecord) error + UpdateRunProgress(context.Context, string, *domain.RunProgress) error + UpdateRunFailure(context.Context, string, *domain.RunFailure) error // AppendNodeExecutionResourceSample pushes one resource sample onto a running // node's resourceSeries in place, so the worker can stream live GPU metrics for // custom-container operators during execution (their full series is otherwise diff --git a/internal/worker/executor.go b/internal/worker/executor.go index e5580f81..1052b110 100644 --- a/internal/worker/executor.go +++ b/internal/worker/executor.go @@ -654,6 +654,9 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, if err != nil { return executionResult{}, nil, err } + if progress, progressErr := latestStructuredProgress(launched.Output); progressErr == nil && progress != nil { + _ = runtime.store.UpdateRunProgress(ctx, run.ID, progress) + } parsed, err := parseExecutionOutput(launched.Output, node.Data.Label) if err == nil { parsed.StageManifestPath = workspace.StageManifestPath diff --git a/internal/worker/http.go b/internal/worker/http.go index 65d50813..1a7d546a 100644 --- a/internal/worker/http.go +++ b/internal/worker/http.go @@ -22,6 +22,7 @@ func NewHTTPHandler(dataStore store.Store, runtime *Runtime) http.Handler { router := chi.NewRouter() router.Get("/worker-api/health", server.handleHealth) + router.Get("/worker-api/ready", server.handleReady) router.Get("/worker-api/stats", server.handleStats) router.Get("/worker-api/metrics", server.handleMetrics) router.Get("/worker-api/capacity", server.handleCapacity) @@ -32,7 +33,33 @@ func NewHTTPHandler(dataStore store.Store, runtime *Runtime) http.Handler { } func (server *HTTPServer) handleHealth(writer http.ResponseWriter, _ *http.Request) { - renderWorkerJSON(writer, http.StatusOK, map[string]string{"status": "ok", "service": "droboticflow-worker"}) + build := BuildInfo{} + runnerImage := "" + if server.runtime != nil { + build = server.runtime.BuildInfo() + runnerImage = server.runtime.operatorRunnerImage + } + renderWorkerJSON(writer, http.StatusOK, map[string]any{ + "status": "ok", + "service": "droboticflow-worker", + "buildCommit": build.BuildCommit, + "buildTime": build.BuildTime, + "image": build.Image, + "runnerImage": runnerImage, + }) +} + +func (server *HTTPServer) handleReady(writer http.ResponseWriter, request *http.Request) { + if server.runtime == nil { + renderWorkerJSON(writer, http.StatusServiceUnavailable, ReadinessSnapshot{Status: "not_ready", Service: "droboticflow-worker", Reason: "worker runtime is not initialized"}) + return + } + snapshot := server.runtime.Readiness(request.Context()) + status := http.StatusOK + if !snapshot.Ready { + status = http.StatusServiceUnavailable + } + renderWorkerJSON(writer, status, snapshot) } func (server *HTTPServer) handleStats(writer http.ResponseWriter, request *http.Request) { @@ -60,6 +87,18 @@ func (server *HTTPServer) handleStats(writer http.ResponseWriter, request *http. "scope": "global", "metrics": server.runtime.MetricsSnapshot(), } + readiness := server.runtime.statusSnapshot() + response["buildCommit"] = readiness.BuildCommit + response["buildTime"] = readiness.BuildTime + response["image"] = readiness.Image + response["runnerImage"] = readiness.RunnerImage + response["lastCycleAt"] = readiness.LastCycleAt + response["lastSuccessfulCycleAt"] = readiness.LastSuccessfulCycleAt + response["cycleDurationMs"] = readiness.CycleDurationMS + response["configuredPollIntervalMs"] = readiness.ConfiguredPollIntervalMS + response["effectivePollIntervalMs"] = readiness.EffectivePollIntervalMS + response["stuckRunCount"] = readiness.StuckRunCount + response["orphanRuntimeCount"] = readiness.OrphanRuntimeCount if organizationID != "" { response["organizationId"] = organizationID response["global"] = map[string]int{ @@ -85,7 +124,7 @@ func (server *HTTPServer) handleCapacity(writer http.ResponseWriter, request *ht renderWorkerJSON(writer, http.StatusBadGateway, map[string]string{"error": err.Error()}) return } - renderWorkerJSON(writer, http.StatusOK, map[string]any{"gpu": gpu}) + renderWorkerJSON(writer, http.StatusOK, map[string]any{"gpu": gpu, "runnerImage": server.runtime.operatorRunnerImage}) } func (server *HTTPServer) handleMetrics(writer http.ResponseWriter, request *http.Request) { @@ -136,6 +175,19 @@ func (server *HTTPServer) handleMetrics(writer http.ResponseWriter, request *htt fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_running_count %d\n", metrics.LastDispatchRunningCount) fmt.Fprintf(writer, "# TYPE droboticflow_worker_last_dispatch_selected_count gauge\n") fmt.Fprintf(writer, "droboticflow_worker_last_dispatch_selected_count %d\n", metrics.LastDispatchSelectedCount) + fmt.Fprintf(writer, "# TYPE droboticflow_worker_cycle_duration_ms gauge\n") + fmt.Fprintf(writer, "droboticflow_worker_cycle_duration_ms %d\n", metrics.CycleDurationMS) + fmt.Fprintf(writer, "# TYPE droboticflow_worker_stuck_runs gauge\n") + fmt.Fprintf(writer, "droboticflow_worker_stuck_runs %d\n", metrics.StuckRunCount) + fmt.Fprintf(writer, "# TYPE droboticflow_worker_orphan_runtimes gauge\n") + fmt.Fprintf(writer, "droboticflow_worker_orphan_runtimes %d\n", metrics.OrphanRuntimeCount) + if server.runtime != nil { + readiness := server.runtime.statusSnapshot() + fmt.Fprintf(writer, "# TYPE droboticflow_worker_configured_poll_interval_ms gauge\n") + fmt.Fprintf(writer, "droboticflow_worker_configured_poll_interval_ms %d\n", readiness.ConfiguredPollIntervalMS) + fmt.Fprintf(writer, "# TYPE droboticflow_worker_effective_poll_interval_ms gauge\n") + fmt.Fprintf(writer, "droboticflow_worker_effective_poll_interval_ms %d\n", readiness.EffectivePollIntervalMS) + } } func writeDurationHistogram(writer http.ResponseWriter, name string, sum uint64, count uint64, buckets []DurationBucketSnapshot) { @@ -178,6 +230,8 @@ func (server *HTTPServer) handleRetry(writer http.ResponseWriter, request *http. run.Artifacts = []domain.ExecutionArtifact{} run.K8sRefs = []domain.K8sRef{} run.Error = "" + run.Failure = nil + run.Progress = nil run.AvailableAt = nowISO() run.UpdatedAt = nowISO() if err := server.store.UpsertRun(request.Context(), *run); err != nil { diff --git a/internal/worker/http_test.go b/internal/worker/http_test.go index e95af850..28d54587 100644 --- a/internal/worker/http_test.go +++ b/internal/worker/http_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "droboticflow/internal/domain" "droboticflow/internal/k8s" @@ -49,6 +50,51 @@ func TestWorkerMetricsEndpoint(t *testing.T) { } } +func TestWorkerReadyReportsBuildAndLiveSchedulerState(t *testing.T) { + dataStore := store.NewMemoryStore() + runtime := &Runtime{store: dataStore, launcher: testLauncher{}, operatorRunnerImage: "registry/runner:abc123"} + runtime.SetBuildInfo(BuildInfo{BuildCommit: "abc123", BuildTime: "2026-07-10T08:00:00Z", Image: "registry/worker:abc123"}) + runtime.SetConfiguredPollInterval(2 * time.Second) + runtime.effectivePollMS.Store(750) + runtime.metrics.RecordProcessAvailable(42, nil) + + handler := NewHTTPHandler(dataStore, runtime) + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/worker-api/ready", nil) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("expected ready 200, got %d body=%s", recorder.Code, recorder.Body.String()) + } + var payload ReadinessSnapshot + if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil { + t.Fatalf("decode readiness: %v", err) + } + if !payload.Ready || payload.BuildCommit != "abc123" || payload.Image != "registry/worker:abc123" || payload.RunnerImage != "registry/runner:abc123" { + t.Fatalf("unexpected readiness payload: %#v", payload) + } + if payload.ConfiguredPollIntervalMS != 2000 || payload.EffectivePollIntervalMS != 750 || payload.CycleDurationMS != 42 { + t.Fatalf("expected scheduler details, got %#v", payload) + } +} + +func TestWorkerReadyFailsBeforeFirstSuccessfulCycleWhileHealthStaysLive(t *testing.T) { + dataStore := store.NewMemoryStore() + runtime := &Runtime{store: dataStore, launcher: testLauncher{}} + handler := NewHTTPHandler(dataStore, runtime) + + readyRecorder := httptest.NewRecorder() + handler.ServeHTTP(readyRecorder, httptest.NewRequest(http.MethodGet, "/worker-api/ready", nil)) + if readyRecorder.Code != http.StatusServiceUnavailable || !strings.Contains(readyRecorder.Body.String(), "has not completed") { + t.Fatalf("expected not-ready response before first cycle, got %d %s", readyRecorder.Code, readyRecorder.Body.String()) + } + + healthRecorder := httptest.NewRecorder() + handler.ServeHTTP(healthRecorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil)) + if healthRecorder.Code != http.StatusOK { + t.Fatalf("liveness must remain healthy, got %d %s", healthRecorder.Code, healthRecorder.Body.String()) + } +} + func TestWorkerStatsCanReturnOrganizationScopedCounts(t *testing.T) { dataStore := store.NewMemoryStore() for _, run := range []domain.RunRecord{ @@ -93,7 +139,8 @@ func TestWorkerStatsCanReturnOrganizationScopedCounts(t *testing.T) { func TestWorkerCapacityEndpointReturnsGPUCapacity(t *testing.T) { dataStore := store.NewMemoryStore() runtime := &Runtime{ - store: dataStore, + store: dataStore, + operatorRunnerImage: "registry/runner:release", launcher: testLauncher{ gpuCapacity: func(_ context.Context) (*k8s.GPUCapacitySnapshot, error) { return &k8s.GPUCapacitySnapshot{ @@ -120,12 +167,13 @@ func TestWorkerCapacityEndpointReturnsGPUCapacity(t *testing.T) { t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String()) } var payload struct { - GPU k8s.GPUCapacitySnapshot `json:"gpu"` + GPU k8s.GPUCapacitySnapshot `json:"gpu"` + RunnerImage string `json:"runnerImage"` } if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil { t.Fatalf("decode capacity payload: %v", err) } - if payload.GPU.Available || len(payload.GPU.Nodes) != 1 || payload.GPU.Nodes[0].Name != "cpu-node" { + if payload.GPU.Available || len(payload.GPU.Nodes) != 1 || payload.GPU.Nodes[0].Name != "cpu-node" || payload.RunnerImage != "registry/runner:release" { t.Fatalf("unexpected gpu capacity payload: %#v", payload.GPU) } } diff --git a/internal/worker/metrics.go b/internal/worker/metrics.go index 8a0ad43f..2eb95f71 100644 --- a/internal/worker/metrics.go +++ b/internal/worker/metrics.go @@ -1,6 +1,9 @@ package worker -import "sync/atomic" +import ( + "sync/atomic" + "time" +) const durationBucketCount = 7 @@ -12,19 +15,24 @@ type DurationBucketSnapshot struct { } type MetricsSnapshot struct { - ProcessAvailableTotal uint64 - ProcessAvailableErrors uint64 - ProcessAvailableDurationMS uint64 - ProcessAvailableDurationBuckets []DurationBucketSnapshot - DispatchedRunsTotal uint64 - NodeExecutionsStartedTotal uint64 - NodeExecutionsCompletedTotal uint64 - NodeExecutionsFailedTotal uint64 - NodeExecutionDurationMS uint64 - NodeExecutionDurationBuckets []DurationBucketSnapshot - LastDispatchQueueDepth uint64 - LastDispatchRunningCount uint64 - LastDispatchSelectedCount uint64 + ProcessAvailableTotal uint64 `json:"processAvailableTotal"` + ProcessAvailableErrors uint64 `json:"processAvailableErrors"` + ProcessAvailableDurationMS uint64 `json:"processAvailableDurationMs"` + ProcessAvailableDurationBuckets []DurationBucketSnapshot `json:"processAvailableDurationBuckets"` + DispatchedRunsTotal uint64 `json:"dispatchedRunsTotal"` + NodeExecutionsStartedTotal uint64 `json:"nodeExecutionsStartedTotal"` + NodeExecutionsCompletedTotal uint64 `json:"nodeExecutionsCompletedTotal"` + NodeExecutionsFailedTotal uint64 `json:"nodeExecutionsFailedTotal"` + NodeExecutionDurationMS uint64 `json:"nodeExecutionDurationMs"` + NodeExecutionDurationBuckets []DurationBucketSnapshot `json:"nodeExecutionDurationBuckets"` + LastDispatchQueueDepth uint64 `json:"lastDispatchQueueDepth"` + LastDispatchRunningCount uint64 `json:"lastDispatchRunningCount"` + LastDispatchSelectedCount uint64 `json:"lastDispatchSelectedCount"` + LastCycleAt string `json:"lastCycleAt,omitempty"` + LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"` + CycleDurationMS uint64 `json:"cycleDurationMs"` + StuckRunCount uint64 `json:"stuckRunCount"` + OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"` } type WorkerMetrics struct { @@ -41,15 +49,25 @@ type WorkerMetrics struct { lastDispatchQueueDepth atomic.Uint64 lastDispatchRunningCount atomic.Uint64 lastDispatchSelectedCount atomic.Uint64 + lastCycleUnixNano atomic.Int64 + lastSuccessfulCycleUnixNano atomic.Int64 + cycleDurationMS atomic.Uint64 + stuckRunCount atomic.Uint64 + orphanRuntimeCount atomic.Uint64 } func (metrics *WorkerMetrics) RecordProcessAvailable(durationMS uint64, err error) { + now := time.Now().UTC() metrics.processAvailableTotal.Add(1) metrics.processAvailableDurationMS.Add(durationMS) recordDurationBucket(metrics.processAvailableDurationBuckets[:], durationMS) + metrics.lastCycleUnixNano.Store(now.UnixNano()) + metrics.cycleDurationMS.Store(durationMS) if err != nil { metrics.processAvailableErrors.Add(1) + return } + metrics.lastSuccessfulCycleUnixNano.Store(now.UnixNano()) } func (metrics *WorkerMetrics) RecordDispatch(queueDepth int, runningCount int, selectedCount int) { @@ -77,6 +95,21 @@ func (metrics *WorkerMetrics) RecordNodeExecutionFailed(durationMS uint64) { recordDurationBucket(metrics.nodeExecutionDurationBuckets[:], durationMS) } +func (metrics *WorkerMetrics) RecordStuckRuns(count int) { + metrics.stuckRunCount.Store(uint64(max(count, 0))) +} + +func (metrics *WorkerMetrics) RecordOrphanRuntimes(count int) { + metrics.orphanRuntimeCount.Store(uint64(max(count, 0))) +} + +func metricTimestamp(value int64) string { + if value <= 0 { + return "" + } + return time.Unix(0, value).UTC().Format(time.RFC3339Nano) +} + func (metrics *WorkerMetrics) Snapshot() MetricsSnapshot { return MetricsSnapshot{ ProcessAvailableTotal: metrics.processAvailableTotal.Load(), @@ -92,6 +125,11 @@ func (metrics *WorkerMetrics) Snapshot() MetricsSnapshot { LastDispatchQueueDepth: metrics.lastDispatchQueueDepth.Load(), LastDispatchRunningCount: metrics.lastDispatchRunningCount.Load(), LastDispatchSelectedCount: metrics.lastDispatchSelectedCount.Load(), + LastCycleAt: metricTimestamp(metrics.lastCycleUnixNano.Load()), + LastSuccessfulCycleAt: metricTimestamp(metrics.lastSuccessfulCycleUnixNano.Load()), + CycleDurationMS: metrics.cycleDurationMS.Load(), + StuckRunCount: metrics.stuckRunCount.Load(), + OrphanRuntimeCount: metrics.orphanRuntimeCount.Load(), } } diff --git a/internal/worker/runtime.go b/internal/worker/runtime.go index e36a0480..1f380d5e 100644 --- a/internal/worker/runtime.go +++ b/internal/worker/runtime.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "droboticflow/internal/domain" @@ -35,17 +36,59 @@ type gpuCapacityProvider interface { GPUCapacity(context.Context) (*k8s.GPUCapacitySnapshot, error) } +type orphanRuntimeReconciler interface { + ReconcileOrphanedRuntimes(context.Context, map[string]struct{}, time.Time, bool) ([]k8s.OrphanRuntime, error) +} + +type nodePodLogTailer interface { + TailNodePodLogs(context.Context, string, string, int64) (string, error) +} + +type BuildInfo struct { + BuildCommit string `json:"buildCommit,omitempty"` + BuildTime string `json:"buildTime,omitempty"` + Image string `json:"image,omitempty"` +} + +type ReadinessSnapshot struct { + Ready bool `json:"ready"` + Status string `json:"status"` + Service string `json:"service"` + BuildCommit string `json:"buildCommit,omitempty"` + BuildTime string `json:"buildTime,omitempty"` + Image string `json:"image,omitempty"` + RunnerImage string `json:"runnerImage,omitempty"` + LastCycleAt string `json:"lastCycleAt,omitempty"` + LastSuccessfulCycleAt string `json:"lastSuccessfulCycleAt,omitempty"` + CycleDurationMS uint64 `json:"cycleDurationMs"` + ConfiguredPollIntervalMS int64 `json:"configuredPollIntervalMs"` + EffectivePollIntervalMS int64 `json:"effectivePollIntervalMs"` + StuckRunCount uint64 `json:"stuckRunCount"` + OrphanRuntimeCount uint64 `json:"orphanRuntimeCount"` + Reason string `json:"reason,omitempty"` +} + type Runtime struct { - store store.Store - operatorRunnerImage string - workspaceRoot string - globalBOSEndpoint string - launcher jobLauncher - metrics WorkerMetrics - processMu sync.Mutex - activeRuns map[string]struct{} - dispatchOwner string - runLeaseTTL time.Duration + store store.Store + operatorRunnerImage string + workspaceRoot string + globalBOSEndpoint string + launcher jobLauncher + metrics WorkerMetrics + processMu sync.Mutex + activeRuns map[string]struct{} + dispatchOwner string + runLeaseTTL time.Duration + buildMu sync.RWMutex + buildInfo BuildInfo + configuredPollMS atomic.Int64 + effectivePollMS atomic.Int64 + orphanCleanup bool + orphanMinimumAge time.Duration + orphanCheckInterval time.Duration + lastOrphanCheckNano atomic.Int64 + pendingFailureAfter time.Duration + missingMaxRecoveries int } type ProcessOnceStatus string @@ -56,6 +99,8 @@ const ( ProcessOnceStatusNoQueuedRun ProcessOnceStatus = "no_queued_run" ) +var errRunLeaseLost = errors.New("run execution lease lost") + type ProcessOnceResult struct { Status ProcessOnceStatus `json:"status"` Run *domain.RunRecord `json:"-"` @@ -133,16 +178,72 @@ func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName str trimmedWorkspaceRoot = "/workspace" } return &Runtime{ - store: dataStore, - operatorRunnerImage: operatorRunnerImage, - workspaceRoot: trimmedWorkspaceRoot, - globalBOSEndpoint: strings.TrimSpace(globalBOSEndpoint), - launcher: launcher, - dispatchOwner: defaultDispatchOwner(), - runLeaseTTL: defaultRunLeaseTTL(), + store: dataStore, + operatorRunnerImage: operatorRunnerImage, + workspaceRoot: trimmedWorkspaceRoot, + globalBOSEndpoint: strings.TrimSpace(globalBOSEndpoint), + launcher: launcher, + dispatchOwner: defaultDispatchOwner(), + runLeaseTTL: defaultRunLeaseTTL(), + orphanCleanup: envBool("WORKER_ORPHAN_CLEANUP_ENABLED", false), + orphanMinimumAge: envDurationSeconds("WORKER_ORPHAN_MIN_AGE_SECONDS", 24*time.Hour), + orphanCheckInterval: envDurationSeconds("WORKER_ORPHAN_CHECK_INTERVAL_SECONDS", 5*time.Minute), + pendingFailureAfter: envDurationSeconds("WORKER_PENDING_FAILURE_SECONDS", 10*time.Minute), + missingMaxRecoveries: envPositiveInt("WORKER_MISSING_JOB_MAX_RECOVERIES", 3), }, nil } +func envBool(key string, fallback bool) bool { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return fallback + } + parsed, err := strconv.ParseBool(raw) + if err != nil { + return fallback + } + return parsed +} + +func envPositiveInt(key string, fallback int) int { + raw := strings.TrimSpace(os.Getenv(key)) + if raw == "" { + return fallback + } + parsed, err := strconv.Atoi(raw) + if err != nil || parsed <= 0 { + return fallback + } + return parsed +} + +func envDurationSeconds(key string, fallback time.Duration) time.Duration { + seconds := envPositiveInt(key, int(fallback/time.Second)) + return time.Duration(seconds) * time.Second +} + +func (runtime *Runtime) SetBuildInfo(info BuildInfo) { + if runtime == nil { + return + } + runtime.buildMu.Lock() + runtime.buildInfo = BuildInfo{ + BuildCommit: strings.TrimSpace(info.BuildCommit), + BuildTime: strings.TrimSpace(info.BuildTime), + Image: strings.TrimSpace(info.Image), + } + runtime.buildMu.Unlock() +} + +func (runtime *Runtime) BuildInfo() BuildInfo { + if runtime == nil { + return BuildInfo{} + } + runtime.buildMu.RLock() + defer runtime.buildMu.RUnlock() + return runtime.buildInfo +} + func runtimeOperatorGeneratedByAI(item domain.OperatorRecord) bool { source := strings.ToLower(strings.TrimSpace(string(item.GenerationSource))) if source == string(domain.OperatorGenerationSourceAI) { @@ -254,6 +355,131 @@ func (runtime *Runtime) MetricsSnapshot() MetricsSnapshot { return runtime.metrics.Snapshot() } +const ( + defaultPollIntervalMS int64 = 2000 + minimumPollIntervalMS int64 = 250 + maximumPollIntervalMS int64 = 60000 +) + +func normalizedPollIntervalMS(value int64) int64 { + if value <= 0 { + return defaultPollIntervalMS + } + if value < minimumPollIntervalMS { + return minimumPollIntervalMS + } + if value > maximumPollIntervalMS { + return maximumPollIntervalMS + } + return value +} + +func (runtime *Runtime) SetConfiguredPollInterval(interval time.Duration) { + if runtime == nil { + return + } + configured := normalizedPollIntervalMS(interval.Milliseconds()) + runtime.configuredPollMS.Store(configured) + if runtime.effectivePollMS.Load() == 0 { + runtime.effectivePollMS.Store(configured) + } +} + +func (runtime *Runtime) updateEffectivePollInterval(settings *domain.SettingsRecord) { + configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load()) + effective := configured + if settings != nil && settings.Worker.PollIntervalMS > 0 { + effective = normalizedPollIntervalMS(int64(settings.Worker.PollIntervalMS)) + } + runtime.configuredPollMS.Store(configured) + runtime.effectivePollMS.Store(effective) +} + +func (runtime *Runtime) EffectivePollInterval() time.Duration { + if runtime == nil { + return time.Duration(defaultPollIntervalMS) * time.Millisecond + } + return time.Duration(normalizedPollIntervalMS(runtime.effectivePollMS.Load())) * time.Millisecond +} + +func (runtime *Runtime) RunScheduler(ctx context.Context, configured time.Duration) { + if runtime == nil { + return + } + runtime.SetConfiguredPollInterval(configured) + timer := time.NewTimer(0) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + if err := runtime.ProcessAvailable(ctx); err != nil && !errors.Is(err, context.Canceled) { + fmt.Printf("[worker] processAvailable failed: %v\n", err) + } + timer.Reset(runtime.EffectivePollInterval()) + } + } +} + +func (runtime *Runtime) statusSnapshot() ReadinessSnapshot { + if runtime == nil { + return ReadinessSnapshot{Status: "not_ready", Service: "droboticflow-worker", Reason: "worker runtime is not initialized"} + } + metrics := runtime.MetricsSnapshot() + build := runtime.BuildInfo() + configured := normalizedPollIntervalMS(runtime.configuredPollMS.Load()) + effective := normalizedPollIntervalMS(runtime.effectivePollMS.Load()) + snapshot := ReadinessSnapshot{ + Ready: false, + Status: "not_ready", + Service: "droboticflow-worker", + BuildCommit: build.BuildCommit, + BuildTime: build.BuildTime, + Image: build.Image, + RunnerImage: runtime.operatorRunnerImage, + LastCycleAt: metrics.LastCycleAt, + LastSuccessfulCycleAt: metrics.LastSuccessfulCycleAt, + CycleDurationMS: metrics.CycleDurationMS, + ConfiguredPollIntervalMS: configured, + EffectivePollIntervalMS: effective, + StuckRunCount: metrics.StuckRunCount, + OrphanRuntimeCount: metrics.OrphanRuntimeCount, + } + return snapshot +} + +func (runtime *Runtime) Readiness(ctx context.Context) ReadinessSnapshot { + snapshot := runtime.statusSnapshot() + if runtime == nil { + return snapshot + } + if runtime.store == nil || runtime.launcher == nil { + snapshot.Reason = "worker runtime is not initialized" + return snapshot + } + if err := runtime.store.Ready(ctx); err != nil { + snapshot.Reason = "mongodb is not ready: " + err.Error() + return snapshot + } + lastSuccessful, ok := parseRFC3339Nano(snapshot.LastSuccessfulCycleAt) + if !ok { + snapshot.Reason = "worker has not completed a successful scheduling cycle" + return snapshot + } + maximumAge := 3 * time.Duration(snapshot.EffectivePollIntervalMS) * time.Millisecond + if maximumAge < 30*time.Second { + maximumAge = 30 * time.Second + } + if time.Since(lastSuccessful) > maximumAge { + snapshot.Reason = "last successful scheduling cycle is stale" + return snapshot + } + snapshot.Ready = true + snapshot.Status = "ready" + return snapshot +} + func (runtime *Runtime) GPUCapacity(ctx context.Context) (*k8s.GPUCapacitySnapshot, error) { if runtime == nil || runtime.launcher == nil { return nil, fmt.Errorf("worker launcher is not configured") @@ -538,7 +764,7 @@ func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error { if err != nil { return err } - runs, err := runtime.store.ListRuns(ctx, "") + runs, err := runtime.store.ListRunSummaries(ctx, "") if err != nil { return err } @@ -546,6 +772,10 @@ func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error { if err != nil { return err } + return runtime.enqueueStreamingWorkflowsWithSnapshots(ctx, workflows, runs, settings) +} + +func (runtime *Runtime) enqueueStreamingWorkflowsWithSnapshots(ctx context.Context, workflows []domain.WorkflowRecord, runs []domain.RunSummary, settings *domain.SettingsRecord) error { now := time.Now().UTC() for _, workflow := range workflows { if !shouldEnqueueScheduledRun(workflow, now) { @@ -591,6 +821,38 @@ func (runtime *Runtime) enqueueStreamingWorkflows(ctx context.Context) error { return nil } +func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord { + runs := make([]domain.RunRecord, 0, len(summaries)) + for _, summary := range summaries { + runs = append(runs, domain.RunRecord{ + ID: summary.ID, + OrganizationID: summary.OrganizationID, + WorkflowID: summary.WorkflowID, + WorkflowName: summary.WorkflowName, + ProjectID: summary.ProjectID, + ProjectName: summary.ProjectName, + Trigger: summary.Trigger, + Attempt: summary.Attempt, + ParentRunID: summary.ParentRunID, + RestartFromNodeID: summary.RestartFromNodeID, + AvailableAt: summary.AvailableAt, + ExecutionLeaseOwner: summary.ExecutionLeaseOwner, + ExecutionLeaseExpiresAt: summary.ExecutionLeaseExpiresAt, + Status: summary.Status, + StopMode: summary.StopMode, + StopRequestedAt: summary.StopRequestedAt, + StartedAt: summary.StartedAt, + EndedAt: summary.EndedAt, + DurationMS: summary.DurationMS, + CurrentNodeID: summary.CurrentNodeID, + Error: summary.Error, + CreatedAt: summary.CreatedAt, + UpdatedAt: summary.UpdatedAt, + }) + } + return runs +} + func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) { if claimed == nil { return @@ -603,10 +865,27 @@ func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.R }() if _, err := runtime.processRun(ctx, claimed); err != nil { fmt.Printf("[worker] process run %s failed: %v\n", claimed.ID, err) + runtime.bestEffortTerminalizeRun(claimed.ID, err) } } -func (runtime *Runtime) renewRunLeaseUntil(ctx context.Context, runID string) { +func (runtime *Runtime) bestEffortTerminalizeRun(runID string, cause error) { + if strings.TrimSpace(runID) == "" || cause == nil || errors.Is(cause, errRunLeaseLost) || errors.Is(cause, context.Canceled) { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + run, err := runtime.store.GetRun(ctx, runID) + if err != nil || run == nil || isTerminalRunStatus(run.Status) { + return + } + message := "Worker failed to process run: " + boundedDiagnostic(cause.Error()) + if err := runtime.failReconciledRun(ctx, run, message); err != nil { + fmt.Printf("[worker] terminalize run %s failed: %v\n", runID, err) + } +} + +func (runtime *Runtime) renewRunLeaseUntil(ctx context.Context, runID string, cancelExecution context.CancelCauseFunc) { ttl := runtime.leaseTTL() interval := ttl / 3 if interval <= 0 || interval > time.Minute { @@ -619,13 +898,22 @@ func (runtime *Runtime) renewRunLeaseUntil(ctx context.Context, runID string) { case <-ctx.Done(): return case <-ticker.C: - ok, err := runtime.store.RenewRunLease(context.Background(), runID, runtime.leaseOwner(), ttl) + renewCtx, cancelRenew := context.WithTimeout(ctx, 10*time.Second) + ok, err := runtime.store.RenewRunLease(renewCtx, runID, runtime.leaseOwner(), ttl) + cancelRenew() if err != nil { + if ctx.Err() != nil { + return + } fmt.Printf("[worker] renew run lease %s failed: %v\n", runID, err) continue } if !ok { fmt.Printf("[worker] renew run lease %s skipped: owner changed or run missing\n", runID) + if cancelExecution != nil { + cancelExecution(errRunLeaseLost) + } + return } } } @@ -653,35 +941,75 @@ func (runtime *Runtime) unmarkRunActive(runID string) { delete(runtime.activeRuns, runID) } +func (runtime *Runtime) orphanMinimumRuntimeAge() time.Duration { + if runtime == nil || runtime.orphanMinimumAge <= 0 { + return 24 * time.Hour + } + return runtime.orphanMinimumAge +} + +func (runtime *Runtime) reconcileOrphanedRuntimes(ctx context.Context, summaries []domain.RunSummary) { + now := time.Now().UTC() + interval := runtime.orphanCheckInterval + if interval <= 0 { + interval = 5 * time.Minute + } + lastCheck := runtime.lastOrphanCheckNano.Load() + if lastCheck > 0 && now.Sub(time.Unix(0, lastCheck)) < interval { + return + } + if !runtime.lastOrphanCheckNano.CompareAndSwap(lastCheck, now.UnixNano()) { + return + } + reconciler, ok := runtime.launcher.(orphanRuntimeReconciler) + if !ok { + runtime.metrics.RecordOrphanRuntimes(0) + return + } + knownRunIDs := make(map[string]struct{}, len(summaries)) + for _, summary := range summaries { + knownRunIDs[summary.ID] = struct{}{} + } + orphans, err := reconciler.ReconcileOrphanedRuntimes(ctx, knownRunIDs, now.Add(-runtime.orphanMinimumRuntimeAge()), runtime.orphanCleanup) + if err != nil { + fmt.Printf("[worker] orphan runtime reconciliation failed: %v\n", err) + return + } + runtime.metrics.RecordOrphanRuntimes(len(orphans)) +} + func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { startedAt := time.Now() var processErr error defer func() { runtime.metrics.RecordProcessAvailable(uint64(time.Since(startedAt).Milliseconds()), processErr) }() - if err := runtime.reconcileActiveRuns(ctx); err != nil { - processErr = err - return processErr - } - if err := runtime.enqueueStreamingWorkflows(ctx); err != nil { - processErr = err - return processErr - } settings, err := runtime.store.GetSettings(ctx) if err != nil { processErr = err return processErr } + runtime.updateEffectivePollInterval(settings) workflows, err := runtime.store.ListAllWorkflows(ctx) if err != nil { processErr = err return processErr } - runs, err := runtime.store.ListRuns(ctx, "") + summaries, err := runtime.store.ListRunSummaries(ctx, "") if err != nil { processErr = err return processErr } + if err := runtime.reconcileActiveRunsWithSummaries(ctx, summaries); err != nil { + processErr = err + return processErr + } + if err := runtime.enqueueStreamingWorkflowsWithSnapshots(ctx, workflows, summaries, settings); err != nil { + processErr = err + return processErr + } + runtime.reconcileOrphanedRuntimes(ctx, summaries) + runs := runRecordsFromSummaries(summaries) selectedRunIDs, queueDepth, runningCount := selectQueuedRunsForDispatchWithStats(runs, workflows, settings) runtime.metrics.RecordDispatch(queueDepth, runningCount, len(selectedRunIDs)) for _, runID := range selectedRunIDs { @@ -691,8 +1019,8 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error { claimed, err := runtime.store.ClaimQueuedRun(ctx, runID, runtime.leaseOwner(), runtime.leaseTTL()) if err != nil { runtime.unmarkRunActive(runID) - processErr = err - return processErr + fmt.Printf("[worker] claim queued run %s failed: %v\n", runID, err) + continue } if claimed == nil { runtime.unmarkRunActive(runID) @@ -753,146 +1081,298 @@ func chooseDataSource(value string, fallback string) (string, error) { } } -func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error { - runs, err := runtime.store.ListRuns(ctx, "") +const missingJobRecoveryLogPrefix = "Kubernetes job is missing; recovery attempt " + +func (runtime *Runtime) missingJobRecoveryLimit() int { + if runtime == nil || runtime.missingMaxRecoveries <= 0 { + return 3 + } + return runtime.missingMaxRecoveries +} + +func (runtime *Runtime) pendingJobFailureThreshold() time.Duration { + if runtime == nil || runtime.pendingFailureAfter <= 0 { + return 10 * time.Minute + } + return runtime.pendingFailureAfter +} + +func missingJobRecoveryCount(execution *domain.NodeExecutionRecord) int { + if execution == nil { + return 0 + } + count := 0 + for _, entry := range execution.Logs { + if strings.HasPrefix(strings.TrimSpace(entry.Message), missingJobRecoveryLogPrefix) { + count++ + } + } + return count +} + +func boundedDiagnostic(value string) string { + trimmed := strings.TrimSpace(value) + const maximum = 4096 + if len(trimmed) <= maximum { + return trimmed + } + return trimmed[:maximum] + "..." +} + +func (runtime *Runtime) failReconciledRun(ctx context.Context, run *domain.RunRecord, message string, structured ...domain.RunFailure) error { + if run == nil { + return nil + } + message = boundedDiagnostic(message) + if strings.TrimSpace(message) == "" { + message = "Kubernetes job failed" + } + if _, execution := findNodeExecution(run, run.CurrentNodeID); execution != nil { + execution.Status = domain.NodeExecutionStatusFailed + execution.EndedAt = nowISO() + execution.Error = message + execution.Logs = append(execution.Logs, logEntry("ERROR", message, execution.NodeLabel)) + applyExecution(run, *execution) + } + run.Status = domain.RunStatusFailed + run.EndedAt = nowISO() + run.Error = message + failure := domain.RunFailure{Code: "worker_execution_failed", Stage: "execution", NodeID: run.CurrentNodeID, Message: message, Retryable: true, Suggestion: "查看节点日志后重试失败子链路"} + if len(structured) > 0 { + failure = structured[0] + failure.Message = message + if strings.TrimSpace(failure.NodeID) == "" { + failure.NodeID = run.CurrentNodeID + } + } + run.Failure = &failure + run.UpdatedAt = nowISO() + if err := runtime.store.UpsertRun(ctx, *run); err != nil { + return err + } + workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) + if err != nil || workflow == nil { + return err + } + return runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusFailed, run.ID, nowISO()) +} + +func (runtime *Runtime) recoverMissingJob(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error { + if run != nil && run.Status == domain.RunStatusStopping { + workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) + if err != nil { + return err + } + executionMap := make(map[string]domain.NodeExecutionRecord, len(run.NodeExecutions)) + for _, item := range run.NodeExecutions { + executionMap[item.NodeID] = item + } + _, err = runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流") + return err + } + if execution == nil { + run.Status = domain.RunStatusQueued + run.StartedAt = "" + run.CurrentNodeID = "" + run.AvailableAt = nowISO() + run.UpdatedAt = nowISO() + return runtime.store.UpsertRun(ctx, *run) + } + terminateCtx, cancelTerminate := context.WithTimeout(context.Background(), 30*time.Second) + err := runtime.launcher.TerminateRun(terminateCtx, "", run.ID) + cancelTerminate() + if err != nil { + return fmt.Errorf("clean up missing job runtime for run %s: %w", run.ID, err) + } + recoveryAttempt := missingJobRecoveryCount(execution) + 1 + if recoveryAttempt > runtime.missingJobRecoveryLimit() { + return runtime.failReconciledRun(ctx, run, fmt.Sprintf("Kubernetes job remained missing after %d recovery attempts", runtime.missingJobRecoveryLimit()), domain.RunFailure{ + Code: "kubernetes_job_missing", Stage: "execution", Retryable: true, Suggestion: "检查 Job 生命周期与集群事件后从失败节点继续", + }) + } + message := fmt.Sprintf("%s%d/%d; node execution was re-queued", missingJobRecoveryLogPrefix, recoveryAttempt, runtime.missingJobRecoveryLimit()) + execution.Status = domain.NodeExecutionStatusFailed + execution.EndedAt = nowISO() + execution.Error = message + execution.Logs = append(execution.Logs, logEntry("WARN", message, execution.NodeLabel)) + applyExecution(run, *execution) + backoff := time.Duration(15*(1<<(recoveryAttempt-1))) * time.Second + run.Status = domain.RunStatusQueued + run.StartedAt = "" + run.CurrentNodeID = "" + run.AvailableAt = time.Now().UTC().Add(backoff).Format(time.RFC3339Nano) + run.UpdatedAt = nowISO() + return runtime.store.UpsertRun(ctx, *run) +} + +func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.RunRecord, inspection *k8s.JobInspection) error { + workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) if err != nil { return err } - for _, candidate := range runs { - if candidate.Status != domain.RunStatusRunning && (candidate.Status != domain.RunStatusQueued || strings.TrimSpace(candidate.CurrentNodeID) == "") { - continue - } - run, err := runtime.store.GetRun(ctx, candidate.ID) - if err != nil { - return err - } - if run == nil { - continue - } - var activeExecution *domain.NodeExecutionRecord - if strings.TrimSpace(run.CurrentNodeID) != "" { - _, activeExecution = findNodeExecution(run, run.CurrentNodeID) - } - if run.Status == domain.RunStatusQueued { - if activeExecution == nil || activeExecution.Status != domain.NodeExecutionStatusRunning || !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) { - continue - } - } - if run.Status == domain.RunStatusRunning { - // The k8s job is created asynchronously inside the dispatch goroutine - // (see processRun), AFTER RunStatusRunning + CurrentNodeID are persisted. - // A freshly-dispatched run therefore has Status=Running with no job yet; - // Inspect would return JobStateMissing and spuriously re-queue it (and - // drop the nodeExecution) while the pod is just about to start — leaving - // an orphaned pod running with the run showing "queued". Skip Inspect - // ONLY while the active node execution is fresh; once it is stale the job - // should exist, so Inspect proceeds (and re-queues via JobStateMissing if - // genuinely absent). Note this is the inverse of the Queued guard above: - // Queued re-queueing is destructive so it requires staleness to proceed, - // whereas Running Inspect is read-only so it requires freshness to skip. - if activeExecution != nil && activeExecution.Status == domain.NodeExecutionStatusRunning && !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) { - continue - } - } - if strings.TrimSpace(run.CurrentNodeID) == "" { - run.Status = domain.RunStatusQueued - run.StartedAt = "" - run.UpdatedAt = nowISO() - if err := runtime.store.UpsertRun(ctx, *run); err != nil { - return err - } - continue - } - inspection, err := runtime.launcher.Inspect(ctx, run.ID, run.CurrentNodeID) - if err != nil { - return err - } - switch inspection.State { - case k8s.JobStateMissing: - // Clean up any orphaned pods/jobs for this run before re-queuing, so the - // re-dispatch does not race with a leftover pod still holding the GPU. - terminateCtx, cancelTerminate := context.WithTimeout(context.Background(), 30*time.Second) - _ = runtime.launcher.TerminateRun(terminateCtx, "", run.ID) - cancelTerminate() - if strings.TrimSpace(run.CurrentNodeID) != "" { - if index, execution := findNodeExecution(run, run.CurrentNodeID); execution != nil && execution.Status == domain.NodeExecutionStatusRunning { - message := "Kubernetes job is missing; node execution was re-queued" - run.Logs = append(run.Logs, logEntry("WARN", message, execution.NodeLabel)) - run.NodeExecutions = append(run.NodeExecutions[:index], run.NodeExecutions[index+1:]...) - } - } - run.Status = domain.RunStatusQueued - run.StartedAt = "" - run.CurrentNodeID = "" - run.UpdatedAt = nowISO() - if err := runtime.store.UpsertRun(ctx, *run); err != nil { - return err - } - case k8s.JobStateSucceeded: - workflow, err := runtime.store.GetWorkflow(ctx, run.WorkflowID) - if err != nil { - return err - } - if workflow == nil { - continue - } - node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID) - if node == nil { - continue - } - executionMap := map[string]domain.NodeExecutionRecord{} - for _, item := range run.NodeExecutions { - executionMap[item.NodeID] = item - } - inputLineage := gatherInputLineage(*workflow, executionMap, node.ID) - inputArtifacts := inputArtifactsFromLineage(inputLineage) - _, existing := findNodeExecution(run, node.ID) - execution := domain.NodeExecutionRecord{ - NodeID: node.ID, - NodeLabel: node.Data.Label, - Attempt: runExecutionAttempt(run), - Status: domain.NodeExecutionStatusRunning, - StartedAt: firstNonEmptyString(run.StartedAt, run.CurrentRunAt, run.CreatedAt), - Logs: []domain.LogEntryRecord{}, - Artifacts: []domain.ExecutionArtifact{}, - DataRefs: []domain.DataRefRecord{}, - ResourceSeries: []domain.ResourceSample{}, - } - if existing != nil { - execution = *existing - } - applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage) - parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label) - if err != nil { - return err - } - execution.Status = domain.NodeExecutionStatusCompleted - execution.EndedAt = nowISO() - execution.Logs = parsed.Logs - execution.Artifacts = parsed.Artifacts - execution.DataRefs = parsed.DataRefs - execution.ResourceSeries = parsed.ResourceSeries - execution.Error = "" - executionMap[node.ID] = execution - applyExecution(run, execution) - if allAllowedNodesCompleted(workflow, run, executionMap) { - if _, err := runtime.finalizeCompletedRun(ctx, workflow, run); err != nil { - return err - } - continue - } - run.Status = domain.RunStatusQueued - run.StartedAt = "" - run.CurrentNodeID = "" - run.UpdatedAt = nowISO() - if err := runtime.store.UpsertRun(ctx, *run); err != nil { - return err - } + if workflow == nil { + return fmt.Errorf("workflow %s not found while reconciling run %s", run.WorkflowID, run.ID) + } + node := domain.GetNodeByID(workflow.Nodes, run.CurrentNodeID) + if node == nil { + return fmt.Errorf("node %s not found while reconciling run %s", run.CurrentNodeID, run.ID) + } + executionMap := map[string]domain.NodeExecutionRecord{} + for _, item := range run.NodeExecutions { + executionMap[item.NodeID] = item + } + inputLineage := gatherInputLineage(*workflow, executionMap, node.ID) + inputArtifacts := inputArtifactsFromLineage(inputLineage) + _, existing := findNodeExecution(run, node.ID) + execution := domain.NodeExecutionRecord{ + NodeID: node.ID, + NodeLabel: node.Data.Label, + Attempt: runExecutionAttempt(run), + Status: domain.NodeExecutionStatusRunning, + StartedAt: firstNonEmptyString(run.StartedAt, run.CurrentRunAt, run.CreatedAt), + Logs: []domain.LogEntryRecord{}, + Artifacts: []domain.ExecutionArtifact{}, + DataRefs: []domain.DataRefRecord{}, + ResourceSeries: []domain.ResourceSample{}, + } + if existing != nil { + execution = *existing + } + applyNodeExecutionSnapshot(&execution, *node, inputArtifacts, inputLineage) + parsed, err := parseExecutionOutput(inspection.Output, node.Data.Label) + if err != nil { + return err + } + execution.Status = domain.NodeExecutionStatusCompleted + execution.EndedAt = nowISO() + execution.Logs = append(execution.Logs, parsed.Logs...) + execution.Artifacts = parsed.Artifacts + execution.DataRefs = parsed.DataRefs + execution.ResourceSeries = parsed.ResourceSeries + execution.Error = "" + executionMap[node.ID] = execution + applyExecution(run, execution) + if run.Status == domain.RunStatusStopping { + _, err := runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流") + return err + } + if allAllowedNodesCompleted(workflow, run, executionMap) { + _, err := runtime.finalizeCompletedRun(ctx, workflow, run) + return err + } + run.Status = domain.RunStatusQueued + run.StartedAt = "" + run.CurrentNodeID = "" + run.AvailableAt = nowISO() + run.UpdatedAt = nowISO() + return runtime.store.UpsertRun(ctx, *run) +} + +func inspectionIsStuck(inspection *k8s.JobInspection, fallbackStartedAt string, threshold time.Duration) bool { + if inspection == nil || strings.TrimSpace(inspection.BlockingReason) == "" { + return false + } + startedAt, ok := parseRFC3339Nano(firstNonEmptyString(inspection.CreatedAt, fallbackStartedAt)) + return ok && time.Since(startedAt) >= threshold +} + +func nodeExecutionStartedAt(execution *domain.NodeExecutionRecord) string { + if execution == nil { + return "" + } + return execution.StartedAt +} + +func (runtime *Runtime) reconcileActiveRun(ctx context.Context, candidate domain.RunSummary) (bool, error) { + run, err := runtime.store.GetRun(ctx, candidate.ID) + if err != nil || run == nil { + return false, err + } + var activeExecution *domain.NodeExecutionRecord + if strings.TrimSpace(run.CurrentNodeID) != "" { + _, activeExecution = findNodeExecution(run, run.CurrentNodeID) + } + if run.Status == domain.RunStatusQueued { + if activeExecution == nil || activeExecution.Status != domain.NodeExecutionStatusRunning || !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) { + return false, nil } } + if run.Status == domain.RunStatusRunning && activeExecution != nil && activeExecution.Status == domain.NodeExecutionStatusRunning && !isStaleQueuedNodeExecution(*activeExecution, 2*time.Minute) { + return false, nil + } + if strings.TrimSpace(run.CurrentNodeID) == "" { + if updatedAt, ok := parseRFC3339Nano(run.UpdatedAt); ok && time.Since(updatedAt) < 2*time.Minute { + return false, nil + } + return false, runtime.recoverMissingJob(ctx, run, nil) + } + inspection, err := runtime.launcher.Inspect(ctx, run.ID, run.CurrentNodeID) + if err != nil { + return false, err + } + switch inspection.State { + case k8s.JobStateMissing: + return true, runtime.recoverMissingJob(ctx, run, activeExecution) + case k8s.JobStateSucceeded: + return false, runtime.recoverSucceededJob(ctx, run, inspection) + case k8s.JobStateFailed: + message := "Kubernetes job failed" + if output := boundedDiagnostic(inspection.Output); output != "" { + message += ": " + output + } + return true, runtime.failReconciledRun(ctx, run, message, domain.RunFailure{ + Code: "kubernetes_job_failed", Stage: "execution", Retryable: true, Suggestion: "查看 Pod 日志和事件后从失败节点继续", + }) + case k8s.JobStatePending: + if inspectionIsStuck(inspection, firstNonEmptyString(nodeExecutionStartedAt(activeExecution), run.StartedAt), runtime.pendingJobFailureThreshold()) { + message := fmt.Sprintf("Kubernetes job remained blocked in %s for at least %s", inspection.BlockingReason, runtime.pendingJobFailureThreshold()) + if details := boundedDiagnostic(inspection.BlockingMessage); details != "" { + message += ": " + details + } + failureCode := "kubernetes_runtime_blocked" + if inspection.BlockingReason == "ImagePullBackOff" || inspection.BlockingReason == "ErrImagePull" || inspection.BlockingReason == "InvalidImageName" { + failureCode = "kubernetes_image_pull_blocked" + } else if inspection.BlockingReason == "Unschedulable" { + failureCode = "kubernetes_unschedulable" + } + return true, runtime.failReconciledRun(ctx, run, message, domain.RunFailure{ + Code: failureCode, Stage: "scheduling", Retryable: true, Suggestion: "修复镜像或调度条件后从失败节点继续", + }) + } + return strings.TrimSpace(inspection.BlockingReason) != "", nil + default: + return false, nil + } +} + +func (runtime *Runtime) reconcileActiveRunsWithSummaries(ctx context.Context, runs []domain.RunSummary) error { + stuckCount := 0 + for _, candidate := range runs { + if candidate.Status != domain.RunStatusRunning && candidate.Status != domain.RunStatusStopping && (candidate.Status != domain.RunStatusQueued || strings.TrimSpace(candidate.CurrentNodeID) == "") { + continue + } + stuck, err := runtime.reconcileActiveRun(ctx, candidate) + if stuck { + stuckCount++ + } + if err != nil { + // A single malformed or temporarily unavailable run must not prevent + // unrelated runs from reconciling and the queue from dispatching. + fmt.Printf("[worker] reconcile run %s failed: %v\n", candidate.ID, err) + } + } + runtime.metrics.RecordStuckRuns(stuckCount) return nil } +func (runtime *Runtime) reconcileActiveRuns(ctx context.Context) error { + runs, err := runtime.store.ListRunSummaries(ctx, "") + if err != nil { + return err + } + return runtime.reconcileActiveRunsWithSummaries(ctx, runs) +} + func upsertNodeHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam { for index := range items { if items[index].Key == key { @@ -1532,6 +2012,10 @@ func syncStopMetadata(target *domain.RunRecord, source *domain.RunRecord) { } target.StopMode = source.StopMode target.StopRequestedAt = source.StopRequestedAt + if source.Progress != nil { + progress := *source.Progress + target.Progress = &progress + } } func finalizeCancelledExecution(execution domain.NodeExecutionRecord, message string) (domain.NodeExecutionRecord, bool) { @@ -1708,9 +2192,10 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor } else if !ok { return claimed, nil } - leaseCtx, stopLeaseRenewal := context.WithCancel(context.Background()) - defer stopLeaseRenewal() - go runtime.renewRunLeaseUntil(leaseCtx, claimed.ID) + leaseCtx, cancelLeaseExecution := context.WithCancelCause(ctx) + defer cancelLeaseExecution(context.Canceled) + go runtime.renewRunLeaseUntil(leaseCtx, claimed.ID, cancelLeaseExecution) + ctx = leaseCtx workflow, err := runtime.store.GetWorkflow(ctx, claimed.WorkflowID) if err != nil { return nil, err @@ -1872,6 +2357,9 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor gracefulStopRequested := false forceStopRequested := false for { + if cause := context.Cause(ctx); cause != nil { + return currentRun, cause + } persistedControl, err = runtime.loadRunControlState(ctx, currentRun.ID) if err != nil { return nil, err @@ -1929,13 +2417,21 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor inputLineage := gatherInputLineage(*workflow, executionMap, node.ID) inputArtifacts := inputArtifactsFromLineage(inputLineage) + executionAttempt := runExecutionAttempt(currentRun) + executionLogs := []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)} + if previous, ok := executionMap[node.ID]; ok && missingJobRecoveryCount(&previous) > 0 { + if previous.Attempt >= executionAttempt { + executionAttempt = previous.Attempt + 1 + } + executionLogs = append(append([]domain.LogEntryRecord{}, previous.Logs...), executionLogs...) + } execution := domain.NodeExecutionRecord{ NodeID: node.ID, NodeLabel: node.Data.Label, - Attempt: runExecutionAttempt(currentRun), + Attempt: executionAttempt, Status: domain.NodeExecutionStatusRunning, StartedAt: nowISO(), - Logs: []domain.LogEntryRecord{logEntry("INFO", "Starting node execution", node.Data.Label)}, + Logs: executionLogs, Artifacts: []domain.ExecutionArtifact{}, DataRefs: []domain.DataRefRecord{}, ResourceSeries: []domain.ResourceSample{}, @@ -1972,8 +2468,9 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor progressCtx, progressCancel := context.WithCancel(nodeExecutionCtx) var progressWG sync.WaitGroup if shouldReportNodeProgress(runtime, nodeSnapshot) { + sampleGPU := runtime.operatorRunnerImage != "" && chooseImage(nodeSnapshot, runtime.operatorRunnerImage) != runtime.operatorRunnerImage progressWG.Add(1) - go runtime.runNodeProgressReporter(progressCtx, runSnapshot.ID, nodeSnapshot.ID, func() { progressWG.Done() }) + go runtime.runNodeProgressReporter(progressCtx, runSnapshot.ID, nodeSnapshot.ID, sampleGPU, func() { progressWG.Done() }) } result, reference, err := runtime.executeNode(nodeExecutionCtx, &runSnapshot, nodeSnapshot, inputSnapshot) progressCancel() @@ -2101,6 +2598,9 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor } execution := executionMap[outcome.nodeID] if outcome.err != nil { + if cause := context.Cause(ctx); cause != nil { + return currentRun, cause + } if forceStopRequested || (currentRun.StopMode == domain.RunStopModeForce && errors.Is(executionCtx.Err(), context.Canceled)) { cancelledExecution, _ := finalizeCancelledExecution(execution, "用户已强制停止工作流") executionMap[outcome.nodeID] = cancelledExecution @@ -2127,6 +2627,14 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor currentRun.CurrentNodeID = outcome.nodeID currentRun.EndedAt = nowISO() currentRun.Error = execution.Error + currentRun.Failure = &domain.RunFailure{ + Code: "node_execution_failed", + Stage: "execution", + NodeID: outcome.nodeID, + Message: execution.Error, + Retryable: true, + Suggestion: "查看节点日志后从失败节点继续", + } currentRun.UpdatedAt = nowISO() if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil { return nil, err @@ -2195,18 +2703,16 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor return runtime.finalizeCompletedRun(ctx, workflow, currentRun) } -// nodeProgressSampleInterval bounds how often the worker pushes live GPU samples for a -// running custom-container node. The launcher already collects the node's full GPU series -// (flushed at completion); this streams samples to the resource monitor DURING execution so -// the GPU tile reflects custom-container usage before the node finishes. +// nodeProgressSampleInterval bounds how often the worker reads structured progress markers +// from long-running pods and, for custom containers, pushes live GPU samples. const nodeProgressSampleInterval = 15 * time.Second -// shouldReportNodeProgress reports whether a node needs platform-side GPU sampling during -// execution. Builtin operator-runner pods self-report resourceSeries via their in-process -// ResourceSampler, so only custom-container operators (resolved image != operator-runner -// image) need the launcher's nvidia-smi sampling — mirroring k8s.Launch's shouldSample. +// shouldReportNodeProgress reports whether a node needs live runtime polling. Custom +// containers need platform-side GPU samples, while long-running builtin and custom nodes +// can emit DROBOTICFLOW_PROGRESS markers that must be persisted before the job finishes. func shouldReportNodeProgress(runtime *Runtime, node domain.WorkflowNode) bool { - return runtime.operatorRunnerImage != "" && chooseImage(node, runtime.operatorRunnerImage) != runtime.operatorRunnerImage + return runtimeContractLongRunning(node.Data.RuntimeContract) || + (runtime.operatorRunnerImage != "" && chooseImage(node, runtime.operatorRunnerImage) != runtime.operatorRunnerImage) } // mapGPUSample maps the launcher's k8s.GPUSample onto the resource monitor's domain @@ -2223,30 +2729,95 @@ func mapGPUSample(sample k8s.GPUSample) domain.ResourceSample { } } -// runNodeProgressReporter periodically samples GPU usage of the pod backing runID+nodeID and -// pushes each sample onto the node's resourceSeries in place (AppendNodeExecutionResourceSample), -// so the resource monitor reflects live GPU usage DURING execution. It stops as soon as ctx is -// canceled (when the node finishes); the caller cancels ctx and waits for this goroutine to -// return BEFORE reporting the outcome, so no live sample can land after the node's final -// (complete) series is persisted at completion. Errors (no Running pod yet, no GPU, RBAC) are -// skipped silently — the next tick retries. -func (runtime *Runtime) runNodeProgressReporter(ctx context.Context, runID string, nodeID string, done func()) { +// runNodeProgressReporter persists DROBOTICFLOW_PROGRESS markers while a long-running node is +// active and optionally samples GPU usage for custom containers. It stops before the final node +// result is persisted, preventing a stale live write from racing with completion. +const progressMarker = "DROBOTICFLOW_PROGRESS " + +func latestStructuredProgress(logs string) (*domain.RunProgress, error) { + var latest *domain.RunProgress + var lastErr error + for _, line := range strings.Split(strings.ReplaceAll(logs, "\r\n", "\n"), "\n") { + markerIndex := strings.Index(line, progressMarker) + if markerIndex < 0 { + continue + } + payload := strings.TrimSpace(line[markerIndex+len(progressMarker):]) + if payload == "" { + continue + } + var progress domain.RunProgress + if err := json.Unmarshal([]byte(payload), &progress); err != nil { + lastErr = fmt.Errorf("decode structured progress marker: %w", err) + continue + } + if strings.TrimSpace(progress.UpdatedAt) == "" { + progress.UpdatedAt = nowISO() + } + copy := progress + latest = © + } + if latest != nil { + return latest, nil + } + return nil, lastErr +} + +func progressFingerprint(progress *domain.RunProgress) string { + if progress == nil { + return "" + } + copy := *progress + copy.UpdatedAt = "" + payload, err := json.Marshal(copy) + if err != nil { + return "" + } + return string(payload) +} + +func (runtime *Runtime) runNodeProgressReporter(ctx context.Context, runID string, nodeID string, sampleGPU bool, done func()) { + runtime.runNodeProgressReporterWithInterval(ctx, runID, nodeID, sampleGPU, nodeProgressSampleInterval, done) +} + +func (runtime *Runtime) runNodeProgressReporterWithInterval(ctx context.Context, runID string, nodeID string, sampleGPU bool, interval time.Duration, done func()) { defer done() - ticker := time.NewTicker(nodeProgressSampleInterval) + if interval <= 0 { + interval = nodeProgressSampleInterval + } + ticker := time.NewTicker(interval) defer ticker.Stop() + lastProgressFingerprint := "" + tailer, canTailLogs := runtime.launcher.(nodePodLogTailer) for { select { case <-ctx.Done(): return case <-ticker.C: } - sampleCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - sample, err := runtime.launcher.SamplePodGPUForRun(sampleCtx, runID, nodeID) - cancel() - if err != nil || sample.Timestamp == "" { + if canTailLogs { + logsCtx, cancelLogs := context.WithTimeout(ctx, 10*time.Second) + logs, err := tailer.TailNodePodLogs(logsCtx, runID, nodeID, 200) + cancelLogs() + if err == nil { + progress, parseErr := latestStructuredProgress(logs) + fingerprint := progressFingerprint(progress) + if parseErr == nil && progress != nil && fingerprint != "" && fingerprint != lastProgressFingerprint { + if err := runtime.store.UpdateRunProgress(ctx, runID, progress); err == nil { + lastProgressFingerprint = fingerprint + } + } + } + } + if !sampleGPU { continue } - _ = runtime.store.AppendNodeExecutionResourceSample(ctx, runID, nodeID, mapGPUSample(sample)) + sampleCtx, cancelSample := context.WithTimeout(ctx, 10*time.Second) + sample, err := runtime.launcher.SamplePodGPUForRun(sampleCtx, runID, nodeID) + cancelSample() + if err == nil && sample.Timestamp != "" { + _ = runtime.store.AppendNodeExecutionResourceSample(ctx, runID, nodeID, mapGPUSample(sample)) + } } } diff --git a/internal/worker/runtime_test.go b/internal/worker/runtime_test.go index 8789db7c..59431676 100644 --- a/internal/worker/runtime_test.go +++ b/internal/worker/runtime_test.go @@ -3,6 +3,7 @@ package worker import ( "context" "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -18,11 +19,19 @@ import ( ) type testLauncher struct { - launch func(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error) - inspect func(context.Context, string, string) (*k8s.JobInspection, error) - gpuCapacity func(context.Context) (*k8s.GPUCapacitySnapshot, error) - terminateRun func(context.Context, string, string) error + launch func(context.Context, k8s.LaunchInput) (*k8s.LaunchResult, error) + inspect func(context.Context, string, string) (*k8s.JobInspection, error) + gpuCapacity func(context.Context) (*k8s.GPUCapacitySnapshot, error) + terminateRun func(context.Context, string, string) error samplePodGPUForRun func(context.Context, string, string) (k8s.GPUSample, error) + tailNodePodLogs func(context.Context, string, string, int64) (string, error) +} + +func (launcher testLauncher) TailNodePodLogs(ctx context.Context, runID string, nodeID string, tailLines int64) (string, error) { + if launcher.tailNodePodLogs == nil { + return "", fmt.Errorf("no pod logs for run %s node %s", runID, nodeID) + } + return launcher.tailNodePodLogs(ctx, runID, nodeID, tailLines) } func (launcher testLauncher) Launch(ctx context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { @@ -3537,8 +3546,17 @@ func TestRuntimeReconcileActiveRunsClearsQueuedRunWithMissingActiveJob(t *testin if recovered.CurrentNodeID != "" { t.Fatalf("expected current node cleared, got %q", recovered.CurrentNodeID) } - if len(recovered.NodeExecutions) != 0 { - t.Fatalf("expected stale source execution removed, got %#v", recovered.NodeExecutions) + if len(recovered.NodeExecutions) != 1 || recovered.NodeExecutions[0].Status != domain.NodeExecutionStatusFailed { + t.Fatalf("expected stale source execution retained as failed recovery evidence, got %#v", recovered.NodeExecutions) + } + if !strings.Contains(recovered.NodeExecutions[0].Error, "recovery attempt 1/3") { + t.Fatalf("expected recovery attempt evidence, got %#v", recovered.NodeExecutions[0]) + } + // Recovery deliberately applies a short backoff in production. Make this + // fixture dispatchable now so the test can verify the resumed execution. + recovered.AvailableAt = nowISO() + if err := dataStore.UpsertRun(ctx, *recovered); err != nil { + t.Fatalf("make recovered run dispatchable: %v", err) } if _, err := runtime.ProcessOnce(ctx); err != nil { t.Fatalf("process once after recovery: %v", err) @@ -4579,6 +4597,317 @@ func TestRuntimeProcessAvailableHonorsRetryBackoff(t *testing.T) { } } +func newReconcileFixture(t *testing.T, dataStore store.Store, name string) (*domain.WorkflowRecord, *domain.RunRecord) { + t.Helper() + ctx := context.Background() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: name + " project", Description: "reconcile"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, + Name: name + " workflow", + Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{{ + ID: "node-1", Type: "operator", Data: domain.WorkflowNodeData{Label: name + " node"}, + }}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + startedAt := time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano) + run.Status = domain.RunStatusRunning + run.CurrentNodeID = "node-1" + run.StartedAt = startedAt + run.UpdatedAt = startedAt + run.NodeExecutions = []domain.NodeExecutionRecord{{ + NodeID: "node-1", NodeLabel: name + " node", Attempt: 1, Status: domain.NodeExecutionStatusRunning, StartedAt: startedAt, + }} + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert run: %v", err) + } + return workflow, run +} + +func TestRuntimeReconcileFailedJobTerminalizesRunAndNode(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + workflow, run := newReconcileFixture(t, dataStore, "failed job") + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(context.Context, string, string) (*k8s.JobInspection, error) { + return &k8s.JobInspection{State: k8s.JobStateFailed, Output: "container exited with code 2"}, nil + }}) + if err := runtime.reconcileActiveRuns(ctx); err != nil { + t.Fatalf("reconcile active runs: %v", err) + } + failed, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if failed.Status != domain.RunStatusFailed || !strings.Contains(failed.Error, "code 2") { + t.Fatalf("expected failed run with diagnostics, got %#v", failed) + } + if failed.Failure == nil || failed.Failure.Code != "kubernetes_job_failed" || failed.Failure.NodeID != "node-1" { + t.Fatalf("expected structured kubernetes failure, got %#v", failed.Failure) + } + if execution := mustNodeExecution(t, failed, "node-1"); execution.Status != domain.NodeExecutionStatusFailed { + t.Fatalf("expected failed node execution, got %#v", execution) + } + storedWorkflow, err := dataStore.GetWorkflow(ctx, workflow.ID) + if err != nil || storedWorkflow.Status != domain.WorkflowStatusFailed { + t.Fatalf("expected failed workflow, got %#v err=%v", storedWorkflow, err) + } +} + +func TestRuntimeReconcileStuckImagePullTerminalizesButHealthyLongJobStaysRunning(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + _, blockedRun := newReconcileFixture(t, dataStore, "blocked image") + _, healthyRun := newReconcileFixture(t, dataStore, "healthy training") + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(_ context.Context, runID string, _ string) (*k8s.JobInspection, error) { + if runID == blockedRun.ID { + return &k8s.JobInspection{ + State: k8s.JobStatePending, CreatedAt: time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano), + BlockingReason: "ImagePullBackOff", BlockingMessage: "back-off pulling image", + }, nil + } + return &k8s.JobInspection{State: k8s.JobStateActive, CreatedAt: time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339Nano)}, nil + }}) + if err := runtime.reconcileActiveRuns(ctx); err != nil { + t.Fatalf("reconcile active runs: %v", err) + } + blocked, _ := dataStore.GetRun(ctx, blockedRun.ID) + healthy, _ := dataStore.GetRun(ctx, healthyRun.ID) + if blocked.Status != domain.RunStatusFailed || !strings.Contains(blocked.Error, "ImagePullBackOff") { + t.Fatalf("expected blocked image run to fail explicitly, got %#v", blocked) + } + if blocked.Failure == nil || blocked.Failure.Code != "kubernetes_image_pull_blocked" || blocked.Failure.Stage != "scheduling" { + t.Fatalf("expected structured image pull failure, got %#v", blocked.Failure) + } + if healthy.Status != domain.RunStatusRunning { + t.Fatalf("expected healthy long-running job unchanged, got %#v", healthy) + } +} + +func TestRuntimeReconcileMissingJobFailsAfterRecoveryLimit(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + _, run := newReconcileFixture(t, dataStore, "missing job") + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) + runtime.missingMaxRecoveries = 1 + if err := runtime.reconcileActiveRuns(ctx); err != nil { + t.Fatalf("first reconcile: %v", err) + } + recovered, _ := dataStore.GetRun(ctx, run.ID) + if recovered.Status != domain.RunStatusQueued || !strings.Contains(mustNodeExecution(t, recovered, "node-1").Error, "recovery attempt 1/1") { + t.Fatalf("expected first recovery to requeue, got %#v", recovered) + } + recovered.Status = domain.RunStatusRunning + recovered.CurrentNodeID = "node-1" + recovered.UpdatedAt = time.Now().UTC().Add(-15 * time.Minute).Format(time.RFC3339Nano) + execution := mustNodeExecution(t, recovered, "node-1") + execution.Status = domain.NodeExecutionStatusRunning + execution.StartedAt = recovered.UpdatedAt + recovered.NodeExecutions = []domain.NodeExecutionRecord{execution} + if err := dataStore.UpsertRun(ctx, *recovered); err != nil { + t.Fatalf("simulate recovered execution: %v", err) + } + if err := runtime.reconcileActiveRuns(ctx); err != nil { + t.Fatalf("second reconcile: %v", err) + } + failed, _ := dataStore.GetRun(ctx, run.ID) + if failed.Status != domain.RunStatusFailed || !strings.Contains(failed.Error, "after 1 recovery attempts") { + t.Fatalf("expected missing job to fail after recovery limit, got %#v", failed) + } + if failed.Failure == nil || failed.Failure.Code != "kubernetes_job_missing" { + t.Fatalf("expected structured missing-job failure, got %#v", failed.Failure) + } +} + +func TestRuntimeReconcileIsolatesPerRunInspectionErrors(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + _, brokenRun := newReconcileFixture(t, dataStore, "inspection error") + _, failedRun := newReconcileFixture(t, dataStore, "independent failure") + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{inspect: func(_ context.Context, runID string, _ string) (*k8s.JobInspection, error) { + if runID == brokenRun.ID { + return nil, fmt.Errorf("temporary kubernetes error") + } + return &k8s.JobInspection{State: k8s.JobStateFailed, Output: "independent failure"}, nil + }}) + if err := runtime.reconcileActiveRuns(ctx); err != nil { + t.Fatalf("per-run error must not fail the cycle: %v", err) + } + broken, _ := dataStore.GetRun(ctx, brokenRun.ID) + failed, _ := dataStore.GetRun(ctx, failedRun.ID) + if broken.Status != domain.RunStatusRunning { + t.Fatalf("expected errored inspection unchanged, got %#v", broken) + } + if failed.Status != domain.RunStatusFailed { + t.Fatalf("expected independent run reconciled, got %#v", failed) + } +} + +func TestRuntimeLeaseOwnerChangeCancelsExecutionContext(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + _, run := newReconcileFixture(t, dataStore, "lease loss") + run.ExecutionLeaseOwner = "other-worker" + run.ExecutionLeaseExpiresAt = time.Now().UTC().Add(time.Hour).Format(time.RFC3339Nano) + if err := dataStore.UpsertRun(ctx, *run); err != nil { + t.Fatalf("upsert foreign lease: %v", err) + } + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) + runtime.dispatchOwner = "this-worker" + runtime.runLeaseTTL = 30 * time.Millisecond + leaseCtx, cancelLease := context.WithCancelCause(ctx) + defer cancelLease(context.Canceled) + go runtime.renewRunLeaseUntil(leaseCtx, run.ID, cancelLease) + deadline := time.After(time.Second) + for !errors.Is(context.Cause(leaseCtx), errRunLeaseLost) { + select { + case <-deadline: + t.Fatalf("expected lease owner change to cancel execution context, cause=%v", context.Cause(leaseCtx)) + case <-time.After(5 * time.Millisecond): + } + } +} + +func TestRuntimeProcessAvailableAppliesPersistedPollInterval(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + settings, err := dataStore.GetSettings(ctx) + if err != nil { + t.Fatalf("get settings: %v", err) + } + settings.Worker.PollIntervalMS = 750 + if err := dataStore.UpsertSettings(ctx, *settings); err != nil { + t.Fatalf("upsert settings: %v", err) + } + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{}) + runtime.SetConfiguredPollInterval(2 * time.Second) + if err := runtime.ProcessAvailable(ctx); err != nil { + t.Fatalf("process available: %v", err) + } + if got := runtime.EffectivePollInterval(); got != 750*time.Millisecond { + t.Fatalf("expected persisted 750ms poll interval, got %s", got) + } +} + +func TestLatestStructuredProgressUsesLatestValidMarker(t *testing.T) { + logs := strings.Join([]string{ + "ordinary training output", + `DROBOTICFLOW_PROGRESS {"stage":"train","epoch":1,"step":20,"maxStep":100,"samplesPerSecond":4.5,"estimatedRemainingSeconds":18,"checkpoint":"checkpoint-20","updatedAt":"2026-07-10T08:00:00Z"}`, + "DROBOTICFLOW_PROGRESS {partial", + `prefix DROBOTICFLOW_PROGRESS {"stage":"train","epoch":2,"step":40,"maxStep":100,"samplesPerSecond":5.5,"estimatedRemainingSeconds":11,"checkpoint":"checkpoint-40","updatedAt":"2026-07-10T08:01:00Z"}`, + }, "\n") + progress, err := latestStructuredProgress(logs) + if err != nil { + t.Fatalf("parse progress: %v", err) + } + if progress == nil || progress.Epoch != 2 || progress.Step != 40 || progress.Checkpoint != "checkpoint-40" { + t.Fatalf("expected latest valid marker, got %#v", progress) + } +} + +func TestNodeProgressReporterPersistsStructuredMarker(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + _, run := newReconcileFixture(t, dataStore, "structured progress") + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{tailNodePodLogs: func(context.Context, string, string, int64) (string, error) { + return `DROBOTICFLOW_PROGRESS {"stage":"train","epoch":3,"step":75,"maxStep":100,"samplesPerSecond":6.25,"estimatedRemainingSeconds":4,"checkpoint":"checkpoint-75"}`, nil + }}) + reporterCtx, cancelReporter := context.WithCancel(ctx) + done := make(chan struct{}) + go runtime.runNodeProgressReporterWithInterval(reporterCtx, run.ID, "node-1", false, 5*time.Millisecond, func() { close(done) }) + deadline := time.Now().Add(time.Second) + for { + stored, err := dataStore.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if stored.Progress != nil { + if stored.Progress.Step != 75 || stored.Progress.Checkpoint != "checkpoint-75" || stored.Progress.UpdatedAt == "" { + t.Fatalf("unexpected structured progress: %#v", stored.Progress) + } + break + } + if time.Now().After(deadline) { + t.Fatal("timed out waiting for structured progress") + } + time.Sleep(5 * time.Millisecond) + } + cancelReporter() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("progress reporter did not stop") + } +} + +func TestSyncStopMetadataPreservesAtomicProgressUpdates(t *testing.T) { + target := &domain.RunRecord{} + source := &domain.RunRecord{Progress: &domain.RunProgress{Stage: "train", Step: 9, UpdatedAt: "2026-07-10T08:00:00Z"}} + syncStopMetadata(target, source) + if target.Progress == nil || target.Progress.Step != 9 { + t.Fatalf("expected progress copied from persisted run, got %#v", target.Progress) + } + source.Progress.Step = 10 + if target.Progress.Step != 9 { + t.Fatalf("expected progress copy to avoid pointer aliasing, got %#v", target.Progress) + } +} + +func TestCompletedNodePersistsTerminalStructuredProgressMarker(t *testing.T) { + ctx := context.Background() + dataStore := store.NewMemoryStore() + project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "terminal progress", Description: "progress"}) + if err != nil { + t.Fatalf("create project: %v", err) + } + workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{ + ProjectID: project.ID, Name: "terminal progress", Type: domain.WorkflowTypeBatch, + Nodes: []domain.WorkflowNode{{ID: "train", Type: "operator", Data: domain.WorkflowNodeData{Label: "train"}}}, + }) + if err != nil { + t.Fatalf("create workflow: %v", err) + } + run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual) + if err != nil { + t.Fatalf("create run: %v", err) + } + runtime := newTestRuntimeWithLauncher(t, dataStore, testLauncher{launch: func(_ context.Context, input k8s.LaunchInput) (*k8s.LaunchResult, error) { + stageManifestPath := input.Env["STAGE_MANIFEST_PATH"] + if err := os.MkdirAll(filepath.Dir(stageManifestPath), 0o755); err != nil { + return nil, err + } + if err := os.WriteFile(stageManifestPath, []byte(`{"dataRefs":[]}`), 0o644); err != nil { + return nil, err + } + return &k8s.LaunchResult{ + Reference: &k8s.JobReference{JobName: "terminal-progress", Namespace: "droboticflow"}, + Output: strings.Join([]string{ + `DROBOTICFLOW_PROGRESS {"stage":"completed","epoch":4,"step":100,"maxStep":100,"samplesPerSecond":6.5,"estimatedRemainingSeconds":0,"checkpoint":"checkpoint-final"}`, + `DROBOTICFLOW_RESULT={"logs":[],"artifacts":[],"resourceSeries":[]}`, + }, "\n"), + }, nil + }}) + result, err := runtime.ProcessOnce(ctx) + if err != nil { + t.Fatalf("process run: %v", err) + } + if result == nil || result.Run == nil || result.Run.ID != run.ID { + t.Fatalf("unexpected process result: %#v", result) + } + completed := waitForRunStatus(t, ctx, dataStore, run.ID, domain.RunStatusCompleted, time.Second) + if completed.Progress == nil || completed.Progress.Stage != "completed" || completed.Progress.Step != 100 || completed.Progress.Checkpoint != "checkpoint-final" { + t.Fatalf("expected terminal structured progress, got %#v", completed.Progress) + } +} + // shouldReportNodeProgress must flag only custom-container operators — builtin // operator-runner pods self-report resourceSeries via their in-process ResourceSampler, // so platform-side GPU sampling is reserved for nodes whose resolved image differs from diff --git a/k8s/droboticflow.template.yaml b/k8s/droboticflow.template.yaml index 961bfb17..03420e52 100644 --- a/k8s/droboticflow.template.yaml +++ b/k8s/droboticflow.template.yaml @@ -28,7 +28,10 @@ rules: resources: ["jobs/status"] verbs: ["get"] - apiGroups: [""] - resources: ["pods", "pods/log", "events"] + resources: ["pods"] + verbs: ["get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["pods/log", "events"] verbs: ["get", "list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 @@ -58,7 +61,10 @@ rules: resources: ["jobs/status"] verbs: ["get"] - apiGroups: [""] - resources: ["pods", "pods/log", "events"] + resources: ["pods"] + verbs: ["get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["pods/log", "events"] verbs: ["get", "list", "watch"] # pods/exec lets the worker sample GPU usage (nvidia-smi) inside custom-container # operator pods, since those bypass operator-runner's in-process ResourceSampler. @@ -114,6 +120,13 @@ data: MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__" MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__" WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__" + DROBOTICFLOW_BUILD_COMMIT: "__DROBOTICFLOW_BUILD_COMMIT__" + DROBOTICFLOW_BUILD_TIME: "__DROBOTICFLOW_BUILD_TIME__" + WORKER_ORPHAN_CLEANUP_ENABLED: "__WORKER_ORPHAN_CLEANUP_ENABLED__" + WORKER_ORPHAN_MIN_AGE_SECONDS: "__WORKER_ORPHAN_MIN_AGE_SECONDS__" + WORKER_ORPHAN_CHECK_INTERVAL_SECONDS: "__WORKER_ORPHAN_CHECK_INTERVAL_SECONDS__" + WORKER_PENDING_FAILURE_SECONDS: "__WORKER_PENDING_FAILURE_SECONDS__" + WORKER_MISSING_JOB_MAX_RECOVERIES: "__WORKER_MISSING_JOB_MAX_RECOVERIES__" WORKER_API_BASE_URL: "__WORKER_API_BASE_URL__" BOS_ENDPOINT: "__BOS_ENDPOINT__" K8S_NAMESPACE: "__NAMESPACE__" @@ -201,6 +214,18 @@ spec: - containerPort: 3001 name: http env: + - name: DROBOTICFLOW_BUILD_COMMIT + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: DROBOTICFLOW_BUILD_COMMIT + - name: DROBOTICFLOW_BUILD_TIME + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: DROBOTICFLOW_BUILD_TIME + - name: DROBOTICFLOW_IMAGE + value: "__API_IMAGE__" - name: PORT valueFrom: configMapKeyRef: @@ -330,7 +355,7 @@ spec: - sleep 5 readinessProbe: httpGet: - path: /api/health + path: /api/ready port: http initialDelaySeconds: 10 periodSeconds: 10 @@ -414,6 +439,18 @@ spec: - containerPort: 3002 name: http env: + - name: DROBOTICFLOW_BUILD_COMMIT + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: DROBOTICFLOW_BUILD_COMMIT + - name: DROBOTICFLOW_BUILD_TIME + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: DROBOTICFLOW_BUILD_TIME + - name: DROBOTICFLOW_IMAGE + value: "__WORKER_IMAGE__" - name: WORKER_PORT valueFrom: configMapKeyRef: @@ -424,6 +461,31 @@ spec: configMapKeyRef: name: droboticflow-config key: WORKER_POLL_INTERVAL_MS + - name: WORKER_ORPHAN_CLEANUP_ENABLED + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: WORKER_ORPHAN_CLEANUP_ENABLED + - name: WORKER_ORPHAN_MIN_AGE_SECONDS + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: WORKER_ORPHAN_MIN_AGE_SECONDS + - name: WORKER_ORPHAN_CHECK_INTERVAL_SECONDS + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: WORKER_ORPHAN_CHECK_INTERVAL_SECONDS + - name: WORKER_PENDING_FAILURE_SECONDS + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: WORKER_PENDING_FAILURE_SECONDS + - name: WORKER_MISSING_JOB_MAX_RECOVERIES + valueFrom: + configMapKeyRef: + name: droboticflow-config + key: WORKER_MISSING_JOB_MAX_RECOVERIES - name: BOS_ENDPOINT valueFrom: configMapKeyRef: @@ -521,7 +583,7 @@ spec: - sleep 5 readinessProbe: httpGet: - path: /worker-api/health + path: /worker-api/ready port: http initialDelaySeconds: 10 periodSeconds: 10 diff --git a/operator-images/builtin/operator-images.json b/operator-images/builtin/operator-images.json index 088fdce8..b7346d0f 100644 --- a/operator-images/builtin/operator-images.json +++ b/operator-images/builtin/operator-images.json @@ -684,8 +684,8 @@ "gpu-a800" ], "dockerfile": "operator-repos/builtin/holobrain_export/Dockerfile", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-export:builtin", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-export:builtin@sha256:6e2f3be59e7094103713ec9113c12381d319a13c785dfed36700be31fe53600c", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-export:a34570f-holobrain-exportrepair5-20260709014631", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-export@sha256:90e90351b00fac0405740b87281608ce51e2171e0a78ed81295a317a5f93e7f6", "operatorId": "operator-builtin-holobrain-export", "operatorSlug": "holobrain_export", "requiresGpu": true, @@ -706,8 +706,8 @@ "gpu-a800" ], "dockerfile": "operator-repos/builtin/holobrain_inference_test/Dockerfile", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-inference-test:builtin", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-inference-test:builtin@sha256:ebb4f8dda26d529d1b738112e3aa03d086cbd00c52c7208f3105419b6b126ed2", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-inference-test:5289510-holobrain-downstream2-202607082304", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-inference-test@sha256:3a79fd20722ce23a8b64eab3cd31e1b0434fef2564b3b6f55c02933d667abf33", "operatorId": "operator-builtin-holobrain-inference-test", "operatorSlug": "holobrain_inference_test", "requiresGpu": true, @@ -728,8 +728,8 @@ "gpu-a800" ], "dockerfile": "operator-repos/builtin/holobrain_robotwin_eval/Dockerfile", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-robotwin-eval:builtin", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-robotwin-eval:builtin@sha256:61440c7d4184f79bc784350e637ef8a2e6cc2e3d78f5958d5e9d6f8f3e7a8357", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-robotwin-eval:bd25938-holobrain-evalsapien-20260709032957", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-holobrain-robotwin-eval@sha256:4f0ca2efcb3d69ad672df06dc28805ce3587c5ad8df4ee790677ea719fc32add", "operatorId": "operator-builtin-holobrain-robotwin-eval", "operatorSlug": "holobrain_robotwin_eval", "requiresGpu": true, diff --git a/operator-repos/builtin/hf_dataset_probe/release.json b/operator-repos/builtin/hf_dataset_probe/release.json index 99807e6e..6db14307 100644 --- a/operator-repos/builtin/hf_dataset_probe/release.json +++ b/operator-repos/builtin/hf_dataset_probe/release.json @@ -25,6 +25,6 @@ "schemaVersion": "droboticflow.operator.release/v1", "slug": "hf_dataset_probe", "sourceCommit": "316b493dba4d3d1a32366d43686656b36fd2d2a1", - "sourceTreeSha256": "9f56a0b8ebaf759e05cc1928253234c9e71f9d0e91262d3e0910d3bbe164fa5a", + "sourceTreeSha256": "9b59218e6b364475a7b557e2f4695243513a2265c337bc2e1ecbf56daa78af2b", "tag": "builtin" } diff --git a/operator-repos/builtin/hf_dataset_probe/repository.json b/operator-repos/builtin/hf_dataset_probe/repository.json index a27fa703..20fa2183 100644 --- a/operator-repos/builtin/hf_dataset_probe/repository.json +++ b/operator-repos/builtin/hf_dataset_probe/repository.json @@ -75,7 +75,7 @@ "runtimeSource": "operator-repos/builtin/hf_dataset_probe/src/operator.py", "schemaVersion": "droboticflow.operator.repository/v1", "sourceRoot": "operator-repos/builtin/hf_dataset_probe", - "sourceTreeSha256": "9f56a0b8ebaf759e05cc1928253234c9e71f9d0e91262d3e0910d3bbe164fa5a", + "sourceTreeSha256": "9b59218e6b364475a7b557e2f4695243513a2265c337bc2e1ecbf56daa78af2b", "strictPocAcceptanceExternals": [], "type": "builtin" } diff --git a/operator-repos/builtin/wilor_hand_pose/release.json b/operator-repos/builtin/wilor_hand_pose/release.json index e2f50b99..341abaf3 100644 --- a/operator-repos/builtin/wilor_hand_pose/release.json +++ b/operator-repos/builtin/wilor_hand_pose/release.json @@ -20,6 +20,7 @@ "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-wilor-hand-pose:wilor-hand-pose-io", "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-wilor-hand-pose@sha256:454fe90ab54513bb3e022ced3f8fddfe6fd78a3e347780d66488ce0d2c05287c", "kanikoJob": "droboticflow-kaniko-op-wilor-hand-pose-wilor-hand-pose-io", + "operatorId": "operator-builtin-wilor-hand-pose", "registry": "ccr-29eug8s3-pub.cnc.bj.baidubce.com", "schemaVersion": "droboticflow.operator.release/v1", "slug": "wilor_hand_pose", diff --git a/scripts/generate_operator_images.py b/scripts/generate_operator_images.py index 819b6c39..ab92e539 100644 --- a/scripts/generate_operator_images.py +++ b/scripts/generate_operator_images.py @@ -139,16 +139,22 @@ STRUCTURED_OPERATOR_OVERRIDES: dict[str, dict[str, Any]] = { ], }, "operator-builtin-holobrain-data-prepare": { - "skillDefinition": "HoloBrain 数据准备算子;校验已预置的官方 RoboTwin 2.0 真实数据,将其映射为 RoboOrchardLab HoloBrain 训练期望的数据目录并输出 manifest。运行时不下载数据,缺少真实数据会直接失败。", + "description": "校验真实 RoboTwin 2.0 数据并生成 HoloBrain manifest;支持 smoke 真实小数据和 full 完整数据模式。", + "skillDefinition": "HoloBrain 数据准备算子;TRAINING_MODE=smoke 使用真实小数据契约(Arrow 默认最多 1 个 episode),full 保持完整数据要求;两种模式都拒绝合成或空输入。", "hyperparams": [ + {"key": "TRAINING_MODE", "value": "full", "description": "full 保持完整训练数据行为;smoke 使用真实小数据完成端到端验证", "type": "string"}, {"key": "DATA_SOURCE", "value": "local", "description": "数据来源;仅支持 local,不支持运行时下载", "type": "string"}, {"key": "ROBOTWIN_SOURCE_PATH", "value": "", "description": "已预置的 RoboTwin/HoloBrain 真实数据根目录;必填", "type": "string"}, - {"key": "EXPECTED_DATASETS_JSON", "value": "[\"robotwin2_0\",\"robotwin2_0_ur5_wsg\"]", "description": "期望发现的数据集目录名", "type": "json"}, + {"key": "EXPECTED_LMDB_DIRS_JSON", "value": "", "description": "可选覆盖 LMDB 目录;空值时 smoke 只要求 aloha_agilex_demo_clean,full 要求三个官方目录", "type": "json"}, + {"key": "SMOKE_MAX_EPISODES", "value": "1", "description": "smoke 从真实 Arrow 数据转换的最大 episode 数", "type": "number"}, + {"key": "ARROW_MAX_EPISODES", "value": "", "description": "显式 Arrow 转换 episode 上限;空值时由模式决定", "type": "number"}, ], }, "operator-builtin-holobrain-train": { - "skillDefinition": "HoloBrain 完整训练算子;使用 RoboOrchardLab 官方 projects/holobrain/scripts/train.py 和 config_holobrain_qwen_common.py,默认执行 100000 step 全量训练并输出训练报告。", + "description": "执行 HoloBrain 训练;full 保留官方长训默认,smoke 用真实小数据短训并输出可恢复 checkpoint 与结构化进度。", + "skillDefinition": "HoloBrain 训练算子;TRAINING_MODE=full 保留官方 100000 step 默认,smoke 默认在真实 RoboTwin 小数据上执行 2 step 并生成 checkpoint。失败重试默认恢复最新有效 Accelerate checkpoint;进度写入 workspace 并以 DROBOTICFLOW_PROGRESS {json} 输出;dtype 在启动前检查。", "hyperparams": [ + {"key": "TRAINING_MODE", "value": "full", "description": "full 或 smoke;full 不覆盖官方长训默认,smoke 使用短步数真实训练", "type": "string"}, {"key": "ROBOORCHARDLAB_ROOT", "value": "/opt/RoboOrchardLab", "description": "镜像内预置的 RoboOrchardLab 仓库路径;缺失时直接失败", "type": "string"}, {"key": "CONFIG_PATH", "value": "projects/holobrain/configs/config_holobrain_qwen_common.py", "description": "HoloBrain Qwen 训练配置", "type": "string"}, {"key": "ROBOTWIN_DATA_ROOT", "value": "", "description": "RoboTwin 数据根目录;为空时消费上游数据准备 manifest", "type": "string"}, @@ -156,25 +162,41 @@ STRUCTURED_OPERATOR_OVERRIDES: dict[str, dict[str, Any]] = { {"key": "LAUNCH_MODE", "value": "single", "description": "single 或 accelerate", "type": "string"}, {"key": "GPU_COUNT", "value": "1", "description": "accelerate 模式使用的进程/GPU 数", "type": "number"}, {"key": "CONFIG_KWARGS_JSON", "value": "{}", "description": "传给 train.py --kwargs 的 JSON;默认不覆盖 max_step=100000", "type": "json"}, + {"key": "SMOKE_MAX_STEP", "value": "2", "description": "smoke 真实训练最大 step;默认 2", "type": "number"}, + {"key": "SMOKE_BATCH_SIZE", "value": "1", "description": "smoke batch size", "type": "number"}, + {"key": "SMOKE_NUM_WORKERS", "value": "0", "description": "smoke 数据加载进程数", "type": "number"}, + {"key": "RESUME_POLICY", "value": "auto", "description": "auto 自动选择 workspace/checkpoints 下最新有效训练状态;explicit 要求 RESUME_FROM;never 禁用恢复", "type": "string"}, + {"key": "RESUME_FROM", "value": "", "description": "显式 Accelerate 训练状态 checkpoint 目录", "type": "string"}, + {"key": "INITIAL_CHECKPOINT_PATH", "value": "", "description": "可选初始模型权重文件;与训练状态恢复分开", "type": "string"}, + {"key": "CHECKPOINT_ARTIFACT", "value": "latest_valid", "description": "latest_valid、explicit 或 none;控制训练报告暴露的模型产物", "type": "string"}, + {"key": "OUTPUT_CHECKPOINT_PATH", "value": "", "description": "CHECKPOINT_ARTIFACT=explicit 时使用的模型 checkpoint", "type": "string"}, + {"key": "PROGRESS_INTERVAL_SECONDS", "value": "15", "description": "结构化进度写入 droboticflow-progress.json、holobrain-progress.json 并输出 DROBOTICFLOW_PROGRESS {json};该值为最长刷新间隔", "type": "number"}, + {"key": "MODEL_DTYPE", "value": "bfloat16", "description": "官方 HoloBrain Qwen 模型 dtype;不兼容值在启动前失败", "type": "string"}, + {"key": "INPUT_DTYPE", "value": "float32", "description": "官方 RoboTwin processor 输入 tensor dtype;不兼容值在启动前失败", "type": "string"}, {"key": "WANDB_MODE", "value": "offline", "description": "训练日志模式", "type": "string"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-holobrain-export": { - "skillDefinition": "HoloBrain 模型导出算子;消费训练 workspace/checkpoint,调用 RoboOrchardLab 官方 export.py 输出可部署模型目录和处理器配置。", + "description": "从显式、上游或训练 workspace 的最新有效 checkpoint 导出可部署 HoloBrain 模型。", + "skillDefinition": "HoloBrain 模型导出算子;按 CHECKPOINT_SELECTION 消费显式、上游或训练 workspace 中最新有效 checkpoint。smoke 模式没有真实训练 checkpoint 时直接失败。", "hyperparams": [ + {"key": "TRAINING_MODE", "value": "full", "description": "full 或 smoke;smoke 强制要求上游真实 checkpoint", "type": "string"}, {"key": "ROBOORCHARDLAB_ROOT", "value": "/opt/RoboOrchardLab", "description": "RoboOrchardLab 仓库路径", "type": "string"}, {"key": "CONFIG_PATH", "value": "projects/holobrain/configs/config_holobrain_qwen_common.py", "description": "HoloBrain Qwen 导出配置", "type": "string"}, {"key": "TRAINING_WORKSPACE", "value": "", "description": "训练 workspace;为空时消费上游训练报告", "type": "string"}, {"key": "CHECKPOINT_PATH", "value": "", "description": "显式 checkpoint 路径;为空时使用最新训练 checkpoint", "type": "string"}, + {"key": "CHECKPOINT_SELECTION", "value": "latest_valid", "description": "latest_valid、upstream 或 explicit", "type": "string"}, {"key": "EXPORT_WORKSPACE", "value": "", "description": "导出目录;为空时使用节点输出目录下 holobrain-export", "type": "string"}, {"key": "CONFIG_KWARGS_JSON", "value": "{}", "description": "额外 export.py --kwargs JSON", "type": "json"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "export reload-test 使用 CUDA", "type": "string"}, ], }, "operator-builtin-holobrain-robotwin-eval": { - "skillDefinition": "HoloBrain RoboTwin 评测算子;消费导出的 HoloBrain 模型,调用官方 robotwin_eval.py 输出评测指标。", + "description": "调用官方 robotwin_eval.py 执行真实 HoloBrain RoboTwin 评测;smoke 为单任务单 episode。", + "skillDefinition": "HoloBrain RoboTwin 评测算子;full 保留官方多任务评测默认,smoke 固定执行 1 个真实 RoboTwin episode,必须由官方 evaluator 产生指标。", "hyperparams": [ + {"key": "TRAINING_MODE", "value": "full", "description": "full 或 smoke;smoke 使用专用单任务单 episode 参数", "type": "string"}, {"key": "ROBOORCHARDLAB_ROOT", "value": "/opt/RoboOrchardLab", "description": "RoboOrchardLab 仓库路径", "type": "string"}, {"key": "MODEL_DIR", "value": "", "description": "导出模型目录;为空时消费上游导出 manifest", "type": "string"}, {"key": "INFERENCE_PREFIX", "value": "robotwin2_0", "description": "导出 pipeline config prefix", "type": "string"}, @@ -184,14 +206,27 @@ STRUCTURED_OPERATOR_OVERRIDES: dict[str, dict[str, Any]] = { {"key": "GPU_IDS_JSON", "value": "[0]", "description": "Ray 评测 GPU id 数组", "type": "json"}, {"key": "WORKERS_PER_GPU", "value": "1", "description": "每张 GPU 的并发 worker 数", "type": "number"}, {"key": "SAVE_VIDEO", "value": "false", "description": "是否保存评测视频", "type": "boolean"}, + {"key": "SMOKE_TASK_NAME", "value": "beat_block_hammer", "description": "smoke 实际执行的 RoboTwin 任务", "type": "string"}, + {"key": "SMOKE_EVAL_EPISODE_NUM", "value": "1", "description": "smoke episode 数", "type": "number"}, + {"key": "SMOKE_EVAL_MODE", "value": "local", "description": "smoke evaluator 模式", "type": "string"}, + {"key": "SMOKE_DEVICE", "value": "cuda", "description": "smoke 模型执行设备", "type": "string"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "推荐 GPU 配置", "type": "string"}, ], }, "operator-builtin-holobrain-inference-test": { - "skillDefinition": "HoloBrain 模型测试算子;消费导出模型和真实样本路径,校验导出 pipeline 文件和样本绑定,作为训练后推理烟测入口。", + "description": "加载导出模型并通过官方 RoboTwin evaluator 执行真实模型推理,不接受文件存在性或硬编码结果冒充成功。", + "skillDefinition": "HoloBrain 模型测试算子;加载导出的真实模型并通过官方 RoboTwin evaluator 执行至少 1 个 episode,不再用文件存在性或硬编码结果冒充推理。", "hyperparams": [ + {"key": "TRAINING_MODE", "value": "full", "description": "记录上游 full/smoke 模式", "type": "string"}, + {"key": "ROBOORCHARDLAB_ROOT", "value": "/opt/RoboOrchardLab", "description": "RoboOrchardLab 仓库路径", "type": "string"}, {"key": "MODEL_DIR", "value": "", "description": "导出模型目录;为空时消费上游导出 manifest", "type": "string"}, - {"key": "SAMPLE_INPUT_PATH", "value": "", "description": "真实样本文件路径;为空时消费上游数据 manifest 的 sampleCandidates", "type": "string"}, + {"key": "INFERENCE_PREFIX", "value": "robotwin2_0", "description": "导出 pipeline config prefix", "type": "string"}, + {"key": "TASK_NAME", "value": "beat_block_hammer", "description": "实际执行的 RoboTwin 推理任务", "type": "string"}, + {"key": "EPISODE_NUM", "value": "1", "description": "真实推理 episode 数,至少为 1", "type": "number"}, + {"key": "INFERENCE_MODE", "value": "local", "description": "local 或 ray", "type": "string"}, + {"key": "DEVICE", "value": "cuda", "description": "模型推理设备", "type": "string"}, + {"key": "GPU_IDS_JSON", "value": "[0]", "description": "ray 模式 GPU id 数组", "type": "json"}, + {"key": "WORKERS_PER_GPU", "value": "1", "description": "ray 模式每卡 worker 数", "type": "number"}, {"key": "RECOMMENDED_GPU", "value": "1x NVIDIA A800 80GB", "description": "完整推理测试推荐 GPU 配置", "type": "string"}, ], }, @@ -538,6 +573,8 @@ def expected_catalog() -> list[dict[str, Any]]: handler = item.get("runtimeHandler") or slug next_item = strip_disallowed_command_params(dict(item)) overrides = STRUCTURED_OPERATOR_OVERRIDES.get(next_item["id"], {}) + if "description" in overrides: + next_item["description"] = overrides["description"] if "skillDefinition" in overrides: next_item["skillDefinition"] = overrides["skillDefinition"] if "hyperparams" in overrides: diff --git a/scripts/generate_operator_repository_manifests.py b/scripts/generate_operator_repository_manifests.py index 11d0712b..94eae67f 100644 --- a/scripts/generate_operator_repository_manifests.py +++ b/scripts/generate_operator_repository_manifests.py @@ -140,11 +140,56 @@ def operator_slug(operator_id: str) -> str: return operator_id.removeprefix("operator-builtin-").replace("-", "_") +def legacy_manifest( + *, + slug: str, + metadata: dict[str, Any], + catalog_item: dict[str, Any], + image: str, + image_id: str, + dependencies: list[str], +) -> dict[str, Any]: + acceptance = dict(metadata["acceptance"]) + manifest: dict[str, Any] = { + "schemaVersion": "droboticflow.operator.repository/v1", + "repositoryId": f"builtin/{slug}", + "operatorId": metadata["id"], + "operatorSlug": slug, + "preferredSlug": operator_slug(metadata["id"]), + "name": metadata["name"], + "category": metadata["category"], + "type": metadata["type"], + "sourceRoot": f"backend/operators/builtin/{slug}", + "entrypoint": "operator.py", + "metadata": "operator.json", + "runtimeSource": metadata["runtimeSource"], + "runtimeHandler": metadata["runtimeHandler"], + "containerEntrypoint": "backend/operator-runner/main.py", + "image": image, + "imageId": image_id, + "containerCommand": catalog_item.get("containerCommand", []), + "containerArgs": catalog_item.get("containerArgs", []), + "runtimeContract": catalog_item.get("runtimeContract", {}), + "pythonImport": f"backend.operators.builtin.{slug}.operator:run", + "runtimeDependencies": dependencies, + "strictPocAcceptanceExternals": STRICT_EXTERNALS.get(slug, []), + "acceptance": { + **acceptance, + "pocFullAcceptanceScript": "scripts/poc-operator-full-acceptance.sh", + "requiresPersistentWorkspaceEvidence": True, + }, + } + if "externalApi" in metadata: + manifest["externalApi"] = metadata["externalApi"] + return manifest + + def expected_manifests() -> dict[pathlib.Path, dict[str, Any]]: catalog_items = { item["id"]: item for item in load_json(CATALOG_PATH) if item.get("type") == "builtin" } manifests: dict[pathlib.Path, dict[str, Any]] = {} + split_operator_ids: set[str] = set() for metadata_path in sorted(OPERATOR_REPO_ROOT.glob("*/operator.json")): metadata = load_json(metadata_path) operator_id = metadata["id"] @@ -199,7 +244,25 @@ def expected_manifests() -> dict[pathlib.Path, dict[str, Any]]: if "externalApi" in metadata: manifest["externalApi"] = metadata["externalApi"] manifests[metadata_path.with_name("repository.json")] = manifest - missing_catalog = sorted(set(catalog_items) - {item["operatorId"] for item in manifests.values()}) + split_operator_ids.add(operator_id) + + legacy_metadata_path = OPERATOR_ROOT / slug / "operator.json" + if not legacy_metadata_path.exists(): + raise RuntimeError(f"{slug} is missing legacy runtime metadata") + legacy_metadata = load_json(legacy_metadata_path) + if legacy_metadata.get("id") != operator_id: + raise RuntimeError( + f"{legacy_metadata_path} operator id {legacy_metadata.get('id')!r} does not match {operator_id!r}" + ) + manifests[legacy_metadata_path.with_name("repository.json")] = legacy_manifest( + slug=slug, + metadata=legacy_metadata, + catalog_item=catalog_item, + image=image, + image_id=image_id, + dependencies=dependencies, + ) + missing_catalog = sorted(set(catalog_items) - split_operator_ids) if missing_catalog: raise RuntimeError(f"catalog operators missing repository manifests: {', '.join(missing_catalog)}") return manifests diff --git a/scripts/holobrain-full-training-workflow-acceptance.sh b/scripts/holobrain-full-training-workflow-acceptance.sh index 1963a5aa..ad8fa21c 100755 --- a/scripts/holobrain-full-training-workflow-acceptance.sh +++ b/scripts/holobrain-full-training-workflow-acceptance.sh @@ -15,7 +15,19 @@ API_TOKEN_SECRET_NAME="${HOLOBRAIN_WORKFLOW_API_TOKEN_SECRET_NAME:-droboticflow- API_TOKEN_SECRET_KEY="${HOLOBRAIN_WORKFLOW_API_TOKEN_SECRET_KEY:-DROBOTICFLOW_API_TOKEN}" DROBOTICFLOW_ACCEPTANCE_EMAIL="${DROBOTICFLOW_ACCEPTANCE_EMAIL:-poc-acceptance-admin@droboticflow.local}" DROBOTICFLOW_ACCEPTANCE_PASSWORD="${DROBOTICFLOW_ACCEPTANCE_PASSWORD:-youbixuanpoc}" -HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS="${HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS:-1800}" +HOLOBRAIN_TRAINING_MODE="${HOLOBRAIN_TRAINING_MODE:-smoke}" +if [[ "$HOLOBRAIN_TRAINING_MODE" != "smoke" && "$HOLOBRAIN_TRAINING_MODE" != "full" ]]; then + echo "ERROR: HOLOBRAIN_TRAINING_MODE must be smoke or full." >&2 + exit 2 +fi +if [[ -z "${HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS:-}" ]]; then + if [[ "$HOLOBRAIN_TRAINING_MODE" == "smoke" ]]; then + HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS=7200 + else + HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS=1800 + fi +fi +export HOLOBRAIN_TRAINING_MODE HOLOBRAIN_WORKFLOW_RUN_TIMEOUT_SECONDS if [[ -z "${API_BASE_URL:-}" && -z "${BASE_URL:-}" ]]; then public_host="$( @@ -114,7 +126,9 @@ api_curl -H 'Content-Type: application/json' -X POST "$API_BASE_URL/api/auth/log project_payload="$( python3 - <<'PY' import json -print(json.dumps({"name": "HoloBrain Full Training Acceptance", "description": "RoboOrchardLab HoloBrain data/train/export/eval/test startup verification", "status": "active"}, ensure_ascii=False)) +import os +mode = os.environ["HOLOBRAIN_TRAINING_MODE"] +print(json.dumps({"name": f"HoloBrain {mode.title()} Training Acceptance", "description": f"RoboOrchardLab HoloBrain {mode} data/train/export/eval/inference verification", "status": "active"}, ensure_ascii=False)) PY )" project_response="$(api_curl_write -X POST "$API_BASE_URL/api/projects" -H 'Content-Type: application/json' -d "$project_payload")" @@ -208,6 +222,7 @@ def with_resource_overrides(data, prefix): return data data_values = { + "TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"), "DATA_SOURCE": env("HOLOBRAIN_DATA_SOURCE", "local"), "ROBOTWIN_SOURCE_PATH": env("HOLOBRAIN_ROBOTWIN_SOURCE_PATH", env("HOLOBRAIN_WORKFLOW_SOURCE_ROOT", "")), "EXPECTED_LMDB_DIRS_JSON": env("HOLOBRAIN_EXPECTED_LMDB_DIRS_JSON", ""), @@ -215,18 +230,29 @@ data_values = { } data_values = {key: value for key, value in data_values.items() if value} train_values = { + "TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"), "ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"), "CONFIG_PATH": "projects/holobrain/configs/config_holobrain_qwen_common.py", "LAUNCH_MODE": env("HOLOBRAIN_LAUNCH_MODE", "single"), "GPU_COUNT": env("HOLOBRAIN_GPU_COUNT", "1"), "CONFIG_KWARGS_JSON": env("HOLOBRAIN_CONFIG_KWARGS_JSON", "{}"), + "SMOKE_MAX_STEP": env("HOLOBRAIN_SMOKE_MAX_STEP", "2"), + "SMOKE_BATCH_SIZE": env("HOLOBRAIN_SMOKE_BATCH_SIZE", "1"), + "SMOKE_NUM_WORKERS": env("HOLOBRAIN_SMOKE_NUM_WORKERS", "0"), + "RESUME_POLICY": env("HOLOBRAIN_RESUME_POLICY", "auto"), + "PROGRESS_INTERVAL_SECONDS": env("HOLOBRAIN_PROGRESS_INTERVAL_SECONDS", "15"), + "MODEL_DTYPE": env("HOLOBRAIN_MODEL_DTYPE", "bfloat16"), + "INPUT_DTYPE": env("HOLOBRAIN_INPUT_DTYPE", "float32"), } export_values = { + "TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"), "ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"), "CONFIG_PATH": "projects/holobrain/configs/config_holobrain_qwen_common.py", "CONFIG_KWARGS_JSON": env("HOLOBRAIN_EXPORT_CONFIG_KWARGS_JSON", env("HOLOBRAIN_CONFIG_KWARGS_JSON", "{}")), + "CHECKPOINT_SELECTION": env("HOLOBRAIN_CHECKPOINT_SELECTION", "latest_valid"), } eval_values = { + "TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"), "ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"), "INFERENCE_PREFIX": env("HOLOBRAIN_EVAL_INFERENCE_PREFIX", "robotwin2_0"), "EVAL_MODE": env("HOLOBRAIN_EVAL_MODE", "ray"), @@ -235,9 +261,19 @@ eval_values = { "GPU_IDS_JSON": env("HOLOBRAIN_EVAL_GPU_IDS_JSON", "[0]"), "WORKERS_PER_GPU": env("HOLOBRAIN_EVAL_WORKERS_PER_GPU", "1"), "SAVE_VIDEO": env("HOLOBRAIN_EVAL_SAVE_VIDEO", "false"), + "SMOKE_TASK_NAME": env("HOLOBRAIN_SMOKE_TASK_NAME", "beat_block_hammer"), + "SMOKE_EVAL_EPISODE_NUM": env("HOLOBRAIN_SMOKE_EVAL_EPISODE_NUM", "1"), + "SMOKE_EVAL_MODE": env("HOLOBRAIN_SMOKE_EVAL_MODE", "local"), + "SMOKE_DEVICE": env("HOLOBRAIN_SMOKE_DEVICE", "cuda"), } test_values = { - "SAMPLE_INPUT_PATH": env("HOLOBRAIN_INFERENCE_SAMPLE_INPUT_PATH", ""), + "TRAINING_MODE": env("HOLOBRAIN_TRAINING_MODE", "smoke"), + "ROBOORCHARDLAB_ROOT": env("HOLOBRAIN_ROBOORCHARDLAB_ROOT", "/opt/RoboOrchardLab"), + "INFERENCE_PREFIX": env("HOLOBRAIN_INFERENCE_PREFIX", "robotwin2_0"), + "TASK_NAME": env("HOLOBRAIN_INFERENCE_TASK_NAME", env("HOLOBRAIN_SMOKE_TASK_NAME", "beat_block_hammer")), + "EPISODE_NUM": env("HOLOBRAIN_INFERENCE_EPISODE_NUM", "1"), + "INFERENCE_MODE": env("HOLOBRAIN_INFERENCE_MODE", "local"), + "DEVICE": env("HOLOBRAIN_INFERENCE_DEVICE", "cuda"), } export_values = {key: value for key, value in export_values.items() if value} eval_values = {key: value for key, value in eval_values.items() if value} @@ -257,8 +293,8 @@ edges = [ {"id": "edge-data-test", "source": "data-1", "sourceHandle": "out-1", "target": "test-1", "targetHandle": "in-2"}, ] print(json.dumps({ - "name": "HoloBrain Full Training Startup Acceptance", - "description": "Full HoloBrain workflow; startup acceptance waits for real data manifest and training node launch evidence.", + "name": f"HoloBrain {env('HOLOBRAIN_TRAINING_MODE', 'smoke').title()} Training Acceptance", + "description": "Smoke requires the complete real data -> train -> export -> eval/inference chain; full preserves long-training startup acceptance.", "type": "batch", "nodeTimeoutSeconds": 0, "runTimeoutSeconds": 0, @@ -285,14 +321,20 @@ while (( SECONDS < deadline )); do printf '%s' "$run_json" | python3 -c 'import json,sys; r=json.load(sys.stdin); print({n.get("nodeId"): n.get("status") for n in r.get("nodeExecutions") or []})' )" printf 'run status: %s nodes: %s\n' "$status" "$node_statuses" - if RUN_JSON="$run_json" python3 - <<'PY' + if RUN_JSON="$run_json" HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 - <<'PY' import json import os import sys run = json.loads(os.environ["RUN_JSON"]) statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []} -if statuses.get("data-1") == "completed" and statuses.get("train-1") in {"queued", "running"}: - raise SystemExit(0) +mode = os.environ["HOLOBRAIN_TRAINING_MODE"] +if mode == "smoke": + expected = {"data-1", "train-1", "export-1", "eval-1", "test-1"} + if expected <= statuses.keys() and all(statuses[node_id] == "completed" for node_id in expected): + raise SystemExit(0) +else: + if statuses.get("data-1") == "completed" and statuses.get("train-1") in {"queued", "running"}: + raise SystemExit(0) if run.get("status") in {"failed", "cancelled"}: raise SystemExit(2) raise SystemExit(1) @@ -313,27 +355,55 @@ if [[ -z "$final_run" ]]; then final_run="$(api_curl "$API_BASE_URL/api/runs/$run_id")" fi -FINAL_RUN="$final_run" python3 - <<'PY' +FINAL_RUN="$final_run" HOLOBRAIN_TRAINING_MODE="$HOLOBRAIN_TRAINING_MODE" python3 - <<'PY' import json import os run = json.loads(os.environ["FINAL_RUN"]) statuses = {item.get("nodeId"): item.get("status") for item in run.get("nodeExecutions") or []} -artifacts = [ - item.get("name", "") +artifact_items = [ + item for node in run.get("nodeExecutions") or [] for item in node.get("artifacts") or [] ] -accepted = statuses.get("data-1") == "completed" and statuses.get("train-1") in {"queued", "running"} +artifacts = [item.get("name", "") for item in artifact_items] +artifact_data = {item.get("name", ""): item.get("data") for item in artifact_items if isinstance(item.get("data"), dict)} +mode = os.environ["HOLOBRAIN_TRAINING_MODE"] +if mode == "smoke": + expected_nodes = {"data-1", "train-1", "export-1", "eval-1", "test-1"} + expected_artifacts = { + "holobrain-data-manifest.json", + "holobrain-training-report.json", + "holobrain-export-manifest.json", + "holobrain-robotwin-eval-report.json", + "holobrain-inference-result.json", + } + inference = artifact_data.get("holobrain-inference-result.json") or {} + evaluation = artifact_data.get("holobrain-robotwin-eval-report.json") or {} + accepted = ( + run.get("status") == "completed" + and expected_nodes <= statuses.keys() + and all(statuses[node_id] == "completed" for node_id in expected_nodes) + and expected_artifacts <= set(artifacts) + and inference.get("execution", {}).get("actualModelExecution") is True + and inference.get("execution", {}).get("syntheticResult") is False + and evaluation.get("execution", {}).get("backend") == "official-robotwin-evaluator" + and evaluation.get("execution", {}).get("syntheticResult") is False + ) +else: + accepted = statuses.get("data-1") == "completed" and statuses.get("train-1") in {"queued", "running"} summary = { + "trainingMode": mode, "projectId": run.get("projectId"), "workflowId": run.get("workflowId"), "runId": run.get("id"), "status": run.get("status"), "nodeStatuses": statuses, "artifacts": sorted(artifacts), - "startupAccepted": accepted, + "accepted": accepted, } print("holobrain workflow acceptance:", json.dumps(summary, ensure_ascii=False)) if not accepted: - raise SystemExit("HoloBrain startup acceptance did not reach data completed + train queued/running") + if mode == "smoke": + raise SystemExit("HoloBrain smoke acceptance did not complete the real data/train/export/eval/inference contract") + raise SystemExit("HoloBrain full startup acceptance did not reach data completed + train queued/running") PY diff --git a/scripts/operator_release_manifest.py b/scripts/operator_release_manifest.py index 8e53acc6..469e7321 100644 --- a/scripts/operator_release_manifest.py +++ b/scripts/operator_release_manifest.py @@ -126,6 +126,11 @@ def validate_release(slug: str, release: dict[str, Any]) -> None: path_label = release_path(slug).relative_to(ROOT) if release.get("schemaVersion") != RELEASE_SCHEMA_VERSION: raise RuntimeError(f"operator_release_manifest_missing: {path_label} schemaVersion must be {RELEASE_SCHEMA_VERSION}") + expected_operator_id = f"operator-builtin-{slug.replace('_', '-')}" + if release.get("operatorId") != expected_operator_id: + raise RuntimeError( + f"operator_release_manifest_missing: {path_label} operatorId must be {expected_operator_id}" + ) if release.get("slug") != slug: raise RuntimeError(f"operator_release_digest_drift: {path_label} slug must be {slug}") image = str(release.get("image") or "").strip() @@ -184,4 +189,3 @@ def release_from_catalog_item(item: dict[str, Any], *, env: str | None = None) - }, } return release - diff --git a/scripts/poc-operator-full-acceptance.sh b/scripts/poc-operator-full-acceptance.sh index bdb9c432..771dd4e0 100755 --- a/scripts/poc-operator-full-acceptance.sh +++ b/scripts/poc-operator-full-acceptance.sh @@ -24,6 +24,43 @@ POC_ACCEPTANCE_DISCOVERY_INCLUDE_ACCEPTANCE_OUTPUT="${POC_ACCEPTANCE_DISCOVERY_I POC_ACCEPTANCE_IMPORT_RESULT="${POC_ACCEPTANCE_IMPORT_RESULT:-true}" POC_ACCEPTANCE_IMPORT_API_BASE_URL="${POC_ACCEPTANCE_IMPORT_API_BASE_URL:-${API_BASE_URL:-${BASE_URL:-}}}" SCRIPT_PATH="${POC_OPERATOR_ACCEPTANCE_SCRIPT_PATH:-$ROOT_DIR/scripts/poc_operator_full_acceptance_job.py}" +POC_ACCEPTANCE_SOURCE_COMMIT="${POC_ACCEPTANCE_SOURCE_COMMIT:-${DROBOTICFLOW_BUILD_COMMIT:-}}" +if [[ -z "$POC_ACCEPTANCE_SOURCE_COMMIT" ]]; then + POC_ACCEPTANCE_SOURCE_COMMIT="$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || true)" +fi +POC_ACCEPTANCE_SOURCE_COMMIT="${POC_ACCEPTANCE_SOURCE_COMMIT:-unknown}" +catalog_metadata="$({ python3 - "$ROOT_DIR/internal/api/builtin_operators_catalog.json" <<'PY' +from __future__ import annotations + +import hashlib +import json +import pathlib +import sys + +catalog_path = pathlib.Path(sys.argv[1]) +catalog = json.loads(catalog_path.read_text(encoding="utf-8")) +if not isinstance(catalog, list) or not catalog: + raise SystemExit("builtin operator catalog must be a non-empty JSON list") +canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":")) +for raw, escaped in ( + ("&", "\\u0026"), + ("<", "\\u003c"), + (">", "\\u003e"), + ("\u2028", "\\u2028"), + ("\u2029", "\\u2029"), +): + canonical_text = canonical_text.replace(raw, escaped) +canonical = canonical_text.encode("utf-8") +print(f"{hashlib.sha256(canonical).hexdigest()}\t{len(catalog)}") +PY +} 2>&1)" || { + echo "ERROR: failed to compute builtin operator catalog provenance: $catalog_metadata" >&2 + exit 1 +} +IFS=$'\t' read -r POC_ACCEPTANCE_CATALOG_HASH POC_ACCEPTANCE_CATALOG_COUNT <<< "$catalog_metadata" +export POC_ACCEPTANCE_SOURCE_COMMIT +export POC_ACCEPTANCE_CATALOG_HASH +export POC_ACCEPTANCE_CATALOG_COUNT validate_dns_label() { local name="$1" @@ -301,6 +338,7 @@ from __future__ import annotations import datetime import json +import os import pathlib import sys from typing import Any @@ -443,6 +481,10 @@ payload = { "status": "failed", "startedAt": now, "endedAt": now, + "sourceCommit": os.environ["POC_ACCEPTANCE_SOURCE_COMMIT"], + "catalogHash": os.environ["POC_ACCEPTANCE_CATALOG_HASH"], + "catalogCount": int(os.environ["POC_ACCEPTANCE_CATALOG_COUNT"]), + "completedAt": now, } print("DROBOTICFLOW_POC_OPERATOR_ACCEPTANCE=" + json.dumps(payload, ensure_ascii=False, sort_keys=True)) PY @@ -605,6 +647,9 @@ segment_duration_json="$(json_string "${POC_ACCEPTANCE_SEGMENT_DURATION_SECONDS: frame_interval_json="$(json_string "${POC_ACCEPTANCE_FRAME_INTERVAL_SECONDS:-}")" audit_rules_json="$(json_string "${POC_ACCEPTANCE_AUDIT_RULES_JSON:-}")" shell_command_json="$(json_string "${POC_ACCEPTANCE_SHELL_COMMAND:-}")" +source_commit_json="$(json_string "$POC_ACCEPTANCE_SOURCE_COMMIT")" +catalog_hash_json="$(json_string "$POC_ACCEPTANCE_CATALOG_HASH")" +catalog_count_json="$(json_string "$POC_ACCEPTANCE_CATALOG_COUNT")" image_pull_secrets_yaml="" if [[ -n "$K8S_IMAGE_PULL_SECRET_NAME" ]]; then image_pull_secret_json="$(json_string "$K8S_IMAGE_PULL_SECRET_NAME")" @@ -647,6 +692,12 @@ $image_pull_secrets_yaml env: - name: POC_ACCEPTANCE_RUN_ID value: $job_name_json + - name: POC_ACCEPTANCE_SOURCE_COMMIT + value: $source_commit_json + - name: POC_ACCEPTANCE_CATALOG_HASH + value: $catalog_hash_json + - name: POC_ACCEPTANCE_CATALOG_COUNT + value: $catalog_count_json - name: WORKSPACE_MOUNT_PATH value: $workspace_mount_json - name: POC_ACCEPTANCE_REQUIRED_EXTERNALS diff --git a/scripts/poc_operator_full_acceptance_job.py b/scripts/poc_operator_full_acceptance_job.py index 1dff8e37..e5654891 100755 --- a/scripts/poc_operator_full_acceptance_job.py +++ b/scripts/poc_operator_full_acceptance_job.py @@ -2,6 +2,7 @@ from __future__ import annotations import datetime as dt +import hashlib import importlib.util import json import os @@ -112,6 +113,94 @@ def now_id() -> str: return dt.datetime.now(dt.timezone.utc).strftime("%Y%m%d%H%M%S") +def now_iso() -> str: + return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") + + +def source_commit() -> str: + for env_name in ("POC_ACCEPTANCE_SOURCE_COMMIT", "DROBOTICFLOW_BUILD_COMMIT"): + value = os.getenv(env_name, "").strip() + if value: + return value + project_root = pathlib.Path(__file__).resolve().parents[1] + try: + return subprocess.run( + ["git", "-C", str(project_root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() or "unknown" + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def builtin_operator_catalog_metadata() -> tuple[str, int]: + injected_hash = os.getenv("POC_ACCEPTANCE_CATALOG_HASH", "").strip().lower() + injected_count = os.getenv("POC_ACCEPTANCE_CATALOG_COUNT", "").strip() + if injected_hash and injected_count: + if len(injected_hash) != 64 or any(char not in "0123456789abcdef" for char in injected_hash): + raise AcceptanceFailure("POC_ACCEPTANCE_CATALOG_HASH must be a lowercase SHA-256 digest") + try: + catalog_count = int(injected_count) + except ValueError as error: + raise AcceptanceFailure("POC_ACCEPTANCE_CATALOG_COUNT must be an integer") from error + if catalog_count <= 0: + raise AcceptanceFailure("POC_ACCEPTANCE_CATALOG_COUNT must be positive") + return injected_hash, catalog_count + + candidates = [ + pathlib.Path(os.getenv("POC_ACCEPTANCE_CATALOG_PATH", "")), + pathlib.Path("/app/internal/api/builtin_operators_catalog.json"), + pathlib.Path(__file__).resolve().parents[1] / "internal" / "api" / "builtin_operators_catalog.json", + ] + catalog_path = next((path for path in candidates if str(path) and path.is_file()), None) + if catalog_path is None: + raise AcceptanceFailure("builtin operator catalog is not available for acceptance provenance") + try: + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AcceptanceFailure(f"builtin operator catalog is not readable: {catalog_path}") from error + if not isinstance(catalog, list) or not catalog: + raise AcceptanceFailure("builtin operator catalog must be a non-empty JSON list") + canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + # Go's encoding/json keeps UTF-8 but escapes HTML-sensitive and line-separator runes. + for raw, escaped in ( + ("&", "\\u0026"), + ("<", "\\u003c"), + (">", "\\u003e"), + ("\u2028", "\\u2028"), + ("\u2029", "\\u2029"), + ): + canonical_text = canonical_text.replace(raw, escaped) + canonical = canonical_text.encode("utf-8") + return hashlib.sha256(canonical).hexdigest(), len(catalog) + + +def acceptance_provenance(completed_at: str | None = None, *, tolerate_errors: bool = False) -> dict[str, Any]: + try: + catalog_hash, catalog_count = builtin_operator_catalog_metadata() + except Exception as error: + if not tolerate_errors: + raise + catalog_hash = os.getenv("POC_ACCEPTANCE_CATALOG_HASH", "").strip().lower() + try: + catalog_count = int(os.getenv("POC_ACCEPTANCE_CATALOG_COUNT", "0")) + except ValueError: + catalog_count = 0 + provenance_error = f"{type(error).__name__}: {error}" + else: + provenance_error = "" + payload = { + "sourceCommit": source_commit(), + "catalogHash": catalog_hash, + "catalogCount": catalog_count, + "completedAt": completed_at or now_iso(), + } + if provenance_error: + payload["provenanceError"] = provenance_error + return payload + + def json_value(value: Any) -> str: if isinstance(value, str): return value @@ -1553,6 +1642,7 @@ def run_acceptance() -> dict[str, Any]: "records": records, "failures": failures, "status": "passed" if not failures else "failed", + **acceptance_provenance(), } write_json(base_dir / "poc-operator-full-acceptance-result.json", summary) return summary @@ -1566,6 +1656,7 @@ def main() -> int: "status": "failed", "fatalError": f"{type(error).__name__}: {error}", "records": [], + **acceptance_provenance(tolerate_errors=True), } print(RESULT_PREFIX + json.dumps(MODULE.json_compatible(summary), ensure_ascii=False), flush=True) return 1 diff --git a/scripts/poc_strict_preflight_result.py b/scripts/poc_strict_preflight_result.py index 767eb98b..35817a42 100644 --- a/scripts/poc_strict_preflight_result.py +++ b/scripts/poc_strict_preflight_result.py @@ -4,10 +4,12 @@ from __future__ import annotations import argparse import ast import datetime as dt +import hashlib import json import os import pathlib import re +import subprocess from typing import Any @@ -50,6 +52,54 @@ def now_iso() -> str: return dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z") +def source_commit() -> str: + for env_name in ("POC_ACCEPTANCE_SOURCE_COMMIT", "DROBOTICFLOW_BUILD_COMMIT"): + value = os.getenv(env_name, "").strip() + if value: + return value + try: + return subprocess.run( + ["git", "-C", str(ROOT), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() or "unknown" + except (OSError, subprocess.CalledProcessError): + return "unknown" + + +def builtin_operator_catalog_metadata() -> tuple[str, int]: + catalog_path = ROOT / "internal" / "api" / "builtin_operators_catalog.json" + try: + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise SystemExit(f"builtin operator catalog is not readable: {catalog_path}") from error + if not isinstance(catalog, list) or not catalog: + raise SystemExit("builtin operator catalog must be a non-empty JSON list") + canonical_text = json.dumps(catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + # Go's encoding/json keeps UTF-8 but escapes HTML-sensitive and line-separator runes. + for raw, escaped in ( + ("&", "\\u0026"), + ("<", "\\u003c"), + (">", "\\u003e"), + ("\u2028", "\\u2028"), + ("\u2029", "\\u2029"), + ): + canonical_text = canonical_text.replace(raw, escaped) + canonical = canonical_text.encode("utf-8") + return hashlib.sha256(canonical).hexdigest(), len(catalog) + + +def acceptance_provenance(completed_at: str | None = None) -> dict[str, Any]: + catalog_hash, catalog_count = builtin_operator_catalog_metadata() + return { + "sourceCommit": source_commit(), + "catalogHash": catalog_hash, + "catalogCount": catalog_count, + "completedAt": completed_at or now_iso(), + } + + def required_externals(raw: str) -> list[str]: return sorted({item.strip().lower() for item in raw.split(",") if item.strip()}) @@ -476,6 +526,7 @@ def build_result(args: argparse.Namespace) -> dict[str, Any]: "namespace": namespace, "startedAt": started_at, "endedAt": started_at, + **acceptance_provenance(started_at), } if fixture_sources: result["fixtureSources"] = fixture_sources diff --git a/scripts/render-manifests.sh b/scripts/render-manifests.sh index a0a10074..39b8986f 100755 --- a/scripts/render-manifests.sh +++ b/scripts/render-manifests.sh @@ -30,6 +30,13 @@ API_SERVICE_TYPE="${API_SERVICE_TYPE:-NodePort}" API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-false}" API_REPLICAS="${API_REPLICAS:-2}" WORKER_REPLICAS="${WORKER_REPLICAS:-2}" +DROBOTICFLOW_BUILD_COMMIT="${DROBOTICFLOW_BUILD_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}" +DROBOTICFLOW_BUILD_TIME="${DROBOTICFLOW_BUILD_TIME:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" +WORKER_ORPHAN_CLEANUP_ENABLED="${WORKER_ORPHAN_CLEANUP_ENABLED:-false}" +WORKER_ORPHAN_MIN_AGE_SECONDS="${WORKER_ORPHAN_MIN_AGE_SECONDS:-86400}" +WORKER_ORPHAN_CHECK_INTERVAL_SECONDS="${WORKER_ORPHAN_CHECK_INTERVAL_SECONDS:-300}" +WORKER_PENDING_FAILURE_SECONDS="${WORKER_PENDING_FAILURE_SECONDS:-600}" +WORKER_MISSING_JOB_MAX_RECOVERIES="${WORKER_MISSING_JOB_MAX_RECOVERIES:-3}" AI_API_BASE_URL="${AI_API_BASE_URL:-}" AI_MODEL="${AI_MODEL:-}" AI_API_KEY="${AI_API_KEY:-}" @@ -104,6 +111,13 @@ export \ API_NODE_PORT_BLOCK \ API_REPLICAS \ WORKER_REPLICAS \ + DROBOTICFLOW_BUILD_COMMIT \ + DROBOTICFLOW_BUILD_TIME \ + WORKER_ORPHAN_CLEANUP_ENABLED \ + WORKER_ORPHAN_MIN_AGE_SECONDS \ + WORKER_ORPHAN_CHECK_INTERVAL_SECONDS \ + WORKER_PENDING_FAILURE_SECONDS \ + WORKER_MISSING_JOB_MAX_RECOVERIES \ AI_API_BASE_URL \ AI_MODEL \ API_IMAGE \ @@ -146,6 +160,13 @@ replacements = { "__API_NODE_PORT_BLOCK__": os.environ["API_NODE_PORT_BLOCK"], "__API_REPLICAS__": os.environ["API_REPLICAS"], "__WORKER_REPLICAS__": os.environ["WORKER_REPLICAS"], + "__DROBOTICFLOW_BUILD_COMMIT__": os.environ["DROBOTICFLOW_BUILD_COMMIT"], + "__DROBOTICFLOW_BUILD_TIME__": os.environ["DROBOTICFLOW_BUILD_TIME"], + "__WORKER_ORPHAN_CLEANUP_ENABLED__": os.environ["WORKER_ORPHAN_CLEANUP_ENABLED"], + "__WORKER_ORPHAN_MIN_AGE_SECONDS__": os.environ["WORKER_ORPHAN_MIN_AGE_SECONDS"], + "__WORKER_ORPHAN_CHECK_INTERVAL_SECONDS__": os.environ["WORKER_ORPHAN_CHECK_INTERVAL_SECONDS"], + "__WORKER_PENDING_FAILURE_SECONDS__": os.environ["WORKER_PENDING_FAILURE_SECONDS"], + "__WORKER_MISSING_JOB_MAX_RECOVERIES__": os.environ["WORKER_MISSING_JOB_MAX_RECOVERIES"], "__AI_API_BASE_URL__": os.environ["AI_API_BASE_URL"], "__AI_MODEL__": os.environ["AI_MODEL"], "__API_IMAGE__": os.environ["API_IMAGE"], diff --git a/scripts/update_operator_release_manifests.py b/scripts/update_operator_release_manifests.py index 1447f9b9..507d455b 100644 --- a/scripts/update_operator_release_manifests.py +++ b/scripts/update_operator_release_manifests.py @@ -46,6 +46,7 @@ def update_single(args: argparse.Namespace, env: str) -> None: digest = image_digest(image_id) release.update( { + "operatorId": f"operator-builtin-{slug.replace('_', '-')}", "sourceCommit": args.source_commit.strip() if args.source_commit else release.get("sourceCommit", ""), "sourceTreeSha256": source_tree_sha256(slug), "image": image, @@ -111,4 +112,3 @@ def main() -> int: if __name__ == "__main__": raise SystemExit(main()) - diff --git a/src/components/ArtifactPreviewPanel.tsx b/src/components/ArtifactPreviewPanel.tsx index 5208b2fb..f08563bc 100644 --- a/src/components/ArtifactPreviewPanel.tsx +++ b/src/components/ArtifactPreviewPanel.tsx @@ -236,7 +236,13 @@ function FileSummaryPreview({ item }: { item: ExecutionData }) { ); } -export function ArtifactPreviewPanel({ item, fullscreen = false }: { item: ExecutionData | null; fullscreen?: boolean }) { +export function ArtifactPreviewPanel({ + item, + fullscreen = false, +}: { + item: ExecutionData | null; + fullscreen?: boolean; +}) { const preview = item ? derivedPreview(item) : null; const previewURL = item && preview ? artifactPreviewURL(item, preview) : ""; const playablePreviewURL = preview?.kind === "video" ? buildBrowserPlayableVideoURL(previewURL) : previewURL; @@ -253,7 +259,12 @@ export function ArtifactPreviewPanel({ item, fullscreen = false }: { item: Execu if (preview.kind === "image" && previewURL) { return ( -
+
@@ -268,7 +279,12 @@ export function ArtifactPreviewPanel({ item, fullscreen = false }: { item: Execu if (preview.kind === "video" && playablePreviewURL) { return ( -
+
@@ -305,7 +321,12 @@ export function ArtifactPreviewPanel({ item, fullscreen = false }: { item: Execu if (preview.kind === "audio" && previewURL) { return ( -
+
@@ -316,11 +337,20 @@ export function ArtifactPreviewPanel({ item, fullscreen = false }: { item: Execu if (preview.kind === "document" && preview.format === "pdf" && previewURL) { return ( -
+
-