droboticflow-new/internal/api/poc_operator_workflow_acceptance.go

1152 lines
49 KiB
Go

package api
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path"
"regexp"
"sort"
"strings"
"droboticflow/internal/domain"
)
const pocOperatorWorkflowAcceptanceProjectNamePrefix = "生产级算子全链路验收项目"
const pocOperatorWorkflowAcceptanceMaxConcurrentRuns = 1
type pocOperatorWorkflowAcceptanceRequest struct {
FixtureManifestPath string `json:"fixtureManifestPath"`
IncludeShellCommand bool `json:"includeShellCommand"`
PreflightOnly bool `json:"preflightOnly"`
}
type pocOperatorWorkflowAcceptanceRun struct {
Batch string `json:"batch"`
WorkflowID string `json:"workflowId"`
Workflow *domain.WorkflowRecord `json:"workflow"`
RunID string `json:"runId"`
Run domain.RunRecord `json:"run"`
OperatorIDs []string `json:"operatorIds"`
ImageEvidence []pocOperatorWorkflowImageEvidence `json:"imageEvidence"`
}
type pocOperatorWorkflowImageEvidence struct {
OperatorID string `json:"operatorId"`
Image string `json:"image"`
ImageID string `json:"imageId"`
ContainerCommand []string `json:"containerCommand"`
ContainerArgs []string `json:"containerArgs"`
RequiresGPU bool `json:"requiresGpu"`
}
type pocOperatorWorkflowAcceptanceSummary struct {
ContractVersion string `json:"contractVersion"`
OperatorCount int `json:"operatorCount"`
CatalogHash string `json:"catalogHash"`
CatalogCount int `json:"catalogCount"`
IndependentImageCoverage int `json:"independentImageCoverage"`
IndependentImageRequired int `json:"independentImageRequired"`
CommandWrapperOnlyOperatorCount int `json:"commandWrapperOnlyOperatorCount"`
GeneratedFixtureCount int `json:"generatedFixtureCount"`
MockFixtureCount int `json:"mockFixtureCount"`
GPUOperatorCount int `json:"gpuOperatorCount"`
StrictReady bool `json:"strictReady"`
TrainingMode string `json:"trainingMode"`
FullTrainingRequired bool `json:"fullTrainingRequired"`
MissingOperatorIDs []string `json:"missingOperatorIds"`
}
type pocOperatorWorkflowAcceptanceResponse struct {
Project *domain.ProjectRecord `json:"project"`
FixtureManifestPath string `json:"fixtureManifestPath"`
FixtureHash string `json:"fixtureHash"`
AcceptanceKey string `json:"acceptanceKey"`
PreflightOnly bool `json:"preflightOnly"`
OperatorCount int `json:"operatorCount"`
Summary pocOperatorWorkflowAcceptanceSummary `json:"summary"`
Runs []pocOperatorWorkflowAcceptanceRun `json:"runs"`
Batches map[string][]string `json:"batches"`
}
type pocOperatorAcceptanceFixturePaths struct {
Manifest string
Root string
XPlatform string
CSV string
JSON string
Training string
LeRobot string
ROSBag string
Model string
YoloONNX string
Shell string
Extra map[string]string
}
type pocOperatorFixtureManifest struct {
Fixtures map[string]string `json:"fixtures"`
ShellCommand string `json:"shellCommand"`
}
func (server *server) defaultPocOperatorFixtureManifestPath() string {
return path.Join(cleanWorkspaceMountPath(server.workspaceMountPath), "datasets", "fixtures", "poc-operator-acceptance", "poc-operator-acceptance.json")
}
func validatePocOperatorFixtureManifestPath(value string, workspaceMountPath string) (string, *validationIssue) {
manifestPath := strings.TrimSpace(value)
if manifestPath == "" {
return "", &validationIssue{Field: "fixtureManifestPath", Message: "fixtureManifestPath is required"}
}
if hasControlCharacter(manifestPath) {
return "", &validationIssue{Field: "fixtureManifestPath", Message: "fixtureManifestPath must not contain control characters"}
}
if !strings.HasPrefix(manifestPath, "/") {
return "", &validationIssue{Field: "fixtureManifestPath", Message: "fixtureManifestPath must be an absolute path inside the workspace mount"}
}
cleaned := path.Clean(manifestPath)
workspaceMount := cleanWorkspaceMountPath(workspaceMountPath)
if cleaned != workspaceMount && !strings.HasPrefix(cleaned, workspaceMount+"/") {
return "", &validationIssue{Field: "fixtureManifestPath", Message: "fixtureManifestPath must stay inside the configured workspace mount"}
}
return cleaned, nil
}
func pocOperatorAcceptanceFixturePathsFromManifest(manifestPath string) pocOperatorAcceptanceFixturePaths {
root := path.Dir(path.Clean(manifestPath))
return pocOperatorAcceptanceFixturePaths{
Manifest: manifestPath,
Root: root,
XPlatform: path.Join(root, "xplatform"),
CSV: path.Join(root, "csv", "records.csv"),
JSON: path.Join(root, "json", "records.json"),
Training: path.Join(root, "json", "training-records.json"),
LeRobot: path.Join(root, "lerobot"),
ROSBag: path.Join(root, "rosbag"),
Model: path.Join(root, "model", "quality-gate.json"),
YoloONNX: path.Join(root, "yolo", "models", "yolov8n.onnx"),
Extra: map[string]string{},
}
}
func includePocOperatorInWorkflowAcceptance(operator domain.OperatorRecord, includeShellCommand bool) bool {
_ = includeShellCommand
if !strings.EqualFold(strings.TrimSpace(operator.Type), "builtin") {
return false
}
return true
}
func loadPocOperatorAcceptanceFixturePaths(manifestPath string) (pocOperatorAcceptanceFixturePaths, error) {
fixtures := pocOperatorAcceptanceFixturePathsFromManifest(manifestPath)
raw, err := os.ReadFile(manifestPath)
if err != nil {
return fixtures, fmt.Errorf("fixture manifest %s is not readable: %w", manifestPath, err)
}
manifest := pocOperatorFixtureManifest{}
if err := json.Unmarshal(raw, &manifest); err != nil {
return fixtures, fmt.Errorf("fixture manifest %s is invalid JSON: %w", manifestPath, err)
}
if len(manifest.Fixtures) == 0 {
return fixtures, fmt.Errorf("fixture manifest %s must declare fixtures", manifestPath)
}
explicitTraining := false
for key, value := range manifest.Fixtures {
normalizedKey := strings.ToLower(strings.TrimSpace(key))
cleanedValue := path.Clean(strings.TrimSpace(value))
if normalizedKey == "" || !strings.HasPrefix(cleanedValue, "/") {
continue
}
switch normalizedKey {
case "xplatform":
fixtures.XPlatform = cleanedValue
case "csv":
fixtures.CSV = cleanedValue
case "json":
fixtures.JSON = cleanedValue
case "training", "training_json", "policy_training", "policy_training_json", "vla_training", "vla_training_json", "openvla_training", "openvla_training_json":
fixtures.Training = cleanedValue
explicitTraining = true
case "lerobot":
fixtures.LeRobot = cleanedValue
case "rosbag":
fixtures.ROSBag = cleanedValue
case "model":
fixtures.Model = cleanedValue
case "yolo_onnx", "yolo_onnx_model", "yolo_model", "dnn_model", "dnn_model_path":
fixtures.YoloONNX = cleanedValue
default:
fixtures.Extra[normalizedKey] = cleanedValue
}
}
if !explicitTraining || strings.TrimSpace(fixtures.Training) == "" {
fixtures.Training = fixtures.JSON
}
fixtures.Shell = strings.TrimSpace(manifest.ShellCommand)
return fixtures, nil
}
func validatePocOperatorAcceptanceFixturePath(value string, workspaceMountPath string, expectedDirectory bool) error {
cleaned := path.Clean(strings.TrimSpace(value))
if cleaned == "." || !strings.HasPrefix(cleaned, "/") {
return fmt.Errorf("fixture path must be an absolute path: %q", value)
}
workspaceMount := cleanWorkspaceMountPath(workspaceMountPath)
if cleaned != workspaceMount && !strings.HasPrefix(cleaned, workspaceMount+"/") {
return fmt.Errorf("fixture path must stay inside %s: %s", workspaceMount, cleaned)
}
if hasControlCharacter(cleaned) {
return fmt.Errorf("fixture path contains control characters: %s", cleaned)
}
info, err := os.Stat(cleaned)
if err != nil {
return fmt.Errorf("fixture path is not readable: %s: %w", cleaned, err)
}
if expectedDirectory && !info.IsDir() {
return fmt.Errorf("fixture path must be a directory: %s", cleaned)
}
if !expectedDirectory && !info.Mode().IsRegular() {
return fmt.Errorf("fixture path must be a regular file: %s", cleaned)
}
return nil
}
func validatePocOperatorAcceptanceFixtures(manifestPath string, workspaceMountPath string) (pocOperatorAcceptanceFixturePaths, string, error) {
fixtures, err := loadPocOperatorAcceptanceFixturePaths(manifestPath)
if err != nil {
return fixtures, "", err
}
required := []struct {
name string
value string
directory bool
}{
{name: "xplatform", value: fixtures.XPlatform, directory: true},
{name: "csv", value: fixtures.CSV},
{name: "json", value: fixtures.JSON},
{name: "lerobot", value: fixtures.LeRobot, directory: true},
{name: "rosbag", value: fixtures.ROSBag, directory: true},
{name: "model", value: fixtures.Model},
}
for _, item := range required {
if strings.TrimSpace(item.value) == "" {
return fixtures, "", fmt.Errorf("fixture %s is required", item.name)
}
if err := validatePocOperatorAcceptanceFixturePath(item.value, workspaceMountPath, item.directory); err != nil {
return fixtures, "", fmt.Errorf("fixture %s: %w", item.name, err)
}
}
if strings.TrimSpace(fixtures.Training) != "" {
if err := validatePocOperatorAcceptanceFixturePath(fixtures.Training, workspaceMountPath, false); err != nil {
return fixtures, "", fmt.Errorf("fixture training: %w", err)
}
}
raw, err := os.ReadFile(manifestPath)
if err != nil {
return fixtures, "", fmt.Errorf("fixture manifest %s is not readable: %w", manifestPath, err)
}
hash := sha256.Sum256(raw)
return fixtures, hex.EncodeToString(hash[:]), nil
}
func (fixtures pocOperatorAcceptanceFixturePaths) pathFor(keys ...string) string {
for _, key := range keys {
normalizedKey := strings.ToLower(strings.TrimSpace(key))
if normalizedKey == "" {
continue
}
if value := strings.TrimSpace(fixtures.Extra[normalizedKey]); value != "" {
return value
}
}
return ""
}
func (server *server) ensurePocOperatorWorkflowAcceptanceProject(ctx context.Context, organizationID string) (*domain.ProjectRecord, error) {
projectName := pocOperatorWorkflowAcceptanceProjectNamePrefix
projects, err := server.store.ListProjectsByOrganization(ctx, organizationID)
if err != nil {
return nil, err
}
for index := range projects {
project := projects[index]
if strings.TrimSpace(project.Name) == projectName {
return &project, nil
}
}
return server.store.CreateProject(ctx, domain.CreateProjectInput{
OrganizationID: organizationID,
Name: projectName,
Description: "中文生产级算子全链路验收项目,使用真实账号、真实 fixture、独立 CCR 镜像和 lineage 证据。",
})
}
func operatorNodeID(operator domain.OperatorRecord) string {
id := strings.TrimSpace(operator.ID)
if id == "" {
id = strings.TrimSpace(operator.Name)
}
id = strings.TrimPrefix(id, "operator-builtin-")
id = regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(strings.ToLower(id), "-")
id = strings.Trim(id, "-")
if id == "" {
id = "operator"
}
if len(id) > 48 {
id = id[:48]
}
return "op-" + id
}
func cloneOperatorHyperparams(operator domain.OperatorRecord) []domain.HyperParam {
items := make([]domain.HyperParam, 0, len(operator.Hyperparams))
for _, item := range operator.Hyperparams {
items = append(items, item)
}
return items
}
func setHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam {
normalized := strings.ToUpper(strings.TrimSpace(key))
for index := range items {
if strings.ToUpper(strings.TrimSpace(items[index].Key)) == normalized {
items[index].Value = value
return items
}
}
return append(items, domain.HyperParam{Key: key, Value: value})
}
func configurePocOperatorHyperparams(operator domain.OperatorRecord, fixtures pocOperatorAcceptanceFixturePaths) []domain.HyperParam {
items := cloneOperatorHyperparams(operator)
handler := pocOperatorHandlerKey(operator)
name := strings.TrimSpace(operator.Name)
setSource := func(value string) {
items = setHyperparam(items, "SOURCE_PATH", value)
}
switch handler {
case "source", "builtin_directory_scan", "builtin_xplatform_index":
setSource(fixtures.XPlatform)
case "builtin_csv_to_json":
setSource(fixtures.CSV)
case "builtin_model_inference":
items = setHyperparam(items, "MODEL_PATH", fixtures.Model)
items = setHyperparam(items, "INPUT_JSON", fmt.Sprintf(`{"items":[{"id":"acceptance-a","score":0.95,"quality":{"clarity":0.95,"shake":0.01}},{"id":"acceptance-b","score":0.42,"quality":{"clarity":0.42,"shake":0.2}}]}`))
case "builtin_vla_model_train":
setSource(firstNonEmptyText(fixtures.pathFor("training_json", "policy_training_json", "vla_training_json"), fixtures.Training, fixtures.JSON))
items = setHyperparam(items, "MIN_SAMPLES", "128")
case "builtin_act_policy_train", "builtin_diffusion_policy_train":
items = setHyperparam(items, "DATASET_PATH", firstNonEmptyText(fixtures.pathFor("training_json", "policy_training_json", "vla_training_json"), fixtures.Training, fixtures.JSON))
items = setHyperparam(items, "STATE_FIELD", "state")
items = setHyperparam(items, "ACTION_FIELD", "action")
items = setHyperparam(items, "EPOCHS", "1")
items = setHyperparam(items, "BATCH_SIZE", "16")
case "builtin_openvla_finetune":
items = setHyperparam(items, "BASE_MODEL", "distilgpt2")
items = setHyperparam(items, "DATASET_PATH", firstNonEmptyText(fixtures.pathFor("openvla_training_json", "training_json", "vla_training_json"), fixtures.Training, fixtures.JSON))
items = setHyperparam(items, "TEXT_FIELD", "instruction")
items = setHyperparam(items, "TARGET_FIELD", "action")
items = setHyperparam(items, "MAX_LENGTH", "64")
items = setHyperparam(items, "EPOCHS", "1")
items = setHyperparam(items, "BATCH_SIZE", "4")
case "builtin_openvla_inference":
items = setHyperparam(items, "BASE_MODEL", "distilgpt2")
items = setHyperparam(items, "PROMPT", "pick the visible target")
items = setHyperparam(items, "MAX_NEW_TOKENS", "16")
case "builtin_yolo_embodied_detection_train":
if value := firstNonEmptyText(
fixtures.pathFor("yolo", "yolo_data_yaml", "yoloDataYaml", "coco128"),
path.Join(fixtures.Root, "yolo", "coco128", "data.yaml"),
); value != "" {
items = setHyperparam(items, "DATA_YAML", value)
}
items = setHyperparam(items, "WORKERS", "0")
case "builtin_json_extract":
items = setHyperparam(items, "FIELD_PATH", "items.0.label")
case "builtin_json_filter", "builtin_condition_router":
items = setHyperparam(items, "FIELD_PATH", "label")
items = setHyperparam(items, "OPERATOR", "eq")
items = setHyperparam(items, "EXPECTED", "keep")
case "builtin_json_map":
items = setHyperparam(items, "FIELD_MAP_JSON", `{"label":"decision","score":"confidence"}`)
case "builtin_text_template":
items = setHyperparam(items, "TEMPLATE_TEXT", "accepted={items.0.id}; decision={items.0.decision}; confidence={items.0.confidence}")
items = setHyperparam(items, "OUTPUT_NAME", "decision.txt")
case "builtin_collection_validate":
items = setHyperparam(items, "REQUIRE_INTRINSICS", "false")
case "builtin_video_quality":
items = setHyperparam(items, "MIN_CLARITY", "0")
items = setHyperparam(items, "MAX_SHAKE", "1")
items = setHyperparam(items, "MAX_HAND_OCCLUSION", "1")
case "builtin_dataset_filter":
items = setHyperparam(items, "MIN_CLARITY", "0")
items = setHyperparam(items, "MAX_SHAKE", "1")
items = setHyperparam(items, "MAX_HAND_OCCLUSION", "1")
case "builtin_dataset_split_manifest":
setSource(fixtures.JSON)
items = setHyperparam(items, "MIN_STEPS", "0")
case "builtin_video_slicing":
items = setHyperparam(items, "SEGMENT_DURATION_SECONDS", "1")
case "builtin_video_annotation":
items = setHyperparam(items, "FRAME_INTERVAL_SECONDS", "0.5")
case "builtin_video_target_clip":
items = setHyperparam(items, "DETECTION_MODE", "motion")
items = setHyperparam(items, "TARGET_LABELS", "motion")
items = setHyperparam(items, "FRAME_INTERVAL_SECONDS", "0.25")
items = setHyperparam(items, "MIN_CONFIDENCE", "0.05")
items = setHyperparam(items, "MIN_MOTION_AREA_RATIO", "0.001")
items = setHyperparam(items, "MIN_SEGMENT_SECONDS", "0.25")
items = setHyperparam(items, "MAX_CLIPS", "5")
case "builtin_yolo_video_target_clip":
items = setHyperparam(items, "DETECTION_MODE", "opencv-dnn")
items = setHyperparam(items, "DNN_MODEL_PATH", firstNonEmptyText(fixtures.pathFor("yolo_onnx_model", "yolo_onnx", "dnn_model_path"), fixtures.YoloONNX))
items = setHyperparam(items, "DNN_INPUT_SIZE", "640")
items = setHyperparam(items, "DNN_NMS_THRESHOLD", "0.45")
items = setHyperparam(items, "DNN_CLASS_NAMES", "person,bicycle,car,motorcycle,airplane,bus,train,truck,boat,traffic light,fire hydrant,stop sign,parking meter,bench,bird,cat,dog,horse,sheep,cow,elephant,bear,zebra,giraffe,backpack,umbrella,handbag,tie,suitcase,frisbee,skis,snowboard,sports ball,kite,baseball bat,baseball glove,skateboard,surfboard,tennis racket,bottle,wine glass,cup,fork,knife,spoon,bowl,banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,cake,chair,couch,potted plant,bed,dining table,toilet,tv,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,sink,refrigerator,book,clock,vase,scissors,teddy bear,hair drier,toothbrush")
items = setHyperparam(items, "TARGET_LABELS", "*")
items = setHyperparam(items, "FRAME_INTERVAL_SECONDS", "0.5")
items = setHyperparam(items, "MIN_CONFIDENCE", "0.05")
items = setHyperparam(items, "MIN_SEGMENT_SECONDS", "0.25")
items = setHyperparam(items, "MAX_CLIPS", "5")
case "builtin_audit_sampling":
items = setHyperparam(items, "RULES_JSON", `[{"id":"all-items","field":"id","operator":"exists","samplePercent":100}]`)
case "builtin_shell_command":
items = setHyperparam(items, "SCRIPT_ID", "fixture_report")
items = setHyperparam(items, "SCRIPT_ARGS_JSON", `{"acceptance":"生产级受控脚本验收"}`)
items = setHyperparam(items, "INPUT_PATHS_JSON", fmt.Sprintf(`[%q,%q]`, fixtures.Manifest, fixtures.JSON))
items = setHyperparam(items, "OUTPUT_CONTRACT_JSON", `{"artifact":"controlled-script-report.json","type":"json"}`)
case "builtin_mongodb_write":
items = setHyperparam(items, "MONGO_TARGET", "platform")
items = setHyperparam(items, "MONGO_COLLECTION", "operatorWorkflowAcceptanceWrites")
items = setHyperparam(items, "MONGO_MODE", "insert_many")
case "builtin_hf_dataset_probe":
items = setHyperparam(items, "DATASET", "lerobot/pusht")
items = setHyperparam(items, "ROW_LENGTH", "1")
items = setHyperparam(items, "INCLUDE_ROWS", "false")
items = setHyperparam(items, "INCLUDE_PARQUET", "false")
items = setHyperparam(items, "INCLUDE_STATISTICS", "false")
case "builtin_bos_incremental_scan":
items = setHyperparam(items, "DATA_SOURCE", "BOS")
items = setHyperparam(items, "SCAN_MODE", "INCREMENTAL")
items = setHyperparam(items, "INCREMENTAL_UNIT", "DIRECTORY")
case "builtin_graspnet_dataset_validate":
setSource(firstNonEmptyText(fixtures.pathFor("graspnet", "graspnet1billion"), path.Join(fixtures.Root, "graspnet")))
items = setHyperparam(items, "CAMERA", "kinect")
items = setHyperparam(items, "MAX_SCENES", "1")
items = setHyperparam(items, "MAX_FRAMES_PER_CAMERA", "1")
case "builtin_ho3d_dataset_validate":
setSource(firstNonEmptyText(fixtures.pathFor("ho3d", "h2o3d"), path.Join(fixtures.Root, "ho3d")))
items = setHyperparam(items, "MAX_SAMPLES", "1")
case "builtin_calvin_episode_validate":
setSource(firstNonEmptyText(fixtures.pathFor("calvin"), path.Join(fixtures.Root, "calvin")))
items = setHyperparam(items, "MAX_FILES", "1")
case "builtin_libero_hdf5_task_validate":
setSource(firstNonEmptyText(fixtures.pathFor("libero"), path.Join(fixtures.Root, "libero")))
items = setHyperparam(items, "MAX_FILES", "1")
items = setHyperparam(items, "MAX_DEMOS", "1")
case "builtin_maniskill_trajectory_validate":
setSource(firstNonEmptyText(fixtures.pathFor("maniskill"), path.Join(fixtures.Root, "maniskill")))
items = setHyperparam(items, "MAX_FILES", "1")
case "builtin_furniturebench_pkl_demonstration_validate":
setSource(firstNonEmptyText(fixtures.pathFor("furniturebench"), path.Join(fixtures.Root, "furniturebench")))
items = setHyperparam(items, "MAX_FILES", "1")
items = setHyperparam(items, "EXPECTED_ACTION_DIM", "8")
case "builtin_partnet_mobility_asset_validate":
setSource(firstNonEmptyText(fixtures.pathFor("partnet", "partnet-mobility"), path.Join(fixtures.Root, "partnet")))
items = setHyperparam(items, "MAX_OBJECTS", "1")
case "builtin_rh20t_scene_validate":
setSource(firstNonEmptyText(fixtures.pathFor("rh20t"), path.Join(fixtures.Root, "rh20t")))
items = setHyperparam(items, "MAX_SCENES", "1")
case "builtin_robocasa_hdf5_demonstration_validate":
setSource(firstNonEmptyText(fixtures.pathFor("robocasa"), path.Join(fixtures.Root, "robocasa")))
items = setHyperparam(items, "MAX_FILES", "1")
items = setHyperparam(items, "MAX_DEMOS", "1")
case "builtin_hdf5_demonstration_summary":
setSource(firstNonEmptyText(fixtures.pathFor("hdf5"), path.Join(fixtures.Root, "hdf5")))
items = setHyperparam(items, "MAX_DEMOS", "1")
case "builtin_umi_zarr_dataset_validate":
setSource(firstNonEmptyText(fixtures.pathFor("zarr", "umi_zarr", "umi"), path.Join(fixtures.Root, "zarr")))
items = setHyperparam(items, "MAX_ARRAYS", "8")
items = setHyperparam(items, "MAX_SAMPLE_ROWS", "8")
items = setHyperparam(items, "MAX_EPISODE_ENDS", "4")
case "builtin_zarr_trajectory_summary":
setSource(firstNonEmptyText(fixtures.pathFor("zarr", "umi_zarr", "umi"), path.Join(fixtures.Root, "zarr")))
items = setHyperparam(items, "MAX_ARRAYS", "8")
case "builtin_joint_limit_validate":
setSource(firstNonEmptyText(fixtures.pathFor("joint_limit", "jointlimit"), path.Join(fixtures.Root, "joint_limit")))
items = setHyperparam(items, "MAX_FILES", "1")
items = setHyperparam(items, "MAX_ROWS", "4")
case "builtin_mcap_topic_summary":
setSource(firstNonEmptyText(fixtures.pathFor("mcap"), path.Join(fixtures.Root, "mcap")))
items = setHyperparam(items, "MAX_MESSAGES", "10")
case "builtin_robot_model_asset_validate":
setSource(firstNonEmptyText(fixtures.pathFor("robot_model", "robotmodel"), path.Join(fixtures.Root, "robot_model")))
items = setHyperparam(items, "MAX_MODELS", "2")
case "builtin_mjcf_actuator_validate":
setSource(firstNonEmptyText(fixtures.pathFor("mjcf", "robot_model"), path.Join(fixtures.Root, "robot_model")))
items = setHyperparam(items, "MAX_MODELS", "1")
case "builtin_webdataset_shard_summary":
setSource(firstNonEmptyText(fixtures.pathFor("webdataset"), path.Join(fixtures.Root, "webdataset")))
items = setHyperparam(items, "MAX_SHARDS", "1")
case "builtin_pointcloud_summary":
setSource(firstNonEmptyText(fixtures.pathFor("pointcloud", "point_cloud"), path.Join(fixtures.Root, "pointcloud")))
items = setHyperparam(items, "MAX_FILES", "2")
items = setHyperparam(items, "MAX_SAMPLE_POINTS", "1000")
case "builtin_rlbench_dataset_validate":
setSource(firstNonEmptyText(fixtures.pathFor("rlbench"), path.Join(fixtures.Root, "rlbench")))
items = setHyperparam(items, "MAX_EPISODES", "1")
items = setHyperparam(items, "CAMERA_DIRS", "front_rgb,left_shoulder_rgb")
case "builtin_rgbd_frame_pair_validate":
setSource(firstNonEmptyText(fixtures.pathFor("rgbd", "rgb_depth"), path.Join(fixtures.Root, "rgbd")))
items = setHyperparam(items, "MAX_PAIRS", "2")
case "builtin_segmentation_mask_validate":
setSource(firstNonEmptyText(fixtures.pathFor("segmentation", "segmentation_mask", "mask"), path.Join(fixtures.Root, "segmentation")))
items = setHyperparam(items, "MAX_PAIRS", "2")
case "builtin_tfrecord_structural_validate":
setSource(firstNonEmptyText(fixtures.pathFor("tfrecord", "tfrecord_structural"), path.Join(fixtures.Root, "tfrecord")))
items = setHyperparam(items, "MAX_RECORDS", "10")
items = setHyperparam(items, "PREVIEW_RECORDS", "2")
case "builtin_lerobot_video_alignment_validate":
setSource(fixtures.LeRobot)
items = setHyperparam(items, "MAX_PARQUET_FILES", "1")
items = setHyperparam(items, "MAX_ROWS", "1")
items = setHyperparam(items, "MAX_VIDEOS", "1")
default:
if needsDexYCBFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("dexycb", "dex-ycb"), path.Join(fixtures.Root, "dexycb")))
} else if needsHO3DFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("ho3d", "h2o3d"), path.Join(fixtures.Root, "ho3d")))
} else if needsCALVINFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("calvin"), path.Join(fixtures.Root, "calvin")))
} else if needsLIBEROFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("libero"), path.Join(fixtures.Root, "libero")))
} else if needsManiSkillFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("maniskill"), path.Join(fixtures.Root, "maniskill")))
} else if needsFurnitureBenchFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("furniturebench"), path.Join(fixtures.Root, "furniturebench")))
} else if needsPartNetFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("partnet", "partnet-mobility"), path.Join(fixtures.Root, "partnet")))
} else if needsRH20TFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("rh20t"), path.Join(fixtures.Root, "rh20t")))
} else if needsRoboCasaFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("robocasa"), path.Join(fixtures.Root, "robocasa")))
} else if needsMJCFFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("mjcf", "robot_model"), path.Join(fixtures.Root, "robot_model")))
} else if needsRobotModelFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("robot_model", "robotmodel"), path.Join(fixtures.Root, "robot_model")))
} else if needsWebDatasetFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("webdataset"), path.Join(fixtures.Root, "webdataset")))
} else if needsPointCloudFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("pointcloud", "point_cloud"), path.Join(fixtures.Root, "pointcloud")))
} else if needsALOHAFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("aloha", "act_hdf5", "hdf5"), path.Join(fixtures.Root, "aloha")))
} else if needsBOPFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("bop"), path.Join(fixtures.Root, "bop")))
} else if needsGraspNetFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("graspnet", "graspnet1billion"), path.Join(fixtures.Root, "graspnet")))
} else if needsHDF5Fixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("hdf5"), path.Join(fixtures.Root, "hdf5")))
} else if needsZarrFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("zarr"), path.Join(fixtures.Root, "zarr")))
} else if needsRLBenchFixture(handler, name) {
setSource(firstNonEmptyText(fixtures.pathFor("rlbench"), path.Join(fixtures.Root, "rlbench")))
} else if needsLeRobotFixture(handler, name) {
setSource(fixtures.LeRobot)
} else if needsROSBagFixture(handler, name) {
setSource(fixtures.ROSBag)
} else if needsJSONFixture(handler, name) {
setSource(fixtures.JSON)
} else {
setSource(fixtures.XPlatform)
}
}
return items
}
func needsDexYCBFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "dexycb") || strings.Contains(value, "dex-ycb")
}
func needsHO3DFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "ho3d") || strings.Contains(value, "h2o3d") || strings.Contains(value, "ho-3d") || strings.Contains(value, "h2o-3d")
}
func needsCALVINFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "calvin")
}
func needsLIBEROFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "libero")
}
func needsManiSkillFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "maniskill")
}
func needsFurnitureBenchFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "furniturebench") || strings.Contains(value, "furniture bench")
}
func needsPartNetFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "partnet") || strings.Contains(value, "partnet-mobility")
}
func needsRH20TFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "rh20t")
}
func needsRoboCasaFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "robocasa")
}
func needsMJCFFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "mjcf")
}
func needsRobotModelFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "robot_model") || strings.Contains(value, "robot model") || strings.Contains(value, "机器人模型")
}
func needsWebDatasetFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "webdataset")
}
func needsPointCloudFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "pointcloud") || strings.Contains(value, "point cloud") || strings.Contains(value, "点云")
}
func needsALOHAFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "aloha") || strings.Contains(value, "act hdf5")
}
func needsBOPFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "bop") || strings.Contains(value, "6d姿态") || strings.Contains(value, "6d pose")
}
func needsGraspNetFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "graspnet") || strings.Contains(value, "grasp net") || strings.Contains(value, "抓取数据集")
}
func needsHDF5Fixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "hdf5") || strings.Contains(value, ".h5")
}
func needsZarrFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "zarr")
}
func needsRLBenchFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "rlbench")
}
func needsLeRobotFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "lerobot") || strings.Contains(value, "parquet") || strings.Contains(value, "agibot")
}
func needsROSBagFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
return strings.Contains(value, "rosbag") || strings.Contains(value, "mcap")
}
func needsJSONFixture(handler string, name string) bool {
value := strings.ToLower(handler + " " + name)
tokens := []string{
"rlds", "bridgedata", "droid", "libero", "robocasa", "furniturebench", "roboturk", "roboset",
"rh20t", "robomind", "hdf5", "robomimic", "aloha", "zarr", "rlbench", "maniskill",
"calvin", "rtx", "embodiment", "force", "trajectory", "action_state", "episode_boundary",
"rgbd", "segmentation", "language", "vla", "camera", "tf_transform", "joint", "graspnet",
"bop", "dexycb", "ho3d", "hand_object", "partnet", "mjcf", "robot_model", "pointcloud",
"sensor", "tfrecord", "coco", "yolo", "split", "media", "schema",
}
for _, token := range tokens {
if strings.Contains(value, token) {
return true
}
}
return false
}
func workflowNodeFromOperator(operator domain.OperatorRecord, fixtures pocOperatorAcceptanceFixturePaths, storageConnection *domain.StorageConnectionRecord, x float64, y float64) domain.WorkflowNode {
hyperparams := configurePocOperatorHyperparams(operator, fixtures)
connectionID := ""
bucket := ""
if pocOperatorHandlerKey(operator) == "builtin_bos_incremental_scan" && storageConnection != nil {
connectionID = storageConnection.ID
bucket = storageConnection.Bucket
if bucket != "" {
prefix := strings.TrimPrefix(strings.TrimPrefix(fixtures.XPlatform, fixtures.Root), "/")
acceptancePrefix := strings.Trim(path.Join("datasets", "fixtures", "poc-operator-acceptance", prefix), "/")
hyperparams = setHyperparam(hyperparams, "SOURCE_PATH", "bos://"+strings.Trim(bucket, "/")+"/"+acceptancePrefix)
}
}
return domain.WorkflowNode{
ID: operatorNodeID(operator),
Type: "operator",
Data: domain.WorkflowNodeData{
Label: operator.Name,
Category: operator.Category,
Description: operator.Description,
Inputs: clonePortDefinitions(operator.Inputs),
Outputs: clonePortDefinitions(operator.Outputs),
OperatorID: operator.ID,
OperatorVersion: operator.Version,
OperatorImageID: operator.ImageID,
Image: operator.Image,
ContainerCommand: cloneStringSlice(operator.ContainerCommand),
ContainerArgs: cloneStringSlice(operator.ContainerArgs),
RuntimeContract: cloneMap(operator.RuntimeContract),
Hyperparams: hyperparams,
ConnectionID: connectionID,
Bucket: bucket,
CPU: operatorResourceCPULimit(operator.Resources),
Memory: operatorResourceMemoryLimit(operator.Resources),
GPU: operatorResourceGPULimit(operator.Resources),
},
Position: domain.Position{X: x, Y: y},
}
}
func operatorResourceCPULimit(resources *domain.OperatorResources) string {
if resources == nil {
return ""
}
return strings.TrimSpace(resources.CPULimit)
}
func operatorResourceMemoryLimit(resources *domain.OperatorResources) string {
if resources == nil {
return ""
}
return strings.TrimSpace(resources.MemoryLimit)
}
func operatorResourceGPULimit(resources *domain.OperatorResources) string {
if resources == nil {
return ""
}
return strings.TrimSpace(resources.GPULimit)
}
func pocOperatorHasCommandWrapper(operator domain.OperatorRecord) bool {
if operator.ID == "operator-builtin-shell-command" {
return false
}
for _, param := range operator.Hyperparams {
key := strings.ToUpper(strings.TrimSpace(param.Key))
if key == "COMMAND" || key == "SCRIPT" {
return true
}
}
return false
}
func pocOperatorRequiresGPU(operator domain.OperatorRecord) bool {
return operator.Resources != nil && strings.TrimSpace(operator.Resources.GPULimit) != ""
}
func pocOperatorHasIndependentImage(operator domain.OperatorRecord) bool {
image := strings.TrimSpace(operator.Image)
imageID := strings.TrimSpace(operator.ImageID)
if image == "" || strings.Contains(image, "droboticflow-operator-runner") || !strings.Contains(image, "droboticflow-operator-") {
return false
}
if !strings.Contains(imageID, "@sha256:") {
return false
}
if len(operator.ContainerCommand) == 0 {
return false
}
return true
}
func buildPocOperatorWorkflowImageEvidence(operator domain.OperatorRecord) pocOperatorWorkflowImageEvidence {
return pocOperatorWorkflowImageEvidence{
OperatorID: operator.ID,
Image: strings.TrimSpace(operator.Image),
ImageID: strings.TrimSpace(operator.ImageID),
ContainerCommand: cloneStringSlice(operator.ContainerCommand),
ContainerArgs: cloneStringSlice(operator.ContainerArgs),
RequiresGPU: pocOperatorRequiresGPU(operator),
}
}
func buildPocOperatorWorkflowAcceptanceSummary(operators []domain.OperatorRecord) pocOperatorWorkflowAcceptanceSummary {
summary := pocOperatorWorkflowAcceptanceSummary{
ContractVersion: "poc-operator-workflows/v4",
OperatorCount: len(operators),
IndependentImageRequired: len(operators),
TrainingMode: "full",
FullTrainingRequired: true,
MissingOperatorIDs: []string{},
}
for _, operator := range operators {
if pocOperatorHasIndependentImage(operator) {
summary.IndependentImageCoverage++
}
if pocOperatorHasCommandWrapper(operator) {
summary.CommandWrapperOnlyOperatorCount++
}
if pocOperatorRequiresGPU(operator) {
summary.GPUOperatorCount++
}
}
summary.StrictReady = summary.OperatorCount > 0 &&
summary.IndependentImageCoverage == summary.IndependentImageRequired &&
summary.CommandWrapperOnlyOperatorCount == 0 &&
summary.GeneratedFixtureCount == 0 &&
summary.MockFixtureCount == 0
return summary
}
func clonePortDefinitions(items []domain.PortDefinition) []domain.PortDefinition {
if len(items) == 0 {
return nil
}
cloned := make([]domain.PortDefinition, 0, len(items))
for _, item := range items {
cloned = append(cloned, item)
}
return cloned
}
func cloneStringSlice(items []string) []string {
if len(items) == 0 {
return nil
}
cloned := make([]string, len(items))
copy(cloned, items)
return cloned
}
func cloneMap(item map[string]any) map[string]any {
if len(item) == 0 {
return nil
}
cloned := make(map[string]any, len(item))
for key, value := range item {
cloned[key] = value
}
return cloned
}
func groupPocOperatorAcceptanceBatches(operators []domain.OperatorRecord) map[string][]domain.OperatorRecord {
batches := map[string][]domain.OperatorRecord{
"数据接入与格式转换链路": {},
"具身数据集结构校验链路一": {},
"具身数据集结构校验链路二": {},
"具身数据集结构校验链路三": {},
"视频检测裁剪与成品链路": {},
"VLA训练推理评测链路": {},
"3D资产生成与预览链路": {},
"外部存储与审计链路": {},
"人工审核与抽检链路": {},
}
operatorByHandler := map[string]domain.OperatorRecord{}
for _, operator := range operators {
handler := pocOperatorHandlerKey(operator)
operatorByHandler[handler] = operator
switch {
case handler == "builtin_human_review":
batches["人工审核与抽检链路"] = append(batches["人工审核与抽检链路"], operator)
case handler == "builtin_audit_sampling":
batches["人工审核与抽检链路"] = append(batches["人工审核与抽检链路"], operator)
case handler == "builtin_vla_model_train" || handler == "builtin_vla_model_inference" || handler == "builtin_vla_model_evaluate" || handler == "builtin_act_policy_train" || handler == "builtin_diffusion_policy_train" || handler == "builtin_openvla_finetune" || handler == "builtin_openvla_inference":
batches["VLA训练推理评测链路"] = append(batches["VLA训练推理评测链路"], operator)
case handler == "builtin_sim3d_asset_generate":
batches["3D资产生成与预览链路"] = append(batches["3D资产生成与预览链路"], operator)
case strings.Contains(handler, "bos") || strings.Contains(handler, "mongodb") || strings.Contains(handler, "shell"):
batches["外部存储与审计链路"] = append(batches["外部存储与审计链路"], operator)
case operator.Category == "VIDEO" || isXPlatformOperator(operator):
batches["视频检测裁剪与成品链路"] = append(batches["视频检测裁剪与成品链路"], operator)
case operator.Category == "DATASET" || operator.Category == "ROBOTICS":
if len(batches["具身数据集结构校验链路一"]) <= len(batches["具身数据集结构校验链路二"]) && len(batches["具身数据集结构校验链路一"]) <= len(batches["具身数据集结构校验链路三"]) {
batches["具身数据集结构校验链路一"] = append(batches["具身数据集结构校验链路一"], operator)
} else if len(batches["具身数据集结构校验链路二"]) <= len(batches["具身数据集结构校验链路三"]) {
batches["具身数据集结构校验链路二"] = append(batches["具身数据集结构校验链路二"], operator)
} else {
batches["具身数据集结构校验链路三"] = append(batches["具身数据集结构校验链路三"], operator)
}
default:
batches["数据接入与格式转换链路"] = append(batches["数据接入与格式转换链路"], operator)
}
}
appendHumanReviewSupport := func(handler string) {
operator, ok := operatorByHandler[handler]
if !ok {
return
}
for _, existing := range batches["人工审核与抽检链路"] {
if existing.ID == operator.ID {
return
}
}
batches["人工审核与抽检链路"] = append(batches["人工审核与抽检链路"], operator)
}
if len(batches["人工审核与抽检链路"]) > 0 {
for _, handler := range []string{
"builtin_xplatform_index",
"builtin_collection_validate",
"builtin_video_quality",
"builtin_dataset_filter",
"builtin_video_slicing",
"builtin_video_annotation",
"builtin_video_target_clip",
"builtin_yolo_video_target_clip",
"builtin_result_package",
} {
appendHumanReviewSupport(handler)
}
}
for name := range batches {
sort.Slice(batches[name], func(i, j int) bool {
return batches[name][i].ID < batches[name][j].ID
})
}
return batches
}
func pocOperatorWorkflowEdge(source string, target string) domain.WorkflowEdge {
return domain.WorkflowEdge{
ID: "edge-" + strings.TrimPrefix(source, "op-") + "-to-" + strings.TrimPrefix(target, "op-"),
Source: source,
Target: target,
}
}
func buildPocOperatorSerialWorkflowEdges(operators []domain.OperatorRecord) []domain.WorkflowEdge {
edges := make([]domain.WorkflowEdge, 0, len(operators)-1)
for index := 1; index < len(operators); index++ {
edges = append(edges, pocOperatorWorkflowEdge(operatorNodeID(operators[index-1]), operatorNodeID(operators[index])))
}
return edges
}
func orderPocOperatorsByHandlers(operators []domain.OperatorRecord, handlers []string) []domain.OperatorRecord {
used := map[string]struct{}{}
ordered := make([]domain.OperatorRecord, 0, len(operators))
for _, handler := range handlers {
for _, operator := range operators {
if _, ok := used[operator.ID]; ok {
continue
}
if pocOperatorHandlerKey(operator) != handler {
continue
}
ordered = append(ordered, operator)
used[operator.ID] = struct{}{}
break
}
}
for _, operator := range operators {
if _, ok := used[operator.ID]; ok {
continue
}
ordered = append(ordered, operator)
used[operator.ID] = struct{}{}
}
return ordered
}
func orderPocOperatorsForBatch(batchName string, operators []domain.OperatorRecord) []domain.OperatorRecord {
switch batchName {
case "外部存储与审计链路":
return orderPocOperatorsByHandlers(operators, []string{
"builtin_shell_command",
"builtin_bos_incremental_scan",
"builtin_mongodb_write",
"builtin_dataset_provenance_audit",
})
case "VLA训练推理评测链路":
return orderPocOperatorsByHandlers(operators, []string{
"builtin_act_policy_train",
"builtin_diffusion_policy_train",
"builtin_openvla_finetune",
"builtin_openvla_inference",
"builtin_vla_model_train",
"builtin_vla_model_inference",
"builtin_vla_model_evaluate",
})
case "视频检测裁剪与成品链路":
return orderPocOperatorsByHandlers(operators, []string{
"builtin_xplatform_index",
"builtin_collection_validate",
"builtin_video_quality",
"builtin_dataset_filter",
"builtin_video_slicing",
"builtin_video_target_clip",
"builtin_yolo_video_target_clip",
"builtin_video_annotation",
"builtin_result_package",
"builtin_audit_sampling",
})
case "人工审核与抽检链路":
return orderPocOperatorsByHandlers(operators, []string{
"builtin_xplatform_index",
"builtin_collection_validate",
"builtin_video_quality",
"builtin_dataset_filter",
"builtin_video_slicing",
"builtin_video_target_clip",
"builtin_yolo_video_target_clip",
"builtin_video_annotation",
"builtin_result_package",
"builtin_audit_sampling",
"builtin_human_review",
})
default:
return operators
}
}
func buildPocOperatorWorkflowEdges(batchName string, operators []domain.OperatorRecord) []domain.WorkflowEdge {
return buildPocOperatorSerialWorkflowEdges(orderPocOperatorsForBatch(batchName, operators))
}
func isXPlatformOperator(operator domain.OperatorRecord) bool {
value := strings.ToLower(strings.TrimSpace(operator.ID) + " " + strings.TrimSpace(operator.Name))
return strings.Contains(value, "xplatform") || strings.Contains(value, "x-platform") || strings.Contains(value, "采集目录")
}
func pocOperatorHandlerKey(operator domain.OperatorRecord) string {
id := strings.TrimPrefix(strings.TrimSpace(operator.ID), "operator-builtin-")
id = strings.ReplaceAll(id, "-", "_")
switch id {
case "real_source":
return "source"
case "bos_incremental_scan":
return "builtin_bos_incremental_scan"
case "xplatform_dataset_index":
return "builtin_xplatform_index"
case "xplatform_dataset_validate":
return "builtin_collection_validate"
case "csv_to_json", "json_to_csv", "json_map_rename", "template_render", "conditional_route":
return "builtin_" + strings.NewReplacer("json_map_rename", "json_map", "template_render", "text_template", "conditional_route", "condition_router").Replace(id)
case "json_field_extract":
return "builtin_json_extract"
case "video_filter":
return "builtin_dataset_filter"
case "video_slice":
return "builtin_video_slicing"
case "video_annotate":
return "builtin_video_annotation"
case "package":
return "builtin_result_package"
case "audit_sample":
return "builtin_audit_sampling"
case "directory_scan", "model_inference", "merge", "upload", "human_review", "mongodb_write", "hf_dataset_probe", "parquet_table_summary", "shell_command":
return "builtin_" + id
default:
return "builtin_" + id
}
}
func buildPocOperatorWorkflowInput(project *domain.ProjectRecord, batchName string, operators []domain.OperatorRecord, fixtures pocOperatorAcceptanceFixturePaths, storageConnection *domain.StorageConnectionRecord) domain.CreateWorkflowInput {
orderedOperators := orderPocOperatorsForBatch(batchName, operators)
nodes := make([]domain.WorkflowNode, 0, len(operators))
for index, operator := range orderedOperators {
nodes = append(nodes, workflowNodeFromOperator(operator, fixtures, storageConnection, float64(index*300), 0))
}
return domain.CreateWorkflowInput{
OrganizationID: normalizedTenantID(project.OrganizationID),
ProjectID: project.ID,
Name: batchName,
Description: "中文生产级 workflow 验收:串联真实数据、独立 CCR 算子镜像、artifact 传输和 lineage 证据。",
Type: domain.WorkflowTypeBatch,
Kind: domain.WorkflowKindStandard,
MaxConcurrentRuns: pocOperatorWorkflowAcceptanceMaxConcurrentRuns,
NodeTimeoutSeconds: 900,
RunTimeoutSeconds: 5400,
MaxRetryAttempts: 0,
RetryBackoffSeconds: 0,
Hyperparams: []domain.HyperParam{
{Key: "POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH", Value: fixtures.Manifest},
{Key: "POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH", Value: fixtures.XPlatform},
{Key: "POC_ACCEPTANCE_CSV_INPUT_PATH", Value: fixtures.CSV},
{Key: "POC_ACCEPTANCE_JSON_INPUT_PATH", Value: fixtures.JSON},
{Key: "POC_ACCEPTANCE_LEROBOT_SOURCE_PATH", Value: fixtures.LeRobot},
{Key: "POC_ACCEPTANCE_ROSBAG_SOURCE_PATH", Value: fixtures.ROSBag},
{Key: "POC_ACCEPTANCE_MODEL_JSON_PATH", Value: fixtures.Model},
},
Nodes: nodes,
Edges: buildPocOperatorWorkflowEdges(batchName, orderedOperators),
}
}
func (server *server) firstPocOperatorAcceptanceStorageConnection(ctx context.Context, organizationID string) (*domain.StorageConnectionRecord, error) {
connections, err := server.store.ListStorageConnectionsByOrganization(ctx, organizationID)
if err != nil {
return nil, err
}
if len(connections) == 0 {
connections, err = server.store.ListStorageConnections(ctx)
if err != nil {
return nil, err
}
}
for index := range connections {
item := connections[index]
if item.Provider == domain.StorageProviderBOS &&
strings.TrimSpace(item.Bucket) != "" &&
strings.TrimSpace(item.AccessKeyID) != "" &&
strings.TrimSpace(item.SecretAccessKey) != "" {
return &item, nil
}
}
return nil, nil
}
func (server *server) upsertPocOperatorWorkflow(ctx context.Context, project *domain.ProjectRecord, input domain.CreateWorkflowInput) (*domain.WorkflowRecord, error) {
workflows, err := server.store.ListWorkflowsByProject(ctx, project.ID)
if err != nil {
return nil, err
}
for index := range workflows {
workflow := workflows[index]
if strings.TrimSpace(workflow.Name) != strings.TrimSpace(input.Name) {
continue
}
workflow.OrganizationID = input.OrganizationID
workflow.ProjectID = input.ProjectID
workflow.Description = input.Description
workflow.Type = input.Type
workflow.Kind = input.Kind
workflow.Hyperparams = input.Hyperparams
workflow.NodeTimeoutSeconds = input.NodeTimeoutSeconds
workflow.RunTimeoutSeconds = input.RunTimeoutSeconds
workflow.MaxRetryAttempts = input.MaxRetryAttempts
workflow.RetryBackoffSeconds = input.RetryBackoffSeconds
workflow.Nodes = input.Nodes
workflow.Edges = input.Edges
workflow.Status = domain.WorkflowStatusDraft
workflow.UpdatedAt = nowISO()
if err := server.store.UpsertWorkflow(ctx, workflow); err != nil {
return nil, err
}
return &workflow, nil
}
return server.store.CreateWorkflow(ctx, input)
}