forked from longtao.wu/droboticflow
1152 lines
49 KiB
Go
1152 lines
49 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
const pocOperatorWorkflowAcceptanceProjectNamePrefix = "生产级算子全链路验收项目"
|
|
const pocOperatorWorkflowAcceptanceMaxConcurrentRuns = 1
|
|
|
|
type pocOperatorWorkflowAcceptanceRequest struct {
|
|
FixtureManifestPath string `json:"fixtureManifestPath"`
|
|
IncludeShellCommand bool `json:"includeShellCommand"`
|
|
PreflightOnly bool `json:"preflightOnly"`
|
|
}
|
|
|
|
type pocOperatorWorkflowAcceptanceRun struct {
|
|
Batch string `json:"batch"`
|
|
WorkflowID string `json:"workflowId"`
|
|
Workflow *domain.WorkflowRecord `json:"workflow"`
|
|
RunID string `json:"runId"`
|
|
Run domain.RunRecord `json:"run"`
|
|
OperatorIDs []string `json:"operatorIds"`
|
|
ImageEvidence []pocOperatorWorkflowImageEvidence `json:"imageEvidence"`
|
|
}
|
|
|
|
type pocOperatorWorkflowImageEvidence struct {
|
|
OperatorID string `json:"operatorId"`
|
|
Image string `json:"image"`
|
|
ImageID string `json:"imageId"`
|
|
ContainerCommand []string `json:"containerCommand"`
|
|
ContainerArgs []string `json:"containerArgs"`
|
|
RequiresGPU bool `json:"requiresGpu"`
|
|
}
|
|
|
|
type pocOperatorWorkflowAcceptanceSummary struct {
|
|
ContractVersion string `json:"contractVersion"`
|
|
OperatorCount int `json:"operatorCount"`
|
|
CatalogHash string `json:"catalogHash"`
|
|
CatalogCount int `json:"catalogCount"`
|
|
IndependentImageCoverage int `json:"independentImageCoverage"`
|
|
IndependentImageRequired int `json:"independentImageRequired"`
|
|
CommandWrapperOnlyOperatorCount int `json:"commandWrapperOnlyOperatorCount"`
|
|
GeneratedFixtureCount int `json:"generatedFixtureCount"`
|
|
MockFixtureCount int `json:"mockFixtureCount"`
|
|
GPUOperatorCount int `json:"gpuOperatorCount"`
|
|
StrictReady bool `json:"strictReady"`
|
|
TrainingMode string `json:"trainingMode"`
|
|
FullTrainingRequired bool `json:"fullTrainingRequired"`
|
|
MissingOperatorIDs []string `json:"missingOperatorIds"`
|
|
}
|
|
|
|
type pocOperatorWorkflowAcceptanceResponse struct {
|
|
Project *domain.ProjectRecord `json:"project"`
|
|
FixtureManifestPath string `json:"fixtureManifestPath"`
|
|
FixtureHash string `json:"fixtureHash"`
|
|
AcceptanceKey string `json:"acceptanceKey"`
|
|
PreflightOnly bool `json:"preflightOnly"`
|
|
OperatorCount int `json:"operatorCount"`
|
|
Summary pocOperatorWorkflowAcceptanceSummary `json:"summary"`
|
|
Runs []pocOperatorWorkflowAcceptanceRun `json:"runs"`
|
|
Batches map[string][]string `json:"batches"`
|
|
}
|
|
|
|
type pocOperatorAcceptanceFixturePaths struct {
|
|
Manifest string
|
|
Root string
|
|
XPlatform string
|
|
CSV string
|
|
JSON string
|
|
Training string
|
|
LeRobot string
|
|
ROSBag string
|
|
Model string
|
|
YoloONNX string
|
|
Shell string
|
|
Extra map[string]string
|
|
}
|
|
|
|
type pocOperatorFixtureManifest struct {
|
|
Fixtures map[string]string `json:"fixtures"`
|
|
ShellCommand string `json:"shellCommand"`
|
|
}
|
|
|
|
func (server *server) defaultPocOperatorFixtureManifestPath() string {
|
|
return path.Join(cleanWorkspaceMountPath(server.workspaceMountPath), "datasets", "fixtures", "poc-operator-acceptance", "poc-operator-acceptance.json")
|
|
}
|
|
|
|
func validatePocOperatorFixtureManifestPath(value string, workspaceMountPath string) (string, *validationIssue) {
|
|
manifestPath := strings.TrimSpace(value)
|
|
if manifestPath == "" {
|
|
return "", &validationIssue{Field: "fixtureManifestPath", Message: "fixtureManifestPath is required"}
|
|
}
|
|
if hasControlCharacter(manifestPath) {
|
|
return "", &validationIssue{Field: "fixtureManifestPath", Message: "fixtureManifestPath must not contain control characters"}
|
|
}
|
|
if !strings.HasPrefix(manifestPath, "/") {
|
|
return "", &validationIssue{Field: "fixtureManifestPath", Message: "fixtureManifestPath must be an absolute path inside the workspace mount"}
|
|
}
|
|
cleaned := path.Clean(manifestPath)
|
|
workspaceMount := cleanWorkspaceMountPath(workspaceMountPath)
|
|
if cleaned != workspaceMount && !strings.HasPrefix(cleaned, workspaceMount+"/") {
|
|
return "", &validationIssue{Field: "fixtureManifestPath", Message: "fixtureManifestPath must stay inside the configured workspace mount"}
|
|
}
|
|
return cleaned, nil
|
|
}
|
|
|
|
func pocOperatorAcceptanceFixturePathsFromManifest(manifestPath string) pocOperatorAcceptanceFixturePaths {
|
|
root := path.Dir(path.Clean(manifestPath))
|
|
return pocOperatorAcceptanceFixturePaths{
|
|
Manifest: manifestPath,
|
|
Root: root,
|
|
XPlatform: path.Join(root, "xplatform"),
|
|
CSV: path.Join(root, "csv", "records.csv"),
|
|
JSON: path.Join(root, "json", "records.json"),
|
|
Training: path.Join(root, "json", "training-records.json"),
|
|
LeRobot: path.Join(root, "lerobot"),
|
|
ROSBag: path.Join(root, "rosbag"),
|
|
Model: path.Join(root, "model", "quality-gate.json"),
|
|
YoloONNX: path.Join(root, "yolo", "models", "yolov8n.onnx"),
|
|
Extra: map[string]string{},
|
|
}
|
|
}
|
|
|
|
func includePocOperatorInWorkflowAcceptance(operator domain.OperatorRecord, includeShellCommand bool) bool {
|
|
_ = includeShellCommand
|
|
if !strings.EqualFold(strings.TrimSpace(operator.Type), "builtin") {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func loadPocOperatorAcceptanceFixturePaths(manifestPath string) (pocOperatorAcceptanceFixturePaths, error) {
|
|
fixtures := pocOperatorAcceptanceFixturePathsFromManifest(manifestPath)
|
|
raw, err := os.ReadFile(manifestPath)
|
|
if err != nil {
|
|
return fixtures, fmt.Errorf("fixture manifest %s is not readable: %w", manifestPath, err)
|
|
}
|
|
manifest := pocOperatorFixtureManifest{}
|
|
if err := json.Unmarshal(raw, &manifest); err != nil {
|
|
return fixtures, fmt.Errorf("fixture manifest %s is invalid JSON: %w", manifestPath, err)
|
|
}
|
|
if len(manifest.Fixtures) == 0 {
|
|
return fixtures, fmt.Errorf("fixture manifest %s must declare fixtures", manifestPath)
|
|
}
|
|
explicitTraining := false
|
|
for key, value := range manifest.Fixtures {
|
|
normalizedKey := strings.ToLower(strings.TrimSpace(key))
|
|
cleanedValue := path.Clean(strings.TrimSpace(value))
|
|
if normalizedKey == "" || !strings.HasPrefix(cleanedValue, "/") {
|
|
continue
|
|
}
|
|
switch normalizedKey {
|
|
case "xplatform":
|
|
fixtures.XPlatform = cleanedValue
|
|
case "csv":
|
|
fixtures.CSV = cleanedValue
|
|
case "json":
|
|
fixtures.JSON = cleanedValue
|
|
case "training", "training_json", "policy_training", "policy_training_json", "vla_training", "vla_training_json", "openvla_training", "openvla_training_json":
|
|
fixtures.Training = cleanedValue
|
|
explicitTraining = true
|
|
case "lerobot":
|
|
fixtures.LeRobot = cleanedValue
|
|
case "rosbag":
|
|
fixtures.ROSBag = cleanedValue
|
|
case "model":
|
|
fixtures.Model = cleanedValue
|
|
case "yolo_onnx", "yolo_onnx_model", "yolo_model", "dnn_model", "dnn_model_path":
|
|
fixtures.YoloONNX = cleanedValue
|
|
default:
|
|
fixtures.Extra[normalizedKey] = cleanedValue
|
|
}
|
|
}
|
|
if !explicitTraining || strings.TrimSpace(fixtures.Training) == "" {
|
|
fixtures.Training = fixtures.JSON
|
|
}
|
|
fixtures.Shell = strings.TrimSpace(manifest.ShellCommand)
|
|
return fixtures, nil
|
|
}
|
|
|
|
func validatePocOperatorAcceptanceFixturePath(value string, workspaceMountPath string, expectedDirectory bool) error {
|
|
cleaned := path.Clean(strings.TrimSpace(value))
|
|
if cleaned == "." || !strings.HasPrefix(cleaned, "/") {
|
|
return fmt.Errorf("fixture path must be an absolute path: %q", value)
|
|
}
|
|
workspaceMount := cleanWorkspaceMountPath(workspaceMountPath)
|
|
if cleaned != workspaceMount && !strings.HasPrefix(cleaned, workspaceMount+"/") {
|
|
return fmt.Errorf("fixture path must stay inside %s: %s", workspaceMount, cleaned)
|
|
}
|
|
if hasControlCharacter(cleaned) {
|
|
return fmt.Errorf("fixture path contains control characters: %s", cleaned)
|
|
}
|
|
info, err := os.Stat(cleaned)
|
|
if err != nil {
|
|
return fmt.Errorf("fixture path is not readable: %s: %w", cleaned, err)
|
|
}
|
|
if expectedDirectory && !info.IsDir() {
|
|
return fmt.Errorf("fixture path must be a directory: %s", cleaned)
|
|
}
|
|
if !expectedDirectory && !info.Mode().IsRegular() {
|
|
return fmt.Errorf("fixture path must be a regular file: %s", cleaned)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validatePocOperatorAcceptanceFixtures(manifestPath string, workspaceMountPath string) (pocOperatorAcceptanceFixturePaths, string, error) {
|
|
fixtures, err := loadPocOperatorAcceptanceFixturePaths(manifestPath)
|
|
if err != nil {
|
|
return fixtures, "", err
|
|
}
|
|
required := []struct {
|
|
name string
|
|
value string
|
|
directory bool
|
|
}{
|
|
{name: "xplatform", value: fixtures.XPlatform, directory: true},
|
|
{name: "csv", value: fixtures.CSV},
|
|
{name: "json", value: fixtures.JSON},
|
|
{name: "lerobot", value: fixtures.LeRobot, directory: true},
|
|
{name: "rosbag", value: fixtures.ROSBag, directory: true},
|
|
{name: "model", value: fixtures.Model},
|
|
}
|
|
for _, item := range required {
|
|
if strings.TrimSpace(item.value) == "" {
|
|
return fixtures, "", fmt.Errorf("fixture %s is required", item.name)
|
|
}
|
|
if err := validatePocOperatorAcceptanceFixturePath(item.value, workspaceMountPath, item.directory); err != nil {
|
|
return fixtures, "", fmt.Errorf("fixture %s: %w", item.name, err)
|
|
}
|
|
}
|
|
if strings.TrimSpace(fixtures.Training) != "" {
|
|
if err := validatePocOperatorAcceptanceFixturePath(fixtures.Training, workspaceMountPath, false); err != nil {
|
|
return fixtures, "", fmt.Errorf("fixture training: %w", err)
|
|
}
|
|
}
|
|
raw, err := os.ReadFile(manifestPath)
|
|
if err != nil {
|
|
return fixtures, "", fmt.Errorf("fixture manifest %s is not readable: %w", manifestPath, err)
|
|
}
|
|
hash := sha256.Sum256(raw)
|
|
return fixtures, hex.EncodeToString(hash[:]), nil
|
|
}
|
|
|
|
func (fixtures pocOperatorAcceptanceFixturePaths) pathFor(keys ...string) string {
|
|
for _, key := range keys {
|
|
normalizedKey := strings.ToLower(strings.TrimSpace(key))
|
|
if normalizedKey == "" {
|
|
continue
|
|
}
|
|
if value := strings.TrimSpace(fixtures.Extra[normalizedKey]); value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (server *server) ensurePocOperatorWorkflowAcceptanceProject(ctx context.Context, organizationID string) (*domain.ProjectRecord, error) {
|
|
projectName := pocOperatorWorkflowAcceptanceProjectNamePrefix
|
|
projects, err := server.store.ListProjectsByOrganization(ctx, organizationID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range projects {
|
|
project := projects[index]
|
|
if strings.TrimSpace(project.Name) == projectName {
|
|
return &project, nil
|
|
}
|
|
}
|
|
return server.store.CreateProject(ctx, domain.CreateProjectInput{
|
|
OrganizationID: organizationID,
|
|
Name: projectName,
|
|
Description: "中文生产级算子全链路验收项目,使用真实账号、真实 fixture、独立 CCR 镜像和 lineage 证据。",
|
|
})
|
|
}
|
|
|
|
func operatorNodeID(operator domain.OperatorRecord) string {
|
|
id := strings.TrimSpace(operator.ID)
|
|
if id == "" {
|
|
id = strings.TrimSpace(operator.Name)
|
|
}
|
|
id = strings.TrimPrefix(id, "operator-builtin-")
|
|
id = regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(strings.ToLower(id), "-")
|
|
id = strings.Trim(id, "-")
|
|
if id == "" {
|
|
id = "operator"
|
|
}
|
|
if len(id) > 48 {
|
|
id = id[:48]
|
|
}
|
|
return "op-" + id
|
|
}
|
|
|
|
func cloneOperatorHyperparams(operator domain.OperatorRecord) []domain.HyperParam {
|
|
items := make([]domain.HyperParam, 0, len(operator.Hyperparams))
|
|
for _, item := range operator.Hyperparams {
|
|
items = append(items, item)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func setHyperparam(items []domain.HyperParam, key string, value string) []domain.HyperParam {
|
|
normalized := strings.ToUpper(strings.TrimSpace(key))
|
|
for index := range items {
|
|
if strings.ToUpper(strings.TrimSpace(items[index].Key)) == normalized {
|
|
items[index].Value = value
|
|
return items
|
|
}
|
|
}
|
|
return append(items, domain.HyperParam{Key: key, Value: value})
|
|
}
|
|
|
|
func configurePocOperatorHyperparams(operator domain.OperatorRecord, fixtures pocOperatorAcceptanceFixturePaths) []domain.HyperParam {
|
|
items := cloneOperatorHyperparams(operator)
|
|
handler := pocOperatorHandlerKey(operator)
|
|
name := strings.TrimSpace(operator.Name)
|
|
setSource := func(value string) {
|
|
items = setHyperparam(items, "SOURCE_PATH", value)
|
|
}
|
|
switch handler {
|
|
case "source", "builtin_directory_scan", "builtin_xplatform_index":
|
|
setSource(fixtures.XPlatform)
|
|
case "builtin_csv_to_json":
|
|
setSource(fixtures.CSV)
|
|
case "builtin_model_inference":
|
|
items = setHyperparam(items, "MODEL_PATH", fixtures.Model)
|
|
items = setHyperparam(items, "INPUT_JSON", fmt.Sprintf(`{"items":[{"id":"acceptance-a","score":0.95,"quality":{"clarity":0.95,"shake":0.01}},{"id":"acceptance-b","score":0.42,"quality":{"clarity":0.42,"shake":0.2}}]}`))
|
|
case "builtin_vla_model_train":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("training_json", "policy_training_json", "vla_training_json"), fixtures.Training, fixtures.JSON))
|
|
items = setHyperparam(items, "MIN_SAMPLES", "128")
|
|
case "builtin_act_policy_train", "builtin_diffusion_policy_train":
|
|
items = setHyperparam(items, "DATASET_PATH", firstNonEmptyText(fixtures.pathFor("training_json", "policy_training_json", "vla_training_json"), fixtures.Training, fixtures.JSON))
|
|
items = setHyperparam(items, "STATE_FIELD", "state")
|
|
items = setHyperparam(items, "ACTION_FIELD", "action")
|
|
items = setHyperparam(items, "EPOCHS", "1")
|
|
items = setHyperparam(items, "BATCH_SIZE", "16")
|
|
case "builtin_openvla_finetune":
|
|
items = setHyperparam(items, "BASE_MODEL", "distilgpt2")
|
|
items = setHyperparam(items, "DATASET_PATH", firstNonEmptyText(fixtures.pathFor("openvla_training_json", "training_json", "vla_training_json"), fixtures.Training, fixtures.JSON))
|
|
items = setHyperparam(items, "TEXT_FIELD", "instruction")
|
|
items = setHyperparam(items, "TARGET_FIELD", "action")
|
|
items = setHyperparam(items, "MAX_LENGTH", "64")
|
|
items = setHyperparam(items, "EPOCHS", "1")
|
|
items = setHyperparam(items, "BATCH_SIZE", "4")
|
|
case "builtin_openvla_inference":
|
|
items = setHyperparam(items, "BASE_MODEL", "distilgpt2")
|
|
items = setHyperparam(items, "PROMPT", "pick the visible target")
|
|
items = setHyperparam(items, "MAX_NEW_TOKENS", "16")
|
|
case "builtin_yolo_embodied_detection_train":
|
|
if value := firstNonEmptyText(
|
|
fixtures.pathFor("yolo", "yolo_data_yaml", "yoloDataYaml", "coco128"),
|
|
path.Join(fixtures.Root, "yolo", "coco128", "data.yaml"),
|
|
); value != "" {
|
|
items = setHyperparam(items, "DATA_YAML", value)
|
|
}
|
|
items = setHyperparam(items, "WORKERS", "0")
|
|
case "builtin_json_extract":
|
|
items = setHyperparam(items, "FIELD_PATH", "items.0.label")
|
|
case "builtin_json_filter", "builtin_condition_router":
|
|
items = setHyperparam(items, "FIELD_PATH", "label")
|
|
items = setHyperparam(items, "OPERATOR", "eq")
|
|
items = setHyperparam(items, "EXPECTED", "keep")
|
|
case "builtin_json_map":
|
|
items = setHyperparam(items, "FIELD_MAP_JSON", `{"label":"decision","score":"confidence"}`)
|
|
case "builtin_text_template":
|
|
items = setHyperparam(items, "TEMPLATE_TEXT", "accepted={items.0.id}; decision={items.0.decision}; confidence={items.0.confidence}")
|
|
items = setHyperparam(items, "OUTPUT_NAME", "decision.txt")
|
|
case "builtin_collection_validate":
|
|
items = setHyperparam(items, "REQUIRE_INTRINSICS", "false")
|
|
case "builtin_video_quality":
|
|
items = setHyperparam(items, "MIN_CLARITY", "0")
|
|
items = setHyperparam(items, "MAX_SHAKE", "1")
|
|
items = setHyperparam(items, "MAX_HAND_OCCLUSION", "1")
|
|
case "builtin_dataset_filter":
|
|
items = setHyperparam(items, "MIN_CLARITY", "0")
|
|
items = setHyperparam(items, "MAX_SHAKE", "1")
|
|
items = setHyperparam(items, "MAX_HAND_OCCLUSION", "1")
|
|
case "builtin_dataset_split_manifest":
|
|
setSource(fixtures.JSON)
|
|
items = setHyperparam(items, "MIN_STEPS", "0")
|
|
case "builtin_video_slicing":
|
|
items = setHyperparam(items, "SEGMENT_DURATION_SECONDS", "1")
|
|
case "builtin_video_annotation":
|
|
items = setHyperparam(items, "FRAME_INTERVAL_SECONDS", "0.5")
|
|
case "builtin_video_target_clip":
|
|
items = setHyperparam(items, "DETECTION_MODE", "motion")
|
|
items = setHyperparam(items, "TARGET_LABELS", "motion")
|
|
items = setHyperparam(items, "FRAME_INTERVAL_SECONDS", "0.25")
|
|
items = setHyperparam(items, "MIN_CONFIDENCE", "0.05")
|
|
items = setHyperparam(items, "MIN_MOTION_AREA_RATIO", "0.001")
|
|
items = setHyperparam(items, "MIN_SEGMENT_SECONDS", "0.25")
|
|
items = setHyperparam(items, "MAX_CLIPS", "5")
|
|
case "builtin_yolo_video_target_clip":
|
|
items = setHyperparam(items, "DETECTION_MODE", "opencv-dnn")
|
|
items = setHyperparam(items, "DNN_MODEL_PATH", firstNonEmptyText(fixtures.pathFor("yolo_onnx_model", "yolo_onnx", "dnn_model_path"), fixtures.YoloONNX))
|
|
items = setHyperparam(items, "DNN_INPUT_SIZE", "640")
|
|
items = setHyperparam(items, "DNN_NMS_THRESHOLD", "0.45")
|
|
items = setHyperparam(items, "DNN_CLASS_NAMES", "person,bicycle,car,motorcycle,airplane,bus,train,truck,boat,traffic light,fire hydrant,stop sign,parking meter,bench,bird,cat,dog,horse,sheep,cow,elephant,bear,zebra,giraffe,backpack,umbrella,handbag,tie,suitcase,frisbee,skis,snowboard,sports ball,kite,baseball bat,baseball glove,skateboard,surfboard,tennis racket,bottle,wine glass,cup,fork,knife,spoon,bowl,banana,apple,sandwich,orange,broccoli,carrot,hot dog,pizza,donut,cake,chair,couch,potted plant,bed,dining table,toilet,tv,laptop,mouse,remote,keyboard,cell phone,microwave,oven,toaster,sink,refrigerator,book,clock,vase,scissors,teddy bear,hair drier,toothbrush")
|
|
items = setHyperparam(items, "TARGET_LABELS", "*")
|
|
items = setHyperparam(items, "FRAME_INTERVAL_SECONDS", "0.5")
|
|
items = setHyperparam(items, "MIN_CONFIDENCE", "0.05")
|
|
items = setHyperparam(items, "MIN_SEGMENT_SECONDS", "0.25")
|
|
items = setHyperparam(items, "MAX_CLIPS", "5")
|
|
case "builtin_audit_sampling":
|
|
items = setHyperparam(items, "RULES_JSON", `[{"id":"all-items","field":"id","operator":"exists","samplePercent":100}]`)
|
|
case "builtin_shell_command":
|
|
items = setHyperparam(items, "SCRIPT_ID", "fixture_report")
|
|
items = setHyperparam(items, "SCRIPT_ARGS_JSON", `{"acceptance":"生产级受控脚本验收"}`)
|
|
items = setHyperparam(items, "INPUT_PATHS_JSON", fmt.Sprintf(`[%q,%q]`, fixtures.Manifest, fixtures.JSON))
|
|
items = setHyperparam(items, "OUTPUT_CONTRACT_JSON", `{"artifact":"controlled-script-report.json","type":"json"}`)
|
|
case "builtin_mongodb_write":
|
|
items = setHyperparam(items, "MONGO_TARGET", "platform")
|
|
items = setHyperparam(items, "MONGO_COLLECTION", "operatorWorkflowAcceptanceWrites")
|
|
items = setHyperparam(items, "MONGO_MODE", "insert_many")
|
|
case "builtin_hf_dataset_probe":
|
|
items = setHyperparam(items, "DATASET", "lerobot/pusht")
|
|
items = setHyperparam(items, "ROW_LENGTH", "1")
|
|
items = setHyperparam(items, "INCLUDE_ROWS", "false")
|
|
items = setHyperparam(items, "INCLUDE_PARQUET", "false")
|
|
items = setHyperparam(items, "INCLUDE_STATISTICS", "false")
|
|
case "builtin_bos_incremental_scan":
|
|
items = setHyperparam(items, "DATA_SOURCE", "BOS")
|
|
items = setHyperparam(items, "SCAN_MODE", "INCREMENTAL")
|
|
items = setHyperparam(items, "INCREMENTAL_UNIT", "DIRECTORY")
|
|
case "builtin_graspnet_dataset_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("graspnet", "graspnet1billion"), path.Join(fixtures.Root, "graspnet")))
|
|
items = setHyperparam(items, "CAMERA", "kinect")
|
|
items = setHyperparam(items, "MAX_SCENES", "1")
|
|
items = setHyperparam(items, "MAX_FRAMES_PER_CAMERA", "1")
|
|
case "builtin_ho3d_dataset_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("ho3d", "h2o3d"), path.Join(fixtures.Root, "ho3d")))
|
|
items = setHyperparam(items, "MAX_SAMPLES", "1")
|
|
case "builtin_calvin_episode_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("calvin"), path.Join(fixtures.Root, "calvin")))
|
|
items = setHyperparam(items, "MAX_FILES", "1")
|
|
case "builtin_libero_hdf5_task_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("libero"), path.Join(fixtures.Root, "libero")))
|
|
items = setHyperparam(items, "MAX_FILES", "1")
|
|
items = setHyperparam(items, "MAX_DEMOS", "1")
|
|
case "builtin_maniskill_trajectory_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("maniskill"), path.Join(fixtures.Root, "maniskill")))
|
|
items = setHyperparam(items, "MAX_FILES", "1")
|
|
case "builtin_furniturebench_pkl_demonstration_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("furniturebench"), path.Join(fixtures.Root, "furniturebench")))
|
|
items = setHyperparam(items, "MAX_FILES", "1")
|
|
items = setHyperparam(items, "EXPECTED_ACTION_DIM", "8")
|
|
case "builtin_partnet_mobility_asset_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("partnet", "partnet-mobility"), path.Join(fixtures.Root, "partnet")))
|
|
items = setHyperparam(items, "MAX_OBJECTS", "1")
|
|
case "builtin_rh20t_scene_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("rh20t"), path.Join(fixtures.Root, "rh20t")))
|
|
items = setHyperparam(items, "MAX_SCENES", "1")
|
|
case "builtin_robocasa_hdf5_demonstration_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("robocasa"), path.Join(fixtures.Root, "robocasa")))
|
|
items = setHyperparam(items, "MAX_FILES", "1")
|
|
items = setHyperparam(items, "MAX_DEMOS", "1")
|
|
case "builtin_hdf5_demonstration_summary":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("hdf5"), path.Join(fixtures.Root, "hdf5")))
|
|
items = setHyperparam(items, "MAX_DEMOS", "1")
|
|
case "builtin_umi_zarr_dataset_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("zarr", "umi_zarr", "umi"), path.Join(fixtures.Root, "zarr")))
|
|
items = setHyperparam(items, "MAX_ARRAYS", "8")
|
|
items = setHyperparam(items, "MAX_SAMPLE_ROWS", "8")
|
|
items = setHyperparam(items, "MAX_EPISODE_ENDS", "4")
|
|
case "builtin_zarr_trajectory_summary":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("zarr", "umi_zarr", "umi"), path.Join(fixtures.Root, "zarr")))
|
|
items = setHyperparam(items, "MAX_ARRAYS", "8")
|
|
case "builtin_joint_limit_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("joint_limit", "jointlimit"), path.Join(fixtures.Root, "joint_limit")))
|
|
items = setHyperparam(items, "MAX_FILES", "1")
|
|
items = setHyperparam(items, "MAX_ROWS", "4")
|
|
case "builtin_mcap_topic_summary":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("mcap"), path.Join(fixtures.Root, "mcap")))
|
|
items = setHyperparam(items, "MAX_MESSAGES", "10")
|
|
case "builtin_robot_model_asset_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("robot_model", "robotmodel"), path.Join(fixtures.Root, "robot_model")))
|
|
items = setHyperparam(items, "MAX_MODELS", "2")
|
|
case "builtin_mjcf_actuator_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("mjcf", "robot_model"), path.Join(fixtures.Root, "robot_model")))
|
|
items = setHyperparam(items, "MAX_MODELS", "1")
|
|
case "builtin_webdataset_shard_summary":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("webdataset"), path.Join(fixtures.Root, "webdataset")))
|
|
items = setHyperparam(items, "MAX_SHARDS", "1")
|
|
case "builtin_pointcloud_summary":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("pointcloud", "point_cloud"), path.Join(fixtures.Root, "pointcloud")))
|
|
items = setHyperparam(items, "MAX_FILES", "2")
|
|
items = setHyperparam(items, "MAX_SAMPLE_POINTS", "1000")
|
|
case "builtin_rlbench_dataset_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("rlbench"), path.Join(fixtures.Root, "rlbench")))
|
|
items = setHyperparam(items, "MAX_EPISODES", "1")
|
|
items = setHyperparam(items, "CAMERA_DIRS", "front_rgb,left_shoulder_rgb")
|
|
case "builtin_rgbd_frame_pair_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("rgbd", "rgb_depth"), path.Join(fixtures.Root, "rgbd")))
|
|
items = setHyperparam(items, "MAX_PAIRS", "2")
|
|
case "builtin_segmentation_mask_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("segmentation", "segmentation_mask", "mask"), path.Join(fixtures.Root, "segmentation")))
|
|
items = setHyperparam(items, "MAX_PAIRS", "2")
|
|
case "builtin_tfrecord_structural_validate":
|
|
setSource(firstNonEmptyText(fixtures.pathFor("tfrecord", "tfrecord_structural"), path.Join(fixtures.Root, "tfrecord")))
|
|
items = setHyperparam(items, "MAX_RECORDS", "10")
|
|
items = setHyperparam(items, "PREVIEW_RECORDS", "2")
|
|
case "builtin_lerobot_video_alignment_validate":
|
|
setSource(fixtures.LeRobot)
|
|
items = setHyperparam(items, "MAX_PARQUET_FILES", "1")
|
|
items = setHyperparam(items, "MAX_ROWS", "1")
|
|
items = setHyperparam(items, "MAX_VIDEOS", "1")
|
|
default:
|
|
if needsDexYCBFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("dexycb", "dex-ycb"), path.Join(fixtures.Root, "dexycb")))
|
|
} else if needsHO3DFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("ho3d", "h2o3d"), path.Join(fixtures.Root, "ho3d")))
|
|
} else if needsCALVINFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("calvin"), path.Join(fixtures.Root, "calvin")))
|
|
} else if needsLIBEROFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("libero"), path.Join(fixtures.Root, "libero")))
|
|
} else if needsManiSkillFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("maniskill"), path.Join(fixtures.Root, "maniskill")))
|
|
} else if needsFurnitureBenchFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("furniturebench"), path.Join(fixtures.Root, "furniturebench")))
|
|
} else if needsPartNetFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("partnet", "partnet-mobility"), path.Join(fixtures.Root, "partnet")))
|
|
} else if needsRH20TFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("rh20t"), path.Join(fixtures.Root, "rh20t")))
|
|
} else if needsRoboCasaFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("robocasa"), path.Join(fixtures.Root, "robocasa")))
|
|
} else if needsMJCFFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("mjcf", "robot_model"), path.Join(fixtures.Root, "robot_model")))
|
|
} else if needsRobotModelFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("robot_model", "robotmodel"), path.Join(fixtures.Root, "robot_model")))
|
|
} else if needsWebDatasetFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("webdataset"), path.Join(fixtures.Root, "webdataset")))
|
|
} else if needsPointCloudFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("pointcloud", "point_cloud"), path.Join(fixtures.Root, "pointcloud")))
|
|
} else if needsALOHAFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("aloha", "act_hdf5", "hdf5"), path.Join(fixtures.Root, "aloha")))
|
|
} else if needsBOPFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("bop"), path.Join(fixtures.Root, "bop")))
|
|
} else if needsGraspNetFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("graspnet", "graspnet1billion"), path.Join(fixtures.Root, "graspnet")))
|
|
} else if needsHDF5Fixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("hdf5"), path.Join(fixtures.Root, "hdf5")))
|
|
} else if needsZarrFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("zarr"), path.Join(fixtures.Root, "zarr")))
|
|
} else if needsRLBenchFixture(handler, name) {
|
|
setSource(firstNonEmptyText(fixtures.pathFor("rlbench"), path.Join(fixtures.Root, "rlbench")))
|
|
} else if needsLeRobotFixture(handler, name) {
|
|
setSource(fixtures.LeRobot)
|
|
} else if needsROSBagFixture(handler, name) {
|
|
setSource(fixtures.ROSBag)
|
|
} else if needsJSONFixture(handler, name) {
|
|
setSource(fixtures.JSON)
|
|
} else {
|
|
setSource(fixtures.XPlatform)
|
|
}
|
|
}
|
|
return items
|
|
}
|
|
|
|
func needsDexYCBFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "dexycb") || strings.Contains(value, "dex-ycb")
|
|
}
|
|
|
|
func needsHO3DFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "ho3d") || strings.Contains(value, "h2o3d") || strings.Contains(value, "ho-3d") || strings.Contains(value, "h2o-3d")
|
|
}
|
|
|
|
func needsCALVINFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "calvin")
|
|
}
|
|
|
|
func needsLIBEROFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "libero")
|
|
}
|
|
|
|
func needsManiSkillFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "maniskill")
|
|
}
|
|
|
|
func needsFurnitureBenchFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "furniturebench") || strings.Contains(value, "furniture bench")
|
|
}
|
|
|
|
func needsPartNetFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "partnet") || strings.Contains(value, "partnet-mobility")
|
|
}
|
|
|
|
func needsRH20TFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "rh20t")
|
|
}
|
|
|
|
func needsRoboCasaFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "robocasa")
|
|
}
|
|
|
|
func needsMJCFFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "mjcf")
|
|
}
|
|
|
|
func needsRobotModelFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "robot_model") || strings.Contains(value, "robot model") || strings.Contains(value, "机器人模型")
|
|
}
|
|
|
|
func needsWebDatasetFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "webdataset")
|
|
}
|
|
|
|
func needsPointCloudFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "pointcloud") || strings.Contains(value, "point cloud") || strings.Contains(value, "点云")
|
|
}
|
|
|
|
func needsALOHAFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "aloha") || strings.Contains(value, "act hdf5")
|
|
}
|
|
|
|
func needsBOPFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "bop") || strings.Contains(value, "6d姿态") || strings.Contains(value, "6d pose")
|
|
}
|
|
|
|
func needsGraspNetFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "graspnet") || strings.Contains(value, "grasp net") || strings.Contains(value, "抓取数据集")
|
|
}
|
|
|
|
func needsHDF5Fixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "hdf5") || strings.Contains(value, ".h5")
|
|
}
|
|
|
|
func needsZarrFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "zarr")
|
|
}
|
|
|
|
func needsRLBenchFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "rlbench")
|
|
}
|
|
|
|
func needsLeRobotFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "lerobot") || strings.Contains(value, "parquet") || strings.Contains(value, "agibot")
|
|
}
|
|
|
|
func needsROSBagFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
return strings.Contains(value, "rosbag") || strings.Contains(value, "mcap")
|
|
}
|
|
|
|
func needsJSONFixture(handler string, name string) bool {
|
|
value := strings.ToLower(handler + " " + name)
|
|
tokens := []string{
|
|
"rlds", "bridgedata", "droid", "libero", "robocasa", "furniturebench", "roboturk", "roboset",
|
|
"rh20t", "robomind", "hdf5", "robomimic", "aloha", "zarr", "rlbench", "maniskill",
|
|
"calvin", "rtx", "embodiment", "force", "trajectory", "action_state", "episode_boundary",
|
|
"rgbd", "segmentation", "language", "vla", "camera", "tf_transform", "joint", "graspnet",
|
|
"bop", "dexycb", "ho3d", "hand_object", "partnet", "mjcf", "robot_model", "pointcloud",
|
|
"sensor", "tfrecord", "coco", "yolo", "split", "media", "schema",
|
|
}
|
|
for _, token := range tokens {
|
|
if strings.Contains(value, token) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func workflowNodeFromOperator(operator domain.OperatorRecord, fixtures pocOperatorAcceptanceFixturePaths, storageConnection *domain.StorageConnectionRecord, x float64, y float64) domain.WorkflowNode {
|
|
hyperparams := configurePocOperatorHyperparams(operator, fixtures)
|
|
connectionID := ""
|
|
bucket := ""
|
|
if pocOperatorHandlerKey(operator) == "builtin_bos_incremental_scan" && storageConnection != nil {
|
|
connectionID = storageConnection.ID
|
|
bucket = storageConnection.Bucket
|
|
if bucket != "" {
|
|
prefix := strings.TrimPrefix(strings.TrimPrefix(fixtures.XPlatform, fixtures.Root), "/")
|
|
acceptancePrefix := strings.Trim(path.Join("datasets", "fixtures", "poc-operator-acceptance", prefix), "/")
|
|
hyperparams = setHyperparam(hyperparams, "SOURCE_PATH", "bos://"+strings.Trim(bucket, "/")+"/"+acceptancePrefix)
|
|
}
|
|
}
|
|
return domain.WorkflowNode{
|
|
ID: operatorNodeID(operator),
|
|
Type: "operator",
|
|
Data: domain.WorkflowNodeData{
|
|
Label: operator.Name,
|
|
Category: operator.Category,
|
|
Description: operator.Description,
|
|
Inputs: clonePortDefinitions(operator.Inputs),
|
|
Outputs: clonePortDefinitions(operator.Outputs),
|
|
OperatorID: operator.ID,
|
|
OperatorVersion: operator.Version,
|
|
OperatorImageID: operator.ImageID,
|
|
Image: operator.Image,
|
|
ContainerCommand: cloneStringSlice(operator.ContainerCommand),
|
|
ContainerArgs: cloneStringSlice(operator.ContainerArgs),
|
|
RuntimeContract: cloneMap(operator.RuntimeContract),
|
|
Hyperparams: hyperparams,
|
|
ConnectionID: connectionID,
|
|
Bucket: bucket,
|
|
CPU: operatorResourceCPULimit(operator.Resources),
|
|
Memory: operatorResourceMemoryLimit(operator.Resources),
|
|
GPU: operatorResourceGPULimit(operator.Resources),
|
|
},
|
|
Position: domain.Position{X: x, Y: y},
|
|
}
|
|
}
|
|
|
|
func operatorResourceCPULimit(resources *domain.OperatorResources) string {
|
|
if resources == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(resources.CPULimit)
|
|
}
|
|
|
|
func operatorResourceMemoryLimit(resources *domain.OperatorResources) string {
|
|
if resources == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(resources.MemoryLimit)
|
|
}
|
|
|
|
func operatorResourceGPULimit(resources *domain.OperatorResources) string {
|
|
if resources == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(resources.GPULimit)
|
|
}
|
|
|
|
func pocOperatorHasCommandWrapper(operator domain.OperatorRecord) bool {
|
|
if operator.ID == "operator-builtin-shell-command" {
|
|
return false
|
|
}
|
|
for _, param := range operator.Hyperparams {
|
|
key := strings.ToUpper(strings.TrimSpace(param.Key))
|
|
if key == "COMMAND" || key == "SCRIPT" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func pocOperatorRequiresGPU(operator domain.OperatorRecord) bool {
|
|
return operator.Resources != nil && strings.TrimSpace(operator.Resources.GPULimit) != ""
|
|
}
|
|
|
|
func pocOperatorHasIndependentImage(operator domain.OperatorRecord) bool {
|
|
image := strings.TrimSpace(operator.Image)
|
|
imageID := strings.TrimSpace(operator.ImageID)
|
|
if image == "" || strings.Contains(image, "droboticflow-operator-runner") || !strings.Contains(image, "droboticflow-operator-") {
|
|
return false
|
|
}
|
|
if !strings.Contains(imageID, "@sha256:") {
|
|
return false
|
|
}
|
|
if len(operator.ContainerCommand) == 0 {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func buildPocOperatorWorkflowImageEvidence(operator domain.OperatorRecord) pocOperatorWorkflowImageEvidence {
|
|
return pocOperatorWorkflowImageEvidence{
|
|
OperatorID: operator.ID,
|
|
Image: strings.TrimSpace(operator.Image),
|
|
ImageID: strings.TrimSpace(operator.ImageID),
|
|
ContainerCommand: cloneStringSlice(operator.ContainerCommand),
|
|
ContainerArgs: cloneStringSlice(operator.ContainerArgs),
|
|
RequiresGPU: pocOperatorRequiresGPU(operator),
|
|
}
|
|
}
|
|
|
|
func buildPocOperatorWorkflowAcceptanceSummary(operators []domain.OperatorRecord) pocOperatorWorkflowAcceptanceSummary {
|
|
summary := pocOperatorWorkflowAcceptanceSummary{
|
|
ContractVersion: "poc-operator-workflows/v4",
|
|
OperatorCount: len(operators),
|
|
IndependentImageRequired: len(operators),
|
|
TrainingMode: "full",
|
|
FullTrainingRequired: true,
|
|
MissingOperatorIDs: []string{},
|
|
}
|
|
for _, operator := range operators {
|
|
if pocOperatorHasIndependentImage(operator) {
|
|
summary.IndependentImageCoverage++
|
|
}
|
|
if pocOperatorHasCommandWrapper(operator) {
|
|
summary.CommandWrapperOnlyOperatorCount++
|
|
}
|
|
if pocOperatorRequiresGPU(operator) {
|
|
summary.GPUOperatorCount++
|
|
}
|
|
}
|
|
summary.StrictReady = summary.OperatorCount > 0 &&
|
|
summary.IndependentImageCoverage == summary.IndependentImageRequired &&
|
|
summary.CommandWrapperOnlyOperatorCount == 0 &&
|
|
summary.GeneratedFixtureCount == 0 &&
|
|
summary.MockFixtureCount == 0
|
|
return summary
|
|
}
|
|
|
|
func clonePortDefinitions(items []domain.PortDefinition) []domain.PortDefinition {
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
cloned := make([]domain.PortDefinition, 0, len(items))
|
|
for _, item := range items {
|
|
cloned = append(cloned, item)
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func cloneStringSlice(items []string) []string {
|
|
if len(items) == 0 {
|
|
return nil
|
|
}
|
|
cloned := make([]string, len(items))
|
|
copy(cloned, items)
|
|
return cloned
|
|
}
|
|
|
|
func cloneMap(item map[string]any) map[string]any {
|
|
if len(item) == 0 {
|
|
return nil
|
|
}
|
|
cloned := make(map[string]any, len(item))
|
|
for key, value := range item {
|
|
cloned[key] = value
|
|
}
|
|
return cloned
|
|
}
|
|
|
|
func groupPocOperatorAcceptanceBatches(operators []domain.OperatorRecord) map[string][]domain.OperatorRecord {
|
|
batches := map[string][]domain.OperatorRecord{
|
|
"数据接入与格式转换链路": {},
|
|
"具身数据集结构校验链路一": {},
|
|
"具身数据集结构校验链路二": {},
|
|
"具身数据集结构校验链路三": {},
|
|
"视频检测裁剪与成品链路": {},
|
|
"VLA训练推理评测链路": {},
|
|
"3D资产生成与预览链路": {},
|
|
"外部存储与审计链路": {},
|
|
"人工审核与抽检链路": {},
|
|
}
|
|
operatorByHandler := map[string]domain.OperatorRecord{}
|
|
for _, operator := range operators {
|
|
handler := pocOperatorHandlerKey(operator)
|
|
operatorByHandler[handler] = operator
|
|
switch {
|
|
case handler == "builtin_human_review":
|
|
batches["人工审核与抽检链路"] = append(batches["人工审核与抽检链路"], operator)
|
|
case handler == "builtin_audit_sampling":
|
|
batches["人工审核与抽检链路"] = append(batches["人工审核与抽检链路"], operator)
|
|
case handler == "builtin_vla_model_train" || handler == "builtin_vla_model_inference" || handler == "builtin_vla_model_evaluate" || handler == "builtin_act_policy_train" || handler == "builtin_diffusion_policy_train" || handler == "builtin_openvla_finetune" || handler == "builtin_openvla_inference":
|
|
batches["VLA训练推理评测链路"] = append(batches["VLA训练推理评测链路"], operator)
|
|
case handler == "builtin_sim3d_asset_generate":
|
|
batches["3D资产生成与预览链路"] = append(batches["3D资产生成与预览链路"], operator)
|
|
case strings.Contains(handler, "bos") || strings.Contains(handler, "mongodb") || strings.Contains(handler, "shell"):
|
|
batches["外部存储与审计链路"] = append(batches["外部存储与审计链路"], operator)
|
|
case operator.Category == "VIDEO" || isXPlatformOperator(operator):
|
|
batches["视频检测裁剪与成品链路"] = append(batches["视频检测裁剪与成品链路"], operator)
|
|
case operator.Category == "DATASET" || operator.Category == "ROBOTICS":
|
|
if len(batches["具身数据集结构校验链路一"]) <= len(batches["具身数据集结构校验链路二"]) && len(batches["具身数据集结构校验链路一"]) <= len(batches["具身数据集结构校验链路三"]) {
|
|
batches["具身数据集结构校验链路一"] = append(batches["具身数据集结构校验链路一"], operator)
|
|
} else if len(batches["具身数据集结构校验链路二"]) <= len(batches["具身数据集结构校验链路三"]) {
|
|
batches["具身数据集结构校验链路二"] = append(batches["具身数据集结构校验链路二"], operator)
|
|
} else {
|
|
batches["具身数据集结构校验链路三"] = append(batches["具身数据集结构校验链路三"], operator)
|
|
}
|
|
default:
|
|
batches["数据接入与格式转换链路"] = append(batches["数据接入与格式转换链路"], operator)
|
|
}
|
|
}
|
|
appendHumanReviewSupport := func(handler string) {
|
|
operator, ok := operatorByHandler[handler]
|
|
if !ok {
|
|
return
|
|
}
|
|
for _, existing := range batches["人工审核与抽检链路"] {
|
|
if existing.ID == operator.ID {
|
|
return
|
|
}
|
|
}
|
|
batches["人工审核与抽检链路"] = append(batches["人工审核与抽检链路"], operator)
|
|
}
|
|
if len(batches["人工审核与抽检链路"]) > 0 {
|
|
for _, handler := range []string{
|
|
"builtin_xplatform_index",
|
|
"builtin_collection_validate",
|
|
"builtin_video_quality",
|
|
"builtin_dataset_filter",
|
|
"builtin_video_slicing",
|
|
"builtin_video_annotation",
|
|
"builtin_video_target_clip",
|
|
"builtin_yolo_video_target_clip",
|
|
"builtin_result_package",
|
|
} {
|
|
appendHumanReviewSupport(handler)
|
|
}
|
|
}
|
|
for name := range batches {
|
|
sort.Slice(batches[name], func(i, j int) bool {
|
|
return batches[name][i].ID < batches[name][j].ID
|
|
})
|
|
}
|
|
return batches
|
|
}
|
|
|
|
func pocOperatorWorkflowEdge(source string, target string) domain.WorkflowEdge {
|
|
return domain.WorkflowEdge{
|
|
ID: "edge-" + strings.TrimPrefix(source, "op-") + "-to-" + strings.TrimPrefix(target, "op-"),
|
|
Source: source,
|
|
Target: target,
|
|
}
|
|
}
|
|
|
|
func buildPocOperatorSerialWorkflowEdges(operators []domain.OperatorRecord) []domain.WorkflowEdge {
|
|
edges := make([]domain.WorkflowEdge, 0, len(operators)-1)
|
|
for index := 1; index < len(operators); index++ {
|
|
edges = append(edges, pocOperatorWorkflowEdge(operatorNodeID(operators[index-1]), operatorNodeID(operators[index])))
|
|
}
|
|
return edges
|
|
}
|
|
|
|
func orderPocOperatorsByHandlers(operators []domain.OperatorRecord, handlers []string) []domain.OperatorRecord {
|
|
used := map[string]struct{}{}
|
|
ordered := make([]domain.OperatorRecord, 0, len(operators))
|
|
for _, handler := range handlers {
|
|
for _, operator := range operators {
|
|
if _, ok := used[operator.ID]; ok {
|
|
continue
|
|
}
|
|
if pocOperatorHandlerKey(operator) != handler {
|
|
continue
|
|
}
|
|
ordered = append(ordered, operator)
|
|
used[operator.ID] = struct{}{}
|
|
break
|
|
}
|
|
}
|
|
for _, operator := range operators {
|
|
if _, ok := used[operator.ID]; ok {
|
|
continue
|
|
}
|
|
ordered = append(ordered, operator)
|
|
used[operator.ID] = struct{}{}
|
|
}
|
|
return ordered
|
|
}
|
|
|
|
func orderPocOperatorsForBatch(batchName string, operators []domain.OperatorRecord) []domain.OperatorRecord {
|
|
switch batchName {
|
|
case "外部存储与审计链路":
|
|
return orderPocOperatorsByHandlers(operators, []string{
|
|
"builtin_shell_command",
|
|
"builtin_bos_incremental_scan",
|
|
"builtin_mongodb_write",
|
|
"builtin_dataset_provenance_audit",
|
|
})
|
|
case "VLA训练推理评测链路":
|
|
return orderPocOperatorsByHandlers(operators, []string{
|
|
"builtin_act_policy_train",
|
|
"builtin_diffusion_policy_train",
|
|
"builtin_openvla_finetune",
|
|
"builtin_openvla_inference",
|
|
"builtin_vla_model_train",
|
|
"builtin_vla_model_inference",
|
|
"builtin_vla_model_evaluate",
|
|
})
|
|
case "视频检测裁剪与成品链路":
|
|
return orderPocOperatorsByHandlers(operators, []string{
|
|
"builtin_xplatform_index",
|
|
"builtin_collection_validate",
|
|
"builtin_video_quality",
|
|
"builtin_dataset_filter",
|
|
"builtin_video_slicing",
|
|
"builtin_video_target_clip",
|
|
"builtin_yolo_video_target_clip",
|
|
"builtin_video_annotation",
|
|
"builtin_result_package",
|
|
"builtin_audit_sampling",
|
|
})
|
|
case "人工审核与抽检链路":
|
|
return orderPocOperatorsByHandlers(operators, []string{
|
|
"builtin_xplatform_index",
|
|
"builtin_collection_validate",
|
|
"builtin_video_quality",
|
|
"builtin_dataset_filter",
|
|
"builtin_video_slicing",
|
|
"builtin_video_target_clip",
|
|
"builtin_yolo_video_target_clip",
|
|
"builtin_video_annotation",
|
|
"builtin_result_package",
|
|
"builtin_audit_sampling",
|
|
"builtin_human_review",
|
|
})
|
|
default:
|
|
return operators
|
|
}
|
|
}
|
|
|
|
func buildPocOperatorWorkflowEdges(batchName string, operators []domain.OperatorRecord) []domain.WorkflowEdge {
|
|
return buildPocOperatorSerialWorkflowEdges(orderPocOperatorsForBatch(batchName, operators))
|
|
}
|
|
|
|
func isXPlatformOperator(operator domain.OperatorRecord) bool {
|
|
value := strings.ToLower(strings.TrimSpace(operator.ID) + " " + strings.TrimSpace(operator.Name))
|
|
return strings.Contains(value, "xplatform") || strings.Contains(value, "x-platform") || strings.Contains(value, "采集目录")
|
|
}
|
|
|
|
func pocOperatorHandlerKey(operator domain.OperatorRecord) string {
|
|
id := strings.TrimPrefix(strings.TrimSpace(operator.ID), "operator-builtin-")
|
|
id = strings.ReplaceAll(id, "-", "_")
|
|
switch id {
|
|
case "real_source":
|
|
return "source"
|
|
case "bos_incremental_scan":
|
|
return "builtin_bos_incremental_scan"
|
|
case "xplatform_dataset_index":
|
|
return "builtin_xplatform_index"
|
|
case "xplatform_dataset_validate":
|
|
return "builtin_collection_validate"
|
|
case "csv_to_json", "json_to_csv", "json_map_rename", "template_render", "conditional_route":
|
|
return "builtin_" + strings.NewReplacer("json_map_rename", "json_map", "template_render", "text_template", "conditional_route", "condition_router").Replace(id)
|
|
case "json_field_extract":
|
|
return "builtin_json_extract"
|
|
case "video_filter":
|
|
return "builtin_dataset_filter"
|
|
case "video_slice":
|
|
return "builtin_video_slicing"
|
|
case "video_annotate":
|
|
return "builtin_video_annotation"
|
|
case "package":
|
|
return "builtin_result_package"
|
|
case "audit_sample":
|
|
return "builtin_audit_sampling"
|
|
case "directory_scan", "model_inference", "merge", "upload", "human_review", "mongodb_write", "hf_dataset_probe", "parquet_table_summary", "shell_command":
|
|
return "builtin_" + id
|
|
default:
|
|
return "builtin_" + id
|
|
}
|
|
}
|
|
|
|
func buildPocOperatorWorkflowInput(project *domain.ProjectRecord, batchName string, operators []domain.OperatorRecord, fixtures pocOperatorAcceptanceFixturePaths, storageConnection *domain.StorageConnectionRecord) domain.CreateWorkflowInput {
|
|
orderedOperators := orderPocOperatorsForBatch(batchName, operators)
|
|
nodes := make([]domain.WorkflowNode, 0, len(operators))
|
|
for index, operator := range orderedOperators {
|
|
nodes = append(nodes, workflowNodeFromOperator(operator, fixtures, storageConnection, float64(index*300), 0))
|
|
}
|
|
return domain.CreateWorkflowInput{
|
|
OrganizationID: normalizedTenantID(project.OrganizationID),
|
|
ProjectID: project.ID,
|
|
Name: batchName,
|
|
Description: "中文生产级 workflow 验收:串联真实数据、独立 CCR 算子镜像、artifact 传输和 lineage 证据。",
|
|
Type: domain.WorkflowTypeBatch,
|
|
Kind: domain.WorkflowKindStandard,
|
|
MaxConcurrentRuns: pocOperatorWorkflowAcceptanceMaxConcurrentRuns,
|
|
NodeTimeoutSeconds: 900,
|
|
RunTimeoutSeconds: 5400,
|
|
MaxRetryAttempts: 0,
|
|
RetryBackoffSeconds: 0,
|
|
Hyperparams: []domain.HyperParam{
|
|
{Key: "POC_ACCEPTANCE_FIXTURE_MANIFEST_PATH", Value: fixtures.Manifest},
|
|
{Key: "POC_ACCEPTANCE_XPLATFORM_SOURCE_PATH", Value: fixtures.XPlatform},
|
|
{Key: "POC_ACCEPTANCE_CSV_INPUT_PATH", Value: fixtures.CSV},
|
|
{Key: "POC_ACCEPTANCE_JSON_INPUT_PATH", Value: fixtures.JSON},
|
|
{Key: "POC_ACCEPTANCE_LEROBOT_SOURCE_PATH", Value: fixtures.LeRobot},
|
|
{Key: "POC_ACCEPTANCE_ROSBAG_SOURCE_PATH", Value: fixtures.ROSBag},
|
|
{Key: "POC_ACCEPTANCE_MODEL_JSON_PATH", Value: fixtures.Model},
|
|
},
|
|
Nodes: nodes,
|
|
Edges: buildPocOperatorWorkflowEdges(batchName, orderedOperators),
|
|
}
|
|
}
|
|
|
|
func (server *server) firstPocOperatorAcceptanceStorageConnection(ctx context.Context, organizationID string) (*domain.StorageConnectionRecord, error) {
|
|
connections, err := server.store.ListStorageConnectionsByOrganization(ctx, organizationID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(connections) == 0 {
|
|
connections, err = server.store.ListStorageConnections(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
for index := range connections {
|
|
item := connections[index]
|
|
if item.Provider == domain.StorageProviderBOS &&
|
|
strings.TrimSpace(item.Bucket) != "" &&
|
|
strings.TrimSpace(item.AccessKeyID) != "" &&
|
|
strings.TrimSpace(item.SecretAccessKey) != "" {
|
|
return &item, nil
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (server *server) upsertPocOperatorWorkflow(ctx context.Context, project *domain.ProjectRecord, input domain.CreateWorkflowInput) (*domain.WorkflowRecord, error) {
|
|
workflows, err := server.store.ListWorkflowsByProject(ctx, project.ID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for index := range workflows {
|
|
workflow := workflows[index]
|
|
if strings.TrimSpace(workflow.Name) != strings.TrimSpace(input.Name) {
|
|
continue
|
|
}
|
|
workflow.OrganizationID = input.OrganizationID
|
|
workflow.ProjectID = input.ProjectID
|
|
workflow.Description = input.Description
|
|
workflow.Type = input.Type
|
|
workflow.Kind = input.Kind
|
|
workflow.Hyperparams = input.Hyperparams
|
|
workflow.NodeTimeoutSeconds = input.NodeTimeoutSeconds
|
|
workflow.RunTimeoutSeconds = input.RunTimeoutSeconds
|
|
workflow.MaxRetryAttempts = input.MaxRetryAttempts
|
|
workflow.RetryBackoffSeconds = input.RetryBackoffSeconds
|
|
workflow.Nodes = input.Nodes
|
|
workflow.Edges = input.Edges
|
|
workflow.Status = domain.WorkflowStatusDraft
|
|
workflow.UpdatedAt = nowISO()
|
|
if err := server.store.UpsertWorkflow(ctx, workflow); err != nil {
|
|
return nil, err
|
|
}
|
|
return &workflow, nil
|
|
}
|
|
return server.store.CreateWorkflow(ctx, input)
|
|
}
|