package api import ( "archive/tar" "archive/zip" "bytes" "compress/gzip" "context" "crypto/sha256" "encoding/base64" "encoding/hex" "encoding/json" "fmt" "io" "math" "mime" "net/http" "net/url" "os" "os/exec" "path" "path/filepath" "reflect" "sort" "strconv" "strings" "time" "droboticflow/internal/domain" "github.com/baidubce/bce-sdk-go/services/bos" "github.com/go-chi/chi/v5" ) type sourceBrowserPreviewRequest struct { FileName string Payload []byte RedirectURL string } type flushResponseWriter struct { writer http.ResponseWriter flusher http.Flusher } func (target flushResponseWriter) Write(payload []byte) (int, error) { count, err := target.writer.Write(payload) if count > 0 { target.flusher.Flush() } return count, err } var sourceBrowserVideoPreviewPath = locateSourceBrowserVideoPreviewPath var sourceBrowserVideoPreviewWriter = streamSourceBrowserVideoPreview var sourceCollectionSignedPreviewURL = buildSourceCollectionSignedPreviewURL type scanConfigRequest struct { Name string `json:"name"` Description string `json:"description"` SourcePath string `json:"sourcePath"` ConnectionID string `json:"connectionId,omitempty"` Bucket string `json:"bucket,omitempty"` Prefix string `json:"prefix,omitempty"` Endpoint string `json:"endpoint,omitempty"` AccessKeyID string `json:"accessKeyId,omitempty"` SecretAccessKey string `json:"secretAccessKey,omitempty"` ScanMode string `json:"scanMode,omitempty"` DataSourceIssue *validationIssue `json:"-"` ScanModeIssue *validationIssue `json:"-"` IncrementalUnit string `json:"incrementalUnit,omitempty"` IncrementalUnitIssue *validationIssue `json:"-"` DirectoryDepth int `json:"directoryDepth,omitempty"` DirectoryDepthIssue *validationIssue `json:"-"` Pattern string `json:"pattern,omitempty"` Interval int `json:"interval"` Enabled bool `json:"enabled"` DataSource string `json:"dataSource,omitempty"` Notifications domain.ScanNotificationSettings `json:"notifications,omitempty"` } type samplingReviewRequest struct { VideoQuality string `json:"videoQuality"` AnnotationAccuracy string `json:"annotationAccuracy"` AnnotationCompleteness string `json:"annotationCompleteness"` OverallGrade string `json:"overallGrade"` Notes string `json:"notes,omitempty"` } func isScanWorkflow(item domain.WorkflowRecord) bool { return item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil } func chooseWorkflowKind(value domain.WorkflowKind) domain.WorkflowKind { if value == "" { return domain.WorkflowKindStandard } return value } func chooseDataSource(value string) string { trimmed := strings.ToUpper(strings.TrimSpace(value)) if trimmed == "" { return "BOS" } return trimmed } func parseWorkflowDataSourceHyperparam(value string) (string, *validationIssue) { trimmed := strings.ToUpper(strings.TrimSpace(value)) if trimmed == "" { return "", nil } if trimmed == "BOS" { return trimmed, nil } return "", &validationIssue{ Field: "dataSource", Message: "DATA_SOURCE must be BOS for scan workflows", } } func validateScanWorkflowDataSource(value string) *validationIssue { trimmed := strings.ToUpper(strings.TrimSpace(value)) if trimmed == "" || trimmed == "BOS" { return nil } return &validationIssue{ Field: "dataSource", Message: "dataSource must be BOS for scan workflows", } } func cleanBOSPrefix(value string) string { return strings.Trim(strings.TrimSpace(value), "/") } func chooseBOSEndpointWithDefault(value string, fallback string) string { if trimmedValue := strings.TrimSpace(value); trimmedValue != "" { return trimmedValue } return strings.TrimSpace(fallback) } func chooseBOSEndpoint(value string) string { return strings.TrimSpace(value) } func (server *server) chooseBOSEndpoint(value string) string { return chooseBOSEndpointWithDefault(value, server.globalBOSEndpoint) } func chooseWorkflowScanMode(value string) string { switch strings.ToUpper(strings.TrimSpace(value)) { case "FULL": return "FULL" default: return "INCREMENTAL" } } func parseWorkflowScanModeHyperparam(value string) (string, *validationIssue) { trimmed := strings.ToUpper(strings.TrimSpace(value)) if trimmed == "" { return "", nil } if trimmed == "FULL" || trimmed == "INCREMENTAL" { return trimmed, nil } return "", &validationIssue{ Field: "scanMode", Message: "SCAN_MODE must be FULL or INCREMENTAL", } } func chooseWorkflowIncrementalUnit(value string) string { switch strings.ToUpper(strings.TrimSpace(value)) { case "DIRECTORY": return "DIRECTORY" default: return "OBJECT" } } func parseWorkflowIncrementalUnitHyperparam(value string) (string, *validationIssue) { trimmed := strings.ToUpper(strings.TrimSpace(value)) if trimmed == "" { return "", nil } if trimmed == "OBJECT" || trimmed == "DIRECTORY" { return trimmed, nil } return "", &validationIssue{ Field: "incrementalUnit", Message: "INCREMENTAL_UNIT must be OBJECT or DIRECTORY", } } func chooseWorkflowDirectoryDepth(value int) int { if value > 0 { return value } return 1 } func parseWorkflowDirectoryDepthHyperparam(value string) (int, *validationIssue) { trimmed := strings.TrimSpace(value) if trimmed == "" { return 0, nil } parsed, err := strconv.Atoi(trimmed) if err != nil || parsed <= 0 { return 0, &validationIssue{ Field: "directoryDepth", Message: "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer", } } return parsed, nil } func scanSourcePath(dataSource string, sourcePath string, bucket string, prefix string) string { if chooseDataSource(dataSource) == "BOS" { if explicit := strings.TrimSpace(sourcePath); explicit != "" { return explicit } if strings.TrimSpace(bucket) == "" { return "" } canonical := "bos://" + strings.TrimSpace(bucket) if cleanedPrefix := cleanBOSPrefix(prefix); cleanedPrefix != "" { canonical += "/" + cleanedPrefix } return canonical } return strings.TrimSpace(sourcePath) } func mergeScanSecrets(existing *domain.ScanConfig, payload scanConfigRequest) (string, string) { accessKeyID := strings.TrimSpace(payload.AccessKeyID) secretAccessKey := strings.TrimSpace(payload.SecretAccessKey) if accessKeyID == "" && existing != nil { accessKeyID = existing.AccessKeyID } if secretAccessKey == "" && existing != nil { secretAccessKey = existing.SecretAccessKey } return accessKeyID, secretAccessKey } func mergeScanConnectionID(existing *domain.ScanConfig, payload scanConfigRequest) string { connectionID := strings.TrimSpace(payload.ConnectionID) if connectionID == "" && existing != nil && strings.TrimSpace(existing.ConnectionID) != "" && strings.TrimSpace(payload.AccessKeyID) == "" && strings.TrimSpace(payload.SecretAccessKey) == "" { connectionID = strings.TrimSpace(existing.ConnectionID) } return connectionID } func (server *server) getStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) { connectionID := strings.TrimSpace(id) if connectionID == "" { return nil, nil } item, err := server.store.GetStorageConnection(ctx, connectionID) if err != nil { return nil, err } if item == nil { return nil, fmt.Errorf("storage connection %s not found", connectionID) } if !server.filterByTenantContext(ctx, item.OrganizationID) { return nil, fmt.Errorf("storage connection %s not found", connectionID) } return item, nil } func (server *server) getMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) { connectionID := strings.TrimSpace(id) if connectionID == "" { return nil, nil } item, err := server.store.GetMongoConnection(ctx, connectionID) if err != nil { return nil, err } if item == nil { return nil, fmt.Errorf("mongo connection %s not found", connectionID) } if !server.filterByTenantContext(ctx, item.OrganizationID) { return nil, fmt.Errorf("mongo connection %s not found", connectionID) } return item, nil } func (server *server) resolveScanConfig(ctx context.Context, scanConfig *domain.ScanConfig) (*domain.ScanConfig, error) { if scanConfig == nil { return nil, nil } copy := *scanConfig connection, err := server.getStorageConnection(ctx, copy.ConnectionID) if err != nil { return nil, err } if connection == nil { if chooseDataSource(copy.DataSource) == "BOS" { copy.Endpoint = server.chooseBOSEndpoint(copy.Endpoint) } return ©, nil } copy.Endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint) if strings.TrimSpace(copy.AccessKeyID) == "" { copy.AccessKeyID = connection.AccessKeyID } if strings.TrimSpace(copy.SecretAccessKey) == "" { copy.SecretAccessKey = connection.SecretAccessKey } return ©, nil } func cloneHyperparams(items []domain.HyperParam) []domain.HyperParam { out := make([]domain.HyperParam, len(items)) copy(out, items) return out } func clonePorts(items []domain.PortDefinition) []domain.PortDefinition { out := make([]domain.PortDefinition, len(items)) copy(out, items) return out } func cloneOperatorIntoNode(operator domain.OperatorRecord, nodeID string, x float64, y float64, overrides map[string]string) domain.WorkflowNode { hyperparams := cloneHyperparams(operator.Hyperparams) seen := make(map[string]int, len(hyperparams)) for index, item := range hyperparams { seen[item.Key] = index } for key, value := range overrides { if index, ok := seen[key]; ok { hyperparams[index].Value = value continue } hyperparams = append(hyperparams, domain.HyperParam{Key: key, Value: value}) } return domain.WorkflowNode{ ID: nodeID, Type: "operator", Data: domain.WorkflowNodeData{ Label: operator.Name, Category: operator.Category, Description: operator.Description, Inputs: clonePorts(operator.Inputs), Outputs: clonePorts(operator.Outputs), Image: operator.Image, SkillDefinition: operator.SkillDefinition, Hyperparams: hyperparams, }, Position: domain.Position{X: x, Y: y}, } } func operatorByName(items []domain.OperatorRecord, name string) (*domain.OperatorRecord, bool) { for _, item := range items { if item.Name == name { copy := item return ©, true } } return nil, false } func (server *server) buildScanWorkflowInput(ctx context.Context, projectID string, payload scanConfigRequest, existing *domain.ScanConfig) (domain.CreateWorkflowInput, error) { operators, err := server.listAvailableOperators(ctx) if err != nil { return domain.CreateWorkflowInput{}, err } dataSource := chooseDataSource(payload.DataSource) sourceOperatorName := "BOS 增量扫描" sourceOperator, ok := operatorByName(operators, sourceOperatorName) if !ok { return domain.CreateWorkflowInput{}, fmt.Errorf("operator %s not found", sourceOperatorName) } indexOperator, ok := operatorByName(operators, "X 平台数据集索引") if !ok { return domain.CreateWorkflowInput{}, fmt.Errorf("operator X 平台数据集索引 not found") } validateOperator, ok := operatorByName(operators, "采集目录校验") if !ok { return domain.CreateWorkflowInput{}, fmt.Errorf("operator 采集目录校验 not found") } sourcePath := scanSourcePath(dataSource, payload.SourcePath, payload.Bucket, payload.Prefix) connectionID := mergeScanConnectionID(existing, payload) var connection *domain.StorageConnectionRecord if connectionID != "" { connection, err = server.getStorageConnection(ctx, connectionID) if err != nil { return domain.CreateWorkflowInput{}, err } } endpoint := "" accessKeyID := "" secretAccessKey := "" if connection != nil { endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint) } else if dataSource == "BOS" { endpoint = server.chooseBOSEndpoint(payload.Endpoint) accessKeyID, secretAccessKey = mergeScanSecrets(existing, payload) } sourceOverrides := map[string]string{ "DATA_SOURCE": dataSource, "SCAN_MODE": chooseWorkflowScanMode(payload.ScanMode), "INCREMENTAL_UNIT": chooseWorkflowIncrementalUnit(payload.IncrementalUnit), "INCREMENTAL_DIRECTORY_DEPTH": strconv.Itoa(chooseWorkflowDirectoryDepth(payload.DirectoryDepth)), } if dataSource == "BOS" { sourceOverrides["SOURCE_PATH"] = sourcePath sourceOverrides["BOS_ENDPOINT"] = endpoint } else { sourceOverrides["SOURCE_PATH"] = sourcePath } sourceNode := cloneOperatorIntoNode(*sourceOperator, "scan-source", 0, 0, sourceOverrides) sourceNode.Data.Bucket = strings.TrimSpace(payload.Bucket) sourceNode.Data.Prefix = cleanBOSPrefix(payload.Prefix) sourceNode.Data.Pattern = strings.TrimSpace(payload.Pattern) indexNode := cloneOperatorIntoNode(*indexOperator, "scan-index", 260, 0, nil) validateNode := cloneOperatorIntoNode(*validateOperator, "scan-validate", 520, 0, nil) schedule := &domain.WorkflowSchedule{SourcePath: sourcePath} workflowType := domain.WorkflowTypeBatch if payload.Enabled && payload.Interval > 0 { schedule.Interval = payload.Interval workflowType = domain.WorkflowTypeStreaming } return domain.CreateWorkflowInput{ ProjectID: projectID, Name: strings.TrimSpace(payload.Name), Description: strings.TrimSpace(payload.Description), Type: workflowType, Kind: domain.WorkflowKindScan, ScanConfig: &domain.ScanConfig{ DataSource: dataSource, SourcePath: sourcePath, ConnectionID: connectionID, Bucket: strings.TrimSpace(payload.Bucket), Prefix: cleanBOSPrefix(payload.Prefix), Endpoint: endpoint, AccessKeyID: accessKeyID, SecretAccessKey: secretAccessKey, ScanMode: chooseWorkflowScanMode(payload.ScanMode), IncrementalUnit: chooseWorkflowIncrementalUnit(payload.IncrementalUnit), DirectoryDepth: chooseWorkflowDirectoryDepth(payload.DirectoryDepth), Pattern: strings.TrimSpace(payload.Pattern), Enabled: payload.Enabled && payload.Interval > 0, Notifications: payload.Notifications, }, Schedule: schedule, Nodes: []domain.WorkflowNode{ sourceNode, indexNode, validateNode, }, Edges: []domain.WorkflowEdge{ {ID: "scan-source-index", Source: sourceNode.ID, SourceHandle: "out-1", Target: indexNode.ID, TargetHandle: "in-1"}, {ID: "scan-index-validate", Source: indexNode.ID, SourceHandle: "out-1", Target: validateNode.ID, TargetHandle: "in-1"}, }, }, nil } func validateScanConfigEndpoint(payload scanConfigRequest, existing *domain.ScanConfig) *validationIssue { if chooseDataSource(payload.DataSource) != "BOS" { return nil } if strings.TrimSpace(mergeScanConnectionID(existing, payload)) != "" { return nil } return validateOptionalHTTPURL("endpoint", payload.Endpoint) } func validateScanConfigDraftPayload(payload scanConfigRequest, existing ...*domain.ScanConfig) []validationIssue { var previous *domain.ScanConfig if len(existing) > 0 { previous = existing[0] } issues := make([]validationIssue, 0, 5) if strings.TrimSpace(payload.Name) == "" { issues = append(issues, validationIssue{Field: "name", Message: "name is required"}) } if payload.Interval < 0 { issues = append(issues, validationIssue{Field: "interval", Message: "interval must be zero or greater"}) } if issue := validateScanWorkflowDataSource(payload.DataSource); issue != nil { issues = append(issues, *issue) } if payload.DataSourceIssue != nil { issues = append(issues, *payload.DataSourceIssue) } if scanMode := strings.ToUpper(strings.TrimSpace(payload.ScanMode)); scanMode != "" && scanMode != "FULL" && scanMode != "INCREMENTAL" { issues = append(issues, validationIssue{Field: "scanMode", Message: "scanMode must be FULL or INCREMENTAL"}) } if payload.ScanModeIssue != nil { issues = append(issues, *payload.ScanModeIssue) } if incrementalUnit := strings.ToUpper(strings.TrimSpace(payload.IncrementalUnit)); incrementalUnit != "" && incrementalUnit != "OBJECT" && incrementalUnit != "DIRECTORY" { issues = append(issues, validationIssue{Field: "incrementalUnit", Message: "incrementalUnit must be OBJECT or DIRECTORY"}) } if payload.IncrementalUnitIssue != nil { issues = append(issues, *payload.IncrementalUnitIssue) } if payload.DirectoryDepthIssue != nil { issues = append(issues, *payload.DirectoryDepthIssue) } if payload.DirectoryDepth < 0 { issues = append(issues, validationIssue{Field: "directoryDepth", Message: "directoryDepth must be zero or greater"}) } if issue := validateScanConfigEndpoint(payload, previous); issue != nil { issues = append(issues, *issue) } return issues } func scanConfigPayloadReady(payload scanConfigRequest, existing *domain.ScanConfig) bool { switch chooseDataSource(payload.DataSource) { case "BOS": connectionID := mergeScanConnectionID(existing, payload) accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload) hasTarget := strings.TrimSpace(payload.SourcePath) != "" || (strings.TrimSpace(payload.Bucket) != "" && cleanBOSPrefix(payload.Prefix) != "") return hasTarget && (strings.TrimSpace(connectionID) != "" || (strings.TrimSpace(accessKeyID) != "" && strings.TrimSpace(secretAccessKey) != "")) default: return strings.TrimSpace(payload.SourcePath) != "" } } func validateScanConfigPayload(payload scanConfigRequest, existing *domain.ScanConfig) []validationIssue { issues := validateScanConfigDraftPayload(payload, existing) switch chooseDataSource(payload.DataSource) { case "BOS": if strings.TrimSpace(payload.SourcePath) == "" { if strings.TrimSpace(payload.Bucket) == "" { issues = append(issues, validationIssue{Field: "bucket", Message: "bucket is required for BOS scan configs when sourcePath is empty"}) } if cleanBOSPrefix(payload.Prefix) == "" { issues = append(issues, validationIssue{Field: "prefix", Message: "prefix is required for BOS scan configs when sourcePath is empty"}) } } connectionID := mergeScanConnectionID(existing, payload) if strings.TrimSpace(connectionID) == "" { accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload) if strings.TrimSpace(accessKeyID) == "" { issues = append(issues, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required for BOS scan configs"}) } if strings.TrimSpace(secretAccessKey) == "" { issues = append(issues, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required for BOS scan configs"}) } } default: if strings.TrimSpace(payload.SourcePath) == "" { issues = append(issues, validationIssue{Field: "sourcePath", Message: "sourcePath is required"}) } } return issues } func (server *server) scanWorkflows(ctx context.Context, request *http.Request) ([]domain.WorkflowRecord, error) { items, err := server.listAllWorkflowsForRequest(ctx, request) if err != nil { return nil, err } filtered := make([]domain.WorkflowRecord, 0) for _, item := range items { if isScanWorkflow(item) { item.Kind = chooseWorkflowKind(item.Kind) filtered = append(filtered, item) } } return filtered, nil } func (server *server) handleScanConfigsList(writer http.ResponseWriter, request *http.Request) { items, err := server.scanWorkflows(request.Context(), request) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } items = server.filterWorkflowsForRequest(request, items) renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)}) } func (server *server) handleScanConfigsCreate(writer http.ResponseWriter, request *http.Request) { var payload scanConfigRequest if !decodeJSON(writer, request, &payload) { return } if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 { renderValidationError(writer, issues...) return } if payload.Interval > 0 && !scanConfigPayloadReady(payload, nil) { renderValidationError(writer, validateScanConfigPayload(payload, nil)...) return } if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(nil, payload)); len(issues) > 0 { renderValidationError(writer, issues...) return } input, err := server.buildScanWorkflowInput(request.Context(), "", payload, nil) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } server.assignTenant(request, &input.OrganizationID) item, err := server.store.CreateWorkflow(request.Context(), input) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item)) } func (server *server) handleScanConfigsGet(writer http.ResponseWriter, request *http.Request) { item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if item == nil || !isScanWorkflow(*item) { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"}) return } if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") { return } item.Kind = chooseWorkflowKind(item.Kind) renderJSON(writer, http.StatusOK, sanitizeWorkflow(item)) } func (server *server) handleScanConfigsUpdate(writer http.ResponseWriter, request *http.Request) { item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if item == nil || !isScanWorkflow(*item) { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"}) return } if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") { return } var payload scanConfigRequest if !decodeJSON(writer, request, &payload) { return } if issues := validateScanConfigDraftPayload(payload, item.ScanConfig); len(issues) > 0 { renderValidationError(writer, issues...) return } if payload.Interval > 0 && !scanConfigPayloadReady(payload, item.ScanConfig) { renderValidationError(writer, validateScanConfigPayload(payload, item.ScanConfig)...) return } if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(item.ScanConfig, payload)); len(issues) > 0 { renderValidationError(writer, issues...) return } input, err := server.buildScanWorkflowInput(request.Context(), item.ProjectID, payload, item.ScanConfig) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } item.Name = input.Name item.Description = input.Description item.Type = input.Type item.Kind = input.Kind item.ScanConfig = input.ScanConfig item.Schedule = input.Schedule item.Nodes = input.Nodes item.Edges = input.Edges item.UpdatedAt = nowISO() if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, sanitizeWorkflow(item)) } func (server *server) handleScanConfigsDelete(writer http.ResponseWriter, request *http.Request) { item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if item == nil || !isScanWorkflow(*item) { writer.WriteHeader(http.StatusNotFound) return } if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") { return } deleted, err := server.store.DeleteWorkflow(request.Context(), item.ID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if !deleted { writer.WriteHeader(http.StatusNotFound) return } writer.WriteHeader(http.StatusNoContent) } func (server *server) handleScanConfigsRun(writer http.ResponseWriter, request *http.Request) { item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if item == nil || !isScanWorkflow(*item) { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"}) return } if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") { return } if issues := server.validateScanConnectionReference(request.Context(), item.ScanConfig); len(issues) > 0 { renderValidationError(writer, issues...) return } run, err := server.createRunWithLightweightManifests(request.Context(), item, domain.RunTriggerManual) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusAccepted, run) } func artifactByName(run domain.RunRecord, name string) *domain.ExecutionArtifact { for index := range run.Artifacts { if run.Artifacts[index].Name == name { return &run.Artifacts[index] } } return nil } func mapValue(value any) map[string]any { item, _ := normalizeStructuredValue(value).(map[string]any) return item } func listValue(value any) []any { items, _ := normalizeStructuredValue(value).([]any) return items } func normalizeStructuredValue(value any) any { switch typed := value.(type) { case nil: return nil case map[string]any: items := make(map[string]any, len(typed)) for key, item := range typed { items[key] = normalizeStructuredValue(item) } return items case []any: items := make([]any, len(typed)) for index, item := range typed { items[index] = normalizeStructuredValue(item) } return items } rv := reflect.ValueOf(value) switch rv.Kind() { case reflect.Map: if rv.Type().Key().Kind() != reflect.String { return value } items := make(map[string]any, rv.Len()) iter := rv.MapRange() for iter.Next() { items[iter.Key().String()] = normalizeStructuredValue(iter.Value().Interface()) } return items case reflect.Slice, reflect.Array: items := make([]any, rv.Len()) for index := 0; index < rv.Len(); index++ { items[index] = normalizeStructuredValue(rv.Index(index).Interface()) } return items default: return value } } func stringValue(value any) string { switch typed := value.(type) { case nil: return "" case string: return typed default: text := strings.TrimSpace(fmt.Sprintf("%v", typed)) if text == "" { return "" } return text } } func boolValue(value any) bool { typed, ok := value.(bool) return ok && typed } func floatValue(value any) float64 { switch typed := value.(type) { case float64: if math.IsNaN(typed) || math.IsInf(typed, 0) { return 0 } return typed case float32: value := float64(typed) if math.IsNaN(value) || math.IsInf(value, 0) { return 0 } return value case int: return float64(typed) case int32: return float64(typed) case int64: return float64(typed) default: return 0 } } func intValue(value any) int { parsed, ok := exactIntegerValue(value) if !ok { return 0 } if strconv.IntSize == 32 { const maxInt32 = int64(1<<31 - 1) const minInt32 = -1 << 31 if parsed < minInt32 || parsed > maxInt32 { return 0 } } return int(parsed) } func int64Value(value any) int64 { parsed, ok := exactIntegerValue(value) if !ok { return 0 } return parsed } func exactIntegerValue(value any) (int64, bool) { switch typed := value.(type) { case int: return int64(typed), true case int8: return int64(typed), true case int16: return int64(typed), true case int32: return int64(typed), true case int64: return typed, true case uint: if uint64(typed) > uint64(math.MaxInt64) { return 0, false } return int64(typed), true case uint8: return int64(typed), true case uint16: return int64(typed), true case uint32: return int64(typed), true case uint64: if typed > uint64(math.MaxInt64) { return 0, false } return int64(typed), true case float32: const maxExactFloat32Integer = 1<<24 - 1 return exactFloatIntegerValue(float64(typed), maxExactFloat32Integer) case float64: const maxExactFloat64Integer = 1<<53 - 1 return exactFloatIntegerValue(typed, maxExactFloat64Integer) case json.Number: parsed, err := typed.Int64() return parsed, err == nil default: return 0, false } } func exactFloatIntegerValue(value float64, maxExactAbs float64) (int64, bool) { if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value { return 0, false } if math.Abs(value) > maxExactAbs { return 0, false } return int64(value), true } func artifactDataRef(artifact *domain.ExecutionArtifact) map[string]any { if artifact == nil || artifact.Data == nil { return nil } return mapValue(artifact.Data["dataRef"]) } func artifactStructuredData(artifact *domain.ExecutionArtifact) map[string]any { if artifact == nil { return nil } if dataRef := artifactDataRef(artifact); dataRef != nil { payloadPath := strings.TrimSpace(stringValue(dataRef["payloadPath"])) if payloadPath != "" && !strings.Contains(payloadPath, "://") { payload, err := os.ReadFile(payloadPath) if err != nil { return nil } var decoded any if err := json.Unmarshal(payload, &decoded); err != nil { return nil } if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok { return structured } return nil } } if artifact.Data != nil { return artifact.Data } if artifact.Type == domain.PortTypeJSON { filePath := strings.TrimSpace(artifact.Path) if filePath != "" && !strings.Contains(filePath, "://") { payload, err := os.ReadFile(filePath) if err == nil { var decoded any if err := json.Unmarshal(payload, &decoded); err == nil { if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok { return structured } } } } } return nil } func sourceArtifactRelativePath(path string) string { normalized := filepath.ToSlash(path) marker := "/_source_tree/" index := strings.Index(normalized, marker) if index < 0 { return "" } return strings.TrimPrefix(normalized[index+len(marker):], "/") } func sourceArtifactPath(artifact domain.ExecutionArtifact) string { if payload := artifactStructuredData(&artifact); payload != nil { if relative := stringValue(payload["relativePath"]); relative != "" { return filepath.ToSlash(strings.Trim(relative, "/")) } } return sourceArtifactRelativePath(artifact.Path) } func packageArtifactRelativePath(path string, packageName string) string { normalized := filepath.ToSlash(path) marker := "/packages/" + packageName + "/" index := strings.Index(normalized, marker) if index < 0 { return "" } return strings.TrimPrefix(normalized[index+len(marker):], "/") } func packageContentURL(runID string, packageName string, relativePath string) string { return fmt.Sprintf( "/api/packages/%s/%s/content?relativePath=%s", url.PathEscape(runID), url.PathEscape(packageName), url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")), ) } func packageArtifactPreviewURL(artifact domain.ExecutionArtifact) string { payload := artifactStructuredData(&artifact) preview := mapValue(payload["preview"]) if preview != nil { if value := strings.TrimSpace(stringValue(preview["url"])); value != "" { return value } } source := mapValue(payload["source"]) if source != nil { if value := strings.TrimSpace(stringValue(source["previewURL"])); value != "" { return value } } return "" } func packageArtifactResolvedRelativePath(artifact domain.ExecutionArtifact, packageName string) string { if relativePath := packageArtifactRelativePath(artifact.Path, packageName); relativePath != "" { return relativePath } payload := artifactStructuredData(&artifact) if payload == nil { return "" } relativePath := strings.Trim(filepath.ToSlash(stringValue(payload["relativePath"])), "/") if relativePath == "" { return "" } packagePrefix := strings.Trim(filepath.ToSlash(path.Join("packages", packageName)), "/") if packagePrefix == "" { return "" } if relativePath == packagePrefix { return "" } if strings.HasPrefix(relativePath, packagePrefix+"/") { return strings.TrimPrefix(relativePath, packagePrefix+"/") } return "" } func packageArtifactContentURL(runID string, packageName string, relativePath string) string { return packageContentURL(runID, packageName, relativePath) } func packageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact { copy := artifact if copy.Data == nil { copy.Data = make(map[string]any) } relativePath := packageArtifactResolvedRelativePath(artifact, packageName) if relativePath == "" { relativePath = filepath.ToSlash(strings.Trim(copy.Path, "/")) } if relativePath != "" { copy.Path = relativePath } copy.Data["contentURL"] = packageArtifactContentURL(runID, packageName, copy.Path) preview := mapValue(copy.Data["preview"]) if preview == nil { preview = make(map[string]any) } kind := strings.TrimSpace(stringValue(preview["kind"])) mimeType := "" if copy.Metadata != nil { mimeType = strings.TrimSpace(copy.Metadata.MimeType) } if kind == "video" || copy.Type == "video" || strings.HasPrefix(mimeType, "video/") { preview["kind"] = "video" preview["url"] = packageContentURL(runID, packageName, copy.Path) copy.Data["preview"] = preview } return copy } const wilorHandPoseSchemaVersion = "droboticflow.wilor.hand_pose_result/v1" func safePackageNamePart(value string) string { trimmed := strings.ToLower(strings.TrimSpace(value)) var builder strings.Builder lastDash := false for _, current := range trimmed { isAlphaNumeric := (current >= 'a' && current <= 'z') || (current >= '0' && current <= '9') if isAlphaNumeric { builder.WriteRune(current) lastDash = false continue } if !lastDash { builder.WriteByte('-') lastDash = true } } return strings.Trim(builder.String(), "-") } func wilorResultPayload(artifact domain.ExecutionArtifact) map[string]any { if artifact.Name != "wilor-result.json" { return nil } payload := artifactStructuredData(&artifact) if payload == nil { return nil } if strings.TrimSpace(stringValue(payload["schemaVersion"])) != wilorHandPoseSchemaVersion { return nil } if strings.TrimSpace(stringValue(payload["relativePath"])) != "" { return nil } return payload } func wilorPackageName(artifact domain.ExecutionArtifact, payload map[string]any) string { source := mapValue(payload["source"]) sourceName := strings.TrimSpace(stringValue(source["name"])) stem := strings.TrimSuffix(path.Base(filepath.ToSlash(sourceName)), path.Ext(sourceName)) sourceSlug := safePackageNamePart(stem) if sourceSlug == "" { sourceSlug = "result" } nodeSlug := safePackageNamePart(artifact.NodeID) if nodeSlug == "" { nodeSlug = "wilor" } return "wilor-" + sourceSlug + "-" + nodeSlug } func wilorResultGroupKey(artifact domain.ExecutionArtifact, payload map[string]any) string { source := mapValue(payload["source"]) sourceName := strings.TrimSpace(stringValue(source["name"])) sourceRef := strings.TrimSpace(stringValue(source["ref"])) mediaKind := strings.TrimSpace(stringValue(payload["mediaKind"])) if sourceName != "" || sourceRef != "" || mediaKind != "" { return strings.ToLower(mediaKind) + "|" + strings.ToLower(sourceName) + "|" + strings.ToLower(sourceRef) } return artifact.ID } func wilorResultArtifactPriority(artifact domain.ExecutionArtifact) int { nodeID := strings.ToLower(strings.TrimSpace(artifact.NodeID)) score := 0 if strings.Contains(nodeID, "upload") || strings.Contains(nodeID, "save") { score += 30 } if strings.Contains(nodeID, "wilor") { score += 20 } if strings.TrimSpace(artifact.Path) != "" { score++ } return score } func wilorResultArtifacts(run domain.RunRecord) []domain.ExecutionArtifact { bestByResult := make(map[string]domain.ExecutionArtifact) for _, artifact := range run.Artifacts { payload := wilorResultPayload(artifact) if payload == nil { continue } key := wilorResultGroupKey(artifact, payload) current, exists := bestByResult[key] if !exists || wilorResultArtifactPriority(artifact) > wilorResultArtifactPriority(current) { bestByResult[key] = artifact } } items := make([]domain.ExecutionArtifact, 0, len(bestByResult)) for _, artifact := range bestByResult { items = append(items, artifact) } sort.Slice(items, func(left int, right int) bool { return wilorPackageName(items[left], wilorResultPayload(items[left])) < wilorPackageName(items[right], wilorResultPayload(items[right])) }) return items } func wilorPackageManifestArtifact(run domain.RunRecord, packageName string) (*domain.ExecutionArtifact, map[string]any) { for index := range run.Artifacts { payload := wilorResultPayload(run.Artifacts[index]) if payload == nil { continue } if wilorPackageName(run.Artifacts[index], payload) == packageName { return &run.Artifacts[index], payload } } return nil, nil } func wilorOutputArtifactNames(manifest domain.ExecutionArtifact, payload map[string]any) map[string]struct{} { names := map[string]struct{}{ manifest.Name: {}, } for _, raw := range listValue(payload["downloadedOutputs"]) { item := mapValue(raw) name := strings.TrimSpace(stringValue(item["name"])) if name == "" { if outputPath := strings.TrimSpace(stringValue(item["path"])); outputPath != "" { name = path.Base(filepath.ToSlash(outputPath)) } } if name != "" { names[name] = struct{}{} } } return names } func wilorPackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact { manifest, payload := wilorPackageManifestArtifact(run, packageName) if manifest == nil || payload == nil { return nil } names := wilorOutputArtifactNames(*manifest, payload) bestByName := make(map[string]domain.ExecutionArtifact) for _, artifact := range run.Artifacts { if _, ok := names[artifact.Name]; !ok { continue } if artifact.NodeID != manifest.NodeID && strings.TrimSpace(stringValue(mapValue(artifactStructuredData(&artifact)["source"])["path"])) == "" { continue } current, exists := bestByName[artifact.Name] if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) { bestByName[artifact.Name] = artifact } } items := make([]domain.ExecutionArtifact, 0, len(bestByName)) for _, artifact := range bestByName { items = append(items, artifact) } sort.Slice(items, func(left int, right int) bool { return items[left].Name < items[right].Name }) return items } func wilorPackageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact { copy := artifact if copy.Data == nil { copy.Data = make(map[string]any) } relativePath := strings.TrimSpace(copy.Name) if relativePath == "" { relativePath = path.Base(filepath.ToSlash(strings.TrimSpace(copy.Path))) } copy.Path = relativePath copy.Data["contentURL"] = packageArtifactContentURL(runID, packageName, relativePath) preview := mapValue(copy.Data["preview"]) if preview == nil { preview = make(map[string]any) } mimeType := "" if copy.Metadata != nil { mimeType = strings.TrimSpace(copy.Metadata.MimeType) } kind := sourceEntryPayloadKind(relativePath, strings.ToLower(mimeType)) if kind == "video" || kind == "image" { preview["kind"] = kind preview["url"] = packageArtifactContentURL(runID, packageName, relativePath) copy.Data["preview"] = preview } return copy } func wilorPackageFilesForResponse(run domain.RunRecord, packageName string) ([]domain.ExecutionArtifact, []domain.ExecutionArtifact) { rawFiles := wilorPackageArtifacts(run, packageName) files := make([]domain.ExecutionArtifact, 0, len(rawFiles)) for _, artifact := range rawFiles { files = append(files, wilorPackageArtifactForResponse(run.ID, packageName, artifact)) } return files, rawFiles } func wilorPackageItem(run domain.RunRecord, manifest domain.ExecutionArtifact, payload map[string]any, rawFiles []domain.ExecutionArtifact) map[string]any { source := mapValue(payload["source"]) result := mapValue(payload["result"]) service := mapValue(payload["service"]) health := mapValue(service["health"]) sourceName := strings.TrimSpace(stringValue(source["name"])) if sourceName == "" { sourceName = "WiLoR result" } mediaKind := strings.TrimSpace(stringValue(payload["mediaKind"])) if mediaKind == "" { mediaKind = "media" } clipID := strings.TrimSuffix(path.Base(filepath.ToSlash(sourceName)), path.Ext(sourceName)) if clipID == "" || clipID == "." { clipID = manifest.NodeID } storagePath := "" for _, artifact := range rawFiles { if candidate := packageArtifactStoragePath(artifact, ""); candidate != "" { storagePath = candidate break } } annotation := map[string]any{ "type": "wilor_hand_pose", "mediaKind": mediaKind, "handCount": intValue(result["handCount"]), "frameCount": intValue(result["frameCount"]), "status": stringValue(payload["status"]), } if task := mapValue(result["wilorTask"]); task != nil { annotation["task"] = task } quality := map[string]any{ "runtime": mapValue(health["runtime"]), "service": map[string]any{ "image": stringValue(service["image"]), "imageId": stringValue(service["imageId"]), }, } return map[string]any{ "packageName": wilorPackageName(manifest, payload), "clipId": clipID, "sourceVideo": sourceName, "clipDuration": 0, "fileCount": len(rawFiles), "pipelineTags": []any{"wilor", "hand_pose", mediaKind}, "storagePath": storagePath, "quality": quality, "annotation": annotation, } } func canonicalBOSPath(bucket string, object string) string { bucket = strings.TrimSpace(bucket) object = strings.Trim(strings.TrimSpace(object), "/") if bucket == "" { return "" } if object == "" { return "bos://" + bucket } return "bos://" + bucket + "/" + object } func packageArtifactPriority(artifact domain.ExecutionArtifact) int { score := 0 if artifact.NodeID == "upload" { score += 40 } artifactPath := strings.TrimSpace(artifact.Path) if strings.HasPrefix(strings.ToLower(artifactPath), "bos://") { score += 30 } else if artifactPath != "" && !strings.Contains(artifactPath, "://") { score += 10 } payload := artifactStructuredData(&artifact) source := mapValue(payload["source"]) sourcePath := strings.TrimSpace(stringValue(source["path"])) if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") { score += 30 } if strings.TrimSpace(packageArtifactPreviewURL(artifact)) != "" { score += 5 } return score } func packageArtifactStoragePath(artifact domain.ExecutionArtifact, packageName string) string { payload := artifactStructuredData(&artifact) source := mapValue(payload["source"]) if sourcePath := strings.TrimSpace(stringValue(source["path"])); sourcePath != "" { if bucket, object, ok := parseBOSPath(sourcePath); ok { return canonicalBOSPath(bucket, path.Dir(object)) } directory := filepath.ToSlash(filepath.Dir(sourcePath)) if directory != "." { return directory } } artifactPath := strings.TrimSpace(artifact.Path) if bucket, object, ok := parseBOSPath(artifactPath); ok { return canonicalBOSPath(bucket, path.Dir(object)) } if artifactPath == "" { return "" } directory := filepath.ToSlash(filepath.Dir(artifactPath)) if directory == "." { return "" } return directory } func dedupePackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact { bestByRelativePath := make(map[string]domain.ExecutionArtifact) for _, artifact := range run.Artifacts { relativePath := packageArtifactResolvedRelativePath(artifact, packageName) if relativePath == "" { continue } current, exists := bestByRelativePath[relativePath] if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) { bestByRelativePath[relativePath] = artifact } } items := make([]domain.ExecutionArtifact, 0, len(bestByRelativePath)) for _, artifact := range bestByRelativePath { items = append(items, artifact) } sort.Slice(items, func(left int, right int) bool { leftPath := packageArtifactResolvedRelativePath(items[left], packageName) rightPath := packageArtifactResolvedRelativePath(items[right], packageName) return leftPath < rightPath }) return items } func packageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact { items := make([]domain.ExecutionArtifact, 0) for _, artifact := range dedupePackageArtifacts(run, packageName) { items = append(items, packageArtifactForResponse(run.ID, packageName, artifact)) } return items } func choosePackagePreview(files []domain.ExecutionArtifact) *domain.ExecutionArtifact { for _, file := range files { if strings.HasSuffix(strings.ToLower(file.Name), ".mp4") { copy := file return © } } if len(files) == 0 { return nil } copy := files[0] return © } func sourceDatasetList(data map[string]any, catalogKey string, fallbackKey string) []any { if _, ok := data[catalogKey]; ok { return listValue(data[catalogKey]) } return listValue(data[fallbackKey]) } func sourceDatasetHasCatalog(data map[string]any) bool { _, hasCollections := data["catalogCollections"] _, hasItems := data["catalogItems"] _, hasEntries := data["catalogEntries"] return hasCollections || hasItems || hasEntries } func sourceDatasetIsDisplayable(data map[string]any) bool { if len(sourceDatasetList(data, "catalogCollections", "collections")) > 0 { return true } if len(sourceDatasetList(data, "catalogEntries", "entries")) > 0 { return true } return intValue(data["catalogCollectionCount"]) > 0 || intValue(data["collectionCount"]) > 0 } func runSourceManifest(run domain.RunRecord) *domain.ExecutionArtifact { manifest := artifactByName(run, "source-manifest.json") if manifest == nil || artifactStructuredData(manifest) == nil { return nil } return manifest } func runDatasetIndex(run domain.RunRecord) *domain.ExecutionArtifact { artifact := artifactByName(run, "dataset-index.json") if artifact == nil || artifactStructuredData(artifact) == nil { return nil } return artifact } func runLooksLikeSourceDataset(workflow domain.WorkflowRecord, run domain.RunRecord) bool { if manifest := runSourceManifest(run); manifest != nil { payload := artifactStructuredData(manifest) if strings.TrimSpace(stringValue(payload["sourcePath"])) != "" || strings.TrimSpace(stringValue(payload["dataSource"])) != "" { return true } } index := runDatasetIndex(run) if index == nil { return false } if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil { return true } indexPayload := artifactStructuredData(index) sourcePath := strings.TrimSpace(stringValue(indexPayload["sourcePath"])) rootPath := strings.TrimSpace(stringValue(indexPayload["rootPath"])) dataSource := strings.TrimSpace(stringValue(indexPayload["dataSource"])) if dataSource != "" || strings.HasPrefix(sourcePath, "bos://") || strings.HasPrefix(rootPath, "bos://") { return true } if len(mapValue(indexPayload["archiveSources"])) > 0 && strings.Contains(rootPath, "_materialized_inputs") { return false } return sourceDatasetHasCatalog(indexPayload) } func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]domain.WorkflowRecord, map[string]domain.RunRecord, error) { workflows, err := server.listAllWorkflowsForContext(ctx) if err != nil { return nil, nil, err } workflowByID := make(map[string]domain.WorkflowRecord, len(workflows)) for _, workflow := range workflows { workflowByID[workflow.ID] = workflow } runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted)) if err != nil { return nil, nil, err } latestByWorkflow := make(map[string]domain.RunRecord) for _, run := range runs { workflow, ok := workflowByID[run.WorkflowID] if !ok { continue } index := runDatasetIndex(run) if index == nil { continue } if !runLooksLikeSourceDataset(workflow, run) { continue } if !sourceDatasetIsDisplayable(artifactStructuredData(index)) { continue } current, exists := latestByWorkflow[run.WorkflowID] if !exists || run.CreatedAt > current.CreatedAt { latestByWorkflow[run.WorkflowID] = run } } return workflowByID, latestByWorkflow, nil } func parseBOSPath(sourcePath string) (string, string, bool) { normalized := strings.TrimSpace(sourcePath) if !strings.HasPrefix(strings.ToLower(normalized), "bos://") { return "", "", false } trimmed := strings.TrimPrefix(normalized, "bos://") trimmed = strings.TrimPrefix(trimmed, "BOS://") bucket, object, found := strings.Cut(trimmed, "/") bucket = strings.TrimSpace(bucket) object = strings.Trim(strings.TrimSpace(object), "/") if !found { object = "" } if bucket == "" { return "", "", false } return bucket, object, true } func normalizeLocalDirectory(value string) string { trimmed := strings.TrimSpace(value) if trimmed == "" { return "" } directory := filepath.ToSlash(filepath.Dir(trimmed)) if directory == "." { return "" } return directory } func assetStorageLocationFromPaths(rootPath string, currentPath string, endpoint string) domain.AssetStorageLocation { location := domain.AssetStorageLocation{ Path: strings.TrimSpace(currentPath), } currentBucket, currentObject, hasCurrentBOS := parseBOSPath(currentPath) rootBucket, rootObject, hasRootBOS := parseBOSPath(rootPath) switch { case hasCurrentBOS: location.Provider = "bos" location.Endpoint = chooseBOSEndpoint(endpoint) location.Bucket = currentBucket if hasRootBOS && rootBucket == currentBucket { location.Prefix = rootObject } else { location.Prefix = path.Dir(currentObject) if location.Prefix == "." { location.Prefix = "" } } case hasRootBOS: location.Provider = "bos" location.Endpoint = chooseBOSEndpoint(endpoint) location.Bucket = rootBucket location.Prefix = rootObject if location.Path == "" { location.Path = strings.TrimSpace(rootPath) } case strings.TrimSpace(currentPath) != "": location.Provider = "k8s-job" location.Prefix = normalizeLocalDirectory(currentPath) case strings.TrimSpace(rootPath) != "": location.Provider = "k8s-job" location.Path = strings.TrimSpace(rootPath) location.Prefix = normalizeLocalDirectory(rootPath) default: location.Provider = "unknown" } if location.Prefix == "." { location.Prefix = "" } return location } func sourceRootPathForRun(run domain.RunRecord, scanConfig *domain.ScanConfig) string { if scanConfig != nil { if value := scanSourcePath(scanConfig.DataSource, scanConfig.SourcePath, scanConfig.Bucket, scanConfig.Prefix); value != "" { return value } } if manifest := runSourceManifest(run); manifest != nil { payload := artifactStructuredData(manifest) if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" { return value } } if index := runDatasetIndex(run); index != nil { payload := artifactStructuredData(index) if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" { return value } if value := strings.TrimSpace(stringValue(payload["rootPath"])); value != "" { return value } } return "" } func (server *server) buildSourceCollectionRecord(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, collection map[string]any) domain.SourceCollectionRecord { meta := mapValue(collection["meta"]) requiredFiles := make([]string, 0) for _, item := range listValue(collection["requiredFiles"]) { requiredFiles = append(requiredFiles, stringValue(item)) } missingFiles := make([]string, 0) for _, item := range listValue(collection["missingRequiredFiles"]) { missingFiles = append(missingFiles, stringValue(item)) } collectionName := stringValue(collection["name"]) if collectionName == "" { if projectID := stringValue(meta["project_id"]); projectID != "" { collectionName = projectID } else if relativePath := strings.Trim(stringValue(collection["relativePath"]), "/"); relativePath != "" && relativePath != "." { collectionName = path.Base(relativePath) } else { collectionName = stringValue(collection["id"]) } } var resolvedScanConfig *domain.ScanConfig if workflow != nil { if scanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig); err == nil { resolvedScanConfig = scanConfig } } storage := assetStorageLocationFromPaths( sourceRootPathForRun(run, resolvedScanConfig), stringValue(collection["path"]), func() string { if resolvedScanConfig == nil { return "" } return resolvedScanConfig.Endpoint }(), ) return domain.SourceCollectionRecord{ RunID: run.ID, WorkflowID: run.WorkflowID, WorkflowName: run.WorkflowName, ProjectID: run.ProjectID, ProjectName: run.ProjectName, CollectionID: stringValue(collection["id"]), CollectionName: collectionName, Path: stringValue(collection["path"]), RelativePath: stringValue(collection["relativePath"]), Meta: meta, HasIntrinsics: boolValue(collection["hasIntrinsics"]), HasVideoMeta: boolValue(collection["hasVideoMeta"]), RequiredFiles: requiredFiles, MissingFiles: missingFiles, VideoCount: intValue(collection["videoCount"]), FileCount: intValue(collection["fileCount"]), TotalBytes: int64Value(collection["totalBytes"]), Valid: boolValue(collection["valid"]), CreatedAt: run.CreatedAt, Storage: storage, } } type sourceCollectionPayloadRecord struct { Run *domain.RunRecord Workflow *domain.WorkflowRecord Collection map[string]any Items []map[string]any Entries []map[string]any ArchiveSource map[string]any } func (server *server) sourceCollectionPayload(ctx context.Context, runID string, collectionID string) (*sourceCollectionPayloadRecord, error) { run, err := server.store.GetRun(ctx, runID) if err != nil || run == nil { return &sourceCollectionPayloadRecord{Run: run}, err } workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID) if err != nil { return &sourceCollectionPayloadRecord{Run: run}, err } artifact := artifactByName(*run, "dataset-index.json") payload := artifactStructuredData(artifact) if artifact == nil || payload == nil { return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil } collections := sourceDatasetList(payload, "catalogCollections", "collections") items := sourceDatasetList(payload, "catalogItems", "items") entries := sourceDatasetList(payload, "catalogEntries", "entries") var matchedCollection map[string]any for _, item := range collections { collection := mapValue(item) if stringValue(collection["id"]) == collectionID { matchedCollection = collection break } } if matchedCollection == nil { return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil } matchedItems := make([]map[string]any, 0) for _, item := range items { row := mapValue(item) if stringValue(row["collectionId"]) == collectionID { matchedItems = append(matchedItems, row) } } matchedEntries := make([]map[string]any, 0) for _, item := range entries { row := mapValue(item) relativePath := strings.Trim(filepath.ToSlash(stringValue(row["relativePath"])), "/") collectionRelativePath := strings.Trim(filepath.ToSlash(stringValue(matchedCollection["relativePath"])), "/") if collectionRelativePath == "" || collectionRelativePath == "." { if relativePath == "" { continue } } else if relativePath != collectionRelativePath && !strings.HasPrefix(relativePath, collectionRelativePath+"/") { continue } matchedEntries = append(matchedEntries, row) } return &sourceCollectionPayloadRecord{ Run: run, Workflow: workflow, Collection: matchedCollection, Items: matchedItems, Entries: matchedEntries, ArchiveSource: mapValue(payload["archiveSources"]), }, nil } func sourceEntryPayloadKind(relativePath string, mimeType string) string { extension := strings.ToLower(path.Ext(relativePath)) switch { case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi": return "video" case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp": return "image" case strings.HasPrefix(mimeType, "audio/") || extension == ".mp3" || extension == ".wav" || extension == ".m4a": return "audio" default: return "binary" } } func sourceEntryArtifactType(relativePath string, mimeType string) domain.PortType { extension := strings.ToLower(path.Ext(relativePath)) switch { case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp": return domain.PortTypeImage case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi": return domain.PortTypeVideo case extension == ".json": return domain.PortTypeJSON default: return domain.PortTypeFile } } func formatArtifactSize(sizeBytes int64) string { switch { case sizeBytes <= 0: return "0 B" case sizeBytes < 1024: return fmt.Sprintf("%d B", sizeBytes) case sizeBytes < 1024*1024: return fmt.Sprintf("%.1f KiB", float64(sizeBytes)/1024) case sizeBytes < 1024*1024*1024: return fmt.Sprintf("%.1f MiB", float64(sizeBytes)/(1024*1024)) default: return fmt.Sprintf("%.1f GiB", float64(sizeBytes)/(1024*1024*1024)) } } func sourceCollectionContentURL(runID string, collectionID string, relativePath string) string { return fmt.Sprintf( "/api/source-data/collections/%s/%s/content?relativePath=%s", url.PathEscape(runID), url.PathEscape(collectionID), url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")), ) } func datasetIndexEntrySource(entry map[string]any, archiveSources map[string]any) map[string]any { source := mapValue(entry["source"]) if len(source) == 0 { source = make(map[string]any) } relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/") if relativePath != "" && stringValue(source["relativePath"]) == "" { source["relativePath"] = relativePath } mimeType := stringValue(entry["mimeType"]) if mimeType == "" { mimeType = stringValue(source["mimeType"]) } if mimeType == "" && relativePath != "" { mimeType = mime.TypeByExtension(path.Ext(relativePath)) } if mimeType != "" { source["mimeType"] = mimeType } if stringValue(source["previewURL"]) == "" { archiveKey := stringValue(source["archiveKey"]) archiveSource := mapValue(archiveSources[archiveKey]) if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" { source["previewURL"] = previewURL } if archiveName := stringValue(archiveSource["name"]); archiveName != "" && stringValue(source["archiveName"]) == "" { source["archiveName"] = archiveName } } if pathValue := stringValue(entry["path"]); pathValue != "" && stringValue(source["path"]) == "" { source["path"] = pathValue } return source } func sourceEntryArtifact(run domain.RunRecord, collectionID string, entry map[string]any, archiveSources map[string]any) *domain.ExecutionArtifact { relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/") if relativePath == "" { relativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/") } if relativePath == "" { return nil } source := datasetIndexEntrySource(entry, archiveSources) mimeType := stringValue(source["mimeType"]) sizeBytes := int64Value(entry["sizeBytes"]) artifactType := sourceEntryArtifactType(relativePath, mimeType) payload := make(map[string]any) if value, ok := entry["data"]; ok { payload["preview"] = map[string]any{ "kind": "json", "value": normalizeStructuredValue(value), } } if previewURL := stringValue(source["previewURL"]); previewURL != "" && len(payload) == 0 { payload["preview"] = map[string]any{ "kind": sourceEntryPayloadKind(relativePath, mimeType), "format": strings.TrimPrefix(strings.ToLower(path.Ext(relativePath)), "."), "url": sourceCollectionContentURL(run.ID, collectionID, relativePath), } } payload["relativePath"] = relativePath if len(source) > 0 { if previewURL := sourceCollectionContentURL(run.ID, collectionID, relativePath); previewURL != "" { source["previewURL"] = previewURL } payload["source"] = source } return &domain.ExecutionArtifact{ ID: fmt.Sprintf("%s-source-%s", run.ID, strings.ReplaceAll(relativePath, "/", "-")), RunID: run.ID, NodeID: "dataset-index", Name: path.Base(relativePath), Type: artifactType, Size: formatArtifactSize(sizeBytes), Path: func() string { if sourcePath := stringValue(source["path"]); sourcePath != "" { return sourcePath } return relativePath }(), Metadata: &domain.ArtifactMetadata{MimeType: mimeType}, Data: payload, } } func contentBytesForDataURL(previewURL string) ([]byte, string, error) { if !strings.HasPrefix(previewURL, "data:") { return nil, "", fmt.Errorf("not a data url") } header, encoded, found := strings.Cut(previewURL, ",") if !found || !strings.Contains(header, ";base64") { return nil, "", fmt.Errorf("invalid data url") } mimeType := strings.TrimPrefix(strings.SplitN(header, ";", 2)[0], "data:") payload, err := base64.StdEncoding.DecodeString(encoded) if err != nil { return nil, "", err } return payload, mimeType, nil } func contentBytesForArchiveMember(archiveName string, archivePreviewURL string, memberPath string) ([]byte, error) { payload, _, err := contentBytesForDataURL(archivePreviewURL) if err != nil { return nil, err } memberPath = strings.Trim(filepath.ToSlash(memberPath), "/") if memberPath == "" { return nil, fmt.Errorf("missing archive member path") } if strings.HasSuffix(strings.ToLower(archiveName), ".zip") { archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload))) if err != nil { return nil, err } for _, file := range archive.File { if strings.Trim(filepath.ToSlash(file.Name), "/") != memberPath { continue } handle, err := file.Open() if err != nil { return nil, err } defer handle.Close() return io.ReadAll(handle) } return nil, os.ErrNotExist } reader := bytes.NewReader(payload) var tarReader *tar.Reader if strings.HasSuffix(strings.ToLower(archiveName), ".tgz") || strings.HasSuffix(strings.ToLower(archiveName), ".tar.gz") || (len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b) { gzipReader, err := gzip.NewReader(reader) if err == nil { defer gzipReader.Close() tarReader = tar.NewReader(gzipReader) } else { reader = bytes.NewReader(payload) tarReader = tar.NewReader(reader) } } else { tarReader = tar.NewReader(reader) } for { header, err := tarReader.Next() if err == io.EOF { return nil, os.ErrNotExist } if err != nil { return nil, err } if header.FileInfo().IsDir() { continue } if strings.Trim(filepath.ToSlash(header.Name), "/") != memberPath { continue } return io.ReadAll(tarReader) } } func sourceCollectionEntryPayload(collection domain.SourceCollectionRecord, payloadEntries []map[string]any, relativePath string) map[string]any { target := strings.Trim(filepath.ToSlash(relativePath), "/") if target == "" { return nil } collectionPrefix := strings.Trim(filepath.ToSlash(collection.RelativePath), "/") fullTarget := target if collectionPrefix != "" && target != collectionPrefix && !strings.HasPrefix(target, collectionPrefix+"/") { fullTarget = collectionPrefix + "/" + target } for _, entry := range payloadEntries { entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/") if entryRelativePath == "" { entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/") } if entryRelativePath == fullTarget || entryRelativePath == target { return entry } } return nil } func writeSourceContent(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, payload []byte) { if mimeType == "" { mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName))) } if mimeType == "" && len(payload) > 0 { mimeType = http.DetectContentType(payload) } if mimeType != "" { writer.Header().Set("Content-Type", mimeType) } writeSourceContentDisposition(writer, request, fileName) http.ServeContent(writer, request, fileName, time.Time{}, bytes.NewReader(payload)) } func sourceContentDownloadRequested(request *http.Request) bool { value := strings.ToLower(strings.TrimSpace(request.URL.Query().Get("download"))) return value == "1" || value == "true" || value == "yes" } func writeSourceContentDisposition(writer http.ResponseWriter, request *http.Request, fileName string) { if !sourceContentDownloadRequested(request) { return } name := strings.TrimSpace(path.Base(fileName)) if name == "" || name == "." || name == "/" { name = "artifact" } writer.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": name})) } func writeSourceContentFile(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, filePath string) error { file, err := os.Open(filePath) if err != nil { return err } defer file.Close() fileInfo, err := file.Stat() if err != nil { return err } if mimeType == "" { mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName))) } if mimeType == "" { header := make([]byte, 512) count, readErr := file.Read(header) if readErr != nil && readErr != io.EOF { return readErr } mimeType = http.DetectContentType(header[:count]) if _, err := file.Seek(0, io.SeekStart); err != nil { return err } } if mimeType != "" { writer.Header().Set("Content-Type", mimeType) } writeSourceContentDisposition(writer, request, fileName) http.ServeContent(writer, request, fileName, fileInfo.ModTime(), file) return nil } func sourceBrowserPreviewCacheKey(request sourceBrowserPreviewRequest) string { hash := sha256.New() hash.Write([]byte("browser-preview-v2")) hash.Write([]byte{0}) hash.Write([]byte(strings.TrimSpace(request.FileName))) hash.Write([]byte{0}) if request.RedirectURL != "" { hash.Write([]byte("redirect")) hash.Write([]byte{0}) hash.Write([]byte(strings.TrimSpace(request.RedirectURL))) } if len(request.Payload) > 0 { payloadHash := sha256.Sum256(request.Payload) hash.Write([]byte("payload")) hash.Write([]byte{0}) hash.Write(payloadHash[:]) } return hex.EncodeToString(hash.Sum(nil)) } func sourceBrowserPreviewCacheDir() (string, error) { cacheDir := filepath.Join(os.TempDir(), "droboticflow", "source-video-previews") if err := os.MkdirAll(cacheDir, 0o755); err != nil { return "", err } return cacheDir, nil } func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string, error) { cacheDir, err := sourceBrowserPreviewCacheDir() if err != nil { return "", err } cacheKey := sourceBrowserPreviewCacheKey(request) return filepath.Join(cacheDir, cacheKey+".mp4"), nil } func parseBOSObjectPath(sourcePath string) (string, string, bool) { bucket, object, ok := parseBOSPath(sourcePath) if !ok || object == "" { return "", "", false } return bucket, object, true } func buildSourceCollectionSignedPreviewURL(scanConfig *domain.ScanConfig, sourcePath string) (string, error) { if scanConfig == nil { return "", fmt.Errorf("scan config is unavailable") } if chooseDataSource(scanConfig.DataSource) != "BOS" { return "", fmt.Errorf("scan config is not BOS") } bucket, object, ok := parseBOSObjectPath(sourcePath) if !ok { return "", fmt.Errorf("invalid BOS source path: %s", sourcePath) } accessKeyID := strings.TrimSpace(scanConfig.AccessKeyID) secretAccessKey := strings.TrimSpace(scanConfig.SecretAccessKey) if accessKeyID == "" || secretAccessKey == "" { return "", fmt.Errorf("BOS credentials are unavailable") } endpoint := chooseBOSEndpoint(scanConfig.Endpoint) if endpoint == "" { return "", fmt.Errorf("BOS endpoint is unavailable") } client, err := bos.NewClient(accessKeyID, secretAccessKey, endpoint) if err != nil { return "", err } previewURL := strings.TrimSpace(client.BasicGeneratePresignedUrl(bucket, object, 3600)) if previewURL == "" { return "", fmt.Errorf("failed to generate BOS preview url") } return previewURL, nil } func locateSourceBrowserVideoPreviewPath(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) { finalPath, err := sourceBrowserPreviewCachePath(request) if err != nil { return "", err } if fileInfo, err := os.Stat(finalPath); err == nil && fileInfo.Size() > 0 { return finalPath, nil } return "", os.ErrNotExist } func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWriter, request sourceBrowserPreviewRequest) error { if strings.TrimSpace(request.RedirectURL) == "" && len(request.Payload) == 0 { return fmt.Errorf("source video content is unavailable") } finalPath, err := sourceBrowserPreviewCachePath(request) if err != nil { return err } inputRef := strings.TrimSpace(request.RedirectURL) var cleanupInput func() if inputRef == "" { extension := path.Ext(request.FileName) if extension == "" { extension = ".bin" } cacheDir := filepath.Dir(finalPath) inputFile, err := os.CreateTemp(cacheDir, "source-video-*"+extension) if err != nil { return err } if _, err := inputFile.Write(request.Payload); err != nil { inputFile.Close() os.Remove(inputFile.Name()) return err } if err := inputFile.Close(); err != nil { os.Remove(inputFile.Name()) return err } inputRef = inputFile.Name() cleanupInput = func() { _ = os.Remove(inputFile.Name()) } } if cleanupInput != nil { defer cleanupInput() } cacheDir := filepath.Dir(finalPath) outputFile, err := os.CreateTemp(cacheDir, sourceBrowserPreviewCacheKey(request)+"-*.mp4") if err != nil { return err } tempOutputPath := outputFile.Name() stderr := &bytes.Buffer{} command := exec.CommandContext( ctx, "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", "-i", inputRef, "-map", "0:v:0", "-map", "0:a?", "-threads", "1", "-probesize", "32M", "-analyzeduration", "0", "-c:v", "libx264", "-preset", "ultrafast", "-tune", "zerolatency", "-crf", "30", "-vf", "fps=12,scale=min(960\\,iw):-2", "-g", "12", "-keyint_min", "12", "-pix_fmt", "yuv420p", "-movflags", "frag_keyframe+empty_moov+default_base_moof", "-c:a", "aac", "-b:a", "96k", "-ac", "1", "-f", "mp4", "pipe:1", ) command.Stderr = stderr stdout, err := command.StdoutPipe() if err != nil { outputFile.Close() _ = os.Remove(tempOutputPath) return err } if err := command.Start(); err != nil { outputFile.Close() _ = os.Remove(tempOutputPath) message := strings.TrimSpace(stderr.String()) if message != "" { return fmt.Errorf("generate browser video preview: %s", message) } return err } streamTarget := io.Writer(writer) if flusher, ok := writer.(http.Flusher); ok { streamTarget = flushResponseWriter{writer: writer, flusher: flusher} } if _, err := io.Copy(io.MultiWriter(streamTarget, outputFile), stdout); err != nil { outputFile.Close() _ = os.Remove(tempOutputPath) _ = command.Wait() return err } if err := outputFile.Close(); err != nil { _ = os.Remove(tempOutputPath) _ = command.Wait() return err } if err := command.Wait(); err != nil { message := strings.TrimSpace(stderr.String()) _ = os.Remove(tempOutputPath) if message != "" { return fmt.Errorf("generate browser video preview: %s", message) } return fmt.Errorf("generate browser video preview: %w", err) } if err := os.Rename(tempOutputPath, finalPath); err != nil { if fileInfo, statErr := os.Stat(finalPath); statErr == nil && fileInfo.Size() > 0 { _ = os.Remove(tempOutputPath) return nil } _ = os.Remove(tempOutputPath) return err } return nil } func derivedSourceEntriesFromIndex(collection domain.SourceCollectionRecord, payloadItems []map[string]any) []map[string]any { entries := make([]map[string]any, 0, len(payloadItems)+2) if len(collection.Meta) > 0 { entries = append(entries, map[string]any{ "collectionId": collection.CollectionID, "relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/meta.json", "/"), "sizeBytes": len(fmt.Sprintf("%v", collection.Meta)), "mimeType": "application/json", "data": collection.Meta, "derivedFromIndex": true, }) } if collection.HasVideoMeta && len(payloadItems) > 0 { videoRows := make([]map[string]any, 0, len(payloadItems)) for _, item := range payloadItems { videoRows = append(videoRows, map[string]any{ "filename": stringValue(item["filename"]), "codec": stringValue(item["codec"]), "resolution": stringValue(item["resolution"]), "frameRate": floatValue(item["frameRate"]), "duration": floatValue(item["duration"]), "startTime": item["startTime"], }) } entries = append(entries, map[string]any{ "collectionId": collection.CollectionID, "relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/video_meta.json", "/"), "mimeType": "application/json", "data": videoRows, "derivedFromIndex": true, }) } for _, item := range payloadItems { entry := map[string]any{ "collectionId": collection.CollectionID, "relativePath": stringValue(item["relativePath"]), "path": stringValue(item["path"]), "sizeBytes": int64Value(item["sizeBytes"]), "mimeType": stringValue(mapValue(item["source"])["mimeType"]), "source": mapValue(item["source"]), "codec": stringValue(item["codec"]), "resolution": stringValue(item["resolution"]), "frameRate": floatValue(item["frameRate"]), "duration": floatValue(item["duration"]), "derivedFromIndex": true, } entries = append(entries, entry) } return entries } func buildSourceEntries(run domain.RunRecord, collection domain.SourceCollectionRecord, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, prefix string) []domain.SourceCollectionEntry { if len(payloadEntries) == 0 { payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems) } videoMetaByRelative := make(map[string]map[string]any, len(payloadItems)) for _, item := range payloadItems { videoMetaByRelative[stringValue(item["relativePath"])] = item } normalizedPrefix := strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/") prefixSlash := normalizedPrefix if prefixSlash != "" { prefixSlash += "/" } directories := make(map[string]domain.SourceCollectionEntry) files := make([]domain.SourceCollectionEntry, 0) collectionPrefix := strings.Trim(collection.RelativePath, "/") if collectionPrefix != "" { collectionPrefix += "/" } for _, entryPayload := range payloadEntries { relativePath := strings.Trim(filepath.ToSlash(stringValue(entryPayload["relativePath"])), "/") if relativePath == "" { relativePath = strings.Trim(filepath.ToSlash(stringValue(entryPayload["path"])), "/") } if relativePath == "" { continue } if collectionPrefix != "" && relativePath != strings.TrimSuffix(collectionPrefix, "/") && !strings.HasPrefix(relativePath, collectionPrefix) { continue } entryPath := relativePath if collectionPrefix != "" { entryPath = strings.TrimPrefix(relativePath, collectionPrefix) } if entryPath == "" { continue } if normalizedPrefix != "" { if entryPath == normalizedPrefix { // exact file match } else if !strings.HasPrefix(entryPath, prefixSlash) { continue } else { entryPath = strings.TrimPrefix(entryPath, prefixSlash) } } if entryPath == "" { continue } parts := strings.Split(entryPath, "/") if len(parts) > 1 { dirPath := parts[0] displayPath := dirPath if normalizedPrefix != "" { displayPath = normalizedPrefix + "/" + dirPath } if _, exists := directories[displayPath]; !exists { directories[displayPath] = domain.SourceCollectionEntry{ Path: displayPath, Name: dirPath, Kind: "directory", CollectionName: collection.CollectionName, DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]), } } else if boolValue(entryPayload["derivedFromIndex"]) { entry := directories[displayPath] entry.DerivedFromIndex = true directories[displayPath] = entry } continue } displayPath := parts[0] if normalizedPrefix != "" && displayPath != normalizedPrefix { displayPath = normalizedPrefix + "/" + displayPath } artifact := sourceEntryArtifact(run, collection.CollectionID, entryPayload, archiveSources) videoMeta := videoMetaByRelative[filepath.ToSlash(strings.Trim(collection.RelativePath, "/")+"/"+displayPath)] entry := domain.SourceCollectionEntry{ Path: displayPath, Name: parts[0], Kind: "file", CollectionName: collection.CollectionName, SizeBytes: int64Value(entryPayload["sizeBytes"]), Artifact: artifact, DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]), } if videoMeta != nil { entry.Kind = "video" entry.Codec = stringValue(videoMeta["codec"]) entry.Resolution = stringValue(videoMeta["resolution"]) entry.FrameRate = floatValue(videoMeta["frameRate"]) entry.Duration = floatValue(videoMeta["duration"]) entry.Metadata = videoMeta if entry.SizeBytes == 0 { entry.SizeBytes = int64Value(videoMeta["sizeBytes"]) } } else if strings.HasSuffix(strings.ToLower(entry.Name), ".json") { entry.Kind = "json" } else if strings.HasSuffix(strings.ToLower(entry.Name), ".mp4") { entry.Kind = "video" } if entry.SizeBytes == 0 { entry.SizeBytes = int64Value(entryPayload["sizeBytes"]) } files = append(files, entry) } items := make([]domain.SourceCollectionEntry, 0, len(directories)+len(files)) for _, item := range directories { items = append(items, item) } sort.Slice(items, func(left int, right int) bool { return items[left].Path < items[right].Path }) sort.Slice(files, func(left int, right int) bool { return files[left].Path < files[right].Path }) items = append(items, files...) return items } func sourceCollectionContentPayload(collection domain.SourceCollectionRecord, scanConfig *domain.ScanConfig, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, relativePath string) ([]byte, string, string, string, error) { if len(payloadEntries) == 0 { payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems) } entry := sourceCollectionEntryPayload(collection, payloadEntries, relativePath) if entry == nil { return nil, "", "", "", os.ErrNotExist } entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/") if entryRelativePath == "" { entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/") } fileName := path.Base(entryRelativePath) source := datasetIndexEntrySource(entry, archiveSources) mimeType := stringValue(source["mimeType"]) if mimeType == "" { mimeType = stringValue(entry["mimeType"]) } if mimeType == "" { mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName))) } if value, ok := entry["data"]; ok { payload, err := json.MarshalIndent(normalizeStructuredValue(value), "", " ") if err != nil { return nil, "", "", "", err } if mimeType == "" { mimeType = "application/json" } return payload, mimeType, fileName, "", nil } if sourcePath := stringValue(source["path"]); sourcePath != "" && !strings.Contains(sourcePath, "://") { if payload, err := os.ReadFile(sourcePath); err == nil { return payload, mimeType, fileName, "", nil } } if sourcePath := stringValue(source["path"]); strings.HasPrefix(strings.ToLower(sourcePath), "bos://") && scanConfig != nil { if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" { source["previewURL"] = previewURL } } archiveKey := stringValue(source["archiveKey"]) archiveMemberPath := stringValue(source["archiveMemberPath"]) if archiveKey != "" && archiveMemberPath != "" { archiveSource := mapValue(archiveSources[archiveKey]) if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" { payload, err := contentBytesForArchiveMember(stringValue(archiveSource["name"]), previewURL, archiveMemberPath) if err == nil { return payload, mimeType, fileName, "", nil } } } if previewURL := stringValue(source["previewURL"]); previewURL != "" { if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") { return nil, "", fileName, previewURL, nil } payload, decodedMimeType, err := contentBytesForDataURL(previewURL) if err == nil { if mimeType == "" { mimeType = decodedMimeType } return payload, mimeType, fileName, "", nil } } return nil, "", "", "", os.ErrNotExist } func (server *server) handleSourceCollectionsList(writer http.ResponseWriter, request *http.Request) { workflowByID, latestRuns, err := server.latestCompletedScanRuns(request.Context()) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } items := make([]domain.SourceCollectionRecord, 0) for _, run := range latestRuns { if !server.filterByTenant(request, run.OrganizationID) { continue } artifact := artifactByName(run, "dataset-index.json") payload := artifactStructuredData(artifact) if artifact == nil || payload == nil { continue } for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") { collection := mapValue(raw) if collection == nil { continue } workflow := workflowByID[run.WorkflowID] items = append(items, server.buildSourceCollectionRecord(request.Context(), run, &workflow, collection)) } } sort.Slice(items, func(left int, right int) bool { if items[left].CreatedAt == items[right].CreatedAt { if items[left].WorkflowName == items[right].WorkflowName { return items[left].CollectionID < items[right].CollectionID } return items[left].WorkflowName < items[right].WorkflowName } return items[left].CreatedAt > items[right].CreatedAt }) renderJSON(writer, http.StatusOK, map[string]any{"items": items}) } func (server *server) handleSourceCollectionDetail(writer http.ResponseWriter, request *http.Request) { payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if payload.Run == nil || payload.Collection == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"}) return } if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") { return } record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection) detail := domain.SourceCollectionDetail{ Collection: record, Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, ""), } renderJSON(writer, http.StatusOK, detail) } func (server *server) handleSourceCollectionEntries(writer http.ResponseWriter, request *http.Request) { payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if payload.Run == nil || payload.Collection == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"}) return } if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") { return } record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection) prefix := request.URL.Query().Get("prefix") detail := domain.SourceCollectionDetail{ Collection: record, Prefix: prefix, Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, prefix), } renderJSON(writer, http.StatusOK, detail) } func (server *server) handleSourceCollectionContent(writer http.ResponseWriter, request *http.Request) { payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if payload.Run == nil || payload.Collection == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"}) return } if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") { return } record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection) relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath")) if relativePath == "" { renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"}) return } var effectiveScanConfig *domain.ScanConfig if payload.Workflow != nil { effectiveScanConfig, err = server.resolveScanConfig(request.Context(), payload.Workflow.ScanConfig) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } } content, mimeType, fileName, redirectURL, err := sourceCollectionContentPayload(record, effectiveScanConfig, payload.Entries, payload.Items, payload.ArchiveSource, relativePath) if err != nil { if os.IsNotExist(err) { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source file not found"}) return } renderError(writer, http.StatusInternalServerError, err) return } if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" { previewRequest := sourceBrowserPreviewRequest{ FileName: fileName, Payload: content, RedirectURL: redirectURL, } if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" { if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil { renderError(writer, http.StatusInternalServerError, err) } return } writer.Header().Set("Content-Type", "video/mp4") writer.Header().Set("Cache-Control", "no-store") writer.WriteHeader(http.StatusOK) if flusher, ok := writer.(http.Flusher); ok { flusher.Flush() } if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil { fmt.Printf("[api] source browser preview stream failed: %v\n", err) } return } if redirectURL != "" { http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) return } writeSourceContent(writer, request, fileName, mimeType, content) } func packageArtifactCandidatesByRelativePath(run domain.RunRecord, packageName string, relativePath string) []domain.ExecutionArtifact { target := strings.Trim(filepath.ToSlash(relativePath), "/") if target == "" { return nil } if _, payload := wilorPackageManifestArtifact(run, packageName); payload != nil { items := make([]domain.ExecutionArtifact, 0) for _, artifact := range wilorPackageArtifacts(run, packageName) { if strings.Trim(filepath.ToSlash(artifact.Name), "/") == target { items = append(items, artifact) } } sort.SliceStable(items, func(left int, right int) bool { return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right]) }) return items } items := make([]domain.ExecutionArtifact, 0) for _, artifact := range run.Artifacts { if packageArtifactResolvedRelativePath(artifact, packageName) == target { items = append(items, artifact) } } sort.SliceStable(items, func(left int, right int) bool { return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right]) }) return items } func workflowNodeHyperparamsMap(node domain.WorkflowNode) map[string]string { values := make(map[string]string, len(node.Data.Hyperparams)) for _, item := range node.Data.Hyperparams { values[item.Key] = item.Value } return values } func (server *server) packageArtifactScanConfig(ctx context.Context, workflow *domain.WorkflowRecord, artifact domain.ExecutionArtifact) (*domain.ScanConfig, error) { node := workflowNodeByID(workflow, artifact.NodeID) if node == nil { return nil, nil } params := workflowNodeHyperparamsMap(*node) dataSource := chooseDataSource(params["DATA_SOURCE"]) sourcePath := strings.TrimSpace(params["SOURCE_PATH"]) if dataSource != "BOS" && strings.HasPrefix(strings.ToLower(sourcePath), "bos://") { dataSource = "BOS" } if dataSource != "BOS" && strings.TrimSpace(node.Data.ConnectionID) != "" { dataSource = "BOS" } if dataSource != "BOS" { return nil, nil } scanConfig := &domain.ScanConfig{ DataSource: "BOS", SourcePath: sourcePath, ConnectionID: strings.TrimSpace(node.Data.ConnectionID), Bucket: strings.TrimSpace(node.Data.Bucket), Prefix: cleanBOSPrefix(node.Data.Prefix), Endpoint: strings.TrimSpace(params["BOS_ENDPOINT"]), AccessKeyID: strings.TrimSpace(params["BOS_ACCESS_KEY_ID"]), SecretAccessKey: strings.TrimSpace(params["BOS_SECRET_ACCESS_KEY"]), } return server.resolveScanConfig(ctx, scanConfig) } func (server *server) packageContentPayload(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, packageName string, relativePath string) ([]byte, string, string, string, string, error) { artifacts := packageArtifactCandidatesByRelativePath(run, packageName, relativePath) if len(artifacts) == 0 { return nil, "", "", "", "", os.ErrNotExist } fileName := path.Base(strings.Trim(filepath.ToSlash(relativePath), "/")) for _, artifact := range artifacts { mimeType := "" if artifact.Metadata != nil { mimeType = strings.TrimSpace(artifact.Metadata.MimeType) } if mimeType == "" { mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName))) } filePath := strings.TrimSpace(artifact.Path) if filePath != "" && !strings.Contains(filePath, "://") { if _, err := os.Stat(filePath); err == nil { return nil, mimeType, fileName, filePath, "", nil } } structuredData := artifactStructuredData(&artifact) if artifact.Type == "json" && structuredData != nil { payload, err := json.MarshalIndent(normalizeStructuredValue(structuredData), "", " ") if err != nil { return nil, "", "", "", "", err } if mimeType == "" { mimeType = "application/json" } return payload, mimeType, fileName, "", "", nil } source := mapValue(structuredData["source"]) sourcePath := stringValue(source["path"]) if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") { scanConfig, err := server.packageArtifactScanConfig(ctx, workflow, artifact) if err != nil { return nil, "", "", "", "", err } if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" { return nil, mimeType, fileName, "", previewURL, nil } } if previewURL := packageArtifactPreviewURL(artifact); previewURL != "" { if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") { return nil, mimeType, fileName, "", previewURL, nil } payload, decodedMimeType, err := contentBytesForDataURL(previewURL) if err == nil { if mimeType == "" { mimeType = decodedMimeType } return payload, mimeType, fileName, "", "", nil } } } return nil, "", "", "", "", os.ErrNotExist } func (server *server) handlePackageContent(writer http.ResponseWriter, request *http.Request) { run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if run == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"}) return } if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Run") { return } packageName := chi.URLParam(request, "packageName") relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath")) if relativePath == "" { renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"}) return } workflow, err := server.store.GetWorkflow(request.Context(), run.WorkflowID) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } content, mimeType, fileName, filePath, redirectURL, err := server.packageContentPayload(request.Context(), *run, workflow, packageName, relativePath) if err != nil { if os.IsNotExist(err) { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package file not found"}) return } renderError(writer, http.StatusInternalServerError, err) return } if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" { previewRequest := sourceBrowserPreviewRequest{ FileName: fileName, Payload: content, } if filePath != "" { previewRequest.RedirectURL = filePath } else { previewRequest.RedirectURL = redirectURL } if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" { if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil { renderError(writer, http.StatusInternalServerError, err) } return } writer.Header().Set("Content-Type", "video/mp4") writer.Header().Set("Cache-Control", "no-store") writer.WriteHeader(http.StatusOK) if flusher, ok := writer.(http.Flusher); ok { flusher.Flush() } if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil { fmt.Printf("[api] package browser preview stream failed: %v\n", err) } return } if filePath != "" { if err := writeSourceContentFile(writer, request, fileName, mimeType, filePath); err != nil { renderError(writer, http.StatusInternalServerError, err) } return } if redirectURL != "" { http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect) return } writeSourceContent(writer, request, fileName, mimeType, content) } func packageRecordFromManifest(run domain.RunRecord, item map[string]any, files []domain.ExecutionArtifact, rawFiles []domain.ExecutionArtifact) domain.PackageRecord { pipelineTags := make([]string, 0) for _, value := range listValue(item["pipelineTags"]) { pipelineTags = append(pipelineTags, stringValue(value)) } storagePath := stringValue(item["storagePath"]) for _, artifact := range rawFiles { if candidate := packageArtifactStoragePath(artifact, stringValue(item["packageName"])); candidate != "" { storagePath = candidate break } } fileCount := intValue(item["fileCount"]) if len(files) > 0 { fileCount = len(files) } return domain.PackageRecord{ RunID: run.ID, WorkflowID: run.WorkflowID, WorkflowName: run.WorkflowName, ProjectID: run.ProjectID, ProjectName: run.ProjectName, PackageName: stringValue(item["packageName"]), ClipID: stringValue(item["clipId"]), SourceVideo: stringValue(item["sourceVideo"]), ClipDuration: floatValue(item["clipDuration"]), FileCount: fileCount, PipelineTags: pipelineTags, StoragePath: storagePath, Quality: mapValue(item["quality"]), Annotation: mapValue(item["annotation"]), CreatedAt: run.CreatedAt, PreviewArtifact: choosePackagePreview(files), Storage: assetStorageLocationFromPaths("", storagePath, ""), } } func (server *server) packageDetail(ctx context.Context, runID string, packageName string) (*domain.PackageDetail, error) { run, err := server.store.GetRun(ctx, runID) if err != nil || run == nil { return nil, err } manifest := artifactByName(*run, "packages-manifest.json") manifestPayload := artifactStructuredData(manifest) if manifest != nil && manifestPayload != nil { rawFiles := dedupePackageArtifacts(*run, packageName) files := make([]domain.ExecutionArtifact, 0, len(rawFiles)) for _, artifact := range rawFiles { files = append(files, packageArtifactForResponse(run.ID, packageName, artifact)) } for _, raw := range listValue(manifestPayload["items"]) { item := mapValue(raw) if stringValue(item["packageName"]) != packageName { continue } record := packageRecordFromManifest(*run, item, files, rawFiles) return &domain.PackageDetail{Package: record, Files: files}, nil } } if manifest, payload := wilorPackageManifestArtifact(*run, packageName); manifest != nil && payload != nil { files, rawFiles := wilorPackageFilesForResponse(*run, packageName) item := wilorPackageItem(*run, *manifest, payload, rawFiles) record := packageRecordFromManifest(*run, item, files, rawFiles) return &domain.PackageDetail{Package: record, Files: files}, nil } return nil, nil } func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) { runs, err := server.listRunArtifactsForRequest(request.Context(), request, string(domain.RunStatusCompleted)) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } items := make([]domain.PackageRecord, 0) latestByWorkflowPackage := make(map[string]domain.PackageRecord) for _, run := range runs { if !server.filterByTenant(request, run.OrganizationID) { continue } manifest := artifactByName(run, "packages-manifest.json") manifestPayload := artifactStructuredData(manifest) if manifest != nil && manifestPayload != nil { for _, raw := range listValue(manifestPayload["items"]) { item := mapValue(raw) if item == nil { continue } packageName := stringValue(item["packageName"]) rawFiles := dedupePackageArtifacts(run, packageName) files := make([]domain.ExecutionArtifact, 0, len(rawFiles)) for _, artifact := range rawFiles { files = append(files, packageArtifactForResponse(run.ID, packageName, artifact)) } record := packageRecordFromManifest(run, item, files, rawFiles) key := run.WorkflowID + ":" + record.PackageName current, exists := latestByWorkflowPackage[key] if !exists || record.CreatedAt > current.CreatedAt { latestByWorkflowPackage[key] = record } } } for _, wilorManifest := range wilorResultArtifacts(run) { payload := wilorResultPayload(wilorManifest) if payload == nil { continue } packageName := wilorPackageName(wilorManifest, payload) files, rawFiles := wilorPackageFilesForResponse(run, packageName) item := wilorPackageItem(run, wilorManifest, payload, rawFiles) record := packageRecordFromManifest(run, item, files, rawFiles) key := run.WorkflowID + ":" + record.PackageName current, exists := latestByWorkflowPackage[key] if !exists || record.CreatedAt > current.CreatedAt { latestByWorkflowPackage[key] = record } } } for _, item := range latestByWorkflowPackage { items = append(items, item) } sort.Slice(items, func(left int, right int) bool { if items[left].CreatedAt == items[right].CreatedAt { return items[left].PackageName < items[right].PackageName } return items[left].CreatedAt > items[right].CreatedAt }) renderJSON(writer, http.StatusOK, map[string]any{"items": items}) } func (server *server) handlePackageDetail(writer http.ResponseWriter, request *http.Request) { run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if run == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"}) return } if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Package") { return } item, err := server.packageDetail(request.Context(), run.ID, chi.URLParam(request, "packageName")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if item == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"}) return } renderJSON(writer, http.StatusOK, item) } var supportedSamplingRuleOperators = map[string]struct{}{ "contains": {}, "eq": {}, "exists": {}, "gt": {}, "gte": {}, "in": {}, "lt": {}, "lte": {}, "missing": {}, "ne": {}, } func samplingRulesFromManifest(manifest map[string]any) ([]domain.SamplingRuleSummary, bool) { rules := make([]domain.SamplingRuleSummary, 0) ruleIDs := make(map[string]struct{}) for _, raw := range listValue(manifest["rules"]) { item := mapValue(raw) if item == nil { return nil, false } id := strings.TrimSpace(stringValue(item["id"])) field := strings.TrimSpace(stringValue(item["field"])) operator := strings.ToLower(strings.TrimSpace(stringValue(item["operator"]))) samplePercent, ok := samplePercentValue(item["samplePercent"]) if id == "" || field == "" || !ok { return nil, false } if _, exists := ruleIDs[id]; exists { return nil, false } if _, supported := supportedSamplingRuleOperators[operator]; !supported { return nil, false } ruleIDs[id] = struct{}{} rules = append(rules, domain.SamplingRuleSummary{ ID: id, Field: field, Operator: operator, Value: item["value"], SamplePercent: samplePercent, MatchedCount: intValue(item["matchedCount"]), SampledCount: intValue(item["sampledCount"]), }) } return rules, len(rules) > 0 } func samplePercentValue(value any) (float64, bool) { switch typed := value.(type) { case int: return boundedSamplePercent(float64(typed)) case int8: return boundedSamplePercent(float64(typed)) case int16: return boundedSamplePercent(float64(typed)) case int32: return boundedSamplePercent(float64(typed)) case int64: return boundedSamplePercent(float64(typed)) case uint: return boundedSamplePercent(float64(typed)) case uint8: return boundedSamplePercent(float64(typed)) case uint16: return boundedSamplePercent(float64(typed)) case uint32: return boundedSamplePercent(float64(typed)) case uint64: return boundedSamplePercent(float64(typed)) case float32: return boundedSamplePercent(float64(typed)) case float64: return boundedSamplePercent(typed) case json.Number: parsed, err := typed.Float64() if err != nil { return 0, false } return boundedSamplePercent(parsed) default: return 0, false } } func boundedSamplePercent(value float64) (float64, bool) { if math.IsNaN(value) || math.IsInf(value, 0) || value <= 0 || value > 100 { return 0, false } return value, true } func buildSamplingItems(run domain.RunRecord, manifest map[string]any) []domain.SamplingItemRecord { items := make([]domain.SamplingItemRecord, 0) previewByPackage := make(map[string]*domain.ExecutionArtifact) for _, raw := range listValue(manifest["items"]) { item := mapValue(raw) packageName := stringValue(item["packageName"]) if _, ok := previewByPackage[packageName]; !ok { previewByPackage[packageName] = choosePackagePreview(packageArtifacts(run, packageName)) } preview := previewByPackage[packageName] items = append(items, domain.SamplingItemRecord{ ID: domain.NewID("sampling-item"), PackageName: packageName, ClipID: stringValue(item["clipId"]), SourceVideo: stringValue(item["sourceVideo"]), SampleRuleID: stringValue(item["sampleRuleId"]), StoragePath: stringValue(item["storagePath"]), Quality: mapValue(item["quality"]), Annotation: mapValue(item["annotation"]), PreviewArtifact: preview, Status: domain.SamplingItemStatusPending, }) } return items } func (server *server) syncSamplingTasks(ctx context.Context) error { existing, err := server.listSamplingTasksForContext(ctx, "") if err != nil { return err } existingByRun := make(map[string]domain.SamplingTaskRecord, len(existing)) for _, task := range existing { existingByRun[task.RunID] = task } runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted)) if err != nil { return err } for _, run := range runs { if !server.filterByTenantContext(ctx, run.OrganizationID) { continue } if _, exists := existingByRun[run.ID]; exists { continue } manifest := artifactByName(run, "audit-samples.json") manifestPayload := artifactStructuredData(manifest) if manifest == nil || manifestPayload == nil || len(listValue(manifestPayload["items"])) == 0 { continue } rules, ok := samplingRulesFromManifest(manifestPayload) if !ok { continue } now := nowISO() items := buildSamplingItems(run, manifestPayload) task := domain.SamplingTaskRecord{ ID: domain.NewID("sampling"), OrganizationID: normalizedTenantID(run.OrganizationID), RunID: run.ID, WorkflowID: run.WorkflowID, WorkflowName: run.WorkflowName, ProjectID: run.ProjectID, ProjectName: run.ProjectName, Name: fmt.Sprintf("%s 抽检任务", run.WorkflowName), Description: fmt.Sprintf("来源于运行 %s 的真实抽检采样结果", run.ID), Status: domain.SamplingTaskStatusPending, Rules: rules, Items: items, CreatedAt: now, UpdatedAt: now, } for index := range task.Items { task.Items[index].TaskID = task.ID } if _, err := server.store.CreateSamplingTask(ctx, task); err != nil { return err } } return nil } func (server *server) handleSamplingTasksList(writer http.ResponseWriter, request *http.Request) { if err := server.syncSamplingTasks(request.Context()); err != nil { renderError(writer, http.StatusInternalServerError, err) return } items, err := server.listSamplingTasksForRequest(request.Context(), request, request.URL.Query().Get("status")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } items = server.filterSamplingTasksForRequest(request, items) renderJSON(writer, http.StatusOK, map[string]any{"items": items}) } func (server *server) handleSamplingTaskGet(writer http.ResponseWriter, request *http.Request) { if err := server.syncSamplingTasks(request.Context()); err != nil { renderError(writer, http.StatusInternalServerError, err) return } item, err := server.store.GetSamplingTask(request.Context(), chi.URLParam(request, "id")) if err != nil { renderError(writer, http.StatusInternalServerError, err) return } if item == nil { renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling task not found"}) return } if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Sampling task") { return } renderJSON(writer, http.StatusOK, item) } func validateSamplingReviewPayload(payload samplingReviewRequest) []validationIssue { issues := make([]validationIssue, 0, 4) if strings.TrimSpace(payload.VideoQuality) == "" { issues = append(issues, validationIssue{Field: "videoQuality", Message: "videoQuality is required"}) } if strings.TrimSpace(payload.AnnotationAccuracy) == "" { issues = append(issues, validationIssue{Field: "annotationAccuracy", Message: "annotationAccuracy is required"}) } if strings.TrimSpace(payload.AnnotationCompleteness) == "" { issues = append(issues, validationIssue{Field: "annotationCompleteness", Message: "annotationCompleteness is required"}) } if strings.TrimSpace(payload.OverallGrade) == "" { issues = append(issues, validationIssue{Field: "overallGrade", Message: "overallGrade is required"}) } return issues } func (server *server) handleSamplingItemReview(writer http.ResponseWriter, request *http.Request) { if err := server.syncSamplingTasks(request.Context()); err != nil { renderError(writer, http.StatusInternalServerError, err) return } var payload samplingReviewRequest if !decodeJSON(writer, request, &payload) { return } if issues := validateSamplingReviewPayload(payload); len(issues) > 0 { renderValidationError(writer, issues...) return } tasks, err := server.listSamplingTasksForRequest(request.Context(), request, "") if err != nil { renderError(writer, http.StatusInternalServerError, err) return } itemID := chi.URLParam(request, "id") for _, task := range tasks { if !server.filterByTenant(request, task.OrganizationID) { continue } updated := false for index := range task.Items { if task.Items[index].ID != itemID { continue } task.Items[index].Status = domain.SamplingItemStatusCompleted task.Items[index].Review = &domain.SamplingReview{ VideoQuality: strings.TrimSpace(payload.VideoQuality), AnnotationAccuracy: strings.TrimSpace(payload.AnnotationAccuracy), AnnotationCompleteness: strings.TrimSpace(payload.AnnotationCompleteness), OverallGrade: strings.TrimSpace(payload.OverallGrade), Notes: strings.TrimSpace(payload.Notes), ReviewedAt: nowISO(), } updated = true break } if !updated { continue } task.Status = domain.SamplingTaskStatusCompleted for _, item := range task.Items { if item.Status != domain.SamplingItemStatusCompleted { task.Status = domain.SamplingTaskStatusPending break } } task.UpdatedAt = nowISO() if err := server.store.UpsertSamplingTask(request.Context(), task); err != nil { renderError(writer, http.StatusInternalServerError, err) return } renderJSON(writer, http.StatusOK, task) return } renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling item not found"}) }