droboticflow/internal/api/data_platform.go

3324 lines
109 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package api
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"math"
"mime"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
"github.com/baidubce/bce-sdk-go/services/bos"
"github.com/go-chi/chi/v5"
)
type sourceBrowserPreviewRequest struct {
FileName string
Payload []byte
RedirectURL string
}
type flushResponseWriter struct {
writer http.ResponseWriter
flusher http.Flusher
}
func (target flushResponseWriter) Write(payload []byte) (int, error) {
count, err := target.writer.Write(payload)
if count > 0 {
target.flusher.Flush()
}
return count, err
}
var sourceBrowserVideoPreviewPath = locateSourceBrowserVideoPreviewPath
var sourceBrowserVideoPreviewWriter = streamSourceBrowserVideoPreview
var sourceCollectionSignedPreviewURL = buildSourceCollectionSignedPreviewURL
type scanConfigRequest struct {
Name string `json:"name"`
Description string `json:"description"`
SourcePath string `json:"sourcePath"`
ConnectionID string `json:"connectionId,omitempty"`
Bucket string `json:"bucket,omitempty"`
Prefix string `json:"prefix,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
AccessKeyID string `json:"accessKeyId,omitempty"`
SecretAccessKey string `json:"secretAccessKey,omitempty"`
ScanMode string `json:"scanMode,omitempty"`
DataSourceIssue *validationIssue `json:"-"`
ScanModeIssue *validationIssue `json:"-"`
IncrementalUnit string `json:"incrementalUnit,omitempty"`
IncrementalUnitIssue *validationIssue `json:"-"`
DirectoryDepth int `json:"directoryDepth,omitempty"`
DirectoryDepthIssue *validationIssue `json:"-"`
Pattern string `json:"pattern,omitempty"`
Interval int `json:"interval"`
Enabled bool `json:"enabled"`
DataSource string `json:"dataSource,omitempty"`
Notifications domain.ScanNotificationSettings `json:"notifications,omitempty"`
}
type samplingReviewRequest struct {
VideoQuality string `json:"videoQuality"`
AnnotationAccuracy string `json:"annotationAccuracy"`
AnnotationCompleteness string `json:"annotationCompleteness"`
OverallGrade string `json:"overallGrade"`
Notes string `json:"notes,omitempty"`
}
func isScanWorkflow(item domain.WorkflowRecord) bool {
return item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil
}
func chooseWorkflowKind(value domain.WorkflowKind) domain.WorkflowKind {
if value == "" {
return domain.WorkflowKindStandard
}
return value
}
func chooseDataSource(value string) string {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "BOS"
}
return trimmed
}
func parseWorkflowDataSourceHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "BOS" {
return trimmed, nil
}
return "", &validationIssue{
Field: "dataSource",
Message: "DATA_SOURCE must be BOS for scan workflows",
}
}
func validateScanWorkflowDataSource(value string) *validationIssue {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" || trimmed == "BOS" {
return nil
}
return &validationIssue{
Field: "dataSource",
Message: "dataSource must be BOS for scan workflows",
}
}
func cleanBOSPrefix(value string) string {
return strings.Trim(strings.TrimSpace(value), "/")
}
func chooseBOSEndpointWithDefault(value string, fallback string) string {
if trimmedValue := strings.TrimSpace(value); trimmedValue != "" {
return trimmedValue
}
return strings.TrimSpace(fallback)
}
func chooseBOSEndpoint(value string) string {
return strings.TrimSpace(value)
}
func (server *server) chooseBOSEndpoint(value string) string {
return chooseBOSEndpointWithDefault(value, server.globalBOSEndpoint)
}
func chooseWorkflowScanMode(value string) string {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "FULL":
return "FULL"
default:
return "INCREMENTAL"
}
}
func parseWorkflowScanModeHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "FULL" || trimmed == "INCREMENTAL" {
return trimmed, nil
}
return "", &validationIssue{
Field: "scanMode",
Message: "SCAN_MODE must be FULL or INCREMENTAL",
}
}
func chooseWorkflowIncrementalUnit(value string) string {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "DIRECTORY":
return "DIRECTORY"
default:
return "OBJECT"
}
}
func parseWorkflowIncrementalUnitHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "OBJECT" || trimmed == "DIRECTORY" {
return trimmed, nil
}
return "", &validationIssue{
Field: "incrementalUnit",
Message: "INCREMENTAL_UNIT must be OBJECT or DIRECTORY",
}
}
func chooseWorkflowDirectoryDepth(value int) int {
if value > 0 {
return value
}
return 1
}
func parseWorkflowDirectoryDepthHyperparam(value string) (int, *validationIssue) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return 0, nil
}
parsed, err := strconv.Atoi(trimmed)
if err != nil || parsed <= 0 {
return 0, &validationIssue{
Field: "directoryDepth",
Message: "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer",
}
}
return parsed, nil
}
func scanSourcePath(dataSource string, sourcePath string, bucket string, prefix string) string {
if chooseDataSource(dataSource) == "BOS" {
if explicit := strings.TrimSpace(sourcePath); explicit != "" {
return explicit
}
if strings.TrimSpace(bucket) == "" {
return ""
}
canonical := "bos://" + strings.TrimSpace(bucket)
if cleanedPrefix := cleanBOSPrefix(prefix); cleanedPrefix != "" {
canonical += "/" + cleanedPrefix
}
return canonical
}
return strings.TrimSpace(sourcePath)
}
func mergeScanSecrets(existing *domain.ScanConfig, payload scanConfigRequest) (string, string) {
accessKeyID := strings.TrimSpace(payload.AccessKeyID)
secretAccessKey := strings.TrimSpace(payload.SecretAccessKey)
if accessKeyID == "" && existing != nil {
accessKeyID = existing.AccessKeyID
}
if secretAccessKey == "" && existing != nil {
secretAccessKey = existing.SecretAccessKey
}
return accessKeyID, secretAccessKey
}
func mergeScanConnectionID(existing *domain.ScanConfig, payload scanConfigRequest) string {
connectionID := strings.TrimSpace(payload.ConnectionID)
if connectionID == "" &&
existing != nil &&
strings.TrimSpace(existing.ConnectionID) != "" &&
strings.TrimSpace(payload.AccessKeyID) == "" &&
strings.TrimSpace(payload.SecretAccessKey) == "" {
connectionID = strings.TrimSpace(existing.ConnectionID)
}
return connectionID
}
func (server *server) getStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) {
connectionID := strings.TrimSpace(id)
if connectionID == "" {
return nil, nil
}
item, err := server.store.GetStorageConnection(ctx, connectionID)
if err != nil {
return nil, err
}
if item == nil {
return nil, fmt.Errorf("storage connection %s not found", connectionID)
}
if !server.filterByTenantContext(ctx, item.OrganizationID) {
return nil, fmt.Errorf("storage connection %s not found", connectionID)
}
return item, nil
}
func (server *server) getMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) {
connectionID := strings.TrimSpace(id)
if connectionID == "" {
return nil, nil
}
item, err := server.store.GetMongoConnection(ctx, connectionID)
if err != nil {
return nil, err
}
if item == nil {
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
}
if !server.filterByTenantContext(ctx, item.OrganizationID) {
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
}
return item, nil
}
func (server *server) resolveScanConfig(ctx context.Context, scanConfig *domain.ScanConfig) (*domain.ScanConfig, error) {
if scanConfig == nil {
return nil, nil
}
copy := *scanConfig
connection, err := server.getStorageConnection(ctx, copy.ConnectionID)
if err != nil {
return nil, err
}
if connection == nil {
if chooseDataSource(copy.DataSource) == "BOS" {
copy.Endpoint = server.chooseBOSEndpoint(copy.Endpoint)
}
return &copy, nil
}
copy.Endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint)
if strings.TrimSpace(copy.AccessKeyID) == "" {
copy.AccessKeyID = connection.AccessKeyID
}
if strings.TrimSpace(copy.SecretAccessKey) == "" {
copy.SecretAccessKey = connection.SecretAccessKey
}
return &copy, nil
}
func cloneHyperparams(items []domain.HyperParam) []domain.HyperParam {
out := make([]domain.HyperParam, len(items))
copy(out, items)
return out
}
func clonePorts(items []domain.PortDefinition) []domain.PortDefinition {
out := make([]domain.PortDefinition, len(items))
copy(out, items)
return out
}
func cloneOperatorIntoNode(operator domain.OperatorRecord, nodeID string, x float64, y float64, overrides map[string]string) domain.WorkflowNode {
hyperparams := cloneHyperparams(operator.Hyperparams)
seen := make(map[string]int, len(hyperparams))
for index, item := range hyperparams {
seen[item.Key] = index
}
for key, value := range overrides {
if index, ok := seen[key]; ok {
hyperparams[index].Value = value
continue
}
hyperparams = append(hyperparams, domain.HyperParam{Key: key, Value: value})
}
return domain.WorkflowNode{
ID: nodeID,
Type: "operator",
Data: domain.WorkflowNodeData{
Label: operator.Name,
Category: operator.Category,
Description: operator.Description,
Inputs: clonePorts(operator.Inputs),
Outputs: clonePorts(operator.Outputs),
Image: operator.Image,
SkillDefinition: operator.SkillDefinition,
Hyperparams: hyperparams,
},
Position: domain.Position{X: x, Y: y},
}
}
func operatorByName(items []domain.OperatorRecord, name string) (*domain.OperatorRecord, bool) {
for _, item := range items {
if item.Name == name {
copy := item
return &copy, true
}
}
return nil, false
}
func (server *server) buildScanWorkflowInput(ctx context.Context, projectID string, payload scanConfigRequest, existing *domain.ScanConfig) (domain.CreateWorkflowInput, error) {
operators, err := server.listAvailableOperators(ctx)
if err != nil {
return domain.CreateWorkflowInput{}, err
}
dataSource := chooseDataSource(payload.DataSource)
sourceOperatorName := "BOS 增量扫描"
sourceOperator, ok := operatorByName(operators, sourceOperatorName)
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator %s not found", sourceOperatorName)
}
indexOperator, ok := operatorByName(operators, "X 平台数据集索引")
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator X 平台数据集索引 not found")
}
validateOperator, ok := operatorByName(operators, "采集目录校验")
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator 采集目录校验 not found")
}
sourcePath := scanSourcePath(dataSource, payload.SourcePath, payload.Bucket, payload.Prefix)
connectionID := mergeScanConnectionID(existing, payload)
var connection *domain.StorageConnectionRecord
if connectionID != "" {
connection, err = server.getStorageConnection(ctx, connectionID)
if err != nil {
return domain.CreateWorkflowInput{}, err
}
}
endpoint := ""
accessKeyID := ""
secretAccessKey := ""
if connection != nil {
endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint)
} else if dataSource == "BOS" {
endpoint = server.chooseBOSEndpoint(payload.Endpoint)
accessKeyID, secretAccessKey = mergeScanSecrets(existing, payload)
}
sourceOverrides := map[string]string{
"DATA_SOURCE": dataSource,
"SCAN_MODE": chooseWorkflowScanMode(payload.ScanMode),
"INCREMENTAL_UNIT": chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
"INCREMENTAL_DIRECTORY_DEPTH": strconv.Itoa(chooseWorkflowDirectoryDepth(payload.DirectoryDepth)),
}
if dataSource == "BOS" {
sourceOverrides["SOURCE_PATH"] = sourcePath
sourceOverrides["BOS_ENDPOINT"] = endpoint
} else {
sourceOverrides["SOURCE_PATH"] = sourcePath
}
sourceNode := cloneOperatorIntoNode(*sourceOperator, "scan-source", 0, 0, sourceOverrides)
sourceNode.Data.Bucket = strings.TrimSpace(payload.Bucket)
sourceNode.Data.Prefix = cleanBOSPrefix(payload.Prefix)
sourceNode.Data.Pattern = strings.TrimSpace(payload.Pattern)
indexNode := cloneOperatorIntoNode(*indexOperator, "scan-index", 260, 0, nil)
validateNode := cloneOperatorIntoNode(*validateOperator, "scan-validate", 520, 0, nil)
schedule := &domain.WorkflowSchedule{SourcePath: sourcePath}
workflowType := domain.WorkflowTypeBatch
if payload.Enabled && payload.Interval > 0 {
schedule.Interval = payload.Interval
workflowType = domain.WorkflowTypeStreaming
}
return domain.CreateWorkflowInput{
ProjectID: projectID,
Name: strings.TrimSpace(payload.Name),
Description: strings.TrimSpace(payload.Description),
Type: workflowType,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: dataSource,
SourcePath: sourcePath,
ConnectionID: connectionID,
Bucket: strings.TrimSpace(payload.Bucket),
Prefix: cleanBOSPrefix(payload.Prefix),
Endpoint: endpoint,
AccessKeyID: accessKeyID,
SecretAccessKey: secretAccessKey,
ScanMode: chooseWorkflowScanMode(payload.ScanMode),
IncrementalUnit: chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
DirectoryDepth: chooseWorkflowDirectoryDepth(payload.DirectoryDepth),
Pattern: strings.TrimSpace(payload.Pattern),
Enabled: payload.Enabled && payload.Interval > 0,
Notifications: payload.Notifications,
},
Schedule: schedule,
Nodes: []domain.WorkflowNode{
sourceNode,
indexNode,
validateNode,
},
Edges: []domain.WorkflowEdge{
{ID: "scan-source-index", Source: sourceNode.ID, SourceHandle: "out-1", Target: indexNode.ID, TargetHandle: "in-1"},
{ID: "scan-index-validate", Source: indexNode.ID, SourceHandle: "out-1", Target: validateNode.ID, TargetHandle: "in-1"},
},
}, nil
}
func validateScanConfigEndpoint(payload scanConfigRequest, existing *domain.ScanConfig) *validationIssue {
if chooseDataSource(payload.DataSource) != "BOS" {
return nil
}
if strings.TrimSpace(mergeScanConnectionID(existing, payload)) != "" {
return nil
}
return validateOptionalHTTPURL("endpoint", payload.Endpoint)
}
func validateScanConfigDraftPayload(payload scanConfigRequest, existing ...*domain.ScanConfig) []validationIssue {
var previous *domain.ScanConfig
if len(existing) > 0 {
previous = existing[0]
}
issues := make([]validationIssue, 0, 5)
if strings.TrimSpace(payload.Name) == "" {
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
}
if payload.Interval < 0 {
issues = append(issues, validationIssue{Field: "interval", Message: "interval must be zero or greater"})
}
if issue := validateScanWorkflowDataSource(payload.DataSource); issue != nil {
issues = append(issues, *issue)
}
if payload.DataSourceIssue != nil {
issues = append(issues, *payload.DataSourceIssue)
}
if scanMode := strings.ToUpper(strings.TrimSpace(payload.ScanMode)); scanMode != "" && scanMode != "FULL" && scanMode != "INCREMENTAL" {
issues = append(issues, validationIssue{Field: "scanMode", Message: "scanMode must be FULL or INCREMENTAL"})
}
if payload.ScanModeIssue != nil {
issues = append(issues, *payload.ScanModeIssue)
}
if incrementalUnit := strings.ToUpper(strings.TrimSpace(payload.IncrementalUnit)); incrementalUnit != "" && incrementalUnit != "OBJECT" && incrementalUnit != "DIRECTORY" {
issues = append(issues, validationIssue{Field: "incrementalUnit", Message: "incrementalUnit must be OBJECT or DIRECTORY"})
}
if payload.IncrementalUnitIssue != nil {
issues = append(issues, *payload.IncrementalUnitIssue)
}
if payload.DirectoryDepthIssue != nil {
issues = append(issues, *payload.DirectoryDepthIssue)
}
if payload.DirectoryDepth < 0 {
issues = append(issues, validationIssue{Field: "directoryDepth", Message: "directoryDepth must be zero or greater"})
}
if issue := validateScanConfigEndpoint(payload, previous); issue != nil {
issues = append(issues, *issue)
}
return issues
}
func scanConfigPayloadReady(payload scanConfigRequest, existing *domain.ScanConfig) bool {
switch chooseDataSource(payload.DataSource) {
case "BOS":
connectionID := mergeScanConnectionID(existing, payload)
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
hasTarget := strings.TrimSpace(payload.SourcePath) != "" ||
(strings.TrimSpace(payload.Bucket) != "" && cleanBOSPrefix(payload.Prefix) != "")
return hasTarget &&
(strings.TrimSpace(connectionID) != "" ||
(strings.TrimSpace(accessKeyID) != "" && strings.TrimSpace(secretAccessKey) != ""))
default:
return strings.TrimSpace(payload.SourcePath) != ""
}
}
func validateScanConfigPayload(payload scanConfigRequest, existing *domain.ScanConfig) []validationIssue {
issues := validateScanConfigDraftPayload(payload, existing)
switch chooseDataSource(payload.DataSource) {
case "BOS":
if strings.TrimSpace(payload.SourcePath) == "" {
if strings.TrimSpace(payload.Bucket) == "" {
issues = append(issues, validationIssue{Field: "bucket", Message: "bucket is required for BOS scan configs when sourcePath is empty"})
}
if cleanBOSPrefix(payload.Prefix) == "" {
issues = append(issues, validationIssue{Field: "prefix", Message: "prefix is required for BOS scan configs when sourcePath is empty"})
}
}
connectionID := mergeScanConnectionID(existing, payload)
if strings.TrimSpace(connectionID) == "" {
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
if strings.TrimSpace(accessKeyID) == "" {
issues = append(issues, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required for BOS scan configs"})
}
if strings.TrimSpace(secretAccessKey) == "" {
issues = append(issues, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required for BOS scan configs"})
}
}
default:
if strings.TrimSpace(payload.SourcePath) == "" {
issues = append(issues, validationIssue{Field: "sourcePath", Message: "sourcePath is required"})
}
}
return issues
}
func (server *server) scanWorkflows(ctx context.Context, request *http.Request) ([]domain.WorkflowRecord, error) {
items, err := server.listAllWorkflowsForRequest(ctx, request)
if err != nil {
return nil, err
}
filtered := make([]domain.WorkflowRecord, 0)
for _, item := range items {
if isScanWorkflow(item) {
item.Kind = chooseWorkflowKind(item.Kind)
filtered = append(filtered, item)
}
}
return filtered, nil
}
func (server *server) handleScanConfigsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.scanWorkflows(request.Context(), request)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items = server.filterWorkflowsForRequest(request, items)
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)})
}
func (server *server) handleScanConfigsCreate(writer http.ResponseWriter, request *http.Request) {
var payload scanConfigRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Interval > 0 && !scanConfigPayloadReady(payload, nil) {
renderValidationError(writer, validateScanConfigPayload(payload, nil)...)
return
}
if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(nil, payload)); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
input, err := server.buildScanWorkflowInput(request.Context(), "", payload, nil)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
server.assignTenant(request, &input.OrganizationID)
item, err := server.store.CreateWorkflow(request.Context(), input)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
item.Kind = chooseWorkflowKind(item.Kind)
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
var payload scanConfigRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateScanConfigDraftPayload(payload, item.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Interval > 0 && !scanConfigPayloadReady(payload, item.ScanConfig) {
renderValidationError(writer, validateScanConfigPayload(payload, item.ScanConfig)...)
return
}
if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(item.ScanConfig, payload)); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
input, err := server.buildScanWorkflowInput(request.Context(), item.ProjectID, payload, item.ScanConfig)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item.Name = input.Name
item.Description = input.Description
item.Type = input.Type
item.Kind = input.Kind
item.ScanConfig = input.ScanConfig
item.Schedule = input.Schedule
item.Nodes = input.Nodes
item.Edges = input.Edges
item.UpdatedAt = nowISO()
if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsDelete(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
writer.WriteHeader(http.StatusNotFound)
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
preview, runs, err := server.buildWorkflowDeletionPreview(request.Context(), item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if preview.BlockingRunCount > 0 {
renderJSON(writer, http.StatusConflict, map[string]any{
"error": "数据扫描配置仍有正在排队或运行的 Run,请先停止这些 Run 后再删除",
"code": "scan_config_run_active",
"preview": preview,
})
return
}
if err := server.deleteWorkflowGeneratedArtifacts(request.Context(), item, runs); err != nil {
renderJSON(writer, http.StatusBadGateway, map[string]string{"error": err.Error(), "code": "scan_config_artifact_cleanup_failed"})
return
}
deleted, err := server.store.DeleteWorkflow(request.Context(), item.ID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func artifactByName(run domain.RunRecord, name string) *domain.ExecutionArtifact {
for index := range run.Artifacts {
if run.Artifacts[index].Name == name {
return &run.Artifacts[index]
}
}
return nil
}
func mapValue(value any) map[string]any {
item, _ := normalizeStructuredValue(value).(map[string]any)
return item
}
func listValue(value any) []any {
items, _ := normalizeStructuredValue(value).([]any)
return items
}
func normalizeStructuredValue(value any) any {
switch typed := value.(type) {
case nil:
return nil
case map[string]any:
items := make(map[string]any, len(typed))
for key, item := range typed {
items[key] = normalizeStructuredValue(item)
}
return items
case []any:
items := make([]any, len(typed))
for index, item := range typed {
items[index] = normalizeStructuredValue(item)
}
return items
}
rv := reflect.ValueOf(value)
switch rv.Kind() {
case reflect.Map:
if rv.Type().Key().Kind() != reflect.String {
return value
}
items := make(map[string]any, rv.Len())
iter := rv.MapRange()
for iter.Next() {
items[iter.Key().String()] = normalizeStructuredValue(iter.Value().Interface())
}
return items
case reflect.Slice, reflect.Array:
items := make([]any, rv.Len())
for index := 0; index < rv.Len(); index++ {
items[index] = normalizeStructuredValue(rv.Index(index).Interface())
}
return items
default:
return value
}
}
func stringValue(value any) string {
switch typed := value.(type) {
case nil:
return ""
case string:
return typed
default:
text := strings.TrimSpace(fmt.Sprintf("%v", typed))
if text == "<nil>" {
return ""
}
return text
}
}
func boolValue(value any) bool {
typed, ok := value.(bool)
return ok && typed
}
func floatValue(value any) float64 {
switch typed := value.(type) {
case float64:
if math.IsNaN(typed) || math.IsInf(typed, 0) {
return 0
}
return typed
case float32:
value := float64(typed)
if math.IsNaN(value) || math.IsInf(value, 0) {
return 0
}
return value
case int:
return float64(typed)
case int32:
return float64(typed)
case int64:
return float64(typed)
default:
return 0
}
}
func intValue(value any) int {
parsed, ok := exactIntegerValue(value)
if !ok {
return 0
}
if strconv.IntSize == 32 {
const maxInt32 = int64(1<<31 - 1)
const minInt32 = -1 << 31
if parsed < minInt32 || parsed > maxInt32 {
return 0
}
}
return int(parsed)
}
func int64Value(value any) int64 {
parsed, ok := exactIntegerValue(value)
if !ok {
return 0
}
return parsed
}
func exactIntegerValue(value any) (int64, bool) {
switch typed := value.(type) {
case int:
return int64(typed), true
case int8:
return int64(typed), true
case int16:
return int64(typed), true
case int32:
return int64(typed), true
case int64:
return typed, true
case uint:
if uint64(typed) > uint64(math.MaxInt64) {
return 0, false
}
return int64(typed), true
case uint8:
return int64(typed), true
case uint16:
return int64(typed), true
case uint32:
return int64(typed), true
case uint64:
if typed > uint64(math.MaxInt64) {
return 0, false
}
return int64(typed), true
case float32:
const maxExactFloat32Integer = 1<<24 - 1
return exactFloatIntegerValue(float64(typed), maxExactFloat32Integer)
case float64:
const maxExactFloat64Integer = 1<<53 - 1
return exactFloatIntegerValue(typed, maxExactFloat64Integer)
case json.Number:
parsed, err := typed.Int64()
return parsed, err == nil
default:
return 0, false
}
}
func exactFloatIntegerValue(value float64, maxExactAbs float64) (int64, bool) {
if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
return 0, false
}
if math.Abs(value) > maxExactAbs {
return 0, false
}
return int64(value), true
}
func artifactDataRef(artifact *domain.ExecutionArtifact) map[string]any {
if artifact == nil || artifact.Data == nil {
return nil
}
return mapValue(artifact.Data["dataRef"])
}
func artifactStructuredData(artifact *domain.ExecutionArtifact) map[string]any {
if artifact == nil {
return nil
}
if dataRef := artifactDataRef(artifact); dataRef != nil {
payloadPath := strings.TrimSpace(stringValue(dataRef["payloadPath"]))
if payloadPath != "" && !strings.Contains(payloadPath, "://") {
payload, err := os.ReadFile(payloadPath)
if err != nil {
return nil
}
var decoded any
if err := json.Unmarshal(payload, &decoded); err != nil {
return nil
}
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
return structured
}
return nil
}
}
if artifact.Data != nil {
return artifact.Data
}
if artifact.Type == domain.PortTypeJSON {
filePath := strings.TrimSpace(artifact.Path)
if filePath != "" && !strings.Contains(filePath, "://") {
payload, err := os.ReadFile(filePath)
if err == nil {
var decoded any
if err := json.Unmarshal(payload, &decoded); err == nil {
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
return structured
}
}
}
}
}
return nil
}
func sourceArtifactRelativePath(path string) string {
normalized := filepath.ToSlash(path)
marker := "/_source_tree/"
index := strings.Index(normalized, marker)
if index < 0 {
return ""
}
return strings.TrimPrefix(normalized[index+len(marker):], "/")
}
func sourceArtifactPath(artifact domain.ExecutionArtifact) string {
if payload := artifactStructuredData(&artifact); payload != nil {
if relative := stringValue(payload["relativePath"]); relative != "" {
return filepath.ToSlash(strings.Trim(relative, "/"))
}
}
return sourceArtifactRelativePath(artifact.Path)
}
func packageArtifactRelativePath(path string, packageName string) string {
normalized := filepath.ToSlash(path)
marker := "/packages/" + packageName + "/"
index := strings.Index(normalized, marker)
if index < 0 {
return ""
}
return strings.TrimPrefix(normalized[index+len(marker):], "/")
}
func packageContentURL(runID string, packageName string, relativePath string) string {
return fmt.Sprintf(
"/api/packages/%s/%s/content?relativePath=%s",
url.PathEscape(runID),
url.PathEscape(packageName),
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
)
}
func packageArtifactPreviewURL(artifact domain.ExecutionArtifact) string {
payload := artifactStructuredData(&artifact)
preview := mapValue(payload["preview"])
if preview != nil {
if value := strings.TrimSpace(stringValue(preview["url"])); value != "" {
return value
}
}
source := mapValue(payload["source"])
if source != nil {
if value := strings.TrimSpace(stringValue(source["previewURL"])); value != "" {
return value
}
}
return ""
}
func packageArtifactResolvedRelativePath(artifact domain.ExecutionArtifact, packageName string) string {
if relativePath := packageArtifactRelativePath(artifact.Path, packageName); relativePath != "" {
return relativePath
}
payload := artifactStructuredData(&artifact)
if payload == nil {
return ""
}
relativePath := strings.Trim(filepath.ToSlash(stringValue(payload["relativePath"])), "/")
if relativePath == "" {
return ""
}
packagePrefix := strings.Trim(filepath.ToSlash(path.Join("packages", packageName)), "/")
if packagePrefix == "" {
return ""
}
if relativePath == packagePrefix {
return ""
}
if strings.HasPrefix(relativePath, packagePrefix+"/") {
return strings.TrimPrefix(relativePath, packagePrefix+"/")
}
return ""
}
func packageArtifactContentURL(runID string, packageName string, relativePath string) string {
return packageContentURL(runID, packageName, relativePath)
}
func packageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
copy := artifact
if copy.Data == nil {
copy.Data = make(map[string]any)
}
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
if relativePath == "" {
relativePath = filepath.ToSlash(strings.Trim(copy.Path, "/"))
}
if relativePath != "" {
copy.Path = relativePath
}
copy.Data["contentURL"] = packageArtifactContentURL(runID, packageName, copy.Path)
preview := mapValue(copy.Data["preview"])
if preview == nil {
preview = make(map[string]any)
}
kind := strings.TrimSpace(stringValue(preview["kind"]))
mimeType := ""
if copy.Metadata != nil {
mimeType = strings.TrimSpace(copy.Metadata.MimeType)
}
if kind == "video" || copy.Type == "video" || strings.HasPrefix(mimeType, "video/") {
preview["kind"] = "video"
preview["url"] = packageContentURL(runID, packageName, copy.Path)
copy.Data["preview"] = preview
}
return copy
}
const wilorHandPoseSchemaVersion = "droboticflow.wilor.hand_pose_result/v1"
func safePackageNamePart(value string) string {
trimmed := strings.ToLower(strings.TrimSpace(value))
var builder strings.Builder
lastDash := false
for _, current := range trimmed {
isAlphaNumeric := (current >= 'a' && current <= 'z') || (current >= '0' && current <= '9')
if isAlphaNumeric {
builder.WriteRune(current)
lastDash = false
continue
}
if !lastDash {
builder.WriteByte('-')
lastDash = true
}
}
return strings.Trim(builder.String(), "-")
}
func wilorResultPayload(artifact domain.ExecutionArtifact) map[string]any {
if artifact.Name != "wilor-result.json" {
return nil
}
payload := artifactStructuredData(&artifact)
if payload == nil {
return nil
}
if strings.TrimSpace(stringValue(payload["schemaVersion"])) != wilorHandPoseSchemaVersion {
return nil
}
if strings.TrimSpace(stringValue(payload["relativePath"])) != "" {
return nil
}
return payload
}
func wilorPackageName(artifact domain.ExecutionArtifact, payload map[string]any) string {
source := mapValue(payload["source"])
sourceName := strings.TrimSpace(stringValue(source["name"]))
stem := strings.TrimSuffix(path.Base(filepath.ToSlash(sourceName)), path.Ext(sourceName))
sourceSlug := safePackageNamePart(stem)
if sourceSlug == "" {
sourceSlug = "result"
}
nodeSlug := safePackageNamePart(artifact.NodeID)
if nodeSlug == "" {
nodeSlug = "wilor"
}
return "wilor-" + sourceSlug + "-" + nodeSlug
}
func wilorResultGroupKey(artifact domain.ExecutionArtifact, payload map[string]any) string {
source := mapValue(payload["source"])
sourceName := strings.TrimSpace(stringValue(source["name"]))
sourceRef := strings.TrimSpace(stringValue(source["ref"]))
mediaKind := strings.TrimSpace(stringValue(payload["mediaKind"]))
if sourceName != "" || sourceRef != "" || mediaKind != "" {
return strings.ToLower(mediaKind) + "|" + strings.ToLower(sourceName) + "|" + strings.ToLower(sourceRef)
}
return artifact.ID
}
func wilorResultArtifactPriority(artifact domain.ExecutionArtifact) int {
nodeID := strings.ToLower(strings.TrimSpace(artifact.NodeID))
score := 0
if strings.Contains(nodeID, "upload") || strings.Contains(nodeID, "save") {
score += 30
}
if strings.Contains(nodeID, "wilor") {
score += 20
}
if strings.TrimSpace(artifact.Path) != "" {
score++
}
return score
}
func wilorResultArtifacts(run domain.RunRecord) []domain.ExecutionArtifact {
bestByResult := make(map[string]domain.ExecutionArtifact)
for _, artifact := range run.Artifacts {
payload := wilorResultPayload(artifact)
if payload == nil {
continue
}
key := wilorResultGroupKey(artifact, payload)
current, exists := bestByResult[key]
if !exists || wilorResultArtifactPriority(artifact) > wilorResultArtifactPriority(current) {
bestByResult[key] = artifact
}
}
items := make([]domain.ExecutionArtifact, 0, len(bestByResult))
for _, artifact := range bestByResult {
items = append(items, artifact)
}
sort.Slice(items, func(left int, right int) bool {
return wilorPackageName(items[left], wilorResultPayload(items[left])) < wilorPackageName(items[right], wilorResultPayload(items[right]))
})
return items
}
func wilorPackageManifestArtifact(run domain.RunRecord, packageName string) (*domain.ExecutionArtifact, map[string]any) {
for index := range run.Artifacts {
payload := wilorResultPayload(run.Artifacts[index])
if payload == nil {
continue
}
if wilorPackageName(run.Artifacts[index], payload) == packageName {
return &run.Artifacts[index], payload
}
}
return nil, nil
}
func wilorOutputArtifactNames(manifest domain.ExecutionArtifact, payload map[string]any) map[string]struct{} {
names := map[string]struct{}{
manifest.Name: {},
}
for _, raw := range listValue(payload["downloadedOutputs"]) {
item := mapValue(raw)
name := strings.TrimSpace(stringValue(item["name"]))
if name == "" {
if outputPath := strings.TrimSpace(stringValue(item["path"])); outputPath != "" {
name = path.Base(filepath.ToSlash(outputPath))
}
}
if name != "" {
names[name] = struct{}{}
}
}
return names
}
func wilorPackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
manifest, payload := wilorPackageManifestArtifact(run, packageName)
if manifest == nil || payload == nil {
return nil
}
names := wilorOutputArtifactNames(*manifest, payload)
bestByName := make(map[string]domain.ExecutionArtifact)
for _, artifact := range run.Artifacts {
if _, ok := names[artifact.Name]; !ok {
continue
}
if artifact.NodeID != manifest.NodeID && strings.TrimSpace(stringValue(mapValue(artifactStructuredData(&artifact)["source"])["path"])) == "" {
continue
}
current, exists := bestByName[artifact.Name]
if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) {
bestByName[artifact.Name] = artifact
}
}
items := make([]domain.ExecutionArtifact, 0, len(bestByName))
for _, artifact := range bestByName {
items = append(items, artifact)
}
sort.Slice(items, func(left int, right int) bool {
return items[left].Name < items[right].Name
})
return items
}
func wilorPackageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
copy := artifact
if copy.Data == nil {
copy.Data = make(map[string]any)
}
relativePath := strings.TrimSpace(copy.Name)
if relativePath == "" {
relativePath = path.Base(filepath.ToSlash(strings.TrimSpace(copy.Path)))
}
copy.Path = relativePath
copy.Data["contentURL"] = packageArtifactContentURL(runID, packageName, relativePath)
preview := mapValue(copy.Data["preview"])
if preview == nil {
preview = make(map[string]any)
}
mimeType := ""
if copy.Metadata != nil {
mimeType = strings.TrimSpace(copy.Metadata.MimeType)
}
kind := sourceEntryPayloadKind(relativePath, strings.ToLower(mimeType))
if kind == "video" || kind == "image" {
preview["kind"] = kind
preview["url"] = packageArtifactContentURL(runID, packageName, relativePath)
copy.Data["preview"] = preview
}
return copy
}
func wilorPackageFilesForResponse(run domain.RunRecord, packageName string) ([]domain.ExecutionArtifact, []domain.ExecutionArtifact) {
rawFiles := wilorPackageArtifacts(run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, wilorPackageArtifactForResponse(run.ID, packageName, artifact))
}
return files, rawFiles
}
func wilorPackageItem(run domain.RunRecord, manifest domain.ExecutionArtifact, payload map[string]any, rawFiles []domain.ExecutionArtifact) map[string]any {
source := mapValue(payload["source"])
result := mapValue(payload["result"])
service := mapValue(payload["service"])
health := mapValue(service["health"])
sourceName := strings.TrimSpace(stringValue(source["name"]))
if sourceName == "" {
sourceName = "WiLoR result"
}
mediaKind := strings.TrimSpace(stringValue(payload["mediaKind"]))
if mediaKind == "" {
mediaKind = "media"
}
clipID := strings.TrimSuffix(path.Base(filepath.ToSlash(sourceName)), path.Ext(sourceName))
if clipID == "" || clipID == "." {
clipID = manifest.NodeID
}
storagePath := ""
for _, artifact := range rawFiles {
if candidate := packageArtifactStoragePath(artifact, ""); candidate != "" {
storagePath = candidate
break
}
}
annotation := map[string]any{
"type": "wilor_hand_pose",
"mediaKind": mediaKind,
"handCount": intValue(result["handCount"]),
"frameCount": intValue(result["frameCount"]),
"status": stringValue(payload["status"]),
"requiresHumanReview": false,
}
if task := mapValue(result["wilorTask"]); task != nil {
annotation["task"] = task
}
quality := map[string]any{
"runtime": mapValue(health["runtime"]),
"service": map[string]any{
"image": stringValue(service["image"]),
"imageId": stringValue(service["imageId"]),
},
}
return map[string]any{
"packageName": wilorPackageName(manifest, payload),
"clipId": clipID,
"sourceVideo": sourceName,
"clipDuration": 0,
"fileCount": len(rawFiles),
"pipelineTags": []any{"wilor", "hand_pose", mediaKind},
"storagePath": storagePath,
"quality": quality,
"annotation": annotation,
}
}
func canonicalBOSPath(bucket string, object string) string {
bucket = strings.TrimSpace(bucket)
object = strings.Trim(strings.TrimSpace(object), "/")
if bucket == "" {
return ""
}
if object == "" {
return "bos://" + bucket
}
return "bos://" + bucket + "/" + object
}
func packageArtifactPriority(artifact domain.ExecutionArtifact) int {
score := 0
if artifact.NodeID == "upload" {
score += 40
}
artifactPath := strings.TrimSpace(artifact.Path)
if strings.HasPrefix(strings.ToLower(artifactPath), "bos://") {
score += 30
} else if artifactPath != "" && !strings.Contains(artifactPath, "://") {
score += 10
}
payload := artifactStructuredData(&artifact)
source := mapValue(payload["source"])
sourcePath := strings.TrimSpace(stringValue(source["path"]))
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
score += 30
}
if strings.TrimSpace(packageArtifactPreviewURL(artifact)) != "" {
score += 5
}
return score
}
func packageArtifactStoragePath(artifact domain.ExecutionArtifact, packageName string) string {
payload := artifactStructuredData(&artifact)
source := mapValue(payload["source"])
if sourcePath := strings.TrimSpace(stringValue(source["path"])); sourcePath != "" {
if bucket, object, ok := parseBOSPath(sourcePath); ok {
return canonicalBOSPath(bucket, path.Dir(object))
}
directory := filepath.ToSlash(filepath.Dir(sourcePath))
if directory != "." {
return directory
}
}
artifactPath := strings.TrimSpace(artifact.Path)
if bucket, object, ok := parseBOSPath(artifactPath); ok {
return canonicalBOSPath(bucket, path.Dir(object))
}
if artifactPath == "" {
return ""
}
directory := filepath.ToSlash(filepath.Dir(artifactPath))
if directory == "." {
return ""
}
return directory
}
func dedupePackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
bestByRelativePath := make(map[string]domain.ExecutionArtifact)
for _, artifact := range run.Artifacts {
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
if relativePath == "" {
continue
}
current, exists := bestByRelativePath[relativePath]
if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) {
bestByRelativePath[relativePath] = artifact
}
}
items := make([]domain.ExecutionArtifact, 0, len(bestByRelativePath))
for _, artifact := range bestByRelativePath {
items = append(items, artifact)
}
sort.Slice(items, func(left int, right int) bool {
leftPath := packageArtifactResolvedRelativePath(items[left], packageName)
rightPath := packageArtifactResolvedRelativePath(items[right], packageName)
return leftPath < rightPath
})
return items
}
func packageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range dedupePackageArtifacts(run, packageName) {
items = append(items, packageArtifactForResponse(run.ID, packageName, artifact))
}
return items
}
func choosePackagePreview(files []domain.ExecutionArtifact) *domain.ExecutionArtifact {
for _, file := range files {
if strings.HasSuffix(strings.ToLower(file.Name), ".mp4") {
copy := file
return &copy
}
}
if len(files) == 0 {
return nil
}
copy := files[0]
return &copy
}
func sourceDatasetList(data map[string]any, catalogKey string, fallbackKey string) []any {
if _, ok := data[catalogKey]; ok {
return listValue(data[catalogKey])
}
return listValue(data[fallbackKey])
}
func sourceDatasetHasCatalog(data map[string]any) bool {
_, hasCollections := data["catalogCollections"]
_, hasItems := data["catalogItems"]
_, hasEntries := data["catalogEntries"]
return hasCollections || hasItems || hasEntries
}
func sourceDatasetIsDisplayable(data map[string]any) bool {
if len(sourceDatasetList(data, "catalogCollections", "collections")) > 0 {
return true
}
if len(sourceDatasetList(data, "catalogEntries", "entries")) > 0 {
return true
}
return intValue(data["catalogCollectionCount"]) > 0 || intValue(data["collectionCount"]) > 0
}
func runSourceManifest(run domain.RunRecord) *domain.ExecutionArtifact {
manifest := artifactByName(run, "source-manifest.json")
if manifest == nil || artifactStructuredData(manifest) == nil {
return nil
}
return manifest
}
func runDatasetIndex(run domain.RunRecord) *domain.ExecutionArtifact {
artifact := artifactByName(run, "dataset-index.json")
if artifact == nil || artifactStructuredData(artifact) == nil {
return nil
}
return artifact
}
func runLooksLikeSourceDataset(workflow domain.WorkflowRecord, run domain.RunRecord) bool {
if manifest := runSourceManifest(run); manifest != nil {
payload := artifactStructuredData(manifest)
if strings.TrimSpace(stringValue(payload["sourcePath"])) != "" || strings.TrimSpace(stringValue(payload["dataSource"])) != "" {
return true
}
}
index := runDatasetIndex(run)
if index == nil {
return false
}
if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil {
return true
}
indexPayload := artifactStructuredData(index)
sourcePath := strings.TrimSpace(stringValue(indexPayload["sourcePath"]))
rootPath := strings.TrimSpace(stringValue(indexPayload["rootPath"]))
dataSource := strings.TrimSpace(stringValue(indexPayload["dataSource"]))
if dataSource != "" || strings.HasPrefix(sourcePath, "bos://") || strings.HasPrefix(rootPath, "bos://") {
return true
}
if len(mapValue(indexPayload["archiveSources"])) > 0 && strings.Contains(rootPath, "_materialized_inputs") {
return false
}
return sourceDatasetHasCatalog(indexPayload)
}
func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]domain.WorkflowRecord, map[string]domain.RunRecord, error) {
workflows, err := server.listAllWorkflowsForContext(ctx)
if err != nil {
return nil, nil, err
}
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
if err != nil {
return nil, nil, err
}
latestByWorkflow := make(map[string]domain.RunRecord)
for _, run := range runs {
workflow, ok := workflowByID[run.WorkflowID]
if !ok {
continue
}
index := runDatasetIndex(run)
if index == nil {
continue
}
if !runLooksLikeSourceDataset(workflow, run) {
continue
}
if !sourceDatasetIsDisplayable(artifactStructuredData(index)) {
continue
}
current, exists := latestByWorkflow[run.WorkflowID]
if !exists || run.CreatedAt > current.CreatedAt {
latestByWorkflow[run.WorkflowID] = run
}
}
return workflowByID, latestByWorkflow, nil
}
func parseBOSPath(sourcePath string) (string, string, bool) {
normalized := strings.TrimSpace(sourcePath)
if !strings.HasPrefix(strings.ToLower(normalized), "bos://") {
return "", "", false
}
trimmed := strings.TrimPrefix(normalized, "bos://")
trimmed = strings.TrimPrefix(trimmed, "BOS://")
bucket, object, found := strings.Cut(trimmed, "/")
bucket = strings.TrimSpace(bucket)
object = strings.Trim(strings.TrimSpace(object), "/")
if !found {
object = ""
}
if bucket == "" {
return "", "", false
}
return bucket, object, true
}
func normalizeLocalDirectory(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
directory := filepath.ToSlash(filepath.Dir(trimmed))
if directory == "." {
return ""
}
return directory
}
func assetStorageLocationFromPaths(rootPath string, currentPath string, endpoint string) domain.AssetStorageLocation {
location := domain.AssetStorageLocation{
Path: strings.TrimSpace(currentPath),
}
currentBucket, currentObject, hasCurrentBOS := parseBOSPath(currentPath)
rootBucket, rootObject, hasRootBOS := parseBOSPath(rootPath)
switch {
case hasCurrentBOS:
location.Provider = "bos"
location.Endpoint = chooseBOSEndpoint(endpoint)
location.Bucket = currentBucket
if hasRootBOS && rootBucket == currentBucket {
location.Prefix = rootObject
} else {
location.Prefix = path.Dir(currentObject)
if location.Prefix == "." {
location.Prefix = ""
}
}
case hasRootBOS:
location.Provider = "bos"
location.Endpoint = chooseBOSEndpoint(endpoint)
location.Bucket = rootBucket
location.Prefix = rootObject
if location.Path == "" {
location.Path = strings.TrimSpace(rootPath)
}
case strings.TrimSpace(currentPath) != "":
location.Provider = "k8s-job"
location.Prefix = normalizeLocalDirectory(currentPath)
case strings.TrimSpace(rootPath) != "":
location.Provider = "k8s-job"
location.Path = strings.TrimSpace(rootPath)
location.Prefix = normalizeLocalDirectory(rootPath)
default:
location.Provider = "unknown"
}
if location.Prefix == "." {
location.Prefix = ""
}
return location
}
func sourceRootPathForRun(run domain.RunRecord, scanConfig *domain.ScanConfig) string {
if scanConfig != nil {
if value := scanSourcePath(scanConfig.DataSource, scanConfig.SourcePath, scanConfig.Bucket, scanConfig.Prefix); value != "" {
return value
}
}
if manifest := runSourceManifest(run); manifest != nil {
payload := artifactStructuredData(manifest)
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
return value
}
}
if index := runDatasetIndex(run); index != nil {
payload := artifactStructuredData(index)
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
return value
}
if value := strings.TrimSpace(stringValue(payload["rootPath"])); value != "" {
return value
}
}
return ""
}
func (server *server) buildSourceCollectionRecord(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, collection map[string]any) domain.SourceCollectionRecord {
meta := mapValue(collection["meta"])
requiredFiles := make([]string, 0)
for _, item := range listValue(collection["requiredFiles"]) {
requiredFiles = append(requiredFiles, stringValue(item))
}
missingFiles := make([]string, 0)
for _, item := range listValue(collection["missingRequiredFiles"]) {
missingFiles = append(missingFiles, stringValue(item))
}
collectionName := stringValue(collection["name"])
if collectionName == "" {
if projectID := stringValue(meta["project_id"]); projectID != "" {
collectionName = projectID
} else if relativePath := strings.Trim(stringValue(collection["relativePath"]), "/"); relativePath != "" && relativePath != "." {
collectionName = path.Base(relativePath)
} else {
collectionName = stringValue(collection["id"])
}
}
var resolvedScanConfig *domain.ScanConfig
if workflow != nil {
if scanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig); err == nil {
resolvedScanConfig = scanConfig
}
}
storage := assetStorageLocationFromPaths(
sourceRootPathForRun(run, resolvedScanConfig),
stringValue(collection["path"]),
func() string {
if resolvedScanConfig == nil {
return ""
}
return resolvedScanConfig.Endpoint
}(),
)
return domain.SourceCollectionRecord{
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
CollectionID: stringValue(collection["id"]),
CollectionName: collectionName,
Path: stringValue(collection["path"]),
RelativePath: stringValue(collection["relativePath"]),
Meta: meta,
HasIntrinsics: boolValue(collection["hasIntrinsics"]),
HasVideoMeta: boolValue(collection["hasVideoMeta"]),
RequiredFiles: requiredFiles,
MissingFiles: missingFiles,
VideoCount: intValue(collection["videoCount"]),
FileCount: intValue(collection["fileCount"]),
TotalBytes: int64Value(collection["totalBytes"]),
Valid: boolValue(collection["valid"]),
CreatedAt: run.CreatedAt,
Storage: storage,
}
}
type sourceCollectionPayloadRecord struct {
Run *domain.RunRecord
Workflow *domain.WorkflowRecord
Collection map[string]any
Items []map[string]any
Entries []map[string]any
ArchiveSource map[string]any
}
func (server *server) sourceCollectionPayload(ctx context.Context, runID string, collectionID string) (*sourceCollectionPayloadRecord, error) {
run, err := server.store.GetRun(ctx, runID)
if err != nil || run == nil {
return &sourceCollectionPayloadRecord{Run: run}, err
}
workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
return &sourceCollectionPayloadRecord{Run: run}, err
}
artifact := artifactByName(*run, "dataset-index.json")
payload := artifactStructuredData(artifact)
if artifact == nil || payload == nil {
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
}
collections := sourceDatasetList(payload, "catalogCollections", "collections")
items := sourceDatasetList(payload, "catalogItems", "items")
entries := sourceDatasetList(payload, "catalogEntries", "entries")
var matchedCollection map[string]any
for _, item := range collections {
collection := mapValue(item)
if stringValue(collection["id"]) == collectionID {
matchedCollection = collection
break
}
}
if matchedCollection == nil {
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
}
matchedItems := make([]map[string]any, 0)
for _, item := range items {
row := mapValue(item)
if stringValue(row["collectionId"]) == collectionID {
matchedItems = append(matchedItems, row)
}
}
matchedEntries := make([]map[string]any, 0)
for _, item := range entries {
row := mapValue(item)
relativePath := strings.Trim(filepath.ToSlash(stringValue(row["relativePath"])), "/")
collectionRelativePath := strings.Trim(filepath.ToSlash(stringValue(matchedCollection["relativePath"])), "/")
if collectionRelativePath == "" || collectionRelativePath == "." {
if relativePath == "" {
continue
}
} else if relativePath != collectionRelativePath && !strings.HasPrefix(relativePath, collectionRelativePath+"/") {
continue
}
matchedEntries = append(matchedEntries, row)
}
return &sourceCollectionPayloadRecord{
Run: run,
Workflow: workflow,
Collection: matchedCollection,
Items: matchedItems,
Entries: matchedEntries,
ArchiveSource: mapValue(payload["archiveSources"]),
}, nil
}
func sourceEntryPayloadKind(relativePath string, mimeType string) string {
extension := strings.ToLower(path.Ext(relativePath))
switch {
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
return "video"
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
return "image"
case strings.HasPrefix(mimeType, "audio/") || extension == ".mp3" || extension == ".wav" || extension == ".m4a":
return "audio"
default:
return "binary"
}
}
func sourceEntryArtifactType(relativePath string, mimeType string) domain.PortType {
extension := strings.ToLower(path.Ext(relativePath))
switch {
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
return domain.PortTypeImage
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
return domain.PortTypeVideo
case extension == ".json":
return domain.PortTypeJSON
default:
return domain.PortTypeFile
}
}
func formatArtifactSize(sizeBytes int64) string {
switch {
case sizeBytes <= 0:
return "0 B"
case sizeBytes < 1024:
return fmt.Sprintf("%d B", sizeBytes)
case sizeBytes < 1024*1024:
return fmt.Sprintf("%.1f KiB", float64(sizeBytes)/1024)
case sizeBytes < 1024*1024*1024:
return fmt.Sprintf("%.1f MiB", float64(sizeBytes)/(1024*1024))
default:
return fmt.Sprintf("%.1f GiB", float64(sizeBytes)/(1024*1024*1024))
}
}
func sourceCollectionContentURL(runID string, collectionID string, relativePath string) string {
return fmt.Sprintf(
"/api/source-data/collections/%s/%s/content?relativePath=%s",
url.PathEscape(runID),
url.PathEscape(collectionID),
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
)
}
func datasetIndexEntrySource(entry map[string]any, archiveSources map[string]any) map[string]any {
source := mapValue(entry["source"])
if len(source) == 0 {
source = make(map[string]any)
}
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if relativePath != "" && stringValue(source["relativePath"]) == "" {
source["relativePath"] = relativePath
}
mimeType := stringValue(entry["mimeType"])
if mimeType == "" {
mimeType = stringValue(source["mimeType"])
}
if mimeType == "" && relativePath != "" {
mimeType = mime.TypeByExtension(path.Ext(relativePath))
}
if mimeType != "" {
source["mimeType"] = mimeType
}
if stringValue(source["previewURL"]) == "" {
archiveKey := stringValue(source["archiveKey"])
archiveSource := mapValue(archiveSources[archiveKey])
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
source["previewURL"] = previewURL
}
if archiveName := stringValue(archiveSource["name"]); archiveName != "" && stringValue(source["archiveName"]) == "" {
source["archiveName"] = archiveName
}
}
if pathValue := stringValue(entry["path"]); pathValue != "" && stringValue(source["path"]) == "" {
source["path"] = pathValue
}
return source
}
func sourceEntryArtifact(run domain.RunRecord, collectionID string, entry map[string]any, archiveSources map[string]any) *domain.ExecutionArtifact {
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if relativePath == "" {
relativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
if relativePath == "" {
return nil
}
source := datasetIndexEntrySource(entry, archiveSources)
mimeType := stringValue(source["mimeType"])
sizeBytes := int64Value(entry["sizeBytes"])
artifactType := sourceEntryArtifactType(relativePath, mimeType)
payload := make(map[string]any)
if value, ok := entry["data"]; ok {
payload["preview"] = map[string]any{
"kind": "json",
"value": normalizeStructuredValue(value),
}
}
if previewURL := stringValue(source["previewURL"]); previewURL != "" && len(payload) == 0 {
payload["preview"] = map[string]any{
"kind": sourceEntryPayloadKind(relativePath, mimeType),
"format": strings.TrimPrefix(strings.ToLower(path.Ext(relativePath)), "."),
"url": sourceCollectionContentURL(run.ID, collectionID, relativePath),
}
}
payload["relativePath"] = relativePath
if len(source) > 0 {
if previewURL := sourceCollectionContentURL(run.ID, collectionID, relativePath); previewURL != "" {
source["previewURL"] = previewURL
}
payload["source"] = source
}
return &domain.ExecutionArtifact{
ID: fmt.Sprintf("%s-source-%s", run.ID, strings.ReplaceAll(relativePath, "/", "-")),
RunID: run.ID,
NodeID: "dataset-index",
Name: path.Base(relativePath),
Type: artifactType,
Size: formatArtifactSize(sizeBytes),
Path: func() string {
if sourcePath := stringValue(source["path"]); sourcePath != "" {
return sourcePath
}
return relativePath
}(),
Metadata: &domain.ArtifactMetadata{MimeType: mimeType},
Data: payload,
}
}
func contentBytesForDataURL(previewURL string) ([]byte, string, error) {
if !strings.HasPrefix(previewURL, "data:") {
return nil, "", fmt.Errorf("not a data url")
}
header, encoded, found := strings.Cut(previewURL, ",")
if !found || !strings.Contains(header, ";base64") {
return nil, "", fmt.Errorf("invalid data url")
}
mimeType := strings.TrimPrefix(strings.SplitN(header, ";", 2)[0], "data:")
payload, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, "", err
}
return payload, mimeType, nil
}
func contentBytesForArchiveMember(archiveName string, archivePreviewURL string, memberPath string) ([]byte, error) {
payload, _, err := contentBytesForDataURL(archivePreviewURL)
if err != nil {
return nil, err
}
memberPath = strings.Trim(filepath.ToSlash(memberPath), "/")
if memberPath == "" {
return nil, fmt.Errorf("missing archive member path")
}
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
if err != nil {
return nil, err
}
for _, file := range archive.File {
if strings.Trim(filepath.ToSlash(file.Name), "/") != memberPath {
continue
}
handle, err := file.Open()
if err != nil {
return nil, err
}
defer handle.Close()
return io.ReadAll(handle)
}
return nil, os.ErrNotExist
}
reader := bytes.NewReader(payload)
var tarReader *tar.Reader
if strings.HasSuffix(strings.ToLower(archiveName), ".tgz") || strings.HasSuffix(strings.ToLower(archiveName), ".tar.gz") || (len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b) {
gzipReader, err := gzip.NewReader(reader)
if err == nil {
defer gzipReader.Close()
tarReader = tar.NewReader(gzipReader)
} else {
reader = bytes.NewReader(payload)
tarReader = tar.NewReader(reader)
}
} else {
tarReader = tar.NewReader(reader)
}
for {
header, err := tarReader.Next()
if err == io.EOF {
return nil, os.ErrNotExist
}
if err != nil {
return nil, err
}
if header.FileInfo().IsDir() {
continue
}
if strings.Trim(filepath.ToSlash(header.Name), "/") != memberPath {
continue
}
return io.ReadAll(tarReader)
}
}
func sourceCollectionEntryPayload(collection domain.SourceCollectionRecord, payloadEntries []map[string]any, relativePath string) map[string]any {
target := strings.Trim(filepath.ToSlash(relativePath), "/")
if target == "" {
return nil
}
collectionPrefix := strings.Trim(filepath.ToSlash(collection.RelativePath), "/")
fullTarget := target
if collectionPrefix != "" && target != collectionPrefix && !strings.HasPrefix(target, collectionPrefix+"/") {
fullTarget = collectionPrefix + "/" + target
}
for _, entry := range payloadEntries {
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if entryRelativePath == "" {
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
if entryRelativePath == fullTarget || entryRelativePath == target {
return entry
}
}
return nil
}
func writeSourceContent(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, payload []byte) {
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if mimeType == "" && len(payload) > 0 {
mimeType = http.DetectContentType(payload)
}
if mimeType != "" {
writer.Header().Set("Content-Type", mimeType)
}
writeSourceContentDisposition(writer, request, fileName)
http.ServeContent(writer, request, fileName, time.Time{}, bytes.NewReader(payload))
}
func sourceContentDownloadRequested(request *http.Request) bool {
value := strings.ToLower(strings.TrimSpace(request.URL.Query().Get("download")))
return value == "1" || value == "true" || value == "yes"
}
func writeSourceContentDisposition(writer http.ResponseWriter, request *http.Request, fileName string) {
if !sourceContentDownloadRequested(request) {
return
}
name := strings.TrimSpace(path.Base(fileName))
if name == "" || name == "." || name == "/" {
name = "artifact"
}
writer.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": name}))
}
func writeSourceContentFile(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, filePath string) error {
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if mimeType == "" {
header := make([]byte, 512)
count, readErr := file.Read(header)
if readErr != nil && readErr != io.EOF {
return readErr
}
mimeType = http.DetectContentType(header[:count])
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
}
if mimeType != "" {
writer.Header().Set("Content-Type", mimeType)
}
writeSourceContentDisposition(writer, request, fileName)
http.ServeContent(writer, request, fileName, fileInfo.ModTime(), file)
return nil
}
func sourceBrowserPreviewCacheKey(request sourceBrowserPreviewRequest) string {
hash := sha256.New()
hash.Write([]byte("browser-preview-v2"))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.FileName)))
hash.Write([]byte{0})
if request.RedirectURL != "" {
hash.Write([]byte("redirect"))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.RedirectURL)))
}
if len(request.Payload) > 0 {
payloadHash := sha256.Sum256(request.Payload)
hash.Write([]byte("payload"))
hash.Write([]byte{0})
hash.Write(payloadHash[:])
}
return hex.EncodeToString(hash.Sum(nil))
}
func sourceBrowserPreviewCacheDir() (string, error) {
cacheDir := filepath.Join(os.TempDir(), "droboticflow", "source-video-previews")
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
return "", err
}
return cacheDir, nil
}
func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string, error) {
cacheDir, err := sourceBrowserPreviewCacheDir()
if err != nil {
return "", err
}
cacheKey := sourceBrowserPreviewCacheKey(request)
return filepath.Join(cacheDir, cacheKey+".mp4"), nil
}
func parseBOSObjectPath(sourcePath string) (string, string, bool) {
bucket, object, ok := parseBOSPath(sourcePath)
if !ok || object == "" {
return "", "", false
}
return bucket, object, true
}
func buildSourceCollectionSignedPreviewURL(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil {
return "", fmt.Errorf("scan config is unavailable")
}
if chooseDataSource(scanConfig.DataSource) != "BOS" {
return "", fmt.Errorf("scan config is not BOS")
}
bucket, object, ok := parseBOSObjectPath(sourcePath)
if !ok {
return "", fmt.Errorf("invalid BOS source path: %s", sourcePath)
}
accessKeyID := strings.TrimSpace(scanConfig.AccessKeyID)
secretAccessKey := strings.TrimSpace(scanConfig.SecretAccessKey)
if accessKeyID == "" || secretAccessKey == "" {
return "", fmt.Errorf("BOS credentials are unavailable")
}
endpoint := chooseBOSEndpoint(scanConfig.Endpoint)
if endpoint == "" {
return "", fmt.Errorf("BOS endpoint is unavailable")
}
client, err := bos.NewClient(accessKeyID, secretAccessKey, endpoint)
if err != nil {
return "", err
}
previewURL := strings.TrimSpace(client.BasicGeneratePresignedUrl(bucket, object, 3600))
if previewURL == "" {
return "", fmt.Errorf("failed to generate BOS preview url")
}
return previewURL, nil
}
func locateSourceBrowserVideoPreviewPath(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
finalPath, err := sourceBrowserPreviewCachePath(request)
if err != nil {
return "", err
}
if fileInfo, err := os.Stat(finalPath); err == nil && fileInfo.Size() > 0 {
return finalPath, nil
}
return "", os.ErrNotExist
}
func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWriter, request sourceBrowserPreviewRequest) error {
if strings.TrimSpace(request.RedirectURL) == "" && len(request.Payload) == 0 {
return fmt.Errorf("source video content is unavailable")
}
finalPath, err := sourceBrowserPreviewCachePath(request)
if err != nil {
return err
}
inputRef := strings.TrimSpace(request.RedirectURL)
var cleanupInput func()
if inputRef == "" {
extension := path.Ext(request.FileName)
if extension == "" {
extension = ".bin"
}
cacheDir := filepath.Dir(finalPath)
inputFile, err := os.CreateTemp(cacheDir, "source-video-*"+extension)
if err != nil {
return err
}
if _, err := inputFile.Write(request.Payload); err != nil {
inputFile.Close()
os.Remove(inputFile.Name())
return err
}
if err := inputFile.Close(); err != nil {
os.Remove(inputFile.Name())
return err
}
inputRef = inputFile.Name()
cleanupInput = func() {
_ = os.Remove(inputFile.Name())
}
}
if cleanupInput != nil {
defer cleanupInput()
}
cacheDir := filepath.Dir(finalPath)
outputFile, err := os.CreateTemp(cacheDir, sourceBrowserPreviewCacheKey(request)+"-*.mp4")
if err != nil {
return err
}
tempOutputPath := outputFile.Name()
stderr := &bytes.Buffer{}
command := exec.CommandContext(
ctx,
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel", "error",
"-i", inputRef,
"-map", "0:v:0",
"-map", "0:a?",
"-threads", "1",
"-probesize", "32M",
"-analyzeduration", "0",
"-c:v", "libx264",
"-preset", "ultrafast",
"-tune", "zerolatency",
"-crf", "30",
"-vf", "fps=12,scale=min(960\\,iw):-2",
"-g", "12",
"-keyint_min", "12",
"-pix_fmt", "yuv420p",
"-movflags", "frag_keyframe+empty_moov+default_base_moof",
"-c:a", "aac",
"-b:a", "96k",
"-ac", "1",
"-f", "mp4",
"pipe:1",
)
command.Stderr = stderr
stdout, err := command.StdoutPipe()
if err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
return err
}
if err := command.Start(); err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
message := strings.TrimSpace(stderr.String())
if message != "" {
return fmt.Errorf("generate browser video preview: %s", message)
}
return err
}
streamTarget := io.Writer(writer)
if flusher, ok := writer.(http.Flusher); ok {
streamTarget = flushResponseWriter{writer: writer, flusher: flusher}
}
if _, err := io.Copy(io.MultiWriter(streamTarget, outputFile), stdout); err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
_ = command.Wait()
return err
}
if err := outputFile.Close(); err != nil {
_ = os.Remove(tempOutputPath)
_ = command.Wait()
return err
}
if err := command.Wait(); err != nil {
message := strings.TrimSpace(stderr.String())
_ = os.Remove(tempOutputPath)
if message != "" {
return fmt.Errorf("generate browser video preview: %s", message)
}
return fmt.Errorf("generate browser video preview: %w", err)
}
if err := os.Rename(tempOutputPath, finalPath); err != nil {
if fileInfo, statErr := os.Stat(finalPath); statErr == nil && fileInfo.Size() > 0 {
_ = os.Remove(tempOutputPath)
return nil
}
_ = os.Remove(tempOutputPath)
return err
}
return nil
}
func derivedSourceEntriesFromIndex(collection domain.SourceCollectionRecord, payloadItems []map[string]any) []map[string]any {
entries := make([]map[string]any, 0, len(payloadItems)+2)
if len(collection.Meta) > 0 {
entries = append(entries, map[string]any{
"collectionId": collection.CollectionID,
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/meta.json", "/"),
"sizeBytes": len(fmt.Sprintf("%v", collection.Meta)),
"mimeType": "application/json",
"data": collection.Meta,
"derivedFromIndex": true,
})
}
if collection.HasVideoMeta && len(payloadItems) > 0 {
videoRows := make([]map[string]any, 0, len(payloadItems))
for _, item := range payloadItems {
videoRows = append(videoRows, map[string]any{
"filename": stringValue(item["filename"]),
"codec": stringValue(item["codec"]),
"resolution": stringValue(item["resolution"]),
"frameRate": floatValue(item["frameRate"]),
"duration": floatValue(item["duration"]),
"startTime": item["startTime"],
})
}
entries = append(entries, map[string]any{
"collectionId": collection.CollectionID,
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/video_meta.json", "/"),
"mimeType": "application/json",
"data": videoRows,
"derivedFromIndex": true,
})
}
for _, item := range payloadItems {
entry := map[string]any{
"collectionId": collection.CollectionID,
"relativePath": stringValue(item["relativePath"]),
"path": stringValue(item["path"]),
"sizeBytes": int64Value(item["sizeBytes"]),
"mimeType": stringValue(mapValue(item["source"])["mimeType"]),
"source": mapValue(item["source"]),
"codec": stringValue(item["codec"]),
"resolution": stringValue(item["resolution"]),
"frameRate": floatValue(item["frameRate"]),
"duration": floatValue(item["duration"]),
"derivedFromIndex": true,
}
entries = append(entries, entry)
}
return entries
}
func buildSourceEntries(run domain.RunRecord, collection domain.SourceCollectionRecord, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, prefix string) []domain.SourceCollectionEntry {
if len(payloadEntries) == 0 {
payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems)
}
videoMetaByRelative := make(map[string]map[string]any, len(payloadItems))
for _, item := range payloadItems {
videoMetaByRelative[stringValue(item["relativePath"])] = item
}
normalizedPrefix := strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/")
prefixSlash := normalizedPrefix
if prefixSlash != "" {
prefixSlash += "/"
}
directories := make(map[string]domain.SourceCollectionEntry)
files := make([]domain.SourceCollectionEntry, 0)
collectionPrefix := strings.Trim(collection.RelativePath, "/")
if collectionPrefix != "" {
collectionPrefix += "/"
}
for _, entryPayload := range payloadEntries {
relativePath := strings.Trim(filepath.ToSlash(stringValue(entryPayload["relativePath"])), "/")
if relativePath == "" {
relativePath = strings.Trim(filepath.ToSlash(stringValue(entryPayload["path"])), "/")
}
if relativePath == "" {
continue
}
if collectionPrefix != "" && relativePath != strings.TrimSuffix(collectionPrefix, "/") && !strings.HasPrefix(relativePath, collectionPrefix) {
continue
}
entryPath := relativePath
if collectionPrefix != "" {
entryPath = strings.TrimPrefix(relativePath, collectionPrefix)
}
if entryPath == "" {
continue
}
if normalizedPrefix != "" {
if entryPath == normalizedPrefix {
// exact file match
} else if !strings.HasPrefix(entryPath, prefixSlash) {
continue
} else {
entryPath = strings.TrimPrefix(entryPath, prefixSlash)
}
}
if entryPath == "" {
continue
}
parts := strings.Split(entryPath, "/")
if len(parts) > 1 {
dirPath := parts[0]
displayPath := dirPath
if normalizedPrefix != "" {
displayPath = normalizedPrefix + "/" + dirPath
}
if _, exists := directories[displayPath]; !exists {
directories[displayPath] = domain.SourceCollectionEntry{
Path: displayPath,
Name: dirPath,
Kind: "directory",
CollectionName: collection.CollectionName,
DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]),
}
} else if boolValue(entryPayload["derivedFromIndex"]) {
entry := directories[displayPath]
entry.DerivedFromIndex = true
directories[displayPath] = entry
}
continue
}
displayPath := parts[0]
if normalizedPrefix != "" && displayPath != normalizedPrefix {
displayPath = normalizedPrefix + "/" + displayPath
}
artifact := sourceEntryArtifact(run, collection.CollectionID, entryPayload, archiveSources)
videoMeta := videoMetaByRelative[filepath.ToSlash(strings.Trim(collection.RelativePath, "/")+"/"+displayPath)]
entry := domain.SourceCollectionEntry{
Path: displayPath,
Name: parts[0],
Kind: "file",
CollectionName: collection.CollectionName,
SizeBytes: int64Value(entryPayload["sizeBytes"]),
Artifact: artifact,
DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]),
}
if videoMeta != nil {
entry.Kind = "video"
entry.Codec = stringValue(videoMeta["codec"])
entry.Resolution = stringValue(videoMeta["resolution"])
entry.FrameRate = floatValue(videoMeta["frameRate"])
entry.Duration = floatValue(videoMeta["duration"])
entry.Metadata = videoMeta
if entry.SizeBytes == 0 {
entry.SizeBytes = int64Value(videoMeta["sizeBytes"])
}
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".json") {
entry.Kind = "json"
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".mp4") {
entry.Kind = "video"
}
if entry.SizeBytes == 0 {
entry.SizeBytes = int64Value(entryPayload["sizeBytes"])
}
files = append(files, entry)
}
items := make([]domain.SourceCollectionEntry, 0, len(directories)+len(files))
for _, item := range directories {
items = append(items, item)
}
sort.Slice(items, func(left int, right int) bool { return items[left].Path < items[right].Path })
sort.Slice(files, func(left int, right int) bool { return files[left].Path < files[right].Path })
items = append(items, files...)
return items
}
func sourceCollectionContentPayload(collection domain.SourceCollectionRecord, scanConfig *domain.ScanConfig, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, relativePath string) ([]byte, string, string, string, error) {
if len(payloadEntries) == 0 {
payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems)
}
entry := sourceCollectionEntryPayload(collection, payloadEntries, relativePath)
if entry == nil {
return nil, "", "", "", os.ErrNotExist
}
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if entryRelativePath == "" {
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
fileName := path.Base(entryRelativePath)
source := datasetIndexEntrySource(entry, archiveSources)
mimeType := stringValue(source["mimeType"])
if mimeType == "" {
mimeType = stringValue(entry["mimeType"])
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if value, ok := entry["data"]; ok {
payload, err := json.MarshalIndent(normalizeStructuredValue(value), "", " ")
if err != nil {
return nil, "", "", "", err
}
if mimeType == "" {
mimeType = "application/json"
}
return payload, mimeType, fileName, "", nil
}
if sourcePath := stringValue(source["path"]); sourcePath != "" && !strings.Contains(sourcePath, "://") {
if payload, err := os.ReadFile(sourcePath); err == nil {
return payload, mimeType, fileName, "", nil
}
}
if sourcePath := stringValue(source["path"]); strings.HasPrefix(strings.ToLower(sourcePath), "bos://") && scanConfig != nil {
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
source["previewURL"] = previewURL
}
}
archiveKey := stringValue(source["archiveKey"])
archiveMemberPath := stringValue(source["archiveMemberPath"])
if archiveKey != "" && archiveMemberPath != "" {
archiveSource := mapValue(archiveSources[archiveKey])
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
payload, err := contentBytesForArchiveMember(stringValue(archiveSource["name"]), previewURL, archiveMemberPath)
if err == nil {
return payload, mimeType, fileName, "", nil
}
}
}
if previewURL := stringValue(source["previewURL"]); previewURL != "" {
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
return nil, "", fileName, previewURL, nil
}
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
if err == nil {
if mimeType == "" {
mimeType = decodedMimeType
}
return payload, mimeType, fileName, "", nil
}
}
return nil, "", "", "", os.ErrNotExist
}
func (server *server) handleSourceCollectionsList(writer http.ResponseWriter, request *http.Request) {
workflowByID, latestRuns, err := server.latestCompletedScanRuns(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items := make([]domain.SourceCollectionRecord, 0)
for _, run := range latestRuns {
if !server.filterByTenant(request, run.OrganizationID) {
continue
}
artifact := artifactByName(run, "dataset-index.json")
payload := artifactStructuredData(artifact)
if artifact == nil || payload == nil {
continue
}
for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") {
collection := mapValue(raw)
if collection == nil {
continue
}
workflow := workflowByID[run.WorkflowID]
items = append(items, server.buildSourceCollectionRecord(request.Context(), run, &workflow, collection))
}
}
sort.Slice(items, func(left int, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
if items[left].WorkflowName == items[right].WorkflowName {
return items[left].CollectionID < items[right].CollectionID
}
return items[left].WorkflowName < items[right].WorkflowName
}
return items[left].CreatedAt > items[right].CreatedAt
})
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleSourceCollectionDetail(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
detail := domain.SourceCollectionDetail{
Collection: record,
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, ""),
}
renderJSON(writer, http.StatusOK, detail)
}
func (server *server) handleSourceCollectionEntries(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
prefix := request.URL.Query().Get("prefix")
detail := domain.SourceCollectionDetail{
Collection: record,
Prefix: prefix,
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, prefix),
}
renderJSON(writer, http.StatusOK, detail)
}
func (server *server) handleSourceCollectionContent(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
if relativePath == "" {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
return
}
var effectiveScanConfig *domain.ScanConfig
if payload.Workflow != nil {
effectiveScanConfig, err = server.resolveScanConfig(request.Context(), payload.Workflow.ScanConfig)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
content, mimeType, fileName, redirectURL, err := sourceCollectionContentPayload(record, effectiveScanConfig, payload.Entries, payload.Items, payload.ArchiveSource, relativePath)
if err != nil {
if os.IsNotExist(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source file not found"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
Payload: content,
RedirectURL: redirectURL,
}
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] source browser preview stream failed: %v\n", err)
}
return
}
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
writeSourceContent(writer, request, fileName, mimeType, content)
}
func packageArtifactCandidatesByRelativePath(run domain.RunRecord, packageName string, relativePath string) []domain.ExecutionArtifact {
target := strings.Trim(filepath.ToSlash(relativePath), "/")
if target == "" {
return nil
}
if _, payload := wilorPackageManifestArtifact(run, packageName); payload != nil {
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range wilorPackageArtifacts(run, packageName) {
if strings.Trim(filepath.ToSlash(artifact.Name), "/") == target {
items = append(items, artifact)
}
}
sort.SliceStable(items, func(left int, right int) bool {
return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right])
})
return items
}
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range run.Artifacts {
if packageArtifactResolvedRelativePath(artifact, packageName) == target {
items = append(items, artifact)
}
}
sort.SliceStable(items, func(left int, right int) bool {
return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right])
})
return items
}
func workflowNodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
values := make(map[string]string, len(node.Data.Hyperparams))
for _, item := range node.Data.Hyperparams {
values[item.Key] = item.Value
}
return values
}
func (server *server) packageArtifactScanConfig(ctx context.Context, workflow *domain.WorkflowRecord, artifact domain.ExecutionArtifact) (*domain.ScanConfig, error) {
node := workflowNodeByID(workflow, artifact.NodeID)
if node == nil {
return nil, nil
}
params := workflowNodeHyperparamsMap(*node)
dataSource := chooseDataSource(params["DATA_SOURCE"])
sourcePath := strings.TrimSpace(params["SOURCE_PATH"])
if dataSource != "BOS" && strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
dataSource = "BOS"
}
if dataSource != "BOS" && strings.TrimSpace(node.Data.ConnectionID) != "" {
dataSource = "BOS"
}
if dataSource != "BOS" {
return nil, nil
}
scanConfig := &domain.ScanConfig{
DataSource: "BOS",
SourcePath: sourcePath,
ConnectionID: strings.TrimSpace(node.Data.ConnectionID),
Bucket: strings.TrimSpace(node.Data.Bucket),
Prefix: cleanBOSPrefix(node.Data.Prefix),
Endpoint: strings.TrimSpace(params["BOS_ENDPOINT"]),
AccessKeyID: strings.TrimSpace(params["BOS_ACCESS_KEY_ID"]),
SecretAccessKey: strings.TrimSpace(params["BOS_SECRET_ACCESS_KEY"]),
}
return server.resolveScanConfig(ctx, scanConfig)
}
func (server *server) packageContentPayload(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, packageName string, relativePath string) ([]byte, string, string, string, string, error) {
artifacts := packageArtifactCandidatesByRelativePath(run, packageName, relativePath)
if len(artifacts) == 0 {
return nil, "", "", "", "", os.ErrNotExist
}
fileName := path.Base(strings.Trim(filepath.ToSlash(relativePath), "/"))
for _, artifact := range artifacts {
mimeType := ""
if artifact.Metadata != nil {
mimeType = strings.TrimSpace(artifact.Metadata.MimeType)
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
filePath := strings.TrimSpace(artifact.Path)
if filePath != "" && !strings.Contains(filePath, "://") {
if _, err := os.Stat(filePath); err == nil {
return nil, mimeType, fileName, filePath, "", nil
}
}
structuredData := artifactStructuredData(&artifact)
if artifact.Type == "json" && structuredData != nil {
payload, err := json.MarshalIndent(normalizeStructuredValue(structuredData), "", " ")
if err != nil {
return nil, "", "", "", "", err
}
if mimeType == "" {
mimeType = "application/json"
}
return payload, mimeType, fileName, "", "", nil
}
source := mapValue(structuredData["source"])
sourcePath := stringValue(source["path"])
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
scanConfig, err := server.packageArtifactScanConfig(ctx, workflow, artifact)
if err != nil {
return nil, "", "", "", "", err
}
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
return nil, mimeType, fileName, "", previewURL, nil
}
}
if previewURL := packageArtifactPreviewURL(artifact); previewURL != "" {
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
return nil, mimeType, fileName, "", previewURL, nil
}
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
if err == nil {
if mimeType == "" {
mimeType = decodedMimeType
}
return payload, mimeType, fileName, "", "", nil
}
}
}
return nil, "", "", "", "", os.ErrNotExist
}
func (server *server) handlePackageContent(writer http.ResponseWriter, request *http.Request) {
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if run == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Run") {
return
}
packageName := chi.URLParam(request, "packageName")
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
if relativePath == "" {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
return
}
workflow, err := server.store.GetWorkflow(request.Context(), run.WorkflowID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
content, mimeType, fileName, filePath, redirectURL, err := server.packageContentPayload(request.Context(), *run, workflow, packageName, relativePath)
if err != nil {
if os.IsNotExist(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package file not found"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
Payload: content,
}
if filePath != "" {
previewRequest.RedirectURL = filePath
} else {
previewRequest.RedirectURL = redirectURL
}
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] package browser preview stream failed: %v\n", err)
}
return
}
if filePath != "" {
if err := writeSourceContentFile(writer, request, fileName, mimeType, filePath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
writeSourceContent(writer, request, fileName, mimeType, content)
}
func packageRecordFromManifest(run domain.RunRecord, item map[string]any, files []domain.ExecutionArtifact, rawFiles []domain.ExecutionArtifact) domain.PackageRecord {
pipelineTags := make([]string, 0)
for _, value := range listValue(item["pipelineTags"]) {
pipelineTags = append(pipelineTags, stringValue(value))
}
storagePath := stringValue(item["storagePath"])
for _, artifact := range rawFiles {
if candidate := packageArtifactStoragePath(artifact, stringValue(item["packageName"])); candidate != "" {
storagePath = candidate
break
}
}
fileCount := intValue(item["fileCount"])
if len(files) > 0 {
fileCount = len(files)
}
sourceVideo := stringValue(item["sourceVideo"])
if sourceVideo == "" {
sourceVideo = stringValue(mapValue(item["annotation"])["sourceVideo"])
}
if sourceVideo == "" {
sourceVideo = stringValue(mapValue(item["source"])["name"])
}
clipDuration := floatValue(item["clipDuration"])
if clipDuration <= 0 {
clipDuration = floatValue(item["duration"])
}
quality := map[string]any{}
for key, value := range mapValue(item["quality"]) {
quality[key] = value
}
if stringValue(quality["source_video"]) == "" && sourceVideo != "" {
quality["source_video"] = sourceVideo
}
return domain.PackageRecord{
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
PackageName: stringValue(item["packageName"]),
ClipID: stringValue(item["clipId"]),
SourceVideo: sourceVideo,
ClipDuration: clipDuration,
FileCount: fileCount,
PipelineTags: pipelineTags,
StoragePath: storagePath,
Quality: quality,
Annotation: mapValue(item["annotation"]),
CreatedAt: run.CreatedAt,
PreviewArtifact: choosePackagePreview(files),
Storage: assetStorageLocationFromPaths("", storagePath, ""),
}
}
func (server *server) packageDetail(ctx context.Context, runID string, packageName string) (*domain.PackageDetail, error) {
run, err := server.store.GetRun(ctx, runID)
if err != nil || run == nil {
return nil, err
}
manifest := artifactByName(*run, "packages-manifest.json")
manifestPayload := artifactStructuredData(manifest)
if manifest != nil && manifestPayload != nil {
rawFiles := dedupePackageArtifacts(*run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
}
for _, raw := range listValue(manifestPayload["items"]) {
item := mapValue(raw)
if stringValue(item["packageName"]) != packageName {
continue
}
record := packageRecordFromManifest(*run, item, files, rawFiles)
return &domain.PackageDetail{Package: record, Files: files}, nil
}
}
if manifest, payload := wilorPackageManifestArtifact(*run, packageName); manifest != nil && payload != nil {
files, rawFiles := wilorPackageFilesForResponse(*run, packageName)
item := wilorPackageItem(*run, *manifest, payload, rawFiles)
record := packageRecordFromManifest(*run, item, files, rawFiles)
return &domain.PackageDetail{Package: record, Files: files}, nil
}
return nil, nil
}
func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) {
runs, err := server.listRunArtifactsForRequest(request.Context(), request, string(domain.RunStatusCompleted))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items := make([]domain.PackageRecord, 0)
latestByWorkflowPackage := make(map[string]domain.PackageRecord)
for _, run := range runs {
if !server.filterByTenant(request, run.OrganizationID) {
continue
}
manifest := artifactByName(run, "packages-manifest.json")
manifestPayload := artifactStructuredData(manifest)
if manifest != nil && manifestPayload != nil {
for _, raw := range listValue(manifestPayload["items"]) {
item := mapValue(raw)
if item == nil {
continue
}
packageName := stringValue(item["packageName"])
rawFiles := dedupePackageArtifacts(run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
}
record := packageRecordFromManifest(run, item, files, rawFiles)
key := run.WorkflowID + ":" + record.PackageName
current, exists := latestByWorkflowPackage[key]
if !exists || record.CreatedAt > current.CreatedAt {
latestByWorkflowPackage[key] = record
}
}
}
for _, wilorManifest := range wilorResultArtifacts(run) {
payload := wilorResultPayload(wilorManifest)
if payload == nil {
continue
}
packageName := wilorPackageName(wilorManifest, payload)
files, rawFiles := wilorPackageFilesForResponse(run, packageName)
item := wilorPackageItem(run, wilorManifest, payload, rawFiles)
record := packageRecordFromManifest(run, item, files, rawFiles)
key := run.WorkflowID + ":" + record.PackageName
current, exists := latestByWorkflowPackage[key]
if !exists || record.CreatedAt > current.CreatedAt {
latestByWorkflowPackage[key] = record
}
}
}
for _, item := range latestByWorkflowPackage {
items = append(items, item)
}
sort.Slice(items, func(left int, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
return items[left].PackageName < items[right].PackageName
}
return items[left].CreatedAt > items[right].CreatedAt
})
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handlePackageDetail(writer http.ResponseWriter, request *http.Request) {
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if run == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
return
}
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Package") {
return
}
item, err := server.packageDetail(request.Context(), run.ID, chi.URLParam(request, "packageName"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
return
}
renderJSON(writer, http.StatusOK, item)
}
var supportedSamplingRuleOperators = map[string]struct{}{
"contains": {},
"eq": {},
"exists": {},
"gt": {},
"gte": {},
"in": {},
"lt": {},
"lte": {},
"missing": {},
"ne": {},
}
func samplingRulesFromManifest(manifest map[string]any) ([]domain.SamplingRuleSummary, bool) {
rules := make([]domain.SamplingRuleSummary, 0)
ruleIDs := make(map[string]struct{})
for _, raw := range listValue(manifest["rules"]) {
item := mapValue(raw)
if item == nil {
return nil, false
}
id := strings.TrimSpace(stringValue(item["id"]))
field := strings.TrimSpace(stringValue(item["field"]))
operator := strings.ToLower(strings.TrimSpace(stringValue(item["operator"])))
samplePercent, ok := samplePercentValue(item["samplePercent"])
if id == "" || field == "" || !ok {
return nil, false
}
if _, exists := ruleIDs[id]; exists {
return nil, false
}
if _, supported := supportedSamplingRuleOperators[operator]; !supported {
return nil, false
}
ruleIDs[id] = struct{}{}
rules = append(rules, domain.SamplingRuleSummary{
ID: id,
Field: field,
Operator: operator,
Value: item["value"],
SamplePercent: samplePercent,
MatchedCount: intValue(item["matchedCount"]),
SampledCount: intValue(item["sampledCount"]),
})
}
return rules, len(rules) > 0
}
func samplePercentValue(value any) (float64, bool) {
switch typed := value.(type) {
case int:
return boundedSamplePercent(float64(typed))
case int8:
return boundedSamplePercent(float64(typed))
case int16:
return boundedSamplePercent(float64(typed))
case int32:
return boundedSamplePercent(float64(typed))
case int64:
return boundedSamplePercent(float64(typed))
case uint:
return boundedSamplePercent(float64(typed))
case uint8:
return boundedSamplePercent(float64(typed))
case uint16:
return boundedSamplePercent(float64(typed))
case uint32:
return boundedSamplePercent(float64(typed))
case uint64:
return boundedSamplePercent(float64(typed))
case float32:
return boundedSamplePercent(float64(typed))
case float64:
return boundedSamplePercent(typed)
case json.Number:
parsed, err := typed.Float64()
if err != nil {
return 0, false
}
return boundedSamplePercent(parsed)
default:
return 0, false
}
}
func boundedSamplePercent(value float64) (float64, bool) {
if math.IsNaN(value) || math.IsInf(value, 0) || value <= 0 || value > 100 {
return 0, false
}
return value, true
}
func buildSamplingItems(run domain.RunRecord, manifest map[string]any) []domain.SamplingItemRecord {
items := make([]domain.SamplingItemRecord, 0)
previewByPackage := make(map[string]*domain.ExecutionArtifact)
for _, raw := range listValue(manifest["items"]) {
item := mapValue(raw)
packageName := stringValue(item["packageName"])
if _, ok := previewByPackage[packageName]; !ok {
previewByPackage[packageName] = choosePackagePreview(packageArtifacts(run, packageName))
}
preview := previewByPackage[packageName]
items = append(items, domain.SamplingItemRecord{
ID: domain.NewID("sampling-item"),
PackageName: packageName,
ClipID: stringValue(item["clipId"]),
SourceVideo: stringValue(item["sourceVideo"]),
SampleRuleID: stringValue(item["sampleRuleId"]),
StoragePath: stringValue(item["storagePath"]),
Quality: mapValue(item["quality"]),
Annotation: mapValue(item["annotation"]),
PreviewArtifact: preview,
Status: domain.SamplingItemStatusPending,
})
}
return items
}
func (server *server) syncSamplingTasks(ctx context.Context) error {
existing, err := server.listSamplingTasksForContext(ctx, "")
if err != nil {
return err
}
existingByRun := make(map[string]domain.SamplingTaskRecord, len(existing))
for _, task := range existing {
existingByRun[task.RunID] = task
}
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
if err != nil {
return err
}
for _, run := range runs {
if !server.filterByTenantContext(ctx, run.OrganizationID) {
continue
}
if _, exists := existingByRun[run.ID]; exists {
continue
}
manifest := artifactByName(run, "audit-samples.json")
manifestPayload := artifactStructuredData(manifest)
if manifest == nil || manifestPayload == nil || len(listValue(manifestPayload["items"])) == 0 {
continue
}
rules, ok := samplingRulesFromManifest(manifestPayload)
if !ok {
continue
}
now := nowISO()
items := buildSamplingItems(run, manifestPayload)
task := domain.SamplingTaskRecord{
ID: domain.NewID("sampling"),
OrganizationID: normalizedTenantID(run.OrganizationID),
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
Name: fmt.Sprintf("%s 抽检任务", run.WorkflowName),
Description: fmt.Sprintf("来源于运行 %s 的真实抽检采样结果", run.ID),
Status: domain.SamplingTaskStatusPending,
Rules: rules,
Items: items,
CreatedAt: now,
UpdatedAt: now,
}
for index := range task.Items {
task.Items[index].TaskID = task.ID
}
if _, err := server.store.CreateSamplingTask(ctx, task); err != nil {
return err
}
}
return nil
}
func (server *server) handleSamplingTasksList(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items, err := server.listSamplingTasksForRequest(request.Context(), request, request.URL.Query().Get("status"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items = server.filterSamplingTasksForRequest(request, items)
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleSamplingTaskGet(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item, err := server.store.GetSamplingTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling task not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Sampling task") {
return
}
renderJSON(writer, http.StatusOK, item)
}
func validateSamplingReviewPayload(payload samplingReviewRequest) []validationIssue {
issues := make([]validationIssue, 0, 4)
if strings.TrimSpace(payload.VideoQuality) == "" {
issues = append(issues, validationIssue{Field: "videoQuality", Message: "videoQuality is required"})
}
if strings.TrimSpace(payload.AnnotationAccuracy) == "" {
issues = append(issues, validationIssue{Field: "annotationAccuracy", Message: "annotationAccuracy is required"})
}
if strings.TrimSpace(payload.AnnotationCompleteness) == "" {
issues = append(issues, validationIssue{Field: "annotationCompleteness", Message: "annotationCompleteness is required"})
}
if strings.TrimSpace(payload.OverallGrade) == "" {
issues = append(issues, validationIssue{Field: "overallGrade", Message: "overallGrade is required"})
}
return issues
}
func (server *server) handleSamplingItemReview(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
var payload samplingReviewRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateSamplingReviewPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
tasks, err := server.listSamplingTasksForRequest(request.Context(), request, "")
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
itemID := chi.URLParam(request, "id")
for _, task := range tasks {
if !server.filterByTenant(request, task.OrganizationID) {
continue
}
updated := false
for index := range task.Items {
if task.Items[index].ID != itemID {
continue
}
task.Items[index].Status = domain.SamplingItemStatusCompleted
task.Items[index].Review = &domain.SamplingReview{
VideoQuality: strings.TrimSpace(payload.VideoQuality),
AnnotationAccuracy: strings.TrimSpace(payload.AnnotationAccuracy),
AnnotationCompleteness: strings.TrimSpace(payload.AnnotationCompleteness),
OverallGrade: strings.TrimSpace(payload.OverallGrade),
Notes: strings.TrimSpace(payload.Notes),
ReviewedAt: nowISO(),
}
updated = true
break
}
if !updated {
continue
}
task.Status = domain.SamplingTaskStatusCompleted
for _, item := range task.Items {
if item.Status != domain.SamplingItemStatusCompleted {
task.Status = domain.SamplingTaskStatusPending
break
}
}
task.UpdatedAt = nowISO()
if err := server.store.UpsertSamplingTask(request.Context(), task); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, task)
return
}
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling item not found"})
}