droboticflow/internal/api/data_platform.go
d-robotics da446d13b5 ⚡ Fix slow page loading: static gzip, list pagination, auth cache, dashboard aggregation, video direct link
- C1/S3: build-time precompressed .gz served by spaHandler (writeBundle plugin), motion-vendor chunk rule fix
- C3/S1: /api/runs default cursor pagination (byte-compatible), /api/human-items limit/cursor + Mongo projection + filter pushdown
- C2/S5: in-process principal TTL cache (45s) with full invalidation paths; /api/users limit+q with org isolation
- C4/S2: dashboard counter pushdown ($group/CountDocuments), 45s response/snapshot caches, production stats snapshot cache
- C5: BOS presigned direct-link video preview (307) with background pre-transcode, /tmp cache TTL+LRU, gzip bypass, sync=1 rollback
- S4/S6: POC readiness point queries, human-batches GET no longer writes (60s sweeper), admin ops page N+1 fan-out removed
- Fixes found en route: BSON cursor self-reference recursion, backfill loop cursor advance, supplier cascade session flush
- Frontend: nextCursor passthrough (human-loop/users), acceptance pool no longer truncated at 500

Co-authored-by: AgentTeams slow-page-fix
2026-10-09 17:45:27 -07:00

3517 lines
116 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package api
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"math"
"mime"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"droboticflow/internal/domain"
"github.com/baidubce/bce-sdk-go/services/bos"
"github.com/go-chi/chi/v5"
)
type sourceBrowserPreviewRequest struct {
FileName string
Payload []byte
RedirectURL string
// ObjectBucket/ObjectKey identify the underlying storage object behind
// RedirectURL. They keep the preview cache stable across presigned URLs,
// which are regenerated (and therefore re-signed) on every request.
ObjectBucket string
ObjectKey string
}
type flushResponseWriter struct {
writer http.ResponseWriter
flusher http.Flusher
}
func (target flushResponseWriter) Write(payload []byte) (int, error) {
count, err := target.writer.Write(payload)
if count > 0 {
target.flusher.Flush()
}
return count, err
}
var sourceBrowserVideoPreviewPath = locateSourceBrowserVideoPreviewPath
var sourceBrowserVideoPreviewWriter = streamSourceBrowserVideoPreview
var sourceCollectionSignedPreviewURL = buildSourceCollectionSignedPreviewURL
var sourceBrowserPreviewLocks sync.Map
func sourceBrowserPreviewLock(key string) *sync.Mutex {
lock := &sync.Mutex{}
actual, _ := sourceBrowserPreviewLocks.LoadOrStore(key, lock)
return actual.(*sync.Mutex)
}
type scanConfigRequest struct {
Name string `json:"name"`
Description string `json:"description"`
SourcePath string `json:"sourcePath"`
ConnectionID string `json:"connectionId,omitempty"`
Bucket string `json:"bucket,omitempty"`
Prefix string `json:"prefix,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
AccessKeyID string `json:"accessKeyId,omitempty"`
SecretAccessKey string `json:"secretAccessKey,omitempty"`
ScanMode string `json:"scanMode,omitempty"`
DataSourceIssue *validationIssue `json:"-"`
ScanModeIssue *validationIssue `json:"-"`
IncrementalUnit string `json:"incrementalUnit,omitempty"`
IncrementalUnitIssue *validationIssue `json:"-"`
DirectoryDepth int `json:"directoryDepth,omitempty"`
DirectoryDepthIssue *validationIssue `json:"-"`
Pattern string `json:"pattern,omitempty"`
Interval int `json:"interval"`
Enabled bool `json:"enabled"`
DataSource string `json:"dataSource,omitempty"`
Notifications domain.ScanNotificationSettings `json:"notifications,omitempty"`
}
type samplingReviewRequest struct {
VideoQuality string `json:"videoQuality"`
AnnotationAccuracy string `json:"annotationAccuracy"`
AnnotationCompleteness string `json:"annotationCompleteness"`
OverallGrade string `json:"overallGrade"`
Notes string `json:"notes,omitempty"`
}
func isScanWorkflow(item domain.WorkflowRecord) bool {
return item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil
}
func chooseWorkflowKind(value domain.WorkflowKind) domain.WorkflowKind {
if value == "" {
return domain.WorkflowKindStandard
}
return value
}
func chooseDataSource(value string) string {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "BOS"
}
return trimmed
}
func parseWorkflowDataSourceHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "BOS" {
return trimmed, nil
}
return "", &validationIssue{
Field: "dataSource",
Message: "DATA_SOURCE must be BOS for scan workflows",
}
}
func validateScanWorkflowDataSource(value string) *validationIssue {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" || trimmed == "BOS" {
return nil
}
return &validationIssue{
Field: "dataSource",
Message: "dataSource must be BOS for scan workflows",
}
}
func cleanBOSPrefix(value string) string {
return strings.Trim(strings.TrimSpace(value), "/")
}
func chooseBOSEndpointWithDefault(value string, fallback string) string {
if trimmedValue := strings.TrimSpace(value); trimmedValue != "" {
return trimmedValue
}
return strings.TrimSpace(fallback)
}
func chooseBOSEndpoint(value string) string {
return strings.TrimSpace(value)
}
func (server *server) chooseBOSEndpoint(value string) string {
return chooseBOSEndpointWithDefault(value, server.globalBOSEndpoint)
}
func chooseWorkflowScanMode(value string) string {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "FULL":
return "FULL"
default:
return "INCREMENTAL"
}
}
func parseWorkflowScanModeHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "FULL" || trimmed == "INCREMENTAL" {
return trimmed, nil
}
return "", &validationIssue{
Field: "scanMode",
Message: "SCAN_MODE must be FULL or INCREMENTAL",
}
}
func chooseWorkflowIncrementalUnit(value string) string {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "DIRECTORY":
return "DIRECTORY"
default:
return "OBJECT"
}
}
func parseWorkflowIncrementalUnitHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "OBJECT" || trimmed == "DIRECTORY" {
return trimmed, nil
}
return "", &validationIssue{
Field: "incrementalUnit",
Message: "INCREMENTAL_UNIT must be OBJECT or DIRECTORY",
}
}
func chooseWorkflowDirectoryDepth(value int) int {
if value > 0 {
return value
}
return 1
}
func parseWorkflowDirectoryDepthHyperparam(value string) (int, *validationIssue) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return 0, nil
}
parsed, err := strconv.Atoi(trimmed)
if err != nil || parsed <= 0 {
return 0, &validationIssue{
Field: "directoryDepth",
Message: "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer",
}
}
return parsed, nil
}
func scanSourcePath(dataSource string, sourcePath string, bucket string, prefix string) string {
if chooseDataSource(dataSource) == "BOS" {
if explicit := strings.TrimSpace(sourcePath); explicit != "" {
return explicit
}
if strings.TrimSpace(bucket) == "" {
return ""
}
canonical := "bos://" + strings.TrimSpace(bucket)
if cleanedPrefix := cleanBOSPrefix(prefix); cleanedPrefix != "" {
canonical += "/" + cleanedPrefix
}
return canonical
}
return strings.TrimSpace(sourcePath)
}
func mergeScanSecrets(existing *domain.ScanConfig, payload scanConfigRequest) (string, string) {
accessKeyID := strings.TrimSpace(payload.AccessKeyID)
secretAccessKey := strings.TrimSpace(payload.SecretAccessKey)
if accessKeyID == "" && existing != nil {
accessKeyID = existing.AccessKeyID
}
if secretAccessKey == "" && existing != nil {
secretAccessKey = existing.SecretAccessKey
}
return accessKeyID, secretAccessKey
}
func mergeScanConnectionID(existing *domain.ScanConfig, payload scanConfigRequest) string {
connectionID := strings.TrimSpace(payload.ConnectionID)
if connectionID == "" &&
existing != nil &&
strings.TrimSpace(existing.ConnectionID) != "" &&
strings.TrimSpace(payload.AccessKeyID) == "" &&
strings.TrimSpace(payload.SecretAccessKey) == "" {
connectionID = strings.TrimSpace(existing.ConnectionID)
}
return connectionID
}
func (server *server) getStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) {
connectionID := strings.TrimSpace(id)
if connectionID == "" {
return nil, nil
}
item, err := server.store.GetStorageConnection(ctx, connectionID)
if err != nil {
return nil, err
}
if item == nil {
return nil, fmt.Errorf("storage connection %s not found", connectionID)
}
if !server.filterByTenantContext(ctx, item.OrganizationID) {
return nil, fmt.Errorf("storage connection %s not found", connectionID)
}
return item, nil
}
func (server *server) getMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) {
connectionID := strings.TrimSpace(id)
if connectionID == "" {
return nil, nil
}
item, err := server.store.GetMongoConnection(ctx, connectionID)
if err != nil {
return nil, err
}
if item == nil {
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
}
if !server.filterByTenantContext(ctx, item.OrganizationID) {
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
}
return item, nil
}
func (server *server) resolveScanConfig(ctx context.Context, scanConfig *domain.ScanConfig) (*domain.ScanConfig, error) {
if scanConfig == nil {
return nil, nil
}
copy := *scanConfig
connection, err := server.getStorageConnection(ctx, copy.ConnectionID)
if err != nil {
return nil, err
}
if connection == nil {
if chooseDataSource(copy.DataSource) == "BOS" {
copy.Endpoint = server.chooseBOSEndpoint(copy.Endpoint)
}
return &copy, nil
}
copy.Endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint)
if strings.TrimSpace(copy.AccessKeyID) == "" {
copy.AccessKeyID = connection.AccessKeyID
}
if strings.TrimSpace(copy.SecretAccessKey) == "" {
copy.SecretAccessKey = connection.SecretAccessKey
}
return &copy, nil
}
func cloneHyperparams(items []domain.HyperParam) []domain.HyperParam {
out := make([]domain.HyperParam, len(items))
copy(out, items)
return out
}
func clonePorts(items []domain.PortDefinition) []domain.PortDefinition {
out := make([]domain.PortDefinition, len(items))
copy(out, items)
return out
}
func cloneOperatorIntoNode(operator domain.OperatorRecord, nodeID string, x float64, y float64, overrides map[string]string) domain.WorkflowNode {
hyperparams := cloneHyperparams(operator.Hyperparams)
seen := make(map[string]int, len(hyperparams))
for index, item := range hyperparams {
seen[item.Key] = index
}
for key, value := range overrides {
if index, ok := seen[key]; ok {
hyperparams[index].Value = value
continue
}
hyperparams = append(hyperparams, domain.HyperParam{Key: key, Value: value})
}
return domain.WorkflowNode{
ID: nodeID,
Type: "operator",
Data: domain.WorkflowNodeData{
Label: operator.Name,
Category: operator.Category,
Description: operator.Description,
Inputs: clonePorts(operator.Inputs),
Outputs: clonePorts(operator.Outputs),
Image: operator.Image,
SkillDefinition: operator.SkillDefinition,
Hyperparams: hyperparams,
},
Position: domain.Position{X: x, Y: y},
}
}
func operatorByName(items []domain.OperatorRecord, name string) (*domain.OperatorRecord, bool) {
for _, item := range items {
if item.Name == name {
copy := item
return &copy, true
}
}
return nil, false
}
func (server *server) buildScanWorkflowInput(ctx context.Context, projectID string, payload scanConfigRequest, existing *domain.ScanConfig) (domain.CreateWorkflowInput, error) {
operators, err := server.listAvailableOperators(ctx)
if err != nil {
return domain.CreateWorkflowInput{}, err
}
dataSource := chooseDataSource(payload.DataSource)
sourceOperatorName := "BOS 增量扫描"
sourceOperator, ok := operatorByName(operators, sourceOperatorName)
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator %s not found", sourceOperatorName)
}
indexOperator, ok := operatorByName(operators, "X 平台数据集索引")
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator X 平台数据集索引 not found")
}
validateOperator, ok := operatorByName(operators, "采集目录校验")
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator 采集目录校验 not found")
}
sourcePath := scanSourcePath(dataSource, payload.SourcePath, payload.Bucket, payload.Prefix)
connectionID := mergeScanConnectionID(existing, payload)
var connection *domain.StorageConnectionRecord
if connectionID != "" {
connection, err = server.getStorageConnection(ctx, connectionID)
if err != nil {
return domain.CreateWorkflowInput{}, err
}
}
endpoint := ""
accessKeyID := ""
secretAccessKey := ""
if connection != nil {
endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint)
} else if dataSource == "BOS" {
endpoint = server.chooseBOSEndpoint(payload.Endpoint)
accessKeyID, secretAccessKey = mergeScanSecrets(existing, payload)
}
sourceOverrides := map[string]string{
"DATA_SOURCE": dataSource,
"SCAN_MODE": chooseWorkflowScanMode(payload.ScanMode),
"INCREMENTAL_UNIT": chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
"INCREMENTAL_DIRECTORY_DEPTH": strconv.Itoa(chooseWorkflowDirectoryDepth(payload.DirectoryDepth)),
}
if dataSource == "BOS" {
sourceOverrides["SOURCE_PATH"] = sourcePath
sourceOverrides["BOS_ENDPOINT"] = endpoint
} else {
sourceOverrides["SOURCE_PATH"] = sourcePath
}
sourceNode := cloneOperatorIntoNode(*sourceOperator, "scan-source", 0, 0, sourceOverrides)
sourceNode.Data.Bucket = strings.TrimSpace(payload.Bucket)
sourceNode.Data.Prefix = cleanBOSPrefix(payload.Prefix)
sourceNode.Data.Pattern = strings.TrimSpace(payload.Pattern)
indexNode := cloneOperatorIntoNode(*indexOperator, "scan-index", 260, 0, nil)
validateNode := cloneOperatorIntoNode(*validateOperator, "scan-validate", 520, 0, nil)
schedule := &domain.WorkflowSchedule{SourcePath: sourcePath}
workflowType := domain.WorkflowTypeBatch
if payload.Enabled && payload.Interval > 0 {
schedule.Interval = payload.Interval
workflowType = domain.WorkflowTypeStreaming
}
return domain.CreateWorkflowInput{
ProjectID: projectID,
Name: strings.TrimSpace(payload.Name),
Description: strings.TrimSpace(payload.Description),
Type: workflowType,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: dataSource,
SourcePath: sourcePath,
ConnectionID: connectionID,
Bucket: strings.TrimSpace(payload.Bucket),
Prefix: cleanBOSPrefix(payload.Prefix),
Endpoint: endpoint,
AccessKeyID: accessKeyID,
SecretAccessKey: secretAccessKey,
ScanMode: chooseWorkflowScanMode(payload.ScanMode),
IncrementalUnit: chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
DirectoryDepth: chooseWorkflowDirectoryDepth(payload.DirectoryDepth),
Pattern: strings.TrimSpace(payload.Pattern),
Enabled: payload.Enabled && payload.Interval > 0,
Notifications: payload.Notifications,
},
Schedule: schedule,
Nodes: []domain.WorkflowNode{
sourceNode,
indexNode,
validateNode,
},
Edges: []domain.WorkflowEdge{
{ID: "scan-source-index", Source: sourceNode.ID, SourceHandle: "out-1", Target: indexNode.ID, TargetHandle: "in-1"},
{ID: "scan-index-validate", Source: indexNode.ID, SourceHandle: "out-1", Target: validateNode.ID, TargetHandle: "in-1"},
},
}, nil
}
func validateScanConfigEndpoint(payload scanConfigRequest, existing *domain.ScanConfig) *validationIssue {
if chooseDataSource(payload.DataSource) != "BOS" {
return nil
}
if strings.TrimSpace(mergeScanConnectionID(existing, payload)) != "" {
return nil
}
return validateOptionalHTTPURL("endpoint", payload.Endpoint)
}
func validateScanConfigDraftPayload(payload scanConfigRequest, existing ...*domain.ScanConfig) []validationIssue {
var previous *domain.ScanConfig
if len(existing) > 0 {
previous = existing[0]
}
issues := make([]validationIssue, 0, 5)
if strings.TrimSpace(payload.Name) == "" {
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
}
if payload.Interval < 0 {
issues = append(issues, validationIssue{Field: "interval", Message: "interval must be zero or greater"})
}
if issue := validateScanWorkflowDataSource(payload.DataSource); issue != nil {
issues = append(issues, *issue)
}
if payload.DataSourceIssue != nil {
issues = append(issues, *payload.DataSourceIssue)
}
if scanMode := strings.ToUpper(strings.TrimSpace(payload.ScanMode)); scanMode != "" && scanMode != "FULL" && scanMode != "INCREMENTAL" {
issues = append(issues, validationIssue{Field: "scanMode", Message: "scanMode must be FULL or INCREMENTAL"})
}
if payload.ScanModeIssue != nil {
issues = append(issues, *payload.ScanModeIssue)
}
if incrementalUnit := strings.ToUpper(strings.TrimSpace(payload.IncrementalUnit)); incrementalUnit != "" && incrementalUnit != "OBJECT" && incrementalUnit != "DIRECTORY" {
issues = append(issues, validationIssue{Field: "incrementalUnit", Message: "incrementalUnit must be OBJECT or DIRECTORY"})
}
if payload.IncrementalUnitIssue != nil {
issues = append(issues, *payload.IncrementalUnitIssue)
}
if payload.DirectoryDepthIssue != nil {
issues = append(issues, *payload.DirectoryDepthIssue)
}
if payload.DirectoryDepth < 0 {
issues = append(issues, validationIssue{Field: "directoryDepth", Message: "directoryDepth must be zero or greater"})
}
if issue := validateScanConfigEndpoint(payload, previous); issue != nil {
issues = append(issues, *issue)
}
return issues
}
func scanConfigPayloadReady(payload scanConfigRequest, existing *domain.ScanConfig) bool {
switch chooseDataSource(payload.DataSource) {
case "BOS":
connectionID := mergeScanConnectionID(existing, payload)
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
hasTarget := strings.TrimSpace(payload.SourcePath) != "" ||
(strings.TrimSpace(payload.Bucket) != "" && cleanBOSPrefix(payload.Prefix) != "")
return hasTarget &&
(strings.TrimSpace(connectionID) != "" ||
(strings.TrimSpace(accessKeyID) != "" && strings.TrimSpace(secretAccessKey) != ""))
default:
return strings.TrimSpace(payload.SourcePath) != ""
}
}
func validateScanConfigPayload(payload scanConfigRequest, existing *domain.ScanConfig) []validationIssue {
issues := validateScanConfigDraftPayload(payload, existing)
switch chooseDataSource(payload.DataSource) {
case "BOS":
if strings.TrimSpace(payload.SourcePath) == "" {
if strings.TrimSpace(payload.Bucket) == "" {
issues = append(issues, validationIssue{Field: "bucket", Message: "bucket is required for BOS scan configs when sourcePath is empty"})
}
if cleanBOSPrefix(payload.Prefix) == "" {
issues = append(issues, validationIssue{Field: "prefix", Message: "prefix is required for BOS scan configs when sourcePath is empty"})
}
}
connectionID := mergeScanConnectionID(existing, payload)
if strings.TrimSpace(connectionID) == "" {
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
if strings.TrimSpace(accessKeyID) == "" {
issues = append(issues, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required for BOS scan configs"})
}
if strings.TrimSpace(secretAccessKey) == "" {
issues = append(issues, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required for BOS scan configs"})
}
}
default:
if strings.TrimSpace(payload.SourcePath) == "" {
issues = append(issues, validationIssue{Field: "sourcePath", Message: "sourcePath is required"})
}
}
return issues
}
func (server *server) scanWorkflows(ctx context.Context, request *http.Request) ([]domain.WorkflowRecord, error) {
items, err := server.listAllWorkflowsForRequest(ctx, request)
if err != nil {
return nil, err
}
filtered := make([]domain.WorkflowRecord, 0)
for _, item := range items {
if isScanWorkflow(item) {
item.Kind = chooseWorkflowKind(item.Kind)
filtered = append(filtered, item)
}
}
return filtered, nil
}
func (server *server) handleScanConfigsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.scanWorkflows(request.Context(), request)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items = server.filterWorkflowsForRequest(request, items)
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)})
}
func (server *server) handleScanConfigsCreate(writer http.ResponseWriter, request *http.Request) {
var payload scanConfigRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Interval > 0 && !scanConfigPayloadReady(payload, nil) {
renderValidationError(writer, validateScanConfigPayload(payload, nil)...)
return
}
if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(nil, payload)); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
input, err := server.buildScanWorkflowInput(request.Context(), "", payload, nil)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
server.assignTenant(request, &input.OrganizationID)
item, err := server.store.CreateWorkflow(request.Context(), input)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
item.Kind = chooseWorkflowKind(item.Kind)
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
var payload scanConfigRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateScanConfigDraftPayload(payload, item.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Interval > 0 && !scanConfigPayloadReady(payload, item.ScanConfig) {
renderValidationError(writer, validateScanConfigPayload(payload, item.ScanConfig)...)
return
}
if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(item.ScanConfig, payload)); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
input, err := server.buildScanWorkflowInput(request.Context(), item.ProjectID, payload, item.ScanConfig)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item.Name = input.Name
item.Description = input.Description
item.Type = input.Type
item.Kind = input.Kind
item.ScanConfig = input.ScanConfig
item.Schedule = input.Schedule
item.Nodes = input.Nodes
item.Edges = input.Edges
item.UpdatedAt = nowISO()
if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsDelete(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
writer.WriteHeader(http.StatusNotFound)
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
preview, runs, err := server.buildWorkflowDeletionPreview(request.Context(), item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if preview.BlockingRunCount > 0 {
renderJSON(writer, http.StatusConflict, map[string]any{
"error": "数据扫描配置仍有正在排队或运行的 Run,请先停止这些 Run 后再删除",
"code": "scan_config_run_active",
"preview": preview,
})
return
}
if err := server.deleteWorkflowGeneratedArtifacts(request.Context(), item, runs); err != nil {
renderJSON(writer, http.StatusBadGateway, map[string]string{"error": err.Error(), "code": "scan_config_artifact_cleanup_failed"})
return
}
deleted, err := server.store.DeleteWorkflow(request.Context(), item.ID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func artifactByName(run domain.RunRecord, name string) *domain.ExecutionArtifact {
for index := range run.Artifacts {
if run.Artifacts[index].Name == name {
return &run.Artifacts[index]
}
}
return nil
}
func mapValue(value any) map[string]any {
item, _ := normalizeStructuredValue(value).(map[string]any)
return item
}
func listValue(value any) []any {
items, _ := normalizeStructuredValue(value).([]any)
return items
}
func normalizeStructuredValue(value any) any {
switch typed := value.(type) {
case nil:
return nil
case map[string]any:
items := make(map[string]any, len(typed))
for key, item := range typed {
items[key] = normalizeStructuredValue(item)
}
return items
case []any:
items := make([]any, len(typed))
for index, item := range typed {
items[index] = normalizeStructuredValue(item)
}
return items
}
rv := reflect.ValueOf(value)
switch rv.Kind() {
case reflect.Map:
if rv.Type().Key().Kind() != reflect.String {
return value
}
items := make(map[string]any, rv.Len())
iter := rv.MapRange()
for iter.Next() {
items[iter.Key().String()] = normalizeStructuredValue(iter.Value().Interface())
}
return items
case reflect.Slice, reflect.Array:
items := make([]any, rv.Len())
for index := 0; index < rv.Len(); index++ {
items[index] = normalizeStructuredValue(rv.Index(index).Interface())
}
return items
default:
return value
}
}
func stringValue(value any) string {
switch typed := value.(type) {
case nil:
return ""
case string:
return typed
default:
text := strings.TrimSpace(fmt.Sprintf("%v", typed))
if text == "<nil>" {
return ""
}
return text
}
}
func boolValue(value any) bool {
typed, ok := value.(bool)
return ok && typed
}
func floatValue(value any) float64 {
switch typed := value.(type) {
case float64:
if math.IsNaN(typed) || math.IsInf(typed, 0) {
return 0
}
return typed
case float32:
value := float64(typed)
if math.IsNaN(value) || math.IsInf(value, 0) {
return 0
}
return value
case int:
return float64(typed)
case int32:
return float64(typed)
case int64:
return float64(typed)
default:
return 0
}
}
func intValue(value any) int {
parsed, ok := exactIntegerValue(value)
if !ok {
return 0
}
if strconv.IntSize == 32 {
const maxInt32 = int64(1<<31 - 1)
const minInt32 = -1 << 31
if parsed < minInt32 || parsed > maxInt32 {
return 0
}
}
return int(parsed)
}
func int64Value(value any) int64 {
parsed, ok := exactIntegerValue(value)
if !ok {
return 0
}
return parsed
}
func exactIntegerValue(value any) (int64, bool) {
switch typed := value.(type) {
case int:
return int64(typed), true
case int8:
return int64(typed), true
case int16:
return int64(typed), true
case int32:
return int64(typed), true
case int64:
return typed, true
case uint:
if uint64(typed) > uint64(math.MaxInt64) {
return 0, false
}
return int64(typed), true
case uint8:
return int64(typed), true
case uint16:
return int64(typed), true
case uint32:
return int64(typed), true
case uint64:
if typed > uint64(math.MaxInt64) {
return 0, false
}
return int64(typed), true
case float32:
const maxExactFloat32Integer = 1<<24 - 1
return exactFloatIntegerValue(float64(typed), maxExactFloat32Integer)
case float64:
const maxExactFloat64Integer = 1<<53 - 1
return exactFloatIntegerValue(typed, maxExactFloat64Integer)
case json.Number:
parsed, err := typed.Int64()
return parsed, err == nil
default:
return 0, false
}
}
func exactFloatIntegerValue(value float64, maxExactAbs float64) (int64, bool) {
if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
return 0, false
}
if math.Abs(value) > maxExactAbs {
return 0, false
}
return int64(value), true
}
func artifactDataRef(artifact *domain.ExecutionArtifact) map[string]any {
if artifact == nil || artifact.Data == nil {
return nil
}
return mapValue(artifact.Data["dataRef"])
}
func artifactStructuredData(artifact *domain.ExecutionArtifact) map[string]any {
if artifact == nil {
return nil
}
if dataRef := artifactDataRef(artifact); dataRef != nil {
payloadPath := strings.TrimSpace(stringValue(dataRef["payloadPath"]))
if payloadPath != "" && !strings.Contains(payloadPath, "://") {
payload, err := os.ReadFile(payloadPath)
if err != nil {
return nil
}
var decoded any
if err := json.Unmarshal(payload, &decoded); err != nil {
return nil
}
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
return structured
}
return nil
}
}
if artifact.Data != nil {
return artifact.Data
}
if artifact.Type == domain.PortTypeJSON {
filePath := strings.TrimSpace(artifact.Path)
if filePath != "" && !strings.Contains(filePath, "://") {
payload, err := os.ReadFile(filePath)
if err == nil {
var decoded any
if err := json.Unmarshal(payload, &decoded); err == nil {
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
return structured
}
}
}
}
}
return nil
}
func sourceArtifactRelativePath(path string) string {
normalized := filepath.ToSlash(path)
marker := "/_source_tree/"
index := strings.Index(normalized, marker)
if index < 0 {
return ""
}
return strings.TrimPrefix(normalized[index+len(marker):], "/")
}
func sourceArtifactPath(artifact domain.ExecutionArtifact) string {
if payload := artifactStructuredData(&artifact); payload != nil {
if relative := stringValue(payload["relativePath"]); relative != "" {
return filepath.ToSlash(strings.Trim(relative, "/"))
}
}
return sourceArtifactRelativePath(artifact.Path)
}
func packageArtifactRelativePath(path string, packageName string) string {
normalized := filepath.ToSlash(path)
marker := "/packages/" + packageName + "/"
index := strings.Index(normalized, marker)
if index < 0 {
return ""
}
return strings.TrimPrefix(normalized[index+len(marker):], "/")
}
func packageContentURL(runID string, packageName string, relativePath string) string {
return fmt.Sprintf(
"/api/packages/%s/%s/content?relativePath=%s",
url.PathEscape(runID),
url.PathEscape(packageName),
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
)
}
func packageArtifactPreviewURL(artifact domain.ExecutionArtifact) string {
payload := artifactStructuredData(&artifact)
preview := mapValue(payload["preview"])
if preview != nil {
if value := strings.TrimSpace(stringValue(preview["url"])); value != "" {
return value
}
}
source := mapValue(payload["source"])
if source != nil {
if value := strings.TrimSpace(stringValue(source["previewURL"])); value != "" {
return value
}
}
return ""
}
func packageArtifactResolvedRelativePath(artifact domain.ExecutionArtifact, packageName string) string {
if relativePath := packageArtifactRelativePath(artifact.Path, packageName); relativePath != "" {
return relativePath
}
payload := artifactStructuredData(&artifact)
if payload == nil {
return ""
}
relativePath := strings.Trim(filepath.ToSlash(stringValue(payload["relativePath"])), "/")
if relativePath == "" {
return ""
}
packagePrefix := strings.Trim(filepath.ToSlash(path.Join("packages", packageName)), "/")
if packagePrefix == "" {
return ""
}
if relativePath == packagePrefix {
return ""
}
if strings.HasPrefix(relativePath, packagePrefix+"/") {
return strings.TrimPrefix(relativePath, packagePrefix+"/")
}
return ""
}
func packageArtifactContentURL(runID string, packageName string, relativePath string) string {
return packageContentURL(runID, packageName, relativePath)
}
func packageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
copy := artifact
if copy.Data == nil {
copy.Data = make(map[string]any)
}
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
if relativePath == "" {
relativePath = filepath.ToSlash(strings.Trim(copy.Path, "/"))
}
if relativePath != "" {
copy.Path = relativePath
}
copy.Data["contentURL"] = packageArtifactContentURL(runID, packageName, copy.Path)
preview := mapValue(copy.Data["preview"])
if preview == nil {
preview = make(map[string]any)
}
kind := strings.TrimSpace(stringValue(preview["kind"]))
mimeType := ""
if copy.Metadata != nil {
mimeType = strings.TrimSpace(copy.Metadata.MimeType)
}
if kind == "video" || copy.Type == "video" || strings.HasPrefix(mimeType, "video/") {
preview["kind"] = "video"
preview["url"] = packageContentURL(runID, packageName, copy.Path)
copy.Data["preview"] = preview
}
return copy
}
const wilorHandPoseSchemaVersion = "droboticflow.wilor.hand_pose_result/v1"
func safePackageNamePart(value string) string {
trimmed := strings.ToLower(strings.TrimSpace(value))
var builder strings.Builder
lastDash := false
for _, current := range trimmed {
isAlphaNumeric := (current >= 'a' && current <= 'z') || (current >= '0' && current <= '9')
if isAlphaNumeric {
builder.WriteRune(current)
lastDash = false
continue
}
if !lastDash {
builder.WriteByte('-')
lastDash = true
}
}
return strings.Trim(builder.String(), "-")
}
func wilorResultPayload(artifact domain.ExecutionArtifact) map[string]any {
if artifact.Name != "wilor-result.json" {
return nil
}
payload := artifactStructuredData(&artifact)
if payload == nil {
return nil
}
if strings.TrimSpace(stringValue(payload["schemaVersion"])) != wilorHandPoseSchemaVersion {
return nil
}
if strings.TrimSpace(stringValue(payload["relativePath"])) != "" {
return nil
}
return payload
}
func wilorPackageName(artifact domain.ExecutionArtifact, payload map[string]any) string {
source := mapValue(payload["source"])
sourceName := strings.TrimSpace(stringValue(source["name"]))
stem := strings.TrimSuffix(path.Base(filepath.ToSlash(sourceName)), path.Ext(sourceName))
sourceSlug := safePackageNamePart(stem)
if sourceSlug == "" {
sourceSlug = "result"
}
nodeSlug := safePackageNamePart(artifact.NodeID)
if nodeSlug == "" {
nodeSlug = "wilor"
}
return "wilor-" + sourceSlug + "-" + nodeSlug
}
func wilorResultGroupKey(artifact domain.ExecutionArtifact, payload map[string]any) string {
source := mapValue(payload["source"])
sourceName := strings.TrimSpace(stringValue(source["name"]))
sourceRef := strings.TrimSpace(stringValue(source["ref"]))
mediaKind := strings.TrimSpace(stringValue(payload["mediaKind"]))
if sourceName != "" || sourceRef != "" || mediaKind != "" {
return strings.ToLower(mediaKind) + "|" + strings.ToLower(sourceName) + "|" + strings.ToLower(sourceRef)
}
return artifact.ID
}
func wilorResultArtifactPriority(artifact domain.ExecutionArtifact) int {
nodeID := strings.ToLower(strings.TrimSpace(artifact.NodeID))
score := 0
if strings.Contains(nodeID, "upload") || strings.Contains(nodeID, "save") {
score += 30
}
if strings.Contains(nodeID, "wilor") {
score += 20
}
if strings.TrimSpace(artifact.Path) != "" {
score++
}
return score
}
func wilorResultArtifacts(run domain.RunRecord) []domain.ExecutionArtifact {
bestByResult := make(map[string]domain.ExecutionArtifact)
for _, artifact := range run.Artifacts {
payload := wilorResultPayload(artifact)
if payload == nil {
continue
}
key := wilorResultGroupKey(artifact, payload)
current, exists := bestByResult[key]
if !exists || wilorResultArtifactPriority(artifact) > wilorResultArtifactPriority(current) {
bestByResult[key] = artifact
}
}
items := make([]domain.ExecutionArtifact, 0, len(bestByResult))
for _, artifact := range bestByResult {
items = append(items, artifact)
}
sort.Slice(items, func(left int, right int) bool {
return wilorPackageName(items[left], wilorResultPayload(items[left])) < wilorPackageName(items[right], wilorResultPayload(items[right]))
})
return items
}
func wilorPackageManifestArtifact(run domain.RunRecord, packageName string) (*domain.ExecutionArtifact, map[string]any) {
for index := range run.Artifacts {
payload := wilorResultPayload(run.Artifacts[index])
if payload == nil {
continue
}
if wilorPackageName(run.Artifacts[index], payload) == packageName {
return &run.Artifacts[index], payload
}
}
return nil, nil
}
func wilorOutputArtifactNames(manifest domain.ExecutionArtifact, payload map[string]any) map[string]struct{} {
names := map[string]struct{}{
manifest.Name: {},
}
for _, raw := range listValue(payload["downloadedOutputs"]) {
item := mapValue(raw)
name := strings.TrimSpace(stringValue(item["name"]))
if name == "" {
if outputPath := strings.TrimSpace(stringValue(item["path"])); outputPath != "" {
name = path.Base(filepath.ToSlash(outputPath))
}
}
if name != "" {
names[name] = struct{}{}
}
}
return names
}
func wilorPackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
manifest, payload := wilorPackageManifestArtifact(run, packageName)
if manifest == nil || payload == nil {
return nil
}
names := wilorOutputArtifactNames(*manifest, payload)
bestByName := make(map[string]domain.ExecutionArtifact)
for _, artifact := range run.Artifacts {
if _, ok := names[artifact.Name]; !ok {
continue
}
if artifact.NodeID != manifest.NodeID && strings.TrimSpace(stringValue(mapValue(artifactStructuredData(&artifact)["source"])["path"])) == "" {
continue
}
current, exists := bestByName[artifact.Name]
if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) {
bestByName[artifact.Name] = artifact
}
}
items := make([]domain.ExecutionArtifact, 0, len(bestByName))
for _, artifact := range bestByName {
items = append(items, artifact)
}
sort.Slice(items, func(left int, right int) bool {
return items[left].Name < items[right].Name
})
return items
}
func wilorPackageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
copy := artifact
if copy.Data == nil {
copy.Data = make(map[string]any)
}
relativePath := strings.TrimSpace(copy.Name)
if relativePath == "" {
relativePath = path.Base(filepath.ToSlash(strings.TrimSpace(copy.Path)))
}
copy.Path = relativePath
copy.Data["contentURL"] = packageArtifactContentURL(runID, packageName, relativePath)
preview := mapValue(copy.Data["preview"])
if preview == nil {
preview = make(map[string]any)
}
mimeType := ""
if copy.Metadata != nil {
mimeType = strings.TrimSpace(copy.Metadata.MimeType)
}
kind := sourceEntryPayloadKind(relativePath, strings.ToLower(mimeType))
if kind == "video" || kind == "image" {
preview["kind"] = kind
preview["url"] = packageArtifactContentURL(runID, packageName, relativePath)
copy.Data["preview"] = preview
}
return copy
}
func wilorPackageFilesForResponse(run domain.RunRecord, packageName string) ([]domain.ExecutionArtifact, []domain.ExecutionArtifact) {
rawFiles := wilorPackageArtifacts(run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, wilorPackageArtifactForResponse(run.ID, packageName, artifact))
}
return files, rawFiles
}
func wilorPackageItem(run domain.RunRecord, manifest domain.ExecutionArtifact, payload map[string]any, rawFiles []domain.ExecutionArtifact) map[string]any {
source := mapValue(payload["source"])
result := mapValue(payload["result"])
service := mapValue(payload["service"])
health := mapValue(service["health"])
sourceName := strings.TrimSpace(stringValue(source["name"]))
if sourceName == "" {
sourceName = "WiLoR result"
}
mediaKind := strings.TrimSpace(stringValue(payload["mediaKind"]))
if mediaKind == "" {
mediaKind = "media"
}
clipID := strings.TrimSuffix(path.Base(filepath.ToSlash(sourceName)), path.Ext(sourceName))
if clipID == "" || clipID == "." {
clipID = manifest.NodeID
}
storagePath := ""
for _, artifact := range rawFiles {
if candidate := packageArtifactStoragePath(artifact, ""); candidate != "" {
storagePath = candidate
break
}
}
annotation := map[string]any{
"type": "wilor_hand_pose",
"mediaKind": mediaKind,
"handCount": intValue(result["handCount"]),
"frameCount": intValue(result["frameCount"]),
"status": stringValue(payload["status"]),
"requiresHumanReview": false,
}
if task := mapValue(result["wilorTask"]); task != nil {
annotation["task"] = task
}
quality := map[string]any{
"runtime": mapValue(health["runtime"]),
"service": map[string]any{
"image": stringValue(service["image"]),
"imageId": stringValue(service["imageId"]),
},
}
return map[string]any{
"packageName": wilorPackageName(manifest, payload),
"clipId": clipID,
"sourceVideo": sourceName,
"clipDuration": 0,
"fileCount": len(rawFiles),
"pipelineTags": []any{"wilor", "hand_pose", mediaKind},
"storagePath": storagePath,
"quality": quality,
"annotation": annotation,
}
}
func canonicalBOSPath(bucket string, object string) string {
bucket = strings.TrimSpace(bucket)
object = strings.Trim(strings.TrimSpace(object), "/")
if bucket == "" {
return ""
}
if object == "" {
return "bos://" + bucket
}
return "bos://" + bucket + "/" + object
}
func packageArtifactPriority(artifact domain.ExecutionArtifact) int {
score := 0
if artifact.NodeID == "upload" {
score += 40
}
artifactPath := strings.TrimSpace(artifact.Path)
if strings.HasPrefix(strings.ToLower(artifactPath), "bos://") {
score += 30
} else if artifactPath != "" && !strings.Contains(artifactPath, "://") {
score += 10
}
payload := artifactStructuredData(&artifact)
source := mapValue(payload["source"])
sourcePath := strings.TrimSpace(stringValue(source["path"]))
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
score += 30
}
if strings.TrimSpace(packageArtifactPreviewURL(artifact)) != "" {
score += 5
}
return score
}
func packageArtifactStoragePath(artifact domain.ExecutionArtifact, packageName string) string {
payload := artifactStructuredData(&artifact)
source := mapValue(payload["source"])
if sourcePath := strings.TrimSpace(stringValue(source["path"])); sourcePath != "" {
if bucket, object, ok := parseBOSPath(sourcePath); ok {
return canonicalBOSPath(bucket, path.Dir(object))
}
directory := filepath.ToSlash(filepath.Dir(sourcePath))
if directory != "." {
return directory
}
}
artifactPath := strings.TrimSpace(artifact.Path)
if bucket, object, ok := parseBOSPath(artifactPath); ok {
return canonicalBOSPath(bucket, path.Dir(object))
}
if artifactPath == "" {
return ""
}
directory := filepath.ToSlash(filepath.Dir(artifactPath))
if directory == "." {
return ""
}
return directory
}
func dedupePackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
bestByRelativePath := make(map[string]domain.ExecutionArtifact)
for _, artifact := range run.Artifacts {
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
if relativePath == "" {
continue
}
current, exists := bestByRelativePath[relativePath]
if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) {
bestByRelativePath[relativePath] = artifact
}
}
items := make([]domain.ExecutionArtifact, 0, len(bestByRelativePath))
for _, artifact := range bestByRelativePath {
items = append(items, artifact)
}
sort.Slice(items, func(left int, right int) bool {
leftPath := packageArtifactResolvedRelativePath(items[left], packageName)
rightPath := packageArtifactResolvedRelativePath(items[right], packageName)
return leftPath < rightPath
})
return items
}
func packageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range dedupePackageArtifacts(run, packageName) {
items = append(items, packageArtifactForResponse(run.ID, packageName, artifact))
}
return items
}
func choosePackagePreview(files []domain.ExecutionArtifact) *domain.ExecutionArtifact {
for _, file := range files {
if strings.HasSuffix(strings.ToLower(file.Name), ".mp4") {
copy := file
return &copy
}
}
if len(files) == 0 {
return nil
}
copy := files[0]
return &copy
}
func sourceDatasetList(data map[string]any, catalogKey string, fallbackKey string) []any {
if _, ok := data[catalogKey]; ok {
return listValue(data[catalogKey])
}
return listValue(data[fallbackKey])
}
func sourceDatasetHasCatalog(data map[string]any) bool {
_, hasCollections := data["catalogCollections"]
_, hasItems := data["catalogItems"]
_, hasEntries := data["catalogEntries"]
return hasCollections || hasItems || hasEntries
}
func sourceDatasetIsDisplayable(data map[string]any) bool {
if len(sourceDatasetList(data, "catalogCollections", "collections")) > 0 {
return true
}
if len(sourceDatasetList(data, "catalogEntries", "entries")) > 0 {
return true
}
return intValue(data["catalogCollectionCount"]) > 0 || intValue(data["collectionCount"]) > 0
}
func runSourceManifest(run domain.RunRecord) *domain.ExecutionArtifact {
manifest := artifactByName(run, "source-manifest.json")
if manifest == nil || artifactStructuredData(manifest) == nil {
return nil
}
return manifest
}
func runDatasetIndex(run domain.RunRecord) *domain.ExecutionArtifact {
artifact := artifactByName(run, "dataset-index.json")
if artifact == nil || artifactStructuredData(artifact) == nil {
return nil
}
return artifact
}
func runLooksLikeSourceDataset(workflow domain.WorkflowRecord, run domain.RunRecord) bool {
if manifest := runSourceManifest(run); manifest != nil {
payload := artifactStructuredData(manifest)
if strings.TrimSpace(stringValue(payload["sourcePath"])) != "" || strings.TrimSpace(stringValue(payload["dataSource"])) != "" {
return true
}
}
index := runDatasetIndex(run)
if index == nil {
return false
}
if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil {
return true
}
indexPayload := artifactStructuredData(index)
sourcePath := strings.TrimSpace(stringValue(indexPayload["sourcePath"]))
rootPath := strings.TrimSpace(stringValue(indexPayload["rootPath"]))
dataSource := strings.TrimSpace(stringValue(indexPayload["dataSource"]))
if dataSource != "" || strings.HasPrefix(sourcePath, "bos://") || strings.HasPrefix(rootPath, "bos://") {
return true
}
if len(mapValue(indexPayload["archiveSources"])) > 0 && strings.Contains(rootPath, "_materialized_inputs") {
return false
}
return sourceDatasetHasCatalog(indexPayload)
}
func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]domain.WorkflowRecord, map[string]domain.RunRecord, error) {
workflows, err := server.listAllWorkflowsForContext(ctx)
if err != nil {
return nil, nil, err
}
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
if err != nil {
return nil, nil, err
}
latestByWorkflow := make(map[string]domain.RunRecord)
for _, run := range runs {
workflow, ok := workflowByID[run.WorkflowID]
if !ok {
continue
}
index := runDatasetIndex(run)
if index == nil {
continue
}
if !runLooksLikeSourceDataset(workflow, run) {
continue
}
if !sourceDatasetIsDisplayable(artifactStructuredData(index)) {
continue
}
current, exists := latestByWorkflow[run.WorkflowID]
if !exists || run.CreatedAt > current.CreatedAt {
latestByWorkflow[run.WorkflowID] = run
}
}
return workflowByID, latestByWorkflow, nil
}
func parseBOSPath(sourcePath string) (string, string, bool) {
normalized := strings.TrimSpace(sourcePath)
if !strings.HasPrefix(strings.ToLower(normalized), "bos://") {
return "", "", false
}
trimmed := strings.TrimPrefix(normalized, "bos://")
trimmed = strings.TrimPrefix(trimmed, "BOS://")
bucket, object, found := strings.Cut(trimmed, "/")
bucket = strings.TrimSpace(bucket)
object = strings.Trim(strings.TrimSpace(object), "/")
if !found {
object = ""
}
if bucket == "" {
return "", "", false
}
return bucket, object, true
}
func normalizeLocalDirectory(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
directory := filepath.ToSlash(filepath.Dir(trimmed))
if directory == "." {
return ""
}
return directory
}
func assetStorageLocationFromPaths(rootPath string, currentPath string, endpoint string) domain.AssetStorageLocation {
location := domain.AssetStorageLocation{
Path: strings.TrimSpace(currentPath),
}
currentBucket, currentObject, hasCurrentBOS := parseBOSPath(currentPath)
rootBucket, rootObject, hasRootBOS := parseBOSPath(rootPath)
switch {
case hasCurrentBOS:
location.Provider = "bos"
location.Endpoint = chooseBOSEndpoint(endpoint)
location.Bucket = currentBucket
if hasRootBOS && rootBucket == currentBucket {
location.Prefix = rootObject
} else {
location.Prefix = path.Dir(currentObject)
if location.Prefix == "." {
location.Prefix = ""
}
}
case hasRootBOS:
location.Provider = "bos"
location.Endpoint = chooseBOSEndpoint(endpoint)
location.Bucket = rootBucket
location.Prefix = rootObject
if location.Path == "" {
location.Path = strings.TrimSpace(rootPath)
}
case strings.TrimSpace(currentPath) != "":
location.Provider = "k8s-job"
location.Prefix = normalizeLocalDirectory(currentPath)
case strings.TrimSpace(rootPath) != "":
location.Provider = "k8s-job"
location.Path = strings.TrimSpace(rootPath)
location.Prefix = normalizeLocalDirectory(rootPath)
default:
location.Provider = "unknown"
}
if location.Prefix == "." {
location.Prefix = ""
}
return location
}
func sourceRootPathForRun(run domain.RunRecord, scanConfig *domain.ScanConfig) string {
if scanConfig != nil {
if value := scanSourcePath(scanConfig.DataSource, scanConfig.SourcePath, scanConfig.Bucket, scanConfig.Prefix); value != "" {
return value
}
}
if manifest := runSourceManifest(run); manifest != nil {
payload := artifactStructuredData(manifest)
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
return value
}
}
if index := runDatasetIndex(run); index != nil {
payload := artifactStructuredData(index)
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
return value
}
if value := strings.TrimSpace(stringValue(payload["rootPath"])); value != "" {
return value
}
}
return ""
}
func (server *server) buildSourceCollectionRecord(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, collection map[string]any) domain.SourceCollectionRecord {
meta := mapValue(collection["meta"])
requiredFiles := make([]string, 0)
for _, item := range listValue(collection["requiredFiles"]) {
requiredFiles = append(requiredFiles, stringValue(item))
}
missingFiles := make([]string, 0)
for _, item := range listValue(collection["missingRequiredFiles"]) {
missingFiles = append(missingFiles, stringValue(item))
}
collectionName := stringValue(collection["name"])
if collectionName == "" {
if projectID := stringValue(meta["project_id"]); projectID != "" {
collectionName = projectID
} else if relativePath := strings.Trim(stringValue(collection["relativePath"]), "/"); relativePath != "" && relativePath != "." {
collectionName = path.Base(relativePath)
} else {
collectionName = stringValue(collection["id"])
}
}
var resolvedScanConfig *domain.ScanConfig
if workflow != nil {
if scanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig); err == nil {
resolvedScanConfig = scanConfig
}
}
storage := assetStorageLocationFromPaths(
sourceRootPathForRun(run, resolvedScanConfig),
stringValue(collection["path"]),
func() string {
if resolvedScanConfig == nil {
return ""
}
return resolvedScanConfig.Endpoint
}(),
)
return domain.SourceCollectionRecord{
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
CollectionID: stringValue(collection["id"]),
CollectionName: collectionName,
Path: stringValue(collection["path"]),
RelativePath: stringValue(collection["relativePath"]),
Meta: meta,
HasIntrinsics: boolValue(collection["hasIntrinsics"]),
HasVideoMeta: boolValue(collection["hasVideoMeta"]),
RequiredFiles: requiredFiles,
MissingFiles: missingFiles,
VideoCount: intValue(collection["videoCount"]),
FileCount: intValue(collection["fileCount"]),
TotalBytes: int64Value(collection["totalBytes"]),
Valid: boolValue(collection["valid"]),
CreatedAt: run.CreatedAt,
Storage: storage,
}
}
type sourceCollectionPayloadRecord struct {
Run *domain.RunRecord
Workflow *domain.WorkflowRecord
Collection map[string]any
Items []map[string]any
Entries []map[string]any
ArchiveSource map[string]any
}
func (server *server) sourceCollectionPayload(ctx context.Context, runID string, collectionID string) (*sourceCollectionPayloadRecord, error) {
run, err := server.store.GetRun(ctx, runID)
if err != nil || run == nil {
return &sourceCollectionPayloadRecord{Run: run}, err
}
workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
return &sourceCollectionPayloadRecord{Run: run}, err
}
artifact := artifactByName(*run, "dataset-index.json")
payload := artifactStructuredData(artifact)
if artifact == nil || payload == nil {
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
}
collections := sourceDatasetList(payload, "catalogCollections", "collections")
items := sourceDatasetList(payload, "catalogItems", "items")
entries := sourceDatasetList(payload, "catalogEntries", "entries")
var matchedCollection map[string]any
for _, item := range collections {
collection := mapValue(item)
if stringValue(collection["id"]) == collectionID {
matchedCollection = collection
break
}
}
if matchedCollection == nil {
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
}
matchedItems := make([]map[string]any, 0)
for _, item := range items {
row := mapValue(item)
if stringValue(row["collectionId"]) == collectionID {
matchedItems = append(matchedItems, row)
}
}
matchedEntries := make([]map[string]any, 0)
for _, item := range entries {
row := mapValue(item)
relativePath := strings.Trim(filepath.ToSlash(stringValue(row["relativePath"])), "/")
collectionRelativePath := strings.Trim(filepath.ToSlash(stringValue(matchedCollection["relativePath"])), "/")
if collectionRelativePath == "" || collectionRelativePath == "." {
if relativePath == "" {
continue
}
} else if relativePath != collectionRelativePath && !strings.HasPrefix(relativePath, collectionRelativePath+"/") {
continue
}
matchedEntries = append(matchedEntries, row)
}
return &sourceCollectionPayloadRecord{
Run: run,
Workflow: workflow,
Collection: matchedCollection,
Items: matchedItems,
Entries: matchedEntries,
ArchiveSource: mapValue(payload["archiveSources"]),
}, nil
}
func sourceEntryPayloadKind(relativePath string, mimeType string) string {
extension := strings.ToLower(path.Ext(relativePath))
switch {
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
return "video"
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
return "image"
case strings.HasPrefix(mimeType, "audio/") || extension == ".mp3" || extension == ".wav" || extension == ".m4a":
return "audio"
default:
return "binary"
}
}
func sourceEntryArtifactType(relativePath string, mimeType string) domain.PortType {
extension := strings.ToLower(path.Ext(relativePath))
switch {
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
return domain.PortTypeImage
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
return domain.PortTypeVideo
case extension == ".json":
return domain.PortTypeJSON
default:
return domain.PortTypeFile
}
}
func formatArtifactSize(sizeBytes int64) string {
switch {
case sizeBytes <= 0:
return "0 B"
case sizeBytes < 1024:
return fmt.Sprintf("%d B", sizeBytes)
case sizeBytes < 1024*1024:
return fmt.Sprintf("%.1f KiB", float64(sizeBytes)/1024)
case sizeBytes < 1024*1024*1024:
return fmt.Sprintf("%.1f MiB", float64(sizeBytes)/(1024*1024))
default:
return fmt.Sprintf("%.1f GiB", float64(sizeBytes)/(1024*1024*1024))
}
}
func sourceCollectionContentURL(runID string, collectionID string, relativePath string) string {
return fmt.Sprintf(
"/api/source-data/collections/%s/%s/content?relativePath=%s",
url.PathEscape(runID),
url.PathEscape(collectionID),
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
)
}
func datasetIndexEntrySource(entry map[string]any, archiveSources map[string]any) map[string]any {
source := mapValue(entry["source"])
if len(source) == 0 {
source = make(map[string]any)
}
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if relativePath != "" && stringValue(source["relativePath"]) == "" {
source["relativePath"] = relativePath
}
mimeType := stringValue(entry["mimeType"])
if mimeType == "" {
mimeType = stringValue(source["mimeType"])
}
if mimeType == "" && relativePath != "" {
mimeType = mime.TypeByExtension(path.Ext(relativePath))
}
if mimeType != "" {
source["mimeType"] = mimeType
}
if stringValue(source["previewURL"]) == "" {
archiveKey := stringValue(source["archiveKey"])
archiveSource := mapValue(archiveSources[archiveKey])
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
source["previewURL"] = previewURL
}
if archiveName := stringValue(archiveSource["name"]); archiveName != "" && stringValue(source["archiveName"]) == "" {
source["archiveName"] = archiveName
}
}
if pathValue := stringValue(entry["path"]); pathValue != "" && stringValue(source["path"]) == "" {
source["path"] = pathValue
}
return source
}
func sourceEntryArtifact(run domain.RunRecord, collectionID string, entry map[string]any, archiveSources map[string]any) *domain.ExecutionArtifact {
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if relativePath == "" {
relativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
if relativePath == "" {
return nil
}
source := datasetIndexEntrySource(entry, archiveSources)
mimeType := stringValue(source["mimeType"])
sizeBytes := int64Value(entry["sizeBytes"])
artifactType := sourceEntryArtifactType(relativePath, mimeType)
payload := make(map[string]any)
if value, ok := entry["data"]; ok {
payload["preview"] = map[string]any{
"kind": "json",
"value": normalizeStructuredValue(value),
}
}
if previewURL := stringValue(source["previewURL"]); previewURL != "" && len(payload) == 0 {
payload["preview"] = map[string]any{
"kind": sourceEntryPayloadKind(relativePath, mimeType),
"format": strings.TrimPrefix(strings.ToLower(path.Ext(relativePath)), "."),
"url": sourceCollectionContentURL(run.ID, collectionID, relativePath),
}
}
payload["relativePath"] = relativePath
if len(source) > 0 {
if previewURL := sourceCollectionContentURL(run.ID, collectionID, relativePath); previewURL != "" {
source["previewURL"] = previewURL
}
payload["source"] = source
}
return &domain.ExecutionArtifact{
ID: fmt.Sprintf("%s-source-%s", run.ID, strings.ReplaceAll(relativePath, "/", "-")),
RunID: run.ID,
NodeID: "dataset-index",
Name: path.Base(relativePath),
Type: artifactType,
Size: formatArtifactSize(sizeBytes),
Path: func() string {
if sourcePath := stringValue(source["path"]); sourcePath != "" {
return sourcePath
}
return relativePath
}(),
Metadata: &domain.ArtifactMetadata{MimeType: mimeType},
Data: payload,
}
}
func contentBytesForDataURL(previewURL string) ([]byte, string, error) {
if !strings.HasPrefix(previewURL, "data:") {
return nil, "", fmt.Errorf("not a data url")
}
header, encoded, found := strings.Cut(previewURL, ",")
if !found || !strings.Contains(header, ";base64") {
return nil, "", fmt.Errorf("invalid data url")
}
mimeType := strings.TrimPrefix(strings.SplitN(header, ";", 2)[0], "data:")
payload, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, "", err
}
return payload, mimeType, nil
}
func contentBytesForArchiveMember(archiveName string, archivePreviewURL string, memberPath string) ([]byte, error) {
payload, _, err := contentBytesForDataURL(archivePreviewURL)
if err != nil {
return nil, err
}
memberPath = strings.Trim(filepath.ToSlash(memberPath), "/")
if memberPath == "" {
return nil, fmt.Errorf("missing archive member path")
}
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
if err != nil {
return nil, err
}
for _, file := range archive.File {
if strings.Trim(filepath.ToSlash(file.Name), "/") != memberPath {
continue
}
handle, err := file.Open()
if err != nil {
return nil, err
}
defer handle.Close()
return io.ReadAll(handle)
}
return nil, os.ErrNotExist
}
reader := bytes.NewReader(payload)
var tarReader *tar.Reader
if strings.HasSuffix(strings.ToLower(archiveName), ".tgz") || strings.HasSuffix(strings.ToLower(archiveName), ".tar.gz") || (len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b) {
gzipReader, err := gzip.NewReader(reader)
if err == nil {
defer gzipReader.Close()
tarReader = tar.NewReader(gzipReader)
} else {
reader = bytes.NewReader(payload)
tarReader = tar.NewReader(reader)
}
} else {
tarReader = tar.NewReader(reader)
}
for {
header, err := tarReader.Next()
if err == io.EOF {
return nil, os.ErrNotExist
}
if err != nil {
return nil, err
}
if header.FileInfo().IsDir() {
continue
}
if strings.Trim(filepath.ToSlash(header.Name), "/") != memberPath {
continue
}
return io.ReadAll(tarReader)
}
}
func sourceCollectionEntryPayload(collection domain.SourceCollectionRecord, payloadEntries []map[string]any, relativePath string) map[string]any {
target := strings.Trim(filepath.ToSlash(relativePath), "/")
if target == "" {
return nil
}
collectionPrefix := strings.Trim(filepath.ToSlash(collection.RelativePath), "/")
fullTarget := target
if collectionPrefix != "" && target != collectionPrefix && !strings.HasPrefix(target, collectionPrefix+"/") {
fullTarget = collectionPrefix + "/" + target
}
for _, entry := range payloadEntries {
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if entryRelativePath == "" {
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
if entryRelativePath == fullTarget || entryRelativePath == target {
return entry
}
}
return nil
}
func writeSourceContent(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, payload []byte) {
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if mimeType == "" && len(payload) > 0 {
mimeType = http.DetectContentType(payload)
}
if mimeType != "" {
writer.Header().Set("Content-Type", mimeType)
}
writeSourceContentDisposition(writer, request, fileName)
http.ServeContent(writer, request, fileName, time.Time{}, bytes.NewReader(payload))
}
func sourceContentDownloadRequested(request *http.Request) bool {
value := strings.ToLower(strings.TrimSpace(request.URL.Query().Get("download")))
return value == "1" || value == "true" || value == "yes"
}
func writeSourceContentDisposition(writer http.ResponseWriter, request *http.Request, fileName string) {
if !sourceContentDownloadRequested(request) {
return
}
name := strings.TrimSpace(path.Base(fileName))
if name == "" || name == "." || name == "/" {
name = "artifact"
}
writer.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": name}))
}
func writeSourceContentFile(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, filePath string) error {
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if mimeType == "" {
header := make([]byte, 512)
count, readErr := file.Read(header)
if readErr != nil && readErr != io.EOF {
return readErr
}
mimeType = http.DetectContentType(header[:count])
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
}
if mimeType != "" {
writer.Header().Set("Content-Type", mimeType)
}
writeSourceContentDisposition(writer, request, fileName)
http.ServeContent(writer, request, fileName, fileInfo.ModTime(), file)
return nil
}
func sourceBrowserPreviewCacheKey(request sourceBrowserPreviewRequest) string {
hash := sha256.New()
hash.Write([]byte("browser-preview-v3"))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.FileName)))
hash.Write([]byte{0})
// Cache on the object coordinates, not on the presigned URL: presigned
// URLs are regenerated per request (timestamped authorization), so a key
// derived from the URL never matches across requests and the expensive
// transcode result never hits the cache. The bucket/object identity is
// embedded in RedirectURL queries or Payload, but the stable object
// coordinates arrive through ObjectBucket/ObjectKey when the input is a
// storage object; otherwise fall back to the URL payload hash.
if request.ObjectBucket != "" || request.ObjectKey != "" {
hash.Write([]byte("object"))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.ObjectBucket)))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.ObjectKey)))
}
if len(request.Payload) > 0 {
payloadHash := sha256.Sum256(request.Payload)
hash.Write([]byte("payload"))
hash.Write([]byte{0})
hash.Write(payloadHash[:])
}
return hex.EncodeToString(hash.Sum(nil))
}
func sourceBrowserPreviewCacheDir() (string, error) {
cacheDir := filepath.Join(os.TempDir(), "droboticflow", "source-video-previews")
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
return "", err
}
return cacheDir, nil
}
func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string, error) {
cacheDir, err := sourceBrowserPreviewCacheDir()
if err != nil {
return "", err
}
maybePruneSourceBrowserPreviewCache(cacheDir)
cacheKey := sourceBrowserPreviewCacheKey(request)
return filepath.Join(cacheDir, cacheKey+".mp4"), nil
}
// The on-disk preview cache is bounded: entries older than the TTL are
// deleted outright, then least-recently-written entries are evicted until
// the directory is under the size cap. Without this bound the /tmp cache
// grows without limit on pods that serve many long videos.
const (
sourceBrowserPreviewCacheMaxBytes = 2 << 30 // 2 GiB total cap
sourceBrowserPreviewCacheMaxAge = 7 * 24 * time.Hour
sourceBrowserPreviewCachePruneEvery = 10 * time.Minute
)
var lastSourceBrowserPreviewCachePrune atomic.Int64
func maybePruneSourceBrowserPreviewCache(dir string) {
interval := sourceBrowserPreviewCachePruneEvery.Nanoseconds()
now := time.Now().UnixNano()
last := lastSourceBrowserPreviewCachePrune.Load()
if now-last < interval {
return
}
if !lastSourceBrowserPreviewCachePrune.CompareAndSwap(last, now) {
return
}
go pruneSourceBrowserPreviewCache(dir, time.Now())
}
// pruneSourceBrowserPreviewCache enforces the TTL and size bounds on the
// preview cache directory. Best effort: unreadable entries are skipped and
// removal failures are ignored (the next prune retries).
func pruneSourceBrowserPreviewCache(dir string, now time.Time) {
pruneSourceBrowserPreviewCacheWithLimit(dir, now, sourceBrowserPreviewCacheMaxBytes)
}
// pruneSourceBrowserPreviewCacheWithLimit is the testable core of the cache
// eviction: delete entries past the TTL, then evict least-recently-written
// entries until the directory is under maxBytes.
func pruneSourceBrowserPreviewCacheWithLimit(dir string, now time.Time, maxBytes int64) {
entries, err := os.ReadDir(dir)
if err != nil {
return
}
type cacheEntry struct {
path string
modTime time.Time
size int64
}
files := make([]cacheEntry, 0, len(entries))
var total int64
for _, entry := range entries {
if entry.IsDir() {
continue
}
info, infoErr := entry.Info()
if infoErr != nil {
continue
}
if now.Sub(info.ModTime()) > sourceBrowserPreviewCacheMaxAge {
_ = os.Remove(filepath.Join(dir, entry.Name()))
continue
}
files = append(files, cacheEntry{
path: filepath.Join(dir, entry.Name()),
modTime: info.ModTime(),
size: info.Size(),
})
total += info.Size()
}
if total <= maxBytes {
return
}
sort.Slice(files, func(left int, right int) bool {
return files[left].modTime.Before(files[right].modTime)
})
for _, file := range files {
if total <= maxBytes {
break
}
if removeErr := os.Remove(file.path); removeErr == nil {
total -= file.size
}
}
}
func parseBOSObjectPath(sourcePath string) (string, string, bool) {
bucket, object, ok := parseBOSPath(sourcePath)
if !ok || object == "" {
return "", "", false
}
return bucket, object, true
}
func buildSourceCollectionSignedPreviewURL(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil {
return "", fmt.Errorf("scan config is unavailable")
}
if chooseDataSource(scanConfig.DataSource) != "BOS" {
return "", fmt.Errorf("scan config is not BOS")
}
bucket, object, ok := parseBOSObjectPath(sourcePath)
if !ok {
return "", fmt.Errorf("invalid BOS source path: %s", sourcePath)
}
accessKeyID := strings.TrimSpace(scanConfig.AccessKeyID)
secretAccessKey := strings.TrimSpace(scanConfig.SecretAccessKey)
if accessKeyID == "" || secretAccessKey == "" {
return "", fmt.Errorf("BOS credentials are unavailable")
}
endpoint := chooseBOSEndpoint(scanConfig.Endpoint)
if endpoint == "" {
return "", fmt.Errorf("BOS endpoint is unavailable")
}
client, err := bos.NewClient(accessKeyID, secretAccessKey, endpoint)
if err != nil {
return "", err
}
previewURL := strings.TrimSpace(client.BasicGeneratePresignedUrl(bucket, object, 3600))
if previewURL == "" {
return "", fmt.Errorf("failed to generate BOS preview url")
}
return previewURL, nil
}
func locateSourceBrowserVideoPreviewPath(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
finalPath, err := sourceBrowserPreviewCachePath(request)
if err != nil {
return "", err
}
if fileInfo, err := os.Stat(finalPath); err == nil && fileInfo.Size() > 0 {
return finalPath, nil
}
return "", os.ErrNotExist
}
func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWriter, request sourceBrowserPreviewRequest) error {
if strings.TrimSpace(request.RedirectURL) == "" && len(request.Payload) == 0 {
return fmt.Errorf("source video content is unavailable")
}
finalPath, err := sourceBrowserPreviewCachePath(request)
if err != nil {
return err
}
lock := sourceBrowserPreviewLock(filepath.Base(finalPath))
lock.Lock()
defer lock.Unlock()
if info, statErr := os.Stat(finalPath); statErr == nil && info.Size() > 0 {
cached, openErr := os.Open(finalPath)
if openErr != nil {
return openErr
}
defer cached.Close()
_, copyErr := io.Copy(writer, cached)
return copyErr
}
inputRef := strings.TrimSpace(request.RedirectURL)
var cleanupInput func()
if inputRef == "" {
extension := path.Ext(request.FileName)
if extension == "" {
extension = ".bin"
}
cacheDir := filepath.Dir(finalPath)
inputFile, err := os.CreateTemp(cacheDir, "source-video-*"+extension)
if err != nil {
return err
}
if _, err := inputFile.Write(request.Payload); err != nil {
inputFile.Close()
os.Remove(inputFile.Name())
return err
}
if err := inputFile.Close(); err != nil {
os.Remove(inputFile.Name())
return err
}
inputRef = inputFile.Name()
cleanupInput = func() {
_ = os.Remove(inputFile.Name())
}
}
if cleanupInput != nil {
defer cleanupInput()
}
cacheDir := filepath.Dir(finalPath)
outputFile, err := os.CreateTemp(cacheDir, sourceBrowserPreviewCacheKey(request)+"-*.mp4")
if err != nil {
return err
}
tempOutputPath := outputFile.Name()
stderr := &bytes.Buffer{}
// A BOS presigned URL can stall mid-stream; without a read timeout the
// transcode hangs until the process is killed and the viewer sees a
// truncated preview. 15 minutes of no-progress terminates the pull.
command := exec.CommandContext(
context.WithoutCancel(ctx),
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel", "error",
"-rw_timeout", "15000000000",
"-reconnect", "1",
"-reconnect_streamed", "1",
"-reconnect_delay_max", "5",
"-i", inputRef,
"-map", "0:v:0",
"-map", "0:a?",
"-threads", "4",
"-probesize", "32M",
"-analyzeduration", "0",
"-c:v", "libx264",
"-preset", "ultrafast",
"-tune", "zerolatency",
"-crf", "30",
"-vf", "fps=12,scale=min(960\\,iw):-2",
"-g", "12",
"-keyint_min", "12",
"-pix_fmt", "yuv420p",
"-movflags", "frag_keyframe+empty_moov+default_base_moof",
"-c:a", "aac",
"-b:a", "96k",
"-ac", "1",
"-f", "mp4",
"pipe:1",
)
command.Stderr = stderr
stdout, err := command.StdoutPipe()
if err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
return err
}
if err := command.Start(); err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
message := strings.TrimSpace(stderr.String())
if message != "" {
return fmt.Errorf("generate browser video preview: %s", message)
}
return err
}
streamTarget := io.Writer(writer)
if flusher, ok := writer.(http.Flusher); ok {
streamTarget = flushResponseWriter{writer: writer, flusher: flusher}
}
var clientErr error
bestEffort := &bestEffortPreviewWriter{target: streamTarget, err: &clientErr}
if _, err := io.Copy(io.MultiWriter(bestEffort, outputFile), stdout); err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
_ = command.Wait()
return err
}
if err := outputFile.Close(); err != nil {
_ = os.Remove(tempOutputPath)
_ = command.Wait()
return err
}
if err := command.Wait(); err != nil {
message := strings.TrimSpace(stderr.String())
_ = os.Remove(tempOutputPath)
if message != "" {
return fmt.Errorf("generate browser video preview: %s", message)
}
return fmt.Errorf("generate browser video preview: %w", err)
}
if err := os.Rename(tempOutputPath, finalPath); err != nil {
if fileInfo, statErr := os.Stat(finalPath); statErr == nil && fileInfo.Size() > 0 {
_ = os.Remove(tempOutputPath)
return nil
}
_ = os.Remove(tempOutputPath)
return err
}
return nil
}
// sourceVideoPreviewSyncMode is the server-side rollback switch for the C5
// video direct-link change: setting DROBOTICFLOW_VIDEO_PREVIEW_SYNC=1 keeps
// every browser-preview request on the legacy blocking ffmpeg stream (no
// background pre-transcode, no redirect to the direct storage URL).
var sourceVideoPreviewSyncMode = sync.OnceValue(func() bool {
return strings.TrimSpace(os.Getenv("DROBOTICFLOW_VIDEO_PREVIEW_SYNC")) == "1"
})
// discardPreviewResponseWriter lets the background pre-transcode reuse
// streamSourceBrowserVideoPreview unchanged: its output is dropped and only
// the cached artifact on disk matters.
type discardPreviewResponseWriter struct{}
func (discardPreviewResponseWriter) Header() http.Header { return http.Header{} }
func (discardPreviewResponseWriter) WriteHeader(int) {}
func (discardPreviewResponseWriter) Write(p []byte) (int, error) { return len(p), nil }
// startSourceBrowserVideoPretranscode warms the preview cache in the
// background. The per-key lock inside streamSourceBrowserVideoPreview
// deduplicates concurrent warmups, and a warm run that finds the cache
// already populated returns immediately without invoking ffmpeg.
func startSourceBrowserVideoPretranscode(previewRequest sourceBrowserPreviewRequest) {
go func() {
// ffmpeg's own rw_timeout (15 minutes without progress) bounds the
// pull; streamSourceBrowserVideoPreview deliberately detaches
// cancellation so a client disconnect cannot truncate the warmup.
// Routed through the writer var so tests can stub the transcode.
if err := sourceBrowserVideoPreviewWriter(context.Background(), discardPreviewResponseWriter{}, previewRequest); err != nil {
fmt.Printf("[api] background browser video preview transcode failed: %v\n", err)
}
}()
}
// writeBrowserVideoPreviewResponse serves the browser-transcoded preview for
// a video entry (query transcode=browser). Order of preference:
// 1. serve the cached pre-transcoded mp4 when present (no ffmpeg at all);
// 2. start a background pre-transcode and immediately redirect the player
// to the direct storage URL (BOS presigned GET link), so the preview
// first frame is never blocked behind server-side ffmpeg;
// 3. legacy blocking ffmpeg stream as fallback (no direct URL available,
// explicit sync=1 request parameter, or the
// DROBOTICFLOW_VIDEO_PREVIEW_SYNC=1 rollback switch).
//
// All video responses set Content-Encoding: identity so the global chi
// Compress(5) middleware skips gzip: mp4 payloads are already compressed
// media and re-compressing them only burned API pod CPU.
func writeBrowserVideoPreviewResponse(writer http.ResponseWriter, request *http.Request, previewRequest sourceBrowserPreviewRequest, directURL string) {
// Videos are already-compressed media; the identity marker makes chi's
// Compress middleware leave the byte stream untouched.
writer.Header().Set("Content-Encoding", "identity")
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(previewRequest.FileName, path.Ext(previewRequest.FileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
direct := strings.TrimSpace(directURL)
forcedSync := sourceVideoPreviewSyncMode() ||
strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("sync")), "1")
if direct != "" && strings.HasPrefix(strings.ToLower(direct), "http") && !forcedSync {
startSourceBrowserVideoPretranscode(previewRequest)
http.Redirect(writer, request, direct, http.StatusTemporaryRedirect)
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] browser preview stream failed: %v\n", err)
}
}
type bestEffortPreviewWriter struct {
target io.Writer
err *error
}
func (w *bestEffortPreviewWriter) Write(payload []byte) (int, error) {
n, err := w.target.Write(payload)
if err != nil {
if w.err != nil && *w.err == nil {
*w.err = err
}
return len(payload), nil
}
return n, nil
}
func derivedSourceEntriesFromIndex(collection domain.SourceCollectionRecord, payloadItems []map[string]any) []map[string]any {
entries := make([]map[string]any, 0, len(payloadItems)+2)
if len(collection.Meta) > 0 {
entries = append(entries, map[string]any{
"collectionId": collection.CollectionID,
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/meta.json", "/"),
"sizeBytes": len(fmt.Sprintf("%v", collection.Meta)),
"mimeType": "application/json",
"data": collection.Meta,
"derivedFromIndex": true,
})
}
if collection.HasVideoMeta && len(payloadItems) > 0 {
videoRows := make([]map[string]any, 0, len(payloadItems))
for _, item := range payloadItems {
videoRows = append(videoRows, map[string]any{
"filename": stringValue(item["filename"]),
"codec": stringValue(item["codec"]),
"resolution": stringValue(item["resolution"]),
"frameRate": floatValue(item["frameRate"]),
"duration": floatValue(item["duration"]),
"startTime": item["startTime"],
})
}
entries = append(entries, map[string]any{
"collectionId": collection.CollectionID,
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/video_meta.json", "/"),
"mimeType": "application/json",
"data": videoRows,
"derivedFromIndex": true,
})
}
for _, item := range payloadItems {
entry := map[string]any{
"collectionId": collection.CollectionID,
"relativePath": stringValue(item["relativePath"]),
"path": stringValue(item["path"]),
"sizeBytes": int64Value(item["sizeBytes"]),
"mimeType": stringValue(mapValue(item["source"])["mimeType"]),
"source": mapValue(item["source"]),
"codec": stringValue(item["codec"]),
"resolution": stringValue(item["resolution"]),
"frameRate": floatValue(item["frameRate"]),
"duration": floatValue(item["duration"]),
"derivedFromIndex": true,
}
entries = append(entries, entry)
}
return entries
}
func buildSourceEntries(run domain.RunRecord, collection domain.SourceCollectionRecord, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, prefix string) []domain.SourceCollectionEntry {
if len(payloadEntries) == 0 {
payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems)
}
videoMetaByRelative := make(map[string]map[string]any, len(payloadItems))
for _, item := range payloadItems {
videoMetaByRelative[stringValue(item["relativePath"])] = item
}
normalizedPrefix := strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/")
prefixSlash := normalizedPrefix
if prefixSlash != "" {
prefixSlash += "/"
}
directories := make(map[string]domain.SourceCollectionEntry)
files := make([]domain.SourceCollectionEntry, 0)
collectionPrefix := strings.Trim(collection.RelativePath, "/")
if collectionPrefix != "" {
collectionPrefix += "/"
}
for _, entryPayload := range payloadEntries {
relativePath := strings.Trim(filepath.ToSlash(stringValue(entryPayload["relativePath"])), "/")
if relativePath == "" {
relativePath = strings.Trim(filepath.ToSlash(stringValue(entryPayload["path"])), "/")
}
if relativePath == "" {
continue
}
if collectionPrefix != "" && relativePath != strings.TrimSuffix(collectionPrefix, "/") && !strings.HasPrefix(relativePath, collectionPrefix) {
continue
}
entryPath := relativePath
if collectionPrefix != "" {
entryPath = strings.TrimPrefix(relativePath, collectionPrefix)
}
if entryPath == "" {
continue
}
if normalizedPrefix != "" {
if entryPath == normalizedPrefix {
// exact file match
} else if !strings.HasPrefix(entryPath, prefixSlash) {
continue
} else {
entryPath = strings.TrimPrefix(entryPath, prefixSlash)
}
}
if entryPath == "" {
continue
}
parts := strings.Split(entryPath, "/")
if len(parts) > 1 {
dirPath := parts[0]
displayPath := dirPath
if normalizedPrefix != "" {
displayPath = normalizedPrefix + "/" + dirPath
}
if _, exists := directories[displayPath]; !exists {
directories[displayPath] = domain.SourceCollectionEntry{
Path: displayPath,
Name: dirPath,
Kind: "directory",
CollectionName: collection.CollectionName,
DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]),
}
} else if boolValue(entryPayload["derivedFromIndex"]) {
entry := directories[displayPath]
entry.DerivedFromIndex = true
directories[displayPath] = entry
}
continue
}
displayPath := parts[0]
if normalizedPrefix != "" && displayPath != normalizedPrefix {
displayPath = normalizedPrefix + "/" + displayPath
}
artifact := sourceEntryArtifact(run, collection.CollectionID, entryPayload, archiveSources)
videoMeta := videoMetaByRelative[filepath.ToSlash(strings.Trim(collection.RelativePath, "/")+"/"+displayPath)]
entry := domain.SourceCollectionEntry{
Path: displayPath,
Name: parts[0],
Kind: "file",
CollectionName: collection.CollectionName,
SizeBytes: int64Value(entryPayload["sizeBytes"]),
Artifact: artifact,
DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]),
}
if videoMeta != nil {
entry.Kind = "video"
entry.Codec = stringValue(videoMeta["codec"])
entry.Resolution = stringValue(videoMeta["resolution"])
entry.FrameRate = floatValue(videoMeta["frameRate"])
entry.Duration = floatValue(videoMeta["duration"])
entry.Metadata = videoMeta
if entry.SizeBytes == 0 {
entry.SizeBytes = int64Value(videoMeta["sizeBytes"])
}
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".json") {
entry.Kind = "json"
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".mp4") {
entry.Kind = "video"
}
if entry.SizeBytes == 0 {
entry.SizeBytes = int64Value(entryPayload["sizeBytes"])
}
files = append(files, entry)
}
items := make([]domain.SourceCollectionEntry, 0, len(directories)+len(files))
for _, item := range directories {
items = append(items, item)
}
sort.Slice(items, func(left int, right int) bool { return items[left].Path < items[right].Path })
sort.Slice(files, func(left int, right int) bool { return files[left].Path < files[right].Path })
items = append(items, files...)
return items
}
func sourceCollectionContentPayload(collection domain.SourceCollectionRecord, scanConfig *domain.ScanConfig, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, relativePath string) ([]byte, string, string, string, error) {
if len(payloadEntries) == 0 {
payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems)
}
entry := sourceCollectionEntryPayload(collection, payloadEntries, relativePath)
if entry == nil {
return nil, "", "", "", os.ErrNotExist
}
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if entryRelativePath == "" {
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
fileName := path.Base(entryRelativePath)
source := datasetIndexEntrySource(entry, archiveSources)
mimeType := stringValue(source["mimeType"])
if mimeType == "" {
mimeType = stringValue(entry["mimeType"])
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if value, ok := entry["data"]; ok {
payload, err := json.MarshalIndent(normalizeStructuredValue(value), "", " ")
if err != nil {
return nil, "", "", "", err
}
if mimeType == "" {
mimeType = "application/json"
}
return payload, mimeType, fileName, "", nil
}
if sourcePath := stringValue(source["path"]); sourcePath != "" && !strings.Contains(sourcePath, "://") {
if payload, err := os.ReadFile(sourcePath); err == nil {
return payload, mimeType, fileName, "", nil
}
}
if sourcePath := stringValue(source["path"]); strings.HasPrefix(strings.ToLower(sourcePath), "bos://") && scanConfig != nil {
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
source["previewURL"] = previewURL
}
}
archiveKey := stringValue(source["archiveKey"])
archiveMemberPath := stringValue(source["archiveMemberPath"])
if archiveKey != "" && archiveMemberPath != "" {
archiveSource := mapValue(archiveSources[archiveKey])
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
payload, err := contentBytesForArchiveMember(stringValue(archiveSource["name"]), previewURL, archiveMemberPath)
if err == nil {
return payload, mimeType, fileName, "", nil
}
}
}
if previewURL := stringValue(source["previewURL"]); previewURL != "" {
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
return nil, "", fileName, previewURL, nil
}
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
if err == nil {
if mimeType == "" {
mimeType = decodedMimeType
}
return payload, mimeType, fileName, "", nil
}
}
return nil, "", "", "", os.ErrNotExist
}
func (server *server) handleSourceCollectionsList(writer http.ResponseWriter, request *http.Request) {
workflowByID, latestRuns, err := server.latestCompletedScanRuns(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items := make([]domain.SourceCollectionRecord, 0)
for _, run := range latestRuns {
if !server.filterByTenant(request, run.OrganizationID) {
continue
}
artifact := artifactByName(run, "dataset-index.json")
payload := artifactStructuredData(artifact)
if artifact == nil || payload == nil {
continue
}
for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") {
collection := mapValue(raw)
if collection == nil {
continue
}
workflow := workflowByID[run.WorkflowID]
items = append(items, server.buildSourceCollectionRecord(request.Context(), run, &workflow, collection))
}
}
sort.Slice(items, func(left int, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
if items[left].WorkflowName == items[right].WorkflowName {
return items[left].CollectionID < items[right].CollectionID
}
return items[left].WorkflowName < items[right].WorkflowName
}
return items[left].CreatedAt > items[right].CreatedAt
})
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleSourceCollectionDetail(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
detail := domain.SourceCollectionDetail{
Collection: record,
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, ""),
}
renderJSON(writer, http.StatusOK, detail)
}
func (server *server) handleSourceCollectionEntries(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
prefix := request.URL.Query().Get("prefix")
detail := domain.SourceCollectionDetail{
Collection: record,
Prefix: prefix,
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, prefix),
}
renderJSON(writer, http.StatusOK, detail)
}
func (server *server) handleSourceCollectionContent(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
if relativePath == "" {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
return
}
var effectiveScanConfig *domain.ScanConfig
if payload.Workflow != nil {
effectiveScanConfig, err = server.resolveScanConfig(request.Context(), payload.Workflow.ScanConfig)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
content, mimeType, fileName, redirectURL, err := sourceCollectionContentPayload(record, effectiveScanConfig, payload.Entries, payload.Items, payload.ArchiveSource, relativePath)
if err != nil {
if os.IsNotExist(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source file not found"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
Payload: content,
RedirectURL: redirectURL,
}
writeBrowserVideoPreviewResponse(writer, request, previewRequest, redirectURL)
return
}
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
writeSourceContent(writer, request, fileName, mimeType, content)
}
func packageArtifactCandidatesByRelativePath(run domain.RunRecord, packageName string, relativePath string) []domain.ExecutionArtifact {
target := strings.Trim(filepath.ToSlash(relativePath), "/")
if target == "" {
return nil
}
if _, payload := wilorPackageManifestArtifact(run, packageName); payload != nil {
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range wilorPackageArtifacts(run, packageName) {
if strings.Trim(filepath.ToSlash(artifact.Name), "/") == target {
items = append(items, artifact)
}
}
sort.SliceStable(items, func(left int, right int) bool {
return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right])
})
return items
}
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range run.Artifacts {
if packageArtifactResolvedRelativePath(artifact, packageName) == target {
items = append(items, artifact)
}
}
sort.SliceStable(items, func(left int, right int) bool {
return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right])
})
return items
}
func workflowNodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
values := make(map[string]string, len(node.Data.Hyperparams))
for _, item := range node.Data.Hyperparams {
values[item.Key] = item.Value
}
return values
}
func (server *server) packageArtifactScanConfig(ctx context.Context, workflow *domain.WorkflowRecord, artifact domain.ExecutionArtifact) (*domain.ScanConfig, error) {
node := workflowNodeByID(workflow, artifact.NodeID)
if node == nil {
return nil, nil
}
params := workflowNodeHyperparamsMap(*node)
dataSource := chooseDataSource(params["DATA_SOURCE"])
sourcePath := strings.TrimSpace(params["SOURCE_PATH"])
if dataSource != "BOS" && strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
dataSource = "BOS"
}
if dataSource != "BOS" && strings.TrimSpace(node.Data.ConnectionID) != "" {
dataSource = "BOS"
}
if dataSource != "BOS" {
return nil, nil
}
scanConfig := &domain.ScanConfig{
DataSource: "BOS",
SourcePath: sourcePath,
ConnectionID: strings.TrimSpace(node.Data.ConnectionID),
Bucket: strings.TrimSpace(node.Data.Bucket),
Prefix: cleanBOSPrefix(node.Data.Prefix),
Endpoint: strings.TrimSpace(params["BOS_ENDPOINT"]),
AccessKeyID: strings.TrimSpace(params["BOS_ACCESS_KEY_ID"]),
SecretAccessKey: strings.TrimSpace(params["BOS_SECRET_ACCESS_KEY"]),
}
return server.resolveScanConfig(ctx, scanConfig)
}
func (server *server) packageContentPayload(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, packageName string, relativePath string) ([]byte, string, string, string, string, error) {
artifacts := packageArtifactCandidatesByRelativePath(run, packageName, relativePath)
if len(artifacts) == 0 {
return nil, "", "", "", "", os.ErrNotExist
}
fileName := path.Base(strings.Trim(filepath.ToSlash(relativePath), "/"))
for _, artifact := range artifacts {
mimeType := ""
if artifact.Metadata != nil {
mimeType = strings.TrimSpace(artifact.Metadata.MimeType)
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
filePath := strings.TrimSpace(artifact.Path)
if filePath != "" && !strings.Contains(filePath, "://") {
if _, err := os.Stat(filePath); err == nil {
return nil, mimeType, fileName, filePath, "", nil
}
}
structuredData := artifactStructuredData(&artifact)
if artifact.Type == "json" && structuredData != nil {
payload, err := json.MarshalIndent(normalizeStructuredValue(structuredData), "", " ")
if err != nil {
return nil, "", "", "", "", err
}
if mimeType == "" {
mimeType = "application/json"
}
return payload, mimeType, fileName, "", "", nil
}
source := mapValue(structuredData["source"])
sourcePath := stringValue(source["path"])
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
scanConfig, err := server.packageArtifactScanConfig(ctx, workflow, artifact)
if err != nil {
return nil, "", "", "", "", err
}
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
return nil, mimeType, fileName, "", previewURL, nil
}
}
if previewURL := packageArtifactPreviewURL(artifact); previewURL != "" {
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
return nil, mimeType, fileName, "", previewURL, nil
}
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
if err == nil {
if mimeType == "" {
mimeType = decodedMimeType
}
return payload, mimeType, fileName, "", "", nil
}
}
}
return nil, "", "", "", "", os.ErrNotExist
}
func (server *server) handlePackageContent(writer http.ResponseWriter, request *http.Request) {
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if run == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Run") {
return
}
packageName := chi.URLParam(request, "packageName")
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
if relativePath == "" {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
return
}
workflow, err := server.store.GetWorkflow(request.Context(), run.WorkflowID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
content, mimeType, fileName, filePath, redirectURL, err := server.packageContentPayload(request.Context(), *run, workflow, packageName, relativePath)
if err != nil {
if os.IsNotExist(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package file not found"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
Payload: content,
}
directURL := ""
if filePath != "" {
previewRequest.RedirectURL = filePath
} else {
previewRequest.RedirectURL = redirectURL
directURL = redirectURL
}
writeBrowserVideoPreviewResponse(writer, request, previewRequest, directURL)
return
}
if filePath != "" {
if err := writeSourceContentFile(writer, request, fileName, mimeType, filePath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
writeSourceContent(writer, request, fileName, mimeType, content)
}
func packageRecordFromManifest(run domain.RunRecord, item map[string]any, files []domain.ExecutionArtifact, rawFiles []domain.ExecutionArtifact) domain.PackageRecord {
pipelineTags := make([]string, 0)
for _, value := range listValue(item["pipelineTags"]) {
pipelineTags = append(pipelineTags, stringValue(value))
}
storagePath := stringValue(item["storagePath"])
for _, artifact := range rawFiles {
if candidate := packageArtifactStoragePath(artifact, stringValue(item["packageName"])); candidate != "" {
storagePath = candidate
break
}
}
fileCount := intValue(item["fileCount"])
if len(files) > 0 {
fileCount = len(files)
}
sourceVideo := stringValue(item["sourceVideo"])
if sourceVideo == "" {
sourceVideo = stringValue(mapValue(item["annotation"])["sourceVideo"])
}
if sourceVideo == "" {
sourceVideo = stringValue(mapValue(item["source"])["name"])
}
clipDuration := floatValue(item["clipDuration"])
if clipDuration <= 0 {
clipDuration = floatValue(item["duration"])
}
quality := map[string]any{}
for key, value := range mapValue(item["quality"]) {
quality[key] = value
}
if stringValue(quality["source_video"]) == "" && sourceVideo != "" {
quality["source_video"] = sourceVideo
}
return domain.PackageRecord{
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
PackageName: stringValue(item["packageName"]),
ClipID: stringValue(item["clipId"]),
SourceVideo: sourceVideo,
ClipDuration: clipDuration,
FileCount: fileCount,
PipelineTags: pipelineTags,
StoragePath: storagePath,
Quality: quality,
Annotation: mapValue(item["annotation"]),
CreatedAt: run.CreatedAt,
PreviewArtifact: choosePackagePreview(files),
Storage: assetStorageLocationFromPaths("", storagePath, ""),
}
}
func (server *server) packageDetail(ctx context.Context, runID string, packageName string) (*domain.PackageDetail, error) {
run, err := server.store.GetRun(ctx, runID)
if err != nil || run == nil {
return nil, err
}
manifest := artifactByName(*run, "packages-manifest.json")
manifestPayload := artifactStructuredData(manifest)
if manifest != nil && manifestPayload != nil {
rawFiles := dedupePackageArtifacts(*run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
}
for _, raw := range listValue(manifestPayload["items"]) {
item := mapValue(raw)
if stringValue(item["packageName"]) != packageName {
continue
}
record := packageRecordFromManifest(*run, item, files, rawFiles)
return &domain.PackageDetail{Package: record, Files: files}, nil
}
}
if manifest, payload := wilorPackageManifestArtifact(*run, packageName); manifest != nil && payload != nil {
files, rawFiles := wilorPackageFilesForResponse(*run, packageName)
item := wilorPackageItem(*run, *manifest, payload, rawFiles)
record := packageRecordFromManifest(*run, item, files, rawFiles)
return &domain.PackageDetail{Package: record, Files: files}, nil
}
return nil, nil
}
func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) {
runs, err := server.listRunArtifactsForRequest(request.Context(), request, string(domain.RunStatusCompleted))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items := make([]domain.PackageRecord, 0)
latestByWorkflowPackage := make(map[string]domain.PackageRecord)
for _, run := range runs {
if !server.filterByTenant(request, run.OrganizationID) {
continue
}
manifest := artifactByName(run, "packages-manifest.json")
manifestPayload := artifactStructuredData(manifest)
if manifest != nil && manifestPayload != nil {
for _, raw := range listValue(manifestPayload["items"]) {
item := mapValue(raw)
if item == nil {
continue
}
packageName := stringValue(item["packageName"])
rawFiles := dedupePackageArtifacts(run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
}
record := packageRecordFromManifest(run, item, files, rawFiles)
key := run.WorkflowID + ":" + record.PackageName
current, exists := latestByWorkflowPackage[key]
if !exists || record.CreatedAt > current.CreatedAt {
latestByWorkflowPackage[key] = record
}
}
}
for _, wilorManifest := range wilorResultArtifacts(run) {
payload := wilorResultPayload(wilorManifest)
if payload == nil {
continue
}
packageName := wilorPackageName(wilorManifest, payload)
files, rawFiles := wilorPackageFilesForResponse(run, packageName)
item := wilorPackageItem(run, wilorManifest, payload, rawFiles)
record := packageRecordFromManifest(run, item, files, rawFiles)
key := run.WorkflowID + ":" + record.PackageName
current, exists := latestByWorkflowPackage[key]
if !exists || record.CreatedAt > current.CreatedAt {
latestByWorkflowPackage[key] = record
}
}
}
for _, item := range latestByWorkflowPackage {
items = append(items, item)
}
sort.Slice(items, func(left int, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
return items[left].PackageName < items[right].PackageName
}
return items[left].CreatedAt > items[right].CreatedAt
})
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handlePackageDetail(writer http.ResponseWriter, request *http.Request) {
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if run == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
return
}
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Package") {
return
}
item, err := server.packageDetail(request.Context(), run.ID, chi.URLParam(request, "packageName"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
return
}
renderJSON(writer, http.StatusOK, item)
}
var supportedSamplingRuleOperators = map[string]struct{}{
"contains": {},
"eq": {},
"exists": {},
"gt": {},
"gte": {},
"in": {},
"lt": {},
"lte": {},
"missing": {},
"ne": {},
}
func samplingRulesFromManifest(manifest map[string]any) ([]domain.SamplingRuleSummary, bool) {
rules := make([]domain.SamplingRuleSummary, 0)
ruleIDs := make(map[string]struct{})
for _, raw := range listValue(manifest["rules"]) {
item := mapValue(raw)
if item == nil {
return nil, false
}
id := strings.TrimSpace(stringValue(item["id"]))
field := strings.TrimSpace(stringValue(item["field"]))
operator := strings.ToLower(strings.TrimSpace(stringValue(item["operator"])))
samplePercent, ok := samplePercentValue(item["samplePercent"])
if id == "" || field == "" || !ok {
return nil, false
}
if _, exists := ruleIDs[id]; exists {
return nil, false
}
if _, supported := supportedSamplingRuleOperators[operator]; !supported {
return nil, false
}
ruleIDs[id] = struct{}{}
rules = append(rules, domain.SamplingRuleSummary{
ID: id,
Field: field,
Operator: operator,
Value: item["value"],
SamplePercent: samplePercent,
MatchedCount: intValue(item["matchedCount"]),
SampledCount: intValue(item["sampledCount"]),
})
}
return rules, len(rules) > 0
}
func samplePercentValue(value any) (float64, bool) {
switch typed := value.(type) {
case int:
return boundedSamplePercent(float64(typed))
case int8:
return boundedSamplePercent(float64(typed))
case int16:
return boundedSamplePercent(float64(typed))
case int32:
return boundedSamplePercent(float64(typed))
case int64:
return boundedSamplePercent(float64(typed))
case uint:
return boundedSamplePercent(float64(typed))
case uint8:
return boundedSamplePercent(float64(typed))
case uint16:
return boundedSamplePercent(float64(typed))
case uint32:
return boundedSamplePercent(float64(typed))
case uint64:
return boundedSamplePercent(float64(typed))
case float32:
return boundedSamplePercent(float64(typed))
case float64:
return boundedSamplePercent(typed)
case json.Number:
parsed, err := typed.Float64()
if err != nil {
return 0, false
}
return boundedSamplePercent(parsed)
default:
return 0, false
}
}
func boundedSamplePercent(value float64) (float64, bool) {
if math.IsNaN(value) || math.IsInf(value, 0) || value <= 0 || value > 100 {
return 0, false
}
return value, true
}
func buildSamplingItems(run domain.RunRecord, manifest map[string]any) []domain.SamplingItemRecord {
items := make([]domain.SamplingItemRecord, 0)
previewByPackage := make(map[string]*domain.ExecutionArtifact)
for _, raw := range listValue(manifest["items"]) {
item := mapValue(raw)
packageName := stringValue(item["packageName"])
if _, ok := previewByPackage[packageName]; !ok {
previewByPackage[packageName] = choosePackagePreview(packageArtifacts(run, packageName))
}
preview := previewByPackage[packageName]
items = append(items, domain.SamplingItemRecord{
ID: domain.NewID("sampling-item"),
PackageName: packageName,
ClipID: stringValue(item["clipId"]),
SourceVideo: stringValue(item["sourceVideo"]),
SampleRuleID: stringValue(item["sampleRuleId"]),
StoragePath: stringValue(item["storagePath"]),
Quality: mapValue(item["quality"]),
Annotation: mapValue(item["annotation"]),
PreviewArtifact: preview,
Status: domain.SamplingItemStatusPending,
})
}
return items
}
func (server *server) syncSamplingTasks(ctx context.Context) error {
existing, err := server.listSamplingTasksForContext(ctx, "")
if err != nil {
return err
}
existingByRun := make(map[string]domain.SamplingTaskRecord, len(existing))
for _, task := range existing {
existingByRun[task.RunID] = task
}
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
if err != nil {
return err
}
for _, run := range runs {
if !server.filterByTenantContext(ctx, run.OrganizationID) {
continue
}
if _, exists := existingByRun[run.ID]; exists {
continue
}
manifest := artifactByName(run, "audit-samples.json")
manifestPayload := artifactStructuredData(manifest)
if manifest == nil || manifestPayload == nil || len(listValue(manifestPayload["items"])) == 0 {
continue
}
rules, ok := samplingRulesFromManifest(manifestPayload)
if !ok {
continue
}
now := nowISO()
items := buildSamplingItems(run, manifestPayload)
task := domain.SamplingTaskRecord{
ID: domain.NewID("sampling"),
OrganizationID: normalizedTenantID(run.OrganizationID),
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
Name: fmt.Sprintf("%s 抽检任务", run.WorkflowName),
Description: fmt.Sprintf("来源于运行 %s 的真实抽检采样结果", run.ID),
Status: domain.SamplingTaskStatusPending,
Rules: rules,
Items: items,
CreatedAt: now,
UpdatedAt: now,
}
for index := range task.Items {
task.Items[index].TaskID = task.ID
}
if _, err := server.store.CreateSamplingTask(ctx, task); err != nil {
return err
}
}
return nil
}
func (server *server) handleSamplingTasksList(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items, err := server.listSamplingTasksForRequest(request.Context(), request, request.URL.Query().Get("status"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items = server.filterSamplingTasksForRequest(request, items)
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleSamplingTaskGet(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item, err := server.store.GetSamplingTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling task not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Sampling task") {
return
}
renderJSON(writer, http.StatusOK, item)
}
func validateSamplingReviewPayload(payload samplingReviewRequest) []validationIssue {
issues := make([]validationIssue, 0, 4)
if strings.TrimSpace(payload.VideoQuality) == "" {
issues = append(issues, validationIssue{Field: "videoQuality", Message: "videoQuality is required"})
}
if strings.TrimSpace(payload.AnnotationAccuracy) == "" {
issues = append(issues, validationIssue{Field: "annotationAccuracy", Message: "annotationAccuracy is required"})
}
if strings.TrimSpace(payload.AnnotationCompleteness) == "" {
issues = append(issues, validationIssue{Field: "annotationCompleteness", Message: "annotationCompleteness is required"})
}
if strings.TrimSpace(payload.OverallGrade) == "" {
issues = append(issues, validationIssue{Field: "overallGrade", Message: "overallGrade is required"})
}
return issues
}
func (server *server) handleSamplingItemReview(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
var payload samplingReviewRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateSamplingReviewPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
tasks, err := server.listSamplingTasksForRequest(request.Context(), request, "")
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
itemID := chi.URLParam(request, "id")
for _, task := range tasks {
if !server.filterByTenant(request, task.OrganizationID) {
continue
}
updated := false
for index := range task.Items {
if task.Items[index].ID != itemID {
continue
}
task.Items[index].Status = domain.SamplingItemStatusCompleted
task.Items[index].Review = &domain.SamplingReview{
VideoQuality: strings.TrimSpace(payload.VideoQuality),
AnnotationAccuracy: strings.TrimSpace(payload.AnnotationAccuracy),
AnnotationCompleteness: strings.TrimSpace(payload.AnnotationCompleteness),
OverallGrade: strings.TrimSpace(payload.OverallGrade),
Notes: strings.TrimSpace(payload.Notes),
ReviewedAt: nowISO(),
}
updated = true
break
}
if !updated {
continue
}
task.Status = domain.SamplingTaskStatusCompleted
for _, item := range task.Items {
if item.Status != domain.SamplingItemStatusCompleted {
task.Status = domain.SamplingTaskStatusPending
break
}
}
task.UpdatedAt = nowISO()
if err := server.store.UpsertSamplingTask(request.Context(), task); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, task)
return
}
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling item not found"})
}