forked from longtao.wu/droboticflow
- C1/S3: build-time precompressed .gz served by spaHandler (writeBundle plugin), motion-vendor chunk rule fix - C3/S1: /api/runs default cursor pagination (byte-compatible), /api/human-items limit/cursor + Mongo projection + filter pushdown - C2/S5: in-process principal TTL cache (45s) with full invalidation paths; /api/users limit+q with org isolation - C4/S2: dashboard counter pushdown ($group/CountDocuments), 45s response/snapshot caches, production stats snapshot cache - C5: BOS presigned direct-link video preview (307) with background pre-transcode, /tmp cache TTL+LRU, gzip bypass, sync=1 rollback - S4/S6: POC readiness point queries, human-batches GET no longer writes (60s sweeper), admin ops page N+1 fan-out removed - Fixes found en route: BSON cursor self-reference recursion, backfill loop cursor advance, supplier cascade session flush - Frontend: nextCursor passthrough (human-loop/users), acceptance pool no longer truncated at 500 Co-authored-by: AgentTeams slow-page-fix
3517 lines
116 KiB
Go
3517 lines
116 KiB
Go
package api
|
||
|
||
import (
|
||
"archive/tar"
|
||
"archive/zip"
|
||
"bytes"
|
||
"compress/gzip"
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/base64"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"math"
|
||
"mime"
|
||
"net/http"
|
||
"net/url"
|
||
"os"
|
||
"os/exec"
|
||
"path"
|
||
"path/filepath"
|
||
"reflect"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"droboticflow/internal/domain"
|
||
|
||
"github.com/baidubce/bce-sdk-go/services/bos"
|
||
"github.com/go-chi/chi/v5"
|
||
)
|
||
|
||
type sourceBrowserPreviewRequest struct {
|
||
FileName string
|
||
Payload []byte
|
||
RedirectURL string
|
||
// ObjectBucket/ObjectKey identify the underlying storage object behind
|
||
// RedirectURL. They keep the preview cache stable across presigned URLs,
|
||
// which are regenerated (and therefore re-signed) on every request.
|
||
ObjectBucket string
|
||
ObjectKey string
|
||
}
|
||
|
||
type flushResponseWriter struct {
|
||
writer http.ResponseWriter
|
||
flusher http.Flusher
|
||
}
|
||
|
||
func (target flushResponseWriter) Write(payload []byte) (int, error) {
|
||
count, err := target.writer.Write(payload)
|
||
if count > 0 {
|
||
target.flusher.Flush()
|
||
}
|
||
return count, err
|
||
}
|
||
|
||
var sourceBrowserVideoPreviewPath = locateSourceBrowserVideoPreviewPath
|
||
var sourceBrowserVideoPreviewWriter = streamSourceBrowserVideoPreview
|
||
var sourceCollectionSignedPreviewURL = buildSourceCollectionSignedPreviewURL
|
||
|
||
var sourceBrowserPreviewLocks sync.Map
|
||
|
||
func sourceBrowserPreviewLock(key string) *sync.Mutex {
|
||
lock := &sync.Mutex{}
|
||
actual, _ := sourceBrowserPreviewLocks.LoadOrStore(key, lock)
|
||
return actual.(*sync.Mutex)
|
||
}
|
||
|
||
type scanConfigRequest struct {
|
||
Name string `json:"name"`
|
||
Description string `json:"description"`
|
||
SourcePath string `json:"sourcePath"`
|
||
ConnectionID string `json:"connectionId,omitempty"`
|
||
Bucket string `json:"bucket,omitempty"`
|
||
Prefix string `json:"prefix,omitempty"`
|
||
Endpoint string `json:"endpoint,omitempty"`
|
||
AccessKeyID string `json:"accessKeyId,omitempty"`
|
||
SecretAccessKey string `json:"secretAccessKey,omitempty"`
|
||
ScanMode string `json:"scanMode,omitempty"`
|
||
DataSourceIssue *validationIssue `json:"-"`
|
||
ScanModeIssue *validationIssue `json:"-"`
|
||
IncrementalUnit string `json:"incrementalUnit,omitempty"`
|
||
IncrementalUnitIssue *validationIssue `json:"-"`
|
||
DirectoryDepth int `json:"directoryDepth,omitempty"`
|
||
DirectoryDepthIssue *validationIssue `json:"-"`
|
||
Pattern string `json:"pattern,omitempty"`
|
||
Interval int `json:"interval"`
|
||
Enabled bool `json:"enabled"`
|
||
DataSource string `json:"dataSource,omitempty"`
|
||
Notifications domain.ScanNotificationSettings `json:"notifications,omitempty"`
|
||
}
|
||
|
||
type samplingReviewRequest struct {
|
||
VideoQuality string `json:"videoQuality"`
|
||
AnnotationAccuracy string `json:"annotationAccuracy"`
|
||
AnnotationCompleteness string `json:"annotationCompleteness"`
|
||
OverallGrade string `json:"overallGrade"`
|
||
Notes string `json:"notes,omitempty"`
|
||
}
|
||
|
||
func isScanWorkflow(item domain.WorkflowRecord) bool {
|
||
return item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil
|
||
}
|
||
|
||
func chooseWorkflowKind(value domain.WorkflowKind) domain.WorkflowKind {
|
||
if value == "" {
|
||
return domain.WorkflowKindStandard
|
||
}
|
||
return value
|
||
}
|
||
|
||
func chooseDataSource(value string) string {
|
||
trimmed := strings.ToUpper(strings.TrimSpace(value))
|
||
if trimmed == "" {
|
||
return "BOS"
|
||
}
|
||
return trimmed
|
||
}
|
||
|
||
func parseWorkflowDataSourceHyperparam(value string) (string, *validationIssue) {
|
||
trimmed := strings.ToUpper(strings.TrimSpace(value))
|
||
if trimmed == "" {
|
||
return "", nil
|
||
}
|
||
if trimmed == "BOS" {
|
||
return trimmed, nil
|
||
}
|
||
return "", &validationIssue{
|
||
Field: "dataSource",
|
||
Message: "DATA_SOURCE must be BOS for scan workflows",
|
||
}
|
||
}
|
||
|
||
func validateScanWorkflowDataSource(value string) *validationIssue {
|
||
trimmed := strings.ToUpper(strings.TrimSpace(value))
|
||
if trimmed == "" || trimmed == "BOS" {
|
||
return nil
|
||
}
|
||
return &validationIssue{
|
||
Field: "dataSource",
|
||
Message: "dataSource must be BOS for scan workflows",
|
||
}
|
||
}
|
||
|
||
func cleanBOSPrefix(value string) string {
|
||
return strings.Trim(strings.TrimSpace(value), "/")
|
||
}
|
||
|
||
func chooseBOSEndpointWithDefault(value string, fallback string) string {
|
||
if trimmedValue := strings.TrimSpace(value); trimmedValue != "" {
|
||
return trimmedValue
|
||
}
|
||
return strings.TrimSpace(fallback)
|
||
}
|
||
|
||
func chooseBOSEndpoint(value string) string {
|
||
return strings.TrimSpace(value)
|
||
}
|
||
|
||
func (server *server) chooseBOSEndpoint(value string) string {
|
||
return chooseBOSEndpointWithDefault(value, server.globalBOSEndpoint)
|
||
}
|
||
|
||
func chooseWorkflowScanMode(value string) string {
|
||
switch strings.ToUpper(strings.TrimSpace(value)) {
|
||
case "FULL":
|
||
return "FULL"
|
||
default:
|
||
return "INCREMENTAL"
|
||
}
|
||
}
|
||
|
||
func parseWorkflowScanModeHyperparam(value string) (string, *validationIssue) {
|
||
trimmed := strings.ToUpper(strings.TrimSpace(value))
|
||
if trimmed == "" {
|
||
return "", nil
|
||
}
|
||
if trimmed == "FULL" || trimmed == "INCREMENTAL" {
|
||
return trimmed, nil
|
||
}
|
||
return "", &validationIssue{
|
||
Field: "scanMode",
|
||
Message: "SCAN_MODE must be FULL or INCREMENTAL",
|
||
}
|
||
}
|
||
|
||
func chooseWorkflowIncrementalUnit(value string) string {
|
||
switch strings.ToUpper(strings.TrimSpace(value)) {
|
||
case "DIRECTORY":
|
||
return "DIRECTORY"
|
||
default:
|
||
return "OBJECT"
|
||
}
|
||
}
|
||
|
||
func parseWorkflowIncrementalUnitHyperparam(value string) (string, *validationIssue) {
|
||
trimmed := strings.ToUpper(strings.TrimSpace(value))
|
||
if trimmed == "" {
|
||
return "", nil
|
||
}
|
||
if trimmed == "OBJECT" || trimmed == "DIRECTORY" {
|
||
return trimmed, nil
|
||
}
|
||
return "", &validationIssue{
|
||
Field: "incrementalUnit",
|
||
Message: "INCREMENTAL_UNIT must be OBJECT or DIRECTORY",
|
||
}
|
||
}
|
||
|
||
func chooseWorkflowDirectoryDepth(value int) int {
|
||
if value > 0 {
|
||
return value
|
||
}
|
||
return 1
|
||
}
|
||
|
||
func parseWorkflowDirectoryDepthHyperparam(value string) (int, *validationIssue) {
|
||
trimmed := strings.TrimSpace(value)
|
||
if trimmed == "" {
|
||
return 0, nil
|
||
}
|
||
parsed, err := strconv.Atoi(trimmed)
|
||
if err != nil || parsed <= 0 {
|
||
return 0, &validationIssue{
|
||
Field: "directoryDepth",
|
||
Message: "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer",
|
||
}
|
||
}
|
||
return parsed, nil
|
||
}
|
||
|
||
func scanSourcePath(dataSource string, sourcePath string, bucket string, prefix string) string {
|
||
if chooseDataSource(dataSource) == "BOS" {
|
||
if explicit := strings.TrimSpace(sourcePath); explicit != "" {
|
||
return explicit
|
||
}
|
||
if strings.TrimSpace(bucket) == "" {
|
||
return ""
|
||
}
|
||
canonical := "bos://" + strings.TrimSpace(bucket)
|
||
if cleanedPrefix := cleanBOSPrefix(prefix); cleanedPrefix != "" {
|
||
canonical += "/" + cleanedPrefix
|
||
}
|
||
return canonical
|
||
}
|
||
return strings.TrimSpace(sourcePath)
|
||
}
|
||
|
||
func mergeScanSecrets(existing *domain.ScanConfig, payload scanConfigRequest) (string, string) {
|
||
accessKeyID := strings.TrimSpace(payload.AccessKeyID)
|
||
secretAccessKey := strings.TrimSpace(payload.SecretAccessKey)
|
||
if accessKeyID == "" && existing != nil {
|
||
accessKeyID = existing.AccessKeyID
|
||
}
|
||
if secretAccessKey == "" && existing != nil {
|
||
secretAccessKey = existing.SecretAccessKey
|
||
}
|
||
return accessKeyID, secretAccessKey
|
||
}
|
||
|
||
func mergeScanConnectionID(existing *domain.ScanConfig, payload scanConfigRequest) string {
|
||
connectionID := strings.TrimSpace(payload.ConnectionID)
|
||
if connectionID == "" &&
|
||
existing != nil &&
|
||
strings.TrimSpace(existing.ConnectionID) != "" &&
|
||
strings.TrimSpace(payload.AccessKeyID) == "" &&
|
||
strings.TrimSpace(payload.SecretAccessKey) == "" {
|
||
connectionID = strings.TrimSpace(existing.ConnectionID)
|
||
}
|
||
return connectionID
|
||
}
|
||
|
||
func (server *server) getStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) {
|
||
connectionID := strings.TrimSpace(id)
|
||
if connectionID == "" {
|
||
return nil, nil
|
||
}
|
||
item, err := server.store.GetStorageConnection(ctx, connectionID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if item == nil {
|
||
return nil, fmt.Errorf("storage connection %s not found", connectionID)
|
||
}
|
||
if !server.filterByTenantContext(ctx, item.OrganizationID) {
|
||
return nil, fmt.Errorf("storage connection %s not found", connectionID)
|
||
}
|
||
return item, nil
|
||
}
|
||
|
||
func (server *server) getMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) {
|
||
connectionID := strings.TrimSpace(id)
|
||
if connectionID == "" {
|
||
return nil, nil
|
||
}
|
||
item, err := server.store.GetMongoConnection(ctx, connectionID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if item == nil {
|
||
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
|
||
}
|
||
if !server.filterByTenantContext(ctx, item.OrganizationID) {
|
||
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
|
||
}
|
||
return item, nil
|
||
}
|
||
|
||
func (server *server) resolveScanConfig(ctx context.Context, scanConfig *domain.ScanConfig) (*domain.ScanConfig, error) {
|
||
if scanConfig == nil {
|
||
return nil, nil
|
||
}
|
||
copy := *scanConfig
|
||
connection, err := server.getStorageConnection(ctx, copy.ConnectionID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if connection == nil {
|
||
if chooseDataSource(copy.DataSource) == "BOS" {
|
||
copy.Endpoint = server.chooseBOSEndpoint(copy.Endpoint)
|
||
}
|
||
return ©, nil
|
||
}
|
||
copy.Endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint)
|
||
if strings.TrimSpace(copy.AccessKeyID) == "" {
|
||
copy.AccessKeyID = connection.AccessKeyID
|
||
}
|
||
if strings.TrimSpace(copy.SecretAccessKey) == "" {
|
||
copy.SecretAccessKey = connection.SecretAccessKey
|
||
}
|
||
return ©, nil
|
||
}
|
||
|
||
func cloneHyperparams(items []domain.HyperParam) []domain.HyperParam {
|
||
out := make([]domain.HyperParam, len(items))
|
||
copy(out, items)
|
||
return out
|
||
}
|
||
|
||
func clonePorts(items []domain.PortDefinition) []domain.PortDefinition {
|
||
out := make([]domain.PortDefinition, len(items))
|
||
copy(out, items)
|
||
return out
|
||
}
|
||
|
||
func cloneOperatorIntoNode(operator domain.OperatorRecord, nodeID string, x float64, y float64, overrides map[string]string) domain.WorkflowNode {
|
||
hyperparams := cloneHyperparams(operator.Hyperparams)
|
||
seen := make(map[string]int, len(hyperparams))
|
||
for index, item := range hyperparams {
|
||
seen[item.Key] = index
|
||
}
|
||
for key, value := range overrides {
|
||
if index, ok := seen[key]; ok {
|
||
hyperparams[index].Value = value
|
||
continue
|
||
}
|
||
hyperparams = append(hyperparams, domain.HyperParam{Key: key, Value: value})
|
||
}
|
||
return domain.WorkflowNode{
|
||
ID: nodeID,
|
||
Type: "operator",
|
||
Data: domain.WorkflowNodeData{
|
||
Label: operator.Name,
|
||
Category: operator.Category,
|
||
Description: operator.Description,
|
||
Inputs: clonePorts(operator.Inputs),
|
||
Outputs: clonePorts(operator.Outputs),
|
||
Image: operator.Image,
|
||
SkillDefinition: operator.SkillDefinition,
|
||
Hyperparams: hyperparams,
|
||
},
|
||
Position: domain.Position{X: x, Y: y},
|
||
}
|
||
}
|
||
|
||
func operatorByName(items []domain.OperatorRecord, name string) (*domain.OperatorRecord, bool) {
|
||
for _, item := range items {
|
||
if item.Name == name {
|
||
copy := item
|
||
return ©, true
|
||
}
|
||
}
|
||
return nil, false
|
||
}
|
||
|
||
func (server *server) buildScanWorkflowInput(ctx context.Context, projectID string, payload scanConfigRequest, existing *domain.ScanConfig) (domain.CreateWorkflowInput, error) {
|
||
operators, err := server.listAvailableOperators(ctx)
|
||
if err != nil {
|
||
return domain.CreateWorkflowInput{}, err
|
||
}
|
||
dataSource := chooseDataSource(payload.DataSource)
|
||
sourceOperatorName := "BOS 增量扫描"
|
||
sourceOperator, ok := operatorByName(operators, sourceOperatorName)
|
||
if !ok {
|
||
return domain.CreateWorkflowInput{}, fmt.Errorf("operator %s not found", sourceOperatorName)
|
||
}
|
||
indexOperator, ok := operatorByName(operators, "X 平台数据集索引")
|
||
if !ok {
|
||
return domain.CreateWorkflowInput{}, fmt.Errorf("operator X 平台数据集索引 not found")
|
||
}
|
||
validateOperator, ok := operatorByName(operators, "采集目录校验")
|
||
if !ok {
|
||
return domain.CreateWorkflowInput{}, fmt.Errorf("operator 采集目录校验 not found")
|
||
}
|
||
|
||
sourcePath := scanSourcePath(dataSource, payload.SourcePath, payload.Bucket, payload.Prefix)
|
||
connectionID := mergeScanConnectionID(existing, payload)
|
||
var connection *domain.StorageConnectionRecord
|
||
if connectionID != "" {
|
||
connection, err = server.getStorageConnection(ctx, connectionID)
|
||
if err != nil {
|
||
return domain.CreateWorkflowInput{}, err
|
||
}
|
||
}
|
||
endpoint := ""
|
||
accessKeyID := ""
|
||
secretAccessKey := ""
|
||
if connection != nil {
|
||
endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint)
|
||
} else if dataSource == "BOS" {
|
||
endpoint = server.chooseBOSEndpoint(payload.Endpoint)
|
||
accessKeyID, secretAccessKey = mergeScanSecrets(existing, payload)
|
||
}
|
||
|
||
sourceOverrides := map[string]string{
|
||
"DATA_SOURCE": dataSource,
|
||
"SCAN_MODE": chooseWorkflowScanMode(payload.ScanMode),
|
||
"INCREMENTAL_UNIT": chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
|
||
"INCREMENTAL_DIRECTORY_DEPTH": strconv.Itoa(chooseWorkflowDirectoryDepth(payload.DirectoryDepth)),
|
||
}
|
||
if dataSource == "BOS" {
|
||
sourceOverrides["SOURCE_PATH"] = sourcePath
|
||
sourceOverrides["BOS_ENDPOINT"] = endpoint
|
||
} else {
|
||
sourceOverrides["SOURCE_PATH"] = sourcePath
|
||
}
|
||
sourceNode := cloneOperatorIntoNode(*sourceOperator, "scan-source", 0, 0, sourceOverrides)
|
||
sourceNode.Data.Bucket = strings.TrimSpace(payload.Bucket)
|
||
sourceNode.Data.Prefix = cleanBOSPrefix(payload.Prefix)
|
||
sourceNode.Data.Pattern = strings.TrimSpace(payload.Pattern)
|
||
indexNode := cloneOperatorIntoNode(*indexOperator, "scan-index", 260, 0, nil)
|
||
validateNode := cloneOperatorIntoNode(*validateOperator, "scan-validate", 520, 0, nil)
|
||
|
||
schedule := &domain.WorkflowSchedule{SourcePath: sourcePath}
|
||
workflowType := domain.WorkflowTypeBatch
|
||
if payload.Enabled && payload.Interval > 0 {
|
||
schedule.Interval = payload.Interval
|
||
workflowType = domain.WorkflowTypeStreaming
|
||
}
|
||
|
||
return domain.CreateWorkflowInput{
|
||
ProjectID: projectID,
|
||
Name: strings.TrimSpace(payload.Name),
|
||
Description: strings.TrimSpace(payload.Description),
|
||
Type: workflowType,
|
||
Kind: domain.WorkflowKindScan,
|
||
ScanConfig: &domain.ScanConfig{
|
||
DataSource: dataSource,
|
||
SourcePath: sourcePath,
|
||
ConnectionID: connectionID,
|
||
Bucket: strings.TrimSpace(payload.Bucket),
|
||
Prefix: cleanBOSPrefix(payload.Prefix),
|
||
Endpoint: endpoint,
|
||
AccessKeyID: accessKeyID,
|
||
SecretAccessKey: secretAccessKey,
|
||
ScanMode: chooseWorkflowScanMode(payload.ScanMode),
|
||
IncrementalUnit: chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
|
||
DirectoryDepth: chooseWorkflowDirectoryDepth(payload.DirectoryDepth),
|
||
Pattern: strings.TrimSpace(payload.Pattern),
|
||
Enabled: payload.Enabled && payload.Interval > 0,
|
||
Notifications: payload.Notifications,
|
||
},
|
||
Schedule: schedule,
|
||
Nodes: []domain.WorkflowNode{
|
||
sourceNode,
|
||
indexNode,
|
||
validateNode,
|
||
},
|
||
Edges: []domain.WorkflowEdge{
|
||
{ID: "scan-source-index", Source: sourceNode.ID, SourceHandle: "out-1", Target: indexNode.ID, TargetHandle: "in-1"},
|
||
{ID: "scan-index-validate", Source: indexNode.ID, SourceHandle: "out-1", Target: validateNode.ID, TargetHandle: "in-1"},
|
||
},
|
||
}, nil
|
||
}
|
||
|
||
func validateScanConfigEndpoint(payload scanConfigRequest, existing *domain.ScanConfig) *validationIssue {
|
||
if chooseDataSource(payload.DataSource) != "BOS" {
|
||
return nil
|
||
}
|
||
if strings.TrimSpace(mergeScanConnectionID(existing, payload)) != "" {
|
||
return nil
|
||
}
|
||
return validateOptionalHTTPURL("endpoint", payload.Endpoint)
|
||
}
|
||
|
||
func validateScanConfigDraftPayload(payload scanConfigRequest, existing ...*domain.ScanConfig) []validationIssue {
|
||
var previous *domain.ScanConfig
|
||
if len(existing) > 0 {
|
||
previous = existing[0]
|
||
}
|
||
issues := make([]validationIssue, 0, 5)
|
||
if strings.TrimSpace(payload.Name) == "" {
|
||
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
|
||
}
|
||
if payload.Interval < 0 {
|
||
issues = append(issues, validationIssue{Field: "interval", Message: "interval must be zero or greater"})
|
||
}
|
||
if issue := validateScanWorkflowDataSource(payload.DataSource); issue != nil {
|
||
issues = append(issues, *issue)
|
||
}
|
||
if payload.DataSourceIssue != nil {
|
||
issues = append(issues, *payload.DataSourceIssue)
|
||
}
|
||
if scanMode := strings.ToUpper(strings.TrimSpace(payload.ScanMode)); scanMode != "" && scanMode != "FULL" && scanMode != "INCREMENTAL" {
|
||
issues = append(issues, validationIssue{Field: "scanMode", Message: "scanMode must be FULL or INCREMENTAL"})
|
||
}
|
||
if payload.ScanModeIssue != nil {
|
||
issues = append(issues, *payload.ScanModeIssue)
|
||
}
|
||
if incrementalUnit := strings.ToUpper(strings.TrimSpace(payload.IncrementalUnit)); incrementalUnit != "" && incrementalUnit != "OBJECT" && incrementalUnit != "DIRECTORY" {
|
||
issues = append(issues, validationIssue{Field: "incrementalUnit", Message: "incrementalUnit must be OBJECT or DIRECTORY"})
|
||
}
|
||
if payload.IncrementalUnitIssue != nil {
|
||
issues = append(issues, *payload.IncrementalUnitIssue)
|
||
}
|
||
if payload.DirectoryDepthIssue != nil {
|
||
issues = append(issues, *payload.DirectoryDepthIssue)
|
||
}
|
||
if payload.DirectoryDepth < 0 {
|
||
issues = append(issues, validationIssue{Field: "directoryDepth", Message: "directoryDepth must be zero or greater"})
|
||
}
|
||
if issue := validateScanConfigEndpoint(payload, previous); issue != nil {
|
||
issues = append(issues, *issue)
|
||
}
|
||
return issues
|
||
}
|
||
|
||
func scanConfigPayloadReady(payload scanConfigRequest, existing *domain.ScanConfig) bool {
|
||
switch chooseDataSource(payload.DataSource) {
|
||
case "BOS":
|
||
connectionID := mergeScanConnectionID(existing, payload)
|
||
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
|
||
hasTarget := strings.TrimSpace(payload.SourcePath) != "" ||
|
||
(strings.TrimSpace(payload.Bucket) != "" && cleanBOSPrefix(payload.Prefix) != "")
|
||
return hasTarget &&
|
||
(strings.TrimSpace(connectionID) != "" ||
|
||
(strings.TrimSpace(accessKeyID) != "" && strings.TrimSpace(secretAccessKey) != ""))
|
||
default:
|
||
return strings.TrimSpace(payload.SourcePath) != ""
|
||
}
|
||
}
|
||
|
||
func validateScanConfigPayload(payload scanConfigRequest, existing *domain.ScanConfig) []validationIssue {
|
||
issues := validateScanConfigDraftPayload(payload, existing)
|
||
switch chooseDataSource(payload.DataSource) {
|
||
case "BOS":
|
||
if strings.TrimSpace(payload.SourcePath) == "" {
|
||
if strings.TrimSpace(payload.Bucket) == "" {
|
||
issues = append(issues, validationIssue{Field: "bucket", Message: "bucket is required for BOS scan configs when sourcePath is empty"})
|
||
}
|
||
if cleanBOSPrefix(payload.Prefix) == "" {
|
||
issues = append(issues, validationIssue{Field: "prefix", Message: "prefix is required for BOS scan configs when sourcePath is empty"})
|
||
}
|
||
}
|
||
connectionID := mergeScanConnectionID(existing, payload)
|
||
if strings.TrimSpace(connectionID) == "" {
|
||
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
|
||
if strings.TrimSpace(accessKeyID) == "" {
|
||
issues = append(issues, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required for BOS scan configs"})
|
||
}
|
||
if strings.TrimSpace(secretAccessKey) == "" {
|
||
issues = append(issues, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required for BOS scan configs"})
|
||
}
|
||
}
|
||
default:
|
||
if strings.TrimSpace(payload.SourcePath) == "" {
|
||
issues = append(issues, validationIssue{Field: "sourcePath", Message: "sourcePath is required"})
|
||
}
|
||
}
|
||
return issues
|
||
}
|
||
|
||
func (server *server) scanWorkflows(ctx context.Context, request *http.Request) ([]domain.WorkflowRecord, error) {
|
||
items, err := server.listAllWorkflowsForRequest(ctx, request)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
filtered := make([]domain.WorkflowRecord, 0)
|
||
for _, item := range items {
|
||
if isScanWorkflow(item) {
|
||
item.Kind = chooseWorkflowKind(item.Kind)
|
||
filtered = append(filtered, item)
|
||
}
|
||
}
|
||
return filtered, nil
|
||
}
|
||
|
||
func (server *server) handleScanConfigsList(writer http.ResponseWriter, request *http.Request) {
|
||
items, err := server.scanWorkflows(request.Context(), request)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
items = server.filterWorkflowsForRequest(request, items)
|
||
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)})
|
||
}
|
||
|
||
func (server *server) handleScanConfigsCreate(writer http.ResponseWriter, request *http.Request) {
|
||
var payload scanConfigRequest
|
||
if !decodeJSON(writer, request, &payload) {
|
||
return
|
||
}
|
||
if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 {
|
||
renderValidationError(writer, issues...)
|
||
return
|
||
}
|
||
if payload.Interval > 0 && !scanConfigPayloadReady(payload, nil) {
|
||
renderValidationError(writer, validateScanConfigPayload(payload, nil)...)
|
||
return
|
||
}
|
||
if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(nil, payload)); len(issues) > 0 {
|
||
renderValidationError(writer, issues...)
|
||
return
|
||
}
|
||
input, err := server.buildScanWorkflowInput(request.Context(), "", payload, nil)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
server.assignTenant(request, &input.OrganizationID)
|
||
item, err := server.store.CreateWorkflow(request.Context(), input)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item))
|
||
}
|
||
|
||
func (server *server) handleScanConfigsGet(writer http.ResponseWriter, request *http.Request) {
|
||
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if item == nil || !isScanWorkflow(*item) {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
|
||
return
|
||
}
|
||
item.Kind = chooseWorkflowKind(item.Kind)
|
||
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
|
||
}
|
||
|
||
func (server *server) handleScanConfigsUpdate(writer http.ResponseWriter, request *http.Request) {
|
||
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if item == nil || !isScanWorkflow(*item) {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
|
||
return
|
||
}
|
||
var payload scanConfigRequest
|
||
if !decodeJSON(writer, request, &payload) {
|
||
return
|
||
}
|
||
if issues := validateScanConfigDraftPayload(payload, item.ScanConfig); len(issues) > 0 {
|
||
renderValidationError(writer, issues...)
|
||
return
|
||
}
|
||
if payload.Interval > 0 && !scanConfigPayloadReady(payload, item.ScanConfig) {
|
||
renderValidationError(writer, validateScanConfigPayload(payload, item.ScanConfig)...)
|
||
return
|
||
}
|
||
if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(item.ScanConfig, payload)); len(issues) > 0 {
|
||
renderValidationError(writer, issues...)
|
||
return
|
||
}
|
||
input, err := server.buildScanWorkflowInput(request.Context(), item.ProjectID, payload, item.ScanConfig)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
item.Name = input.Name
|
||
item.Description = input.Description
|
||
item.Type = input.Type
|
||
item.Kind = input.Kind
|
||
item.ScanConfig = input.ScanConfig
|
||
item.Schedule = input.Schedule
|
||
item.Nodes = input.Nodes
|
||
item.Edges = input.Edges
|
||
item.UpdatedAt = nowISO()
|
||
if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
|
||
}
|
||
|
||
func (server *server) handleScanConfigsDelete(writer http.ResponseWriter, request *http.Request) {
|
||
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if item == nil || !isScanWorkflow(*item) {
|
||
writer.WriteHeader(http.StatusNotFound)
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
|
||
return
|
||
}
|
||
preview, runs, err := server.buildWorkflowDeletionPreview(request.Context(), item)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if preview.BlockingRunCount > 0 {
|
||
renderJSON(writer, http.StatusConflict, map[string]any{
|
||
"error": "数据扫描配置仍有正在排队或运行的 Run,请先停止这些 Run 后再删除",
|
||
"code": "scan_config_run_active",
|
||
"preview": preview,
|
||
})
|
||
return
|
||
}
|
||
if err := server.deleteWorkflowGeneratedArtifacts(request.Context(), item, runs); err != nil {
|
||
renderJSON(writer, http.StatusBadGateway, map[string]string{"error": err.Error(), "code": "scan_config_artifact_cleanup_failed"})
|
||
return
|
||
}
|
||
deleted, err := server.store.DeleteWorkflow(request.Context(), item.ID)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if !deleted {
|
||
writer.WriteHeader(http.StatusNotFound)
|
||
return
|
||
}
|
||
writer.WriteHeader(http.StatusNoContent)
|
||
}
|
||
|
||
func artifactByName(run domain.RunRecord, name string) *domain.ExecutionArtifact {
|
||
for index := range run.Artifacts {
|
||
if run.Artifacts[index].Name == name {
|
||
return &run.Artifacts[index]
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func mapValue(value any) map[string]any {
|
||
item, _ := normalizeStructuredValue(value).(map[string]any)
|
||
return item
|
||
}
|
||
|
||
func listValue(value any) []any {
|
||
items, _ := normalizeStructuredValue(value).([]any)
|
||
return items
|
||
}
|
||
|
||
func normalizeStructuredValue(value any) any {
|
||
switch typed := value.(type) {
|
||
case nil:
|
||
return nil
|
||
case map[string]any:
|
||
items := make(map[string]any, len(typed))
|
||
for key, item := range typed {
|
||
items[key] = normalizeStructuredValue(item)
|
||
}
|
||
return items
|
||
case []any:
|
||
items := make([]any, len(typed))
|
||
for index, item := range typed {
|
||
items[index] = normalizeStructuredValue(item)
|
||
}
|
||
return items
|
||
}
|
||
|
||
rv := reflect.ValueOf(value)
|
||
switch rv.Kind() {
|
||
case reflect.Map:
|
||
if rv.Type().Key().Kind() != reflect.String {
|
||
return value
|
||
}
|
||
items := make(map[string]any, rv.Len())
|
||
iter := rv.MapRange()
|
||
for iter.Next() {
|
||
items[iter.Key().String()] = normalizeStructuredValue(iter.Value().Interface())
|
||
}
|
||
return items
|
||
case reflect.Slice, reflect.Array:
|
||
items := make([]any, rv.Len())
|
||
for index := 0; index < rv.Len(); index++ {
|
||
items[index] = normalizeStructuredValue(rv.Index(index).Interface())
|
||
}
|
||
return items
|
||
default:
|
||
return value
|
||
}
|
||
}
|
||
|
||
func stringValue(value any) string {
|
||
switch typed := value.(type) {
|
||
case nil:
|
||
return ""
|
||
case string:
|
||
return typed
|
||
default:
|
||
text := strings.TrimSpace(fmt.Sprintf("%v", typed))
|
||
if text == "<nil>" {
|
||
return ""
|
||
}
|
||
return text
|
||
}
|
||
}
|
||
|
||
func boolValue(value any) bool {
|
||
typed, ok := value.(bool)
|
||
return ok && typed
|
||
}
|
||
|
||
func floatValue(value any) float64 {
|
||
switch typed := value.(type) {
|
||
case float64:
|
||
if math.IsNaN(typed) || math.IsInf(typed, 0) {
|
||
return 0
|
||
}
|
||
return typed
|
||
case float32:
|
||
value := float64(typed)
|
||
if math.IsNaN(value) || math.IsInf(value, 0) {
|
||
return 0
|
||
}
|
||
return value
|
||
case int:
|
||
return float64(typed)
|
||
case int32:
|
||
return float64(typed)
|
||
case int64:
|
||
return float64(typed)
|
||
default:
|
||
return 0
|
||
}
|
||
}
|
||
|
||
func intValue(value any) int {
|
||
parsed, ok := exactIntegerValue(value)
|
||
if !ok {
|
||
return 0
|
||
}
|
||
if strconv.IntSize == 32 {
|
||
const maxInt32 = int64(1<<31 - 1)
|
||
const minInt32 = -1 << 31
|
||
if parsed < minInt32 || parsed > maxInt32 {
|
||
return 0
|
||
}
|
||
}
|
||
return int(parsed)
|
||
}
|
||
|
||
func int64Value(value any) int64 {
|
||
parsed, ok := exactIntegerValue(value)
|
||
if !ok {
|
||
return 0
|
||
}
|
||
return parsed
|
||
}
|
||
|
||
func exactIntegerValue(value any) (int64, bool) {
|
||
switch typed := value.(type) {
|
||
case int:
|
||
return int64(typed), true
|
||
case int8:
|
||
return int64(typed), true
|
||
case int16:
|
||
return int64(typed), true
|
||
case int32:
|
||
return int64(typed), true
|
||
case int64:
|
||
return typed, true
|
||
case uint:
|
||
if uint64(typed) > uint64(math.MaxInt64) {
|
||
return 0, false
|
||
}
|
||
return int64(typed), true
|
||
case uint8:
|
||
return int64(typed), true
|
||
case uint16:
|
||
return int64(typed), true
|
||
case uint32:
|
||
return int64(typed), true
|
||
case uint64:
|
||
if typed > uint64(math.MaxInt64) {
|
||
return 0, false
|
||
}
|
||
return int64(typed), true
|
||
case float32:
|
||
const maxExactFloat32Integer = 1<<24 - 1
|
||
return exactFloatIntegerValue(float64(typed), maxExactFloat32Integer)
|
||
case float64:
|
||
const maxExactFloat64Integer = 1<<53 - 1
|
||
return exactFloatIntegerValue(typed, maxExactFloat64Integer)
|
||
case json.Number:
|
||
parsed, err := typed.Int64()
|
||
return parsed, err == nil
|
||
default:
|
||
return 0, false
|
||
}
|
||
}
|
||
|
||
func exactFloatIntegerValue(value float64, maxExactAbs float64) (int64, bool) {
|
||
if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
|
||
return 0, false
|
||
}
|
||
if math.Abs(value) > maxExactAbs {
|
||
return 0, false
|
||
}
|
||
return int64(value), true
|
||
}
|
||
|
||
func artifactDataRef(artifact *domain.ExecutionArtifact) map[string]any {
|
||
if artifact == nil || artifact.Data == nil {
|
||
return nil
|
||
}
|
||
return mapValue(artifact.Data["dataRef"])
|
||
}
|
||
|
||
func artifactStructuredData(artifact *domain.ExecutionArtifact) map[string]any {
|
||
if artifact == nil {
|
||
return nil
|
||
}
|
||
if dataRef := artifactDataRef(artifact); dataRef != nil {
|
||
payloadPath := strings.TrimSpace(stringValue(dataRef["payloadPath"]))
|
||
if payloadPath != "" && !strings.Contains(payloadPath, "://") {
|
||
payload, err := os.ReadFile(payloadPath)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
var decoded any
|
||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||
return nil
|
||
}
|
||
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
|
||
return structured
|
||
}
|
||
return nil
|
||
}
|
||
}
|
||
if artifact.Data != nil {
|
||
return artifact.Data
|
||
}
|
||
if artifact.Type == domain.PortTypeJSON {
|
||
filePath := strings.TrimSpace(artifact.Path)
|
||
if filePath != "" && !strings.Contains(filePath, "://") {
|
||
payload, err := os.ReadFile(filePath)
|
||
if err == nil {
|
||
var decoded any
|
||
if err := json.Unmarshal(payload, &decoded); err == nil {
|
||
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
|
||
return structured
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func sourceArtifactRelativePath(path string) string {
|
||
normalized := filepath.ToSlash(path)
|
||
marker := "/_source_tree/"
|
||
index := strings.Index(normalized, marker)
|
||
if index < 0 {
|
||
return ""
|
||
}
|
||
return strings.TrimPrefix(normalized[index+len(marker):], "/")
|
||
}
|
||
|
||
func sourceArtifactPath(artifact domain.ExecutionArtifact) string {
|
||
if payload := artifactStructuredData(&artifact); payload != nil {
|
||
if relative := stringValue(payload["relativePath"]); relative != "" {
|
||
return filepath.ToSlash(strings.Trim(relative, "/"))
|
||
}
|
||
}
|
||
return sourceArtifactRelativePath(artifact.Path)
|
||
}
|
||
|
||
func packageArtifactRelativePath(path string, packageName string) string {
|
||
normalized := filepath.ToSlash(path)
|
||
marker := "/packages/" + packageName + "/"
|
||
index := strings.Index(normalized, marker)
|
||
if index < 0 {
|
||
return ""
|
||
}
|
||
return strings.TrimPrefix(normalized[index+len(marker):], "/")
|
||
}
|
||
|
||
func packageContentURL(runID string, packageName string, relativePath string) string {
|
||
return fmt.Sprintf(
|
||
"/api/packages/%s/%s/content?relativePath=%s",
|
||
url.PathEscape(runID),
|
||
url.PathEscape(packageName),
|
||
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
|
||
)
|
||
}
|
||
|
||
func packageArtifactPreviewURL(artifact domain.ExecutionArtifact) string {
|
||
payload := artifactStructuredData(&artifact)
|
||
preview := mapValue(payload["preview"])
|
||
if preview != nil {
|
||
if value := strings.TrimSpace(stringValue(preview["url"])); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
source := mapValue(payload["source"])
|
||
if source != nil {
|
||
if value := strings.TrimSpace(stringValue(source["previewURL"])); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func packageArtifactResolvedRelativePath(artifact domain.ExecutionArtifact, packageName string) string {
|
||
if relativePath := packageArtifactRelativePath(artifact.Path, packageName); relativePath != "" {
|
||
return relativePath
|
||
}
|
||
payload := artifactStructuredData(&artifact)
|
||
if payload == nil {
|
||
return ""
|
||
}
|
||
relativePath := strings.Trim(filepath.ToSlash(stringValue(payload["relativePath"])), "/")
|
||
if relativePath == "" {
|
||
return ""
|
||
}
|
||
packagePrefix := strings.Trim(filepath.ToSlash(path.Join("packages", packageName)), "/")
|
||
if packagePrefix == "" {
|
||
return ""
|
||
}
|
||
if relativePath == packagePrefix {
|
||
return ""
|
||
}
|
||
if strings.HasPrefix(relativePath, packagePrefix+"/") {
|
||
return strings.TrimPrefix(relativePath, packagePrefix+"/")
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func packageArtifactContentURL(runID string, packageName string, relativePath string) string {
|
||
return packageContentURL(runID, packageName, relativePath)
|
||
}
|
||
|
||
func packageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
|
||
copy := artifact
|
||
if copy.Data == nil {
|
||
copy.Data = make(map[string]any)
|
||
}
|
||
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
|
||
if relativePath == "" {
|
||
relativePath = filepath.ToSlash(strings.Trim(copy.Path, "/"))
|
||
}
|
||
if relativePath != "" {
|
||
copy.Path = relativePath
|
||
}
|
||
copy.Data["contentURL"] = packageArtifactContentURL(runID, packageName, copy.Path)
|
||
preview := mapValue(copy.Data["preview"])
|
||
if preview == nil {
|
||
preview = make(map[string]any)
|
||
}
|
||
kind := strings.TrimSpace(stringValue(preview["kind"]))
|
||
mimeType := ""
|
||
if copy.Metadata != nil {
|
||
mimeType = strings.TrimSpace(copy.Metadata.MimeType)
|
||
}
|
||
if kind == "video" || copy.Type == "video" || strings.HasPrefix(mimeType, "video/") {
|
||
preview["kind"] = "video"
|
||
preview["url"] = packageContentURL(runID, packageName, copy.Path)
|
||
copy.Data["preview"] = preview
|
||
}
|
||
return copy
|
||
}
|
||
|
||
const wilorHandPoseSchemaVersion = "droboticflow.wilor.hand_pose_result/v1"
|
||
|
||
func safePackageNamePart(value string) string {
|
||
trimmed := strings.ToLower(strings.TrimSpace(value))
|
||
var builder strings.Builder
|
||
lastDash := false
|
||
for _, current := range trimmed {
|
||
isAlphaNumeric := (current >= 'a' && current <= 'z') || (current >= '0' && current <= '9')
|
||
if isAlphaNumeric {
|
||
builder.WriteRune(current)
|
||
lastDash = false
|
||
continue
|
||
}
|
||
if !lastDash {
|
||
builder.WriteByte('-')
|
||
lastDash = true
|
||
}
|
||
}
|
||
return strings.Trim(builder.String(), "-")
|
||
}
|
||
|
||
func wilorResultPayload(artifact domain.ExecutionArtifact) map[string]any {
|
||
if artifact.Name != "wilor-result.json" {
|
||
return nil
|
||
}
|
||
payload := artifactStructuredData(&artifact)
|
||
if payload == nil {
|
||
return nil
|
||
}
|
||
if strings.TrimSpace(stringValue(payload["schemaVersion"])) != wilorHandPoseSchemaVersion {
|
||
return nil
|
||
}
|
||
if strings.TrimSpace(stringValue(payload["relativePath"])) != "" {
|
||
return nil
|
||
}
|
||
return payload
|
||
}
|
||
|
||
func wilorPackageName(artifact domain.ExecutionArtifact, payload map[string]any) string {
|
||
source := mapValue(payload["source"])
|
||
sourceName := strings.TrimSpace(stringValue(source["name"]))
|
||
stem := strings.TrimSuffix(path.Base(filepath.ToSlash(sourceName)), path.Ext(sourceName))
|
||
sourceSlug := safePackageNamePart(stem)
|
||
if sourceSlug == "" {
|
||
sourceSlug = "result"
|
||
}
|
||
nodeSlug := safePackageNamePart(artifact.NodeID)
|
||
if nodeSlug == "" {
|
||
nodeSlug = "wilor"
|
||
}
|
||
return "wilor-" + sourceSlug + "-" + nodeSlug
|
||
}
|
||
|
||
func wilorResultGroupKey(artifact domain.ExecutionArtifact, payload map[string]any) string {
|
||
source := mapValue(payload["source"])
|
||
sourceName := strings.TrimSpace(stringValue(source["name"]))
|
||
sourceRef := strings.TrimSpace(stringValue(source["ref"]))
|
||
mediaKind := strings.TrimSpace(stringValue(payload["mediaKind"]))
|
||
if sourceName != "" || sourceRef != "" || mediaKind != "" {
|
||
return strings.ToLower(mediaKind) + "|" + strings.ToLower(sourceName) + "|" + strings.ToLower(sourceRef)
|
||
}
|
||
return artifact.ID
|
||
}
|
||
|
||
func wilorResultArtifactPriority(artifact domain.ExecutionArtifact) int {
|
||
nodeID := strings.ToLower(strings.TrimSpace(artifact.NodeID))
|
||
score := 0
|
||
if strings.Contains(nodeID, "upload") || strings.Contains(nodeID, "save") {
|
||
score += 30
|
||
}
|
||
if strings.Contains(nodeID, "wilor") {
|
||
score += 20
|
||
}
|
||
if strings.TrimSpace(artifact.Path) != "" {
|
||
score++
|
||
}
|
||
return score
|
||
}
|
||
|
||
func wilorResultArtifacts(run domain.RunRecord) []domain.ExecutionArtifact {
|
||
bestByResult := make(map[string]domain.ExecutionArtifact)
|
||
for _, artifact := range run.Artifacts {
|
||
payload := wilorResultPayload(artifact)
|
||
if payload == nil {
|
||
continue
|
||
}
|
||
key := wilorResultGroupKey(artifact, payload)
|
||
current, exists := bestByResult[key]
|
||
if !exists || wilorResultArtifactPriority(artifact) > wilorResultArtifactPriority(current) {
|
||
bestByResult[key] = artifact
|
||
}
|
||
}
|
||
items := make([]domain.ExecutionArtifact, 0, len(bestByResult))
|
||
for _, artifact := range bestByResult {
|
||
items = append(items, artifact)
|
||
}
|
||
sort.Slice(items, func(left int, right int) bool {
|
||
return wilorPackageName(items[left], wilorResultPayload(items[left])) < wilorPackageName(items[right], wilorResultPayload(items[right]))
|
||
})
|
||
return items
|
||
}
|
||
|
||
func wilorPackageManifestArtifact(run domain.RunRecord, packageName string) (*domain.ExecutionArtifact, map[string]any) {
|
||
for index := range run.Artifacts {
|
||
payload := wilorResultPayload(run.Artifacts[index])
|
||
if payload == nil {
|
||
continue
|
||
}
|
||
if wilorPackageName(run.Artifacts[index], payload) == packageName {
|
||
return &run.Artifacts[index], payload
|
||
}
|
||
}
|
||
return nil, nil
|
||
}
|
||
|
||
func wilorOutputArtifactNames(manifest domain.ExecutionArtifact, payload map[string]any) map[string]struct{} {
|
||
names := map[string]struct{}{
|
||
manifest.Name: {},
|
||
}
|
||
for _, raw := range listValue(payload["downloadedOutputs"]) {
|
||
item := mapValue(raw)
|
||
name := strings.TrimSpace(stringValue(item["name"]))
|
||
if name == "" {
|
||
if outputPath := strings.TrimSpace(stringValue(item["path"])); outputPath != "" {
|
||
name = path.Base(filepath.ToSlash(outputPath))
|
||
}
|
||
}
|
||
if name != "" {
|
||
names[name] = struct{}{}
|
||
}
|
||
}
|
||
return names
|
||
}
|
||
|
||
func wilorPackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
|
||
manifest, payload := wilorPackageManifestArtifact(run, packageName)
|
||
if manifest == nil || payload == nil {
|
||
return nil
|
||
}
|
||
names := wilorOutputArtifactNames(*manifest, payload)
|
||
bestByName := make(map[string]domain.ExecutionArtifact)
|
||
for _, artifact := range run.Artifacts {
|
||
if _, ok := names[artifact.Name]; !ok {
|
||
continue
|
||
}
|
||
if artifact.NodeID != manifest.NodeID && strings.TrimSpace(stringValue(mapValue(artifactStructuredData(&artifact)["source"])["path"])) == "" {
|
||
continue
|
||
}
|
||
current, exists := bestByName[artifact.Name]
|
||
if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) {
|
||
bestByName[artifact.Name] = artifact
|
||
}
|
||
}
|
||
items := make([]domain.ExecutionArtifact, 0, len(bestByName))
|
||
for _, artifact := range bestByName {
|
||
items = append(items, artifact)
|
||
}
|
||
sort.Slice(items, func(left int, right int) bool {
|
||
return items[left].Name < items[right].Name
|
||
})
|
||
return items
|
||
}
|
||
|
||
func wilorPackageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
|
||
copy := artifact
|
||
if copy.Data == nil {
|
||
copy.Data = make(map[string]any)
|
||
}
|
||
relativePath := strings.TrimSpace(copy.Name)
|
||
if relativePath == "" {
|
||
relativePath = path.Base(filepath.ToSlash(strings.TrimSpace(copy.Path)))
|
||
}
|
||
copy.Path = relativePath
|
||
copy.Data["contentURL"] = packageArtifactContentURL(runID, packageName, relativePath)
|
||
preview := mapValue(copy.Data["preview"])
|
||
if preview == nil {
|
||
preview = make(map[string]any)
|
||
}
|
||
mimeType := ""
|
||
if copy.Metadata != nil {
|
||
mimeType = strings.TrimSpace(copy.Metadata.MimeType)
|
||
}
|
||
kind := sourceEntryPayloadKind(relativePath, strings.ToLower(mimeType))
|
||
if kind == "video" || kind == "image" {
|
||
preview["kind"] = kind
|
||
preview["url"] = packageArtifactContentURL(runID, packageName, relativePath)
|
||
copy.Data["preview"] = preview
|
||
}
|
||
return copy
|
||
}
|
||
|
||
func wilorPackageFilesForResponse(run domain.RunRecord, packageName string) ([]domain.ExecutionArtifact, []domain.ExecutionArtifact) {
|
||
rawFiles := wilorPackageArtifacts(run, packageName)
|
||
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
|
||
for _, artifact := range rawFiles {
|
||
files = append(files, wilorPackageArtifactForResponse(run.ID, packageName, artifact))
|
||
}
|
||
return files, rawFiles
|
||
}
|
||
|
||
func wilorPackageItem(run domain.RunRecord, manifest domain.ExecutionArtifact, payload map[string]any, rawFiles []domain.ExecutionArtifact) map[string]any {
|
||
source := mapValue(payload["source"])
|
||
result := mapValue(payload["result"])
|
||
service := mapValue(payload["service"])
|
||
health := mapValue(service["health"])
|
||
sourceName := strings.TrimSpace(stringValue(source["name"]))
|
||
if sourceName == "" {
|
||
sourceName = "WiLoR result"
|
||
}
|
||
mediaKind := strings.TrimSpace(stringValue(payload["mediaKind"]))
|
||
if mediaKind == "" {
|
||
mediaKind = "media"
|
||
}
|
||
clipID := strings.TrimSuffix(path.Base(filepath.ToSlash(sourceName)), path.Ext(sourceName))
|
||
if clipID == "" || clipID == "." {
|
||
clipID = manifest.NodeID
|
||
}
|
||
storagePath := ""
|
||
for _, artifact := range rawFiles {
|
||
if candidate := packageArtifactStoragePath(artifact, ""); candidate != "" {
|
||
storagePath = candidate
|
||
break
|
||
}
|
||
}
|
||
annotation := map[string]any{
|
||
"type": "wilor_hand_pose",
|
||
"mediaKind": mediaKind,
|
||
"handCount": intValue(result["handCount"]),
|
||
"frameCount": intValue(result["frameCount"]),
|
||
"status": stringValue(payload["status"]),
|
||
"requiresHumanReview": false,
|
||
}
|
||
if task := mapValue(result["wilorTask"]); task != nil {
|
||
annotation["task"] = task
|
||
}
|
||
quality := map[string]any{
|
||
"runtime": mapValue(health["runtime"]),
|
||
"service": map[string]any{
|
||
"image": stringValue(service["image"]),
|
||
"imageId": stringValue(service["imageId"]),
|
||
},
|
||
}
|
||
return map[string]any{
|
||
"packageName": wilorPackageName(manifest, payload),
|
||
"clipId": clipID,
|
||
"sourceVideo": sourceName,
|
||
"clipDuration": 0,
|
||
"fileCount": len(rawFiles),
|
||
"pipelineTags": []any{"wilor", "hand_pose", mediaKind},
|
||
"storagePath": storagePath,
|
||
"quality": quality,
|
||
"annotation": annotation,
|
||
}
|
||
}
|
||
|
||
func canonicalBOSPath(bucket string, object string) string {
|
||
bucket = strings.TrimSpace(bucket)
|
||
object = strings.Trim(strings.TrimSpace(object), "/")
|
||
if bucket == "" {
|
||
return ""
|
||
}
|
||
if object == "" {
|
||
return "bos://" + bucket
|
||
}
|
||
return "bos://" + bucket + "/" + object
|
||
}
|
||
|
||
func packageArtifactPriority(artifact domain.ExecutionArtifact) int {
|
||
score := 0
|
||
if artifact.NodeID == "upload" {
|
||
score += 40
|
||
}
|
||
artifactPath := strings.TrimSpace(artifact.Path)
|
||
if strings.HasPrefix(strings.ToLower(artifactPath), "bos://") {
|
||
score += 30
|
||
} else if artifactPath != "" && !strings.Contains(artifactPath, "://") {
|
||
score += 10
|
||
}
|
||
payload := artifactStructuredData(&artifact)
|
||
source := mapValue(payload["source"])
|
||
sourcePath := strings.TrimSpace(stringValue(source["path"]))
|
||
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
|
||
score += 30
|
||
}
|
||
if strings.TrimSpace(packageArtifactPreviewURL(artifact)) != "" {
|
||
score += 5
|
||
}
|
||
return score
|
||
}
|
||
|
||
func packageArtifactStoragePath(artifact domain.ExecutionArtifact, packageName string) string {
|
||
payload := artifactStructuredData(&artifact)
|
||
source := mapValue(payload["source"])
|
||
if sourcePath := strings.TrimSpace(stringValue(source["path"])); sourcePath != "" {
|
||
if bucket, object, ok := parseBOSPath(sourcePath); ok {
|
||
return canonicalBOSPath(bucket, path.Dir(object))
|
||
}
|
||
directory := filepath.ToSlash(filepath.Dir(sourcePath))
|
||
if directory != "." {
|
||
return directory
|
||
}
|
||
}
|
||
artifactPath := strings.TrimSpace(artifact.Path)
|
||
if bucket, object, ok := parseBOSPath(artifactPath); ok {
|
||
return canonicalBOSPath(bucket, path.Dir(object))
|
||
}
|
||
if artifactPath == "" {
|
||
return ""
|
||
}
|
||
directory := filepath.ToSlash(filepath.Dir(artifactPath))
|
||
if directory == "." {
|
||
return ""
|
||
}
|
||
return directory
|
||
}
|
||
|
||
func dedupePackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
|
||
bestByRelativePath := make(map[string]domain.ExecutionArtifact)
|
||
for _, artifact := range run.Artifacts {
|
||
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
|
||
if relativePath == "" {
|
||
continue
|
||
}
|
||
current, exists := bestByRelativePath[relativePath]
|
||
if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) {
|
||
bestByRelativePath[relativePath] = artifact
|
||
}
|
||
}
|
||
items := make([]domain.ExecutionArtifact, 0, len(bestByRelativePath))
|
||
for _, artifact := range bestByRelativePath {
|
||
items = append(items, artifact)
|
||
}
|
||
sort.Slice(items, func(left int, right int) bool {
|
||
leftPath := packageArtifactResolvedRelativePath(items[left], packageName)
|
||
rightPath := packageArtifactResolvedRelativePath(items[right], packageName)
|
||
return leftPath < rightPath
|
||
})
|
||
return items
|
||
}
|
||
|
||
func packageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
|
||
items := make([]domain.ExecutionArtifact, 0)
|
||
for _, artifact := range dedupePackageArtifacts(run, packageName) {
|
||
items = append(items, packageArtifactForResponse(run.ID, packageName, artifact))
|
||
}
|
||
return items
|
||
}
|
||
|
||
func choosePackagePreview(files []domain.ExecutionArtifact) *domain.ExecutionArtifact {
|
||
for _, file := range files {
|
||
if strings.HasSuffix(strings.ToLower(file.Name), ".mp4") {
|
||
copy := file
|
||
return ©
|
||
}
|
||
}
|
||
if len(files) == 0 {
|
||
return nil
|
||
}
|
||
copy := files[0]
|
||
return ©
|
||
}
|
||
|
||
func sourceDatasetList(data map[string]any, catalogKey string, fallbackKey string) []any {
|
||
if _, ok := data[catalogKey]; ok {
|
||
return listValue(data[catalogKey])
|
||
}
|
||
return listValue(data[fallbackKey])
|
||
}
|
||
|
||
func sourceDatasetHasCatalog(data map[string]any) bool {
|
||
_, hasCollections := data["catalogCollections"]
|
||
_, hasItems := data["catalogItems"]
|
||
_, hasEntries := data["catalogEntries"]
|
||
return hasCollections || hasItems || hasEntries
|
||
}
|
||
|
||
func sourceDatasetIsDisplayable(data map[string]any) bool {
|
||
if len(sourceDatasetList(data, "catalogCollections", "collections")) > 0 {
|
||
return true
|
||
}
|
||
if len(sourceDatasetList(data, "catalogEntries", "entries")) > 0 {
|
||
return true
|
||
}
|
||
return intValue(data["catalogCollectionCount"]) > 0 || intValue(data["collectionCount"]) > 0
|
||
}
|
||
|
||
func runSourceManifest(run domain.RunRecord) *domain.ExecutionArtifact {
|
||
manifest := artifactByName(run, "source-manifest.json")
|
||
if manifest == nil || artifactStructuredData(manifest) == nil {
|
||
return nil
|
||
}
|
||
return manifest
|
||
}
|
||
|
||
func runDatasetIndex(run domain.RunRecord) *domain.ExecutionArtifact {
|
||
artifact := artifactByName(run, "dataset-index.json")
|
||
if artifact == nil || artifactStructuredData(artifact) == nil {
|
||
return nil
|
||
}
|
||
return artifact
|
||
}
|
||
|
||
func runLooksLikeSourceDataset(workflow domain.WorkflowRecord, run domain.RunRecord) bool {
|
||
if manifest := runSourceManifest(run); manifest != nil {
|
||
payload := artifactStructuredData(manifest)
|
||
if strings.TrimSpace(stringValue(payload["sourcePath"])) != "" || strings.TrimSpace(stringValue(payload["dataSource"])) != "" {
|
||
return true
|
||
}
|
||
}
|
||
index := runDatasetIndex(run)
|
||
if index == nil {
|
||
return false
|
||
}
|
||
if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil {
|
||
return true
|
||
}
|
||
indexPayload := artifactStructuredData(index)
|
||
sourcePath := strings.TrimSpace(stringValue(indexPayload["sourcePath"]))
|
||
rootPath := strings.TrimSpace(stringValue(indexPayload["rootPath"]))
|
||
dataSource := strings.TrimSpace(stringValue(indexPayload["dataSource"]))
|
||
if dataSource != "" || strings.HasPrefix(sourcePath, "bos://") || strings.HasPrefix(rootPath, "bos://") {
|
||
return true
|
||
}
|
||
if len(mapValue(indexPayload["archiveSources"])) > 0 && strings.Contains(rootPath, "_materialized_inputs") {
|
||
return false
|
||
}
|
||
return sourceDatasetHasCatalog(indexPayload)
|
||
}
|
||
|
||
func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]domain.WorkflowRecord, map[string]domain.RunRecord, error) {
|
||
workflows, err := server.listAllWorkflowsForContext(ctx)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
|
||
for _, workflow := range workflows {
|
||
workflowByID[workflow.ID] = workflow
|
||
}
|
||
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
latestByWorkflow := make(map[string]domain.RunRecord)
|
||
for _, run := range runs {
|
||
workflow, ok := workflowByID[run.WorkflowID]
|
||
if !ok {
|
||
continue
|
||
}
|
||
index := runDatasetIndex(run)
|
||
if index == nil {
|
||
continue
|
||
}
|
||
if !runLooksLikeSourceDataset(workflow, run) {
|
||
continue
|
||
}
|
||
if !sourceDatasetIsDisplayable(artifactStructuredData(index)) {
|
||
continue
|
||
}
|
||
current, exists := latestByWorkflow[run.WorkflowID]
|
||
if !exists || run.CreatedAt > current.CreatedAt {
|
||
latestByWorkflow[run.WorkflowID] = run
|
||
}
|
||
}
|
||
return workflowByID, latestByWorkflow, nil
|
||
}
|
||
|
||
func parseBOSPath(sourcePath string) (string, string, bool) {
|
||
normalized := strings.TrimSpace(sourcePath)
|
||
if !strings.HasPrefix(strings.ToLower(normalized), "bos://") {
|
||
return "", "", false
|
||
}
|
||
trimmed := strings.TrimPrefix(normalized, "bos://")
|
||
trimmed = strings.TrimPrefix(trimmed, "BOS://")
|
||
bucket, object, found := strings.Cut(trimmed, "/")
|
||
bucket = strings.TrimSpace(bucket)
|
||
object = strings.Trim(strings.TrimSpace(object), "/")
|
||
if !found {
|
||
object = ""
|
||
}
|
||
if bucket == "" {
|
||
return "", "", false
|
||
}
|
||
return bucket, object, true
|
||
}
|
||
|
||
func normalizeLocalDirectory(value string) string {
|
||
trimmed := strings.TrimSpace(value)
|
||
if trimmed == "" {
|
||
return ""
|
||
}
|
||
directory := filepath.ToSlash(filepath.Dir(trimmed))
|
||
if directory == "." {
|
||
return ""
|
||
}
|
||
return directory
|
||
}
|
||
|
||
func assetStorageLocationFromPaths(rootPath string, currentPath string, endpoint string) domain.AssetStorageLocation {
|
||
location := domain.AssetStorageLocation{
|
||
Path: strings.TrimSpace(currentPath),
|
||
}
|
||
|
||
currentBucket, currentObject, hasCurrentBOS := parseBOSPath(currentPath)
|
||
rootBucket, rootObject, hasRootBOS := parseBOSPath(rootPath)
|
||
|
||
switch {
|
||
case hasCurrentBOS:
|
||
location.Provider = "bos"
|
||
location.Endpoint = chooseBOSEndpoint(endpoint)
|
||
location.Bucket = currentBucket
|
||
if hasRootBOS && rootBucket == currentBucket {
|
||
location.Prefix = rootObject
|
||
} else {
|
||
location.Prefix = path.Dir(currentObject)
|
||
if location.Prefix == "." {
|
||
location.Prefix = ""
|
||
}
|
||
}
|
||
case hasRootBOS:
|
||
location.Provider = "bos"
|
||
location.Endpoint = chooseBOSEndpoint(endpoint)
|
||
location.Bucket = rootBucket
|
||
location.Prefix = rootObject
|
||
if location.Path == "" {
|
||
location.Path = strings.TrimSpace(rootPath)
|
||
}
|
||
case strings.TrimSpace(currentPath) != "":
|
||
location.Provider = "k8s-job"
|
||
location.Prefix = normalizeLocalDirectory(currentPath)
|
||
case strings.TrimSpace(rootPath) != "":
|
||
location.Provider = "k8s-job"
|
||
location.Path = strings.TrimSpace(rootPath)
|
||
location.Prefix = normalizeLocalDirectory(rootPath)
|
||
default:
|
||
location.Provider = "unknown"
|
||
}
|
||
|
||
if location.Prefix == "." {
|
||
location.Prefix = ""
|
||
}
|
||
return location
|
||
}
|
||
|
||
func sourceRootPathForRun(run domain.RunRecord, scanConfig *domain.ScanConfig) string {
|
||
if scanConfig != nil {
|
||
if value := scanSourcePath(scanConfig.DataSource, scanConfig.SourcePath, scanConfig.Bucket, scanConfig.Prefix); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
if manifest := runSourceManifest(run); manifest != nil {
|
||
payload := artifactStructuredData(manifest)
|
||
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
if index := runDatasetIndex(run); index != nil {
|
||
payload := artifactStructuredData(index)
|
||
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
|
||
return value
|
||
}
|
||
if value := strings.TrimSpace(stringValue(payload["rootPath"])); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func (server *server) buildSourceCollectionRecord(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, collection map[string]any) domain.SourceCollectionRecord {
|
||
meta := mapValue(collection["meta"])
|
||
requiredFiles := make([]string, 0)
|
||
for _, item := range listValue(collection["requiredFiles"]) {
|
||
requiredFiles = append(requiredFiles, stringValue(item))
|
||
}
|
||
missingFiles := make([]string, 0)
|
||
for _, item := range listValue(collection["missingRequiredFiles"]) {
|
||
missingFiles = append(missingFiles, stringValue(item))
|
||
}
|
||
collectionName := stringValue(collection["name"])
|
||
if collectionName == "" {
|
||
if projectID := stringValue(meta["project_id"]); projectID != "" {
|
||
collectionName = projectID
|
||
} else if relativePath := strings.Trim(stringValue(collection["relativePath"]), "/"); relativePath != "" && relativePath != "." {
|
||
collectionName = path.Base(relativePath)
|
||
} else {
|
||
collectionName = stringValue(collection["id"])
|
||
}
|
||
}
|
||
var resolvedScanConfig *domain.ScanConfig
|
||
if workflow != nil {
|
||
if scanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig); err == nil {
|
||
resolvedScanConfig = scanConfig
|
||
}
|
||
}
|
||
storage := assetStorageLocationFromPaths(
|
||
sourceRootPathForRun(run, resolvedScanConfig),
|
||
stringValue(collection["path"]),
|
||
func() string {
|
||
if resolvedScanConfig == nil {
|
||
return ""
|
||
}
|
||
return resolvedScanConfig.Endpoint
|
||
}(),
|
||
)
|
||
return domain.SourceCollectionRecord{
|
||
RunID: run.ID,
|
||
WorkflowID: run.WorkflowID,
|
||
WorkflowName: run.WorkflowName,
|
||
ProjectID: run.ProjectID,
|
||
ProjectName: run.ProjectName,
|
||
CollectionID: stringValue(collection["id"]),
|
||
CollectionName: collectionName,
|
||
Path: stringValue(collection["path"]),
|
||
RelativePath: stringValue(collection["relativePath"]),
|
||
Meta: meta,
|
||
HasIntrinsics: boolValue(collection["hasIntrinsics"]),
|
||
HasVideoMeta: boolValue(collection["hasVideoMeta"]),
|
||
RequiredFiles: requiredFiles,
|
||
MissingFiles: missingFiles,
|
||
VideoCount: intValue(collection["videoCount"]),
|
||
FileCount: intValue(collection["fileCount"]),
|
||
TotalBytes: int64Value(collection["totalBytes"]),
|
||
Valid: boolValue(collection["valid"]),
|
||
CreatedAt: run.CreatedAt,
|
||
Storage: storage,
|
||
}
|
||
}
|
||
|
||
type sourceCollectionPayloadRecord struct {
|
||
Run *domain.RunRecord
|
||
Workflow *domain.WorkflowRecord
|
||
Collection map[string]any
|
||
Items []map[string]any
|
||
Entries []map[string]any
|
||
ArchiveSource map[string]any
|
||
}
|
||
|
||
func (server *server) sourceCollectionPayload(ctx context.Context, runID string, collectionID string) (*sourceCollectionPayloadRecord, error) {
|
||
run, err := server.store.GetRun(ctx, runID)
|
||
if err != nil || run == nil {
|
||
return &sourceCollectionPayloadRecord{Run: run}, err
|
||
}
|
||
workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID)
|
||
if err != nil {
|
||
return &sourceCollectionPayloadRecord{Run: run}, err
|
||
}
|
||
artifact := artifactByName(*run, "dataset-index.json")
|
||
payload := artifactStructuredData(artifact)
|
||
if artifact == nil || payload == nil {
|
||
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
|
||
}
|
||
collections := sourceDatasetList(payload, "catalogCollections", "collections")
|
||
items := sourceDatasetList(payload, "catalogItems", "items")
|
||
entries := sourceDatasetList(payload, "catalogEntries", "entries")
|
||
var matchedCollection map[string]any
|
||
for _, item := range collections {
|
||
collection := mapValue(item)
|
||
if stringValue(collection["id"]) == collectionID {
|
||
matchedCollection = collection
|
||
break
|
||
}
|
||
}
|
||
if matchedCollection == nil {
|
||
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
|
||
}
|
||
matchedItems := make([]map[string]any, 0)
|
||
for _, item := range items {
|
||
row := mapValue(item)
|
||
if stringValue(row["collectionId"]) == collectionID {
|
||
matchedItems = append(matchedItems, row)
|
||
}
|
||
}
|
||
matchedEntries := make([]map[string]any, 0)
|
||
for _, item := range entries {
|
||
row := mapValue(item)
|
||
relativePath := strings.Trim(filepath.ToSlash(stringValue(row["relativePath"])), "/")
|
||
collectionRelativePath := strings.Trim(filepath.ToSlash(stringValue(matchedCollection["relativePath"])), "/")
|
||
if collectionRelativePath == "" || collectionRelativePath == "." {
|
||
if relativePath == "" {
|
||
continue
|
||
}
|
||
} else if relativePath != collectionRelativePath && !strings.HasPrefix(relativePath, collectionRelativePath+"/") {
|
||
continue
|
||
}
|
||
matchedEntries = append(matchedEntries, row)
|
||
}
|
||
return &sourceCollectionPayloadRecord{
|
||
Run: run,
|
||
Workflow: workflow,
|
||
Collection: matchedCollection,
|
||
Items: matchedItems,
|
||
Entries: matchedEntries,
|
||
ArchiveSource: mapValue(payload["archiveSources"]),
|
||
}, nil
|
||
}
|
||
|
||
func sourceEntryPayloadKind(relativePath string, mimeType string) string {
|
||
extension := strings.ToLower(path.Ext(relativePath))
|
||
switch {
|
||
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
|
||
return "video"
|
||
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
|
||
return "image"
|
||
case strings.HasPrefix(mimeType, "audio/") || extension == ".mp3" || extension == ".wav" || extension == ".m4a":
|
||
return "audio"
|
||
default:
|
||
return "binary"
|
||
}
|
||
}
|
||
|
||
func sourceEntryArtifactType(relativePath string, mimeType string) domain.PortType {
|
||
extension := strings.ToLower(path.Ext(relativePath))
|
||
switch {
|
||
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
|
||
return domain.PortTypeImage
|
||
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
|
||
return domain.PortTypeVideo
|
||
case extension == ".json":
|
||
return domain.PortTypeJSON
|
||
default:
|
||
return domain.PortTypeFile
|
||
}
|
||
}
|
||
|
||
func formatArtifactSize(sizeBytes int64) string {
|
||
switch {
|
||
case sizeBytes <= 0:
|
||
return "0 B"
|
||
case sizeBytes < 1024:
|
||
return fmt.Sprintf("%d B", sizeBytes)
|
||
case sizeBytes < 1024*1024:
|
||
return fmt.Sprintf("%.1f KiB", float64(sizeBytes)/1024)
|
||
case sizeBytes < 1024*1024*1024:
|
||
return fmt.Sprintf("%.1f MiB", float64(sizeBytes)/(1024*1024))
|
||
default:
|
||
return fmt.Sprintf("%.1f GiB", float64(sizeBytes)/(1024*1024*1024))
|
||
}
|
||
}
|
||
|
||
func sourceCollectionContentURL(runID string, collectionID string, relativePath string) string {
|
||
return fmt.Sprintf(
|
||
"/api/source-data/collections/%s/%s/content?relativePath=%s",
|
||
url.PathEscape(runID),
|
||
url.PathEscape(collectionID),
|
||
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
|
||
)
|
||
}
|
||
|
||
func datasetIndexEntrySource(entry map[string]any, archiveSources map[string]any) map[string]any {
|
||
source := mapValue(entry["source"])
|
||
if len(source) == 0 {
|
||
source = make(map[string]any)
|
||
}
|
||
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
|
||
if relativePath != "" && stringValue(source["relativePath"]) == "" {
|
||
source["relativePath"] = relativePath
|
||
}
|
||
mimeType := stringValue(entry["mimeType"])
|
||
if mimeType == "" {
|
||
mimeType = stringValue(source["mimeType"])
|
||
}
|
||
if mimeType == "" && relativePath != "" {
|
||
mimeType = mime.TypeByExtension(path.Ext(relativePath))
|
||
}
|
||
if mimeType != "" {
|
||
source["mimeType"] = mimeType
|
||
}
|
||
if stringValue(source["previewURL"]) == "" {
|
||
archiveKey := stringValue(source["archiveKey"])
|
||
archiveSource := mapValue(archiveSources[archiveKey])
|
||
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
|
||
source["previewURL"] = previewURL
|
||
}
|
||
if archiveName := stringValue(archiveSource["name"]); archiveName != "" && stringValue(source["archiveName"]) == "" {
|
||
source["archiveName"] = archiveName
|
||
}
|
||
}
|
||
if pathValue := stringValue(entry["path"]); pathValue != "" && stringValue(source["path"]) == "" {
|
||
source["path"] = pathValue
|
||
}
|
||
return source
|
||
}
|
||
|
||
func sourceEntryArtifact(run domain.RunRecord, collectionID string, entry map[string]any, archiveSources map[string]any) *domain.ExecutionArtifact {
|
||
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
|
||
if relativePath == "" {
|
||
relativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
|
||
}
|
||
if relativePath == "" {
|
||
return nil
|
||
}
|
||
source := datasetIndexEntrySource(entry, archiveSources)
|
||
mimeType := stringValue(source["mimeType"])
|
||
sizeBytes := int64Value(entry["sizeBytes"])
|
||
artifactType := sourceEntryArtifactType(relativePath, mimeType)
|
||
payload := make(map[string]any)
|
||
if value, ok := entry["data"]; ok {
|
||
payload["preview"] = map[string]any{
|
||
"kind": "json",
|
||
"value": normalizeStructuredValue(value),
|
||
}
|
||
}
|
||
if previewURL := stringValue(source["previewURL"]); previewURL != "" && len(payload) == 0 {
|
||
payload["preview"] = map[string]any{
|
||
"kind": sourceEntryPayloadKind(relativePath, mimeType),
|
||
"format": strings.TrimPrefix(strings.ToLower(path.Ext(relativePath)), "."),
|
||
"url": sourceCollectionContentURL(run.ID, collectionID, relativePath),
|
||
}
|
||
}
|
||
payload["relativePath"] = relativePath
|
||
if len(source) > 0 {
|
||
if previewURL := sourceCollectionContentURL(run.ID, collectionID, relativePath); previewURL != "" {
|
||
source["previewURL"] = previewURL
|
||
}
|
||
payload["source"] = source
|
||
}
|
||
return &domain.ExecutionArtifact{
|
||
ID: fmt.Sprintf("%s-source-%s", run.ID, strings.ReplaceAll(relativePath, "/", "-")),
|
||
RunID: run.ID,
|
||
NodeID: "dataset-index",
|
||
Name: path.Base(relativePath),
|
||
Type: artifactType,
|
||
Size: formatArtifactSize(sizeBytes),
|
||
Path: func() string {
|
||
if sourcePath := stringValue(source["path"]); sourcePath != "" {
|
||
return sourcePath
|
||
}
|
||
return relativePath
|
||
}(),
|
||
Metadata: &domain.ArtifactMetadata{MimeType: mimeType},
|
||
Data: payload,
|
||
}
|
||
}
|
||
|
||
func contentBytesForDataURL(previewURL string) ([]byte, string, error) {
|
||
if !strings.HasPrefix(previewURL, "data:") {
|
||
return nil, "", fmt.Errorf("not a data url")
|
||
}
|
||
header, encoded, found := strings.Cut(previewURL, ",")
|
||
if !found || !strings.Contains(header, ";base64") {
|
||
return nil, "", fmt.Errorf("invalid data url")
|
||
}
|
||
mimeType := strings.TrimPrefix(strings.SplitN(header, ";", 2)[0], "data:")
|
||
payload, err := base64.StdEncoding.DecodeString(encoded)
|
||
if err != nil {
|
||
return nil, "", err
|
||
}
|
||
return payload, mimeType, nil
|
||
}
|
||
|
||
func contentBytesForArchiveMember(archiveName string, archivePreviewURL string, memberPath string) ([]byte, error) {
|
||
payload, _, err := contentBytesForDataURL(archivePreviewURL)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
memberPath = strings.Trim(filepath.ToSlash(memberPath), "/")
|
||
if memberPath == "" {
|
||
return nil, fmt.Errorf("missing archive member path")
|
||
}
|
||
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
|
||
archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, file := range archive.File {
|
||
if strings.Trim(filepath.ToSlash(file.Name), "/") != memberPath {
|
||
continue
|
||
}
|
||
handle, err := file.Open()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer handle.Close()
|
||
return io.ReadAll(handle)
|
||
}
|
||
return nil, os.ErrNotExist
|
||
}
|
||
reader := bytes.NewReader(payload)
|
||
var tarReader *tar.Reader
|
||
if strings.HasSuffix(strings.ToLower(archiveName), ".tgz") || strings.HasSuffix(strings.ToLower(archiveName), ".tar.gz") || (len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b) {
|
||
gzipReader, err := gzip.NewReader(reader)
|
||
if err == nil {
|
||
defer gzipReader.Close()
|
||
tarReader = tar.NewReader(gzipReader)
|
||
} else {
|
||
reader = bytes.NewReader(payload)
|
||
tarReader = tar.NewReader(reader)
|
||
}
|
||
} else {
|
||
tarReader = tar.NewReader(reader)
|
||
}
|
||
for {
|
||
header, err := tarReader.Next()
|
||
if err == io.EOF {
|
||
return nil, os.ErrNotExist
|
||
}
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if header.FileInfo().IsDir() {
|
||
continue
|
||
}
|
||
if strings.Trim(filepath.ToSlash(header.Name), "/") != memberPath {
|
||
continue
|
||
}
|
||
return io.ReadAll(tarReader)
|
||
}
|
||
}
|
||
|
||
func sourceCollectionEntryPayload(collection domain.SourceCollectionRecord, payloadEntries []map[string]any, relativePath string) map[string]any {
|
||
target := strings.Trim(filepath.ToSlash(relativePath), "/")
|
||
if target == "" {
|
||
return nil
|
||
}
|
||
collectionPrefix := strings.Trim(filepath.ToSlash(collection.RelativePath), "/")
|
||
fullTarget := target
|
||
if collectionPrefix != "" && target != collectionPrefix && !strings.HasPrefix(target, collectionPrefix+"/") {
|
||
fullTarget = collectionPrefix + "/" + target
|
||
}
|
||
for _, entry := range payloadEntries {
|
||
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
|
||
if entryRelativePath == "" {
|
||
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
|
||
}
|
||
if entryRelativePath == fullTarget || entryRelativePath == target {
|
||
return entry
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func writeSourceContent(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, payload []byte) {
|
||
if mimeType == "" {
|
||
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
|
||
}
|
||
if mimeType == "" && len(payload) > 0 {
|
||
mimeType = http.DetectContentType(payload)
|
||
}
|
||
if mimeType != "" {
|
||
writer.Header().Set("Content-Type", mimeType)
|
||
}
|
||
writeSourceContentDisposition(writer, request, fileName)
|
||
http.ServeContent(writer, request, fileName, time.Time{}, bytes.NewReader(payload))
|
||
}
|
||
|
||
func sourceContentDownloadRequested(request *http.Request) bool {
|
||
value := strings.ToLower(strings.TrimSpace(request.URL.Query().Get("download")))
|
||
return value == "1" || value == "true" || value == "yes"
|
||
}
|
||
|
||
func writeSourceContentDisposition(writer http.ResponseWriter, request *http.Request, fileName string) {
|
||
if !sourceContentDownloadRequested(request) {
|
||
return
|
||
}
|
||
name := strings.TrimSpace(path.Base(fileName))
|
||
if name == "" || name == "." || name == "/" {
|
||
name = "artifact"
|
||
}
|
||
writer.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": name}))
|
||
}
|
||
|
||
func writeSourceContentFile(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, filePath string) error {
|
||
file, err := os.Open(filePath)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
defer file.Close()
|
||
|
||
fileInfo, err := file.Stat()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
if mimeType == "" {
|
||
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
|
||
}
|
||
if mimeType == "" {
|
||
header := make([]byte, 512)
|
||
count, readErr := file.Read(header)
|
||
if readErr != nil && readErr != io.EOF {
|
||
return readErr
|
||
}
|
||
mimeType = http.DetectContentType(header[:count])
|
||
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
if mimeType != "" {
|
||
writer.Header().Set("Content-Type", mimeType)
|
||
}
|
||
writeSourceContentDisposition(writer, request, fileName)
|
||
http.ServeContent(writer, request, fileName, fileInfo.ModTime(), file)
|
||
return nil
|
||
}
|
||
|
||
func sourceBrowserPreviewCacheKey(request sourceBrowserPreviewRequest) string {
|
||
hash := sha256.New()
|
||
hash.Write([]byte("browser-preview-v3"))
|
||
hash.Write([]byte{0})
|
||
hash.Write([]byte(strings.TrimSpace(request.FileName)))
|
||
hash.Write([]byte{0})
|
||
// Cache on the object coordinates, not on the presigned URL: presigned
|
||
// URLs are regenerated per request (timestamped authorization), so a key
|
||
// derived from the URL never matches across requests and the expensive
|
||
// transcode result never hits the cache. The bucket/object identity is
|
||
// embedded in RedirectURL queries or Payload, but the stable object
|
||
// coordinates arrive through ObjectBucket/ObjectKey when the input is a
|
||
// storage object; otherwise fall back to the URL payload hash.
|
||
if request.ObjectBucket != "" || request.ObjectKey != "" {
|
||
hash.Write([]byte("object"))
|
||
hash.Write([]byte{0})
|
||
hash.Write([]byte(strings.TrimSpace(request.ObjectBucket)))
|
||
hash.Write([]byte{0})
|
||
hash.Write([]byte(strings.TrimSpace(request.ObjectKey)))
|
||
}
|
||
if len(request.Payload) > 0 {
|
||
payloadHash := sha256.Sum256(request.Payload)
|
||
hash.Write([]byte("payload"))
|
||
hash.Write([]byte{0})
|
||
hash.Write(payloadHash[:])
|
||
}
|
||
return hex.EncodeToString(hash.Sum(nil))
|
||
}
|
||
|
||
func sourceBrowserPreviewCacheDir() (string, error) {
|
||
cacheDir := filepath.Join(os.TempDir(), "droboticflow", "source-video-previews")
|
||
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
|
||
return "", err
|
||
}
|
||
return cacheDir, nil
|
||
}
|
||
|
||
func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string, error) {
|
||
cacheDir, err := sourceBrowserPreviewCacheDir()
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
maybePruneSourceBrowserPreviewCache(cacheDir)
|
||
cacheKey := sourceBrowserPreviewCacheKey(request)
|
||
return filepath.Join(cacheDir, cacheKey+".mp4"), nil
|
||
}
|
||
|
||
// The on-disk preview cache is bounded: entries older than the TTL are
|
||
// deleted outright, then least-recently-written entries are evicted until
|
||
// the directory is under the size cap. Without this bound the /tmp cache
|
||
// grows without limit on pods that serve many long videos.
|
||
const (
|
||
sourceBrowserPreviewCacheMaxBytes = 2 << 30 // 2 GiB total cap
|
||
sourceBrowserPreviewCacheMaxAge = 7 * 24 * time.Hour
|
||
sourceBrowserPreviewCachePruneEvery = 10 * time.Minute
|
||
)
|
||
|
||
var lastSourceBrowserPreviewCachePrune atomic.Int64
|
||
|
||
func maybePruneSourceBrowserPreviewCache(dir string) {
|
||
interval := sourceBrowserPreviewCachePruneEvery.Nanoseconds()
|
||
now := time.Now().UnixNano()
|
||
last := lastSourceBrowserPreviewCachePrune.Load()
|
||
if now-last < interval {
|
||
return
|
||
}
|
||
if !lastSourceBrowserPreviewCachePrune.CompareAndSwap(last, now) {
|
||
return
|
||
}
|
||
go pruneSourceBrowserPreviewCache(dir, time.Now())
|
||
}
|
||
|
||
// pruneSourceBrowserPreviewCache enforces the TTL and size bounds on the
|
||
// preview cache directory. Best effort: unreadable entries are skipped and
|
||
// removal failures are ignored (the next prune retries).
|
||
func pruneSourceBrowserPreviewCache(dir string, now time.Time) {
|
||
pruneSourceBrowserPreviewCacheWithLimit(dir, now, sourceBrowserPreviewCacheMaxBytes)
|
||
}
|
||
|
||
// pruneSourceBrowserPreviewCacheWithLimit is the testable core of the cache
|
||
// eviction: delete entries past the TTL, then evict least-recently-written
|
||
// entries until the directory is under maxBytes.
|
||
func pruneSourceBrowserPreviewCacheWithLimit(dir string, now time.Time, maxBytes int64) {
|
||
entries, err := os.ReadDir(dir)
|
||
if err != nil {
|
||
return
|
||
}
|
||
type cacheEntry struct {
|
||
path string
|
||
modTime time.Time
|
||
size int64
|
||
}
|
||
files := make([]cacheEntry, 0, len(entries))
|
||
var total int64
|
||
for _, entry := range entries {
|
||
if entry.IsDir() {
|
||
continue
|
||
}
|
||
info, infoErr := entry.Info()
|
||
if infoErr != nil {
|
||
continue
|
||
}
|
||
if now.Sub(info.ModTime()) > sourceBrowserPreviewCacheMaxAge {
|
||
_ = os.Remove(filepath.Join(dir, entry.Name()))
|
||
continue
|
||
}
|
||
files = append(files, cacheEntry{
|
||
path: filepath.Join(dir, entry.Name()),
|
||
modTime: info.ModTime(),
|
||
size: info.Size(),
|
||
})
|
||
total += info.Size()
|
||
}
|
||
if total <= maxBytes {
|
||
return
|
||
}
|
||
sort.Slice(files, func(left int, right int) bool {
|
||
return files[left].modTime.Before(files[right].modTime)
|
||
})
|
||
for _, file := range files {
|
||
if total <= maxBytes {
|
||
break
|
||
}
|
||
if removeErr := os.Remove(file.path); removeErr == nil {
|
||
total -= file.size
|
||
}
|
||
}
|
||
}
|
||
|
||
func parseBOSObjectPath(sourcePath string) (string, string, bool) {
|
||
bucket, object, ok := parseBOSPath(sourcePath)
|
||
if !ok || object == "" {
|
||
return "", "", false
|
||
}
|
||
return bucket, object, true
|
||
}
|
||
|
||
func buildSourceCollectionSignedPreviewURL(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
|
||
if scanConfig == nil {
|
||
return "", fmt.Errorf("scan config is unavailable")
|
||
}
|
||
if chooseDataSource(scanConfig.DataSource) != "BOS" {
|
||
return "", fmt.Errorf("scan config is not BOS")
|
||
}
|
||
bucket, object, ok := parseBOSObjectPath(sourcePath)
|
||
if !ok {
|
||
return "", fmt.Errorf("invalid BOS source path: %s", sourcePath)
|
||
}
|
||
accessKeyID := strings.TrimSpace(scanConfig.AccessKeyID)
|
||
secretAccessKey := strings.TrimSpace(scanConfig.SecretAccessKey)
|
||
if accessKeyID == "" || secretAccessKey == "" {
|
||
return "", fmt.Errorf("BOS credentials are unavailable")
|
||
}
|
||
endpoint := chooseBOSEndpoint(scanConfig.Endpoint)
|
||
if endpoint == "" {
|
||
return "", fmt.Errorf("BOS endpoint is unavailable")
|
||
}
|
||
client, err := bos.NewClient(accessKeyID, secretAccessKey, endpoint)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
previewURL := strings.TrimSpace(client.BasicGeneratePresignedUrl(bucket, object, 3600))
|
||
if previewURL == "" {
|
||
return "", fmt.Errorf("failed to generate BOS preview url")
|
||
}
|
||
return previewURL, nil
|
||
}
|
||
|
||
func locateSourceBrowserVideoPreviewPath(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
|
||
finalPath, err := sourceBrowserPreviewCachePath(request)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
if fileInfo, err := os.Stat(finalPath); err == nil && fileInfo.Size() > 0 {
|
||
return finalPath, nil
|
||
}
|
||
return "", os.ErrNotExist
|
||
}
|
||
|
||
func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWriter, request sourceBrowserPreviewRequest) error {
|
||
if strings.TrimSpace(request.RedirectURL) == "" && len(request.Payload) == 0 {
|
||
return fmt.Errorf("source video content is unavailable")
|
||
}
|
||
|
||
finalPath, err := sourceBrowserPreviewCachePath(request)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
lock := sourceBrowserPreviewLock(filepath.Base(finalPath))
|
||
lock.Lock()
|
||
defer lock.Unlock()
|
||
if info, statErr := os.Stat(finalPath); statErr == nil && info.Size() > 0 {
|
||
cached, openErr := os.Open(finalPath)
|
||
if openErr != nil {
|
||
return openErr
|
||
}
|
||
defer cached.Close()
|
||
_, copyErr := io.Copy(writer, cached)
|
||
return copyErr
|
||
}
|
||
inputRef := strings.TrimSpace(request.RedirectURL)
|
||
var cleanupInput func()
|
||
if inputRef == "" {
|
||
extension := path.Ext(request.FileName)
|
||
if extension == "" {
|
||
extension = ".bin"
|
||
}
|
||
cacheDir := filepath.Dir(finalPath)
|
||
inputFile, err := os.CreateTemp(cacheDir, "source-video-*"+extension)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if _, err := inputFile.Write(request.Payload); err != nil {
|
||
inputFile.Close()
|
||
os.Remove(inputFile.Name())
|
||
return err
|
||
}
|
||
if err := inputFile.Close(); err != nil {
|
||
os.Remove(inputFile.Name())
|
||
return err
|
||
}
|
||
inputRef = inputFile.Name()
|
||
cleanupInput = func() {
|
||
_ = os.Remove(inputFile.Name())
|
||
}
|
||
}
|
||
if cleanupInput != nil {
|
||
defer cleanupInput()
|
||
}
|
||
|
||
cacheDir := filepath.Dir(finalPath)
|
||
outputFile, err := os.CreateTemp(cacheDir, sourceBrowserPreviewCacheKey(request)+"-*.mp4")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
tempOutputPath := outputFile.Name()
|
||
stderr := &bytes.Buffer{}
|
||
// A BOS presigned URL can stall mid-stream; without a read timeout the
|
||
// transcode hangs until the process is killed and the viewer sees a
|
||
// truncated preview. 15 minutes of no-progress terminates the pull.
|
||
command := exec.CommandContext(
|
||
context.WithoutCancel(ctx),
|
||
"ffmpeg",
|
||
"-y",
|
||
"-hide_banner",
|
||
"-loglevel", "error",
|
||
"-rw_timeout", "15000000000",
|
||
"-reconnect", "1",
|
||
"-reconnect_streamed", "1",
|
||
"-reconnect_delay_max", "5",
|
||
"-i", inputRef,
|
||
"-map", "0:v:0",
|
||
"-map", "0:a?",
|
||
"-threads", "4",
|
||
"-probesize", "32M",
|
||
"-analyzeduration", "0",
|
||
"-c:v", "libx264",
|
||
"-preset", "ultrafast",
|
||
"-tune", "zerolatency",
|
||
"-crf", "30",
|
||
"-vf", "fps=12,scale=min(960\\,iw):-2",
|
||
"-g", "12",
|
||
"-keyint_min", "12",
|
||
"-pix_fmt", "yuv420p",
|
||
"-movflags", "frag_keyframe+empty_moov+default_base_moof",
|
||
"-c:a", "aac",
|
||
"-b:a", "96k",
|
||
"-ac", "1",
|
||
"-f", "mp4",
|
||
"pipe:1",
|
||
)
|
||
command.Stderr = stderr
|
||
stdout, err := command.StdoutPipe()
|
||
if err != nil {
|
||
outputFile.Close()
|
||
_ = os.Remove(tempOutputPath)
|
||
return err
|
||
}
|
||
if err := command.Start(); err != nil {
|
||
outputFile.Close()
|
||
_ = os.Remove(tempOutputPath)
|
||
message := strings.TrimSpace(stderr.String())
|
||
if message != "" {
|
||
return fmt.Errorf("generate browser video preview: %s", message)
|
||
}
|
||
return err
|
||
}
|
||
streamTarget := io.Writer(writer)
|
||
if flusher, ok := writer.(http.Flusher); ok {
|
||
streamTarget = flushResponseWriter{writer: writer, flusher: flusher}
|
||
}
|
||
var clientErr error
|
||
bestEffort := &bestEffortPreviewWriter{target: streamTarget, err: &clientErr}
|
||
if _, err := io.Copy(io.MultiWriter(bestEffort, outputFile), stdout); err != nil {
|
||
outputFile.Close()
|
||
_ = os.Remove(tempOutputPath)
|
||
_ = command.Wait()
|
||
return err
|
||
}
|
||
if err := outputFile.Close(); err != nil {
|
||
_ = os.Remove(tempOutputPath)
|
||
_ = command.Wait()
|
||
return err
|
||
}
|
||
if err := command.Wait(); err != nil {
|
||
message := strings.TrimSpace(stderr.String())
|
||
_ = os.Remove(tempOutputPath)
|
||
if message != "" {
|
||
return fmt.Errorf("generate browser video preview: %s", message)
|
||
}
|
||
return fmt.Errorf("generate browser video preview: %w", err)
|
||
}
|
||
|
||
if err := os.Rename(tempOutputPath, finalPath); err != nil {
|
||
if fileInfo, statErr := os.Stat(finalPath); statErr == nil && fileInfo.Size() > 0 {
|
||
_ = os.Remove(tempOutputPath)
|
||
return nil
|
||
}
|
||
_ = os.Remove(tempOutputPath)
|
||
return err
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// sourceVideoPreviewSyncMode is the server-side rollback switch for the C5
|
||
// video direct-link change: setting DROBOTICFLOW_VIDEO_PREVIEW_SYNC=1 keeps
|
||
// every browser-preview request on the legacy blocking ffmpeg stream (no
|
||
// background pre-transcode, no redirect to the direct storage URL).
|
||
var sourceVideoPreviewSyncMode = sync.OnceValue(func() bool {
|
||
return strings.TrimSpace(os.Getenv("DROBOTICFLOW_VIDEO_PREVIEW_SYNC")) == "1"
|
||
})
|
||
|
||
// discardPreviewResponseWriter lets the background pre-transcode reuse
|
||
// streamSourceBrowserVideoPreview unchanged: its output is dropped and only
|
||
// the cached artifact on disk matters.
|
||
type discardPreviewResponseWriter struct{}
|
||
|
||
func (discardPreviewResponseWriter) Header() http.Header { return http.Header{} }
|
||
func (discardPreviewResponseWriter) WriteHeader(int) {}
|
||
func (discardPreviewResponseWriter) Write(p []byte) (int, error) { return len(p), nil }
|
||
|
||
// startSourceBrowserVideoPretranscode warms the preview cache in the
|
||
// background. The per-key lock inside streamSourceBrowserVideoPreview
|
||
// deduplicates concurrent warmups, and a warm run that finds the cache
|
||
// already populated returns immediately without invoking ffmpeg.
|
||
func startSourceBrowserVideoPretranscode(previewRequest sourceBrowserPreviewRequest) {
|
||
go func() {
|
||
// ffmpeg's own rw_timeout (15 minutes without progress) bounds the
|
||
// pull; streamSourceBrowserVideoPreview deliberately detaches
|
||
// cancellation so a client disconnect cannot truncate the warmup.
|
||
// Routed through the writer var so tests can stub the transcode.
|
||
if err := sourceBrowserVideoPreviewWriter(context.Background(), discardPreviewResponseWriter{}, previewRequest); err != nil {
|
||
fmt.Printf("[api] background browser video preview transcode failed: %v\n", err)
|
||
}
|
||
}()
|
||
}
|
||
|
||
// writeBrowserVideoPreviewResponse serves the browser-transcoded preview for
|
||
// a video entry (query transcode=browser). Order of preference:
|
||
// 1. serve the cached pre-transcoded mp4 when present (no ffmpeg at all);
|
||
// 2. start a background pre-transcode and immediately redirect the player
|
||
// to the direct storage URL (BOS presigned GET link), so the preview
|
||
// first frame is never blocked behind server-side ffmpeg;
|
||
// 3. legacy blocking ffmpeg stream as fallback (no direct URL available,
|
||
// explicit sync=1 request parameter, or the
|
||
// DROBOTICFLOW_VIDEO_PREVIEW_SYNC=1 rollback switch).
|
||
//
|
||
// All video responses set Content-Encoding: identity so the global chi
|
||
// Compress(5) middleware skips gzip: mp4 payloads are already compressed
|
||
// media and re-compressing them only burned API pod CPU.
|
||
func writeBrowserVideoPreviewResponse(writer http.ResponseWriter, request *http.Request, previewRequest sourceBrowserPreviewRequest, directURL string) {
|
||
// Videos are already-compressed media; the identity marker makes chi's
|
||
// Compress middleware leave the byte stream untouched.
|
||
writer.Header().Set("Content-Encoding", "identity")
|
||
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
|
||
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(previewRequest.FileName, path.Ext(previewRequest.FileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
}
|
||
return
|
||
}
|
||
direct := strings.TrimSpace(directURL)
|
||
forcedSync := sourceVideoPreviewSyncMode() ||
|
||
strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("sync")), "1")
|
||
if direct != "" && strings.HasPrefix(strings.ToLower(direct), "http") && !forcedSync {
|
||
startSourceBrowserVideoPretranscode(previewRequest)
|
||
http.Redirect(writer, request, direct, http.StatusTemporaryRedirect)
|
||
return
|
||
}
|
||
writer.Header().Set("Content-Type", "video/mp4")
|
||
writer.Header().Set("Cache-Control", "no-store")
|
||
writer.WriteHeader(http.StatusOK)
|
||
if flusher, ok := writer.(http.Flusher); ok {
|
||
flusher.Flush()
|
||
}
|
||
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
|
||
fmt.Printf("[api] browser preview stream failed: %v\n", err)
|
||
}
|
||
}
|
||
|
||
type bestEffortPreviewWriter struct {
|
||
target io.Writer
|
||
err *error
|
||
}
|
||
|
||
func (w *bestEffortPreviewWriter) Write(payload []byte) (int, error) {
|
||
n, err := w.target.Write(payload)
|
||
if err != nil {
|
||
if w.err != nil && *w.err == nil {
|
||
*w.err = err
|
||
}
|
||
return len(payload), nil
|
||
}
|
||
return n, nil
|
||
}
|
||
|
||
func derivedSourceEntriesFromIndex(collection domain.SourceCollectionRecord, payloadItems []map[string]any) []map[string]any {
|
||
entries := make([]map[string]any, 0, len(payloadItems)+2)
|
||
if len(collection.Meta) > 0 {
|
||
entries = append(entries, map[string]any{
|
||
"collectionId": collection.CollectionID,
|
||
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/meta.json", "/"),
|
||
"sizeBytes": len(fmt.Sprintf("%v", collection.Meta)),
|
||
"mimeType": "application/json",
|
||
"data": collection.Meta,
|
||
"derivedFromIndex": true,
|
||
})
|
||
}
|
||
if collection.HasVideoMeta && len(payloadItems) > 0 {
|
||
videoRows := make([]map[string]any, 0, len(payloadItems))
|
||
for _, item := range payloadItems {
|
||
videoRows = append(videoRows, map[string]any{
|
||
"filename": stringValue(item["filename"]),
|
||
"codec": stringValue(item["codec"]),
|
||
"resolution": stringValue(item["resolution"]),
|
||
"frameRate": floatValue(item["frameRate"]),
|
||
"duration": floatValue(item["duration"]),
|
||
"startTime": item["startTime"],
|
||
})
|
||
}
|
||
entries = append(entries, map[string]any{
|
||
"collectionId": collection.CollectionID,
|
||
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/video_meta.json", "/"),
|
||
"mimeType": "application/json",
|
||
"data": videoRows,
|
||
"derivedFromIndex": true,
|
||
})
|
||
}
|
||
for _, item := range payloadItems {
|
||
entry := map[string]any{
|
||
"collectionId": collection.CollectionID,
|
||
"relativePath": stringValue(item["relativePath"]),
|
||
"path": stringValue(item["path"]),
|
||
"sizeBytes": int64Value(item["sizeBytes"]),
|
||
"mimeType": stringValue(mapValue(item["source"])["mimeType"]),
|
||
"source": mapValue(item["source"]),
|
||
"codec": stringValue(item["codec"]),
|
||
"resolution": stringValue(item["resolution"]),
|
||
"frameRate": floatValue(item["frameRate"]),
|
||
"duration": floatValue(item["duration"]),
|
||
"derivedFromIndex": true,
|
||
}
|
||
entries = append(entries, entry)
|
||
}
|
||
return entries
|
||
}
|
||
|
||
func buildSourceEntries(run domain.RunRecord, collection domain.SourceCollectionRecord, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, prefix string) []domain.SourceCollectionEntry {
|
||
if len(payloadEntries) == 0 {
|
||
payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems)
|
||
}
|
||
videoMetaByRelative := make(map[string]map[string]any, len(payloadItems))
|
||
for _, item := range payloadItems {
|
||
videoMetaByRelative[stringValue(item["relativePath"])] = item
|
||
}
|
||
normalizedPrefix := strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/")
|
||
prefixSlash := normalizedPrefix
|
||
if prefixSlash != "" {
|
||
prefixSlash += "/"
|
||
}
|
||
|
||
directories := make(map[string]domain.SourceCollectionEntry)
|
||
files := make([]domain.SourceCollectionEntry, 0)
|
||
collectionPrefix := strings.Trim(collection.RelativePath, "/")
|
||
if collectionPrefix != "" {
|
||
collectionPrefix += "/"
|
||
}
|
||
|
||
for _, entryPayload := range payloadEntries {
|
||
relativePath := strings.Trim(filepath.ToSlash(stringValue(entryPayload["relativePath"])), "/")
|
||
if relativePath == "" {
|
||
relativePath = strings.Trim(filepath.ToSlash(stringValue(entryPayload["path"])), "/")
|
||
}
|
||
if relativePath == "" {
|
||
continue
|
||
}
|
||
if collectionPrefix != "" && relativePath != strings.TrimSuffix(collectionPrefix, "/") && !strings.HasPrefix(relativePath, collectionPrefix) {
|
||
continue
|
||
}
|
||
entryPath := relativePath
|
||
if collectionPrefix != "" {
|
||
entryPath = strings.TrimPrefix(relativePath, collectionPrefix)
|
||
}
|
||
if entryPath == "" {
|
||
continue
|
||
}
|
||
if normalizedPrefix != "" {
|
||
if entryPath == normalizedPrefix {
|
||
// exact file match
|
||
} else if !strings.HasPrefix(entryPath, prefixSlash) {
|
||
continue
|
||
} else {
|
||
entryPath = strings.TrimPrefix(entryPath, prefixSlash)
|
||
}
|
||
}
|
||
if entryPath == "" {
|
||
continue
|
||
}
|
||
parts := strings.Split(entryPath, "/")
|
||
if len(parts) > 1 {
|
||
dirPath := parts[0]
|
||
displayPath := dirPath
|
||
if normalizedPrefix != "" {
|
||
displayPath = normalizedPrefix + "/" + dirPath
|
||
}
|
||
if _, exists := directories[displayPath]; !exists {
|
||
directories[displayPath] = domain.SourceCollectionEntry{
|
||
Path: displayPath,
|
||
Name: dirPath,
|
||
Kind: "directory",
|
||
CollectionName: collection.CollectionName,
|
||
DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]),
|
||
}
|
||
} else if boolValue(entryPayload["derivedFromIndex"]) {
|
||
entry := directories[displayPath]
|
||
entry.DerivedFromIndex = true
|
||
directories[displayPath] = entry
|
||
}
|
||
continue
|
||
}
|
||
displayPath := parts[0]
|
||
if normalizedPrefix != "" && displayPath != normalizedPrefix {
|
||
displayPath = normalizedPrefix + "/" + displayPath
|
||
}
|
||
artifact := sourceEntryArtifact(run, collection.CollectionID, entryPayload, archiveSources)
|
||
videoMeta := videoMetaByRelative[filepath.ToSlash(strings.Trim(collection.RelativePath, "/")+"/"+displayPath)]
|
||
entry := domain.SourceCollectionEntry{
|
||
Path: displayPath,
|
||
Name: parts[0],
|
||
Kind: "file",
|
||
CollectionName: collection.CollectionName,
|
||
SizeBytes: int64Value(entryPayload["sizeBytes"]),
|
||
Artifact: artifact,
|
||
DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]),
|
||
}
|
||
if videoMeta != nil {
|
||
entry.Kind = "video"
|
||
entry.Codec = stringValue(videoMeta["codec"])
|
||
entry.Resolution = stringValue(videoMeta["resolution"])
|
||
entry.FrameRate = floatValue(videoMeta["frameRate"])
|
||
entry.Duration = floatValue(videoMeta["duration"])
|
||
entry.Metadata = videoMeta
|
||
if entry.SizeBytes == 0 {
|
||
entry.SizeBytes = int64Value(videoMeta["sizeBytes"])
|
||
}
|
||
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".json") {
|
||
entry.Kind = "json"
|
||
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".mp4") {
|
||
entry.Kind = "video"
|
||
}
|
||
if entry.SizeBytes == 0 {
|
||
entry.SizeBytes = int64Value(entryPayload["sizeBytes"])
|
||
}
|
||
files = append(files, entry)
|
||
}
|
||
|
||
items := make([]domain.SourceCollectionEntry, 0, len(directories)+len(files))
|
||
for _, item := range directories {
|
||
items = append(items, item)
|
||
}
|
||
sort.Slice(items, func(left int, right int) bool { return items[left].Path < items[right].Path })
|
||
sort.Slice(files, func(left int, right int) bool { return files[left].Path < files[right].Path })
|
||
items = append(items, files...)
|
||
return items
|
||
}
|
||
|
||
func sourceCollectionContentPayload(collection domain.SourceCollectionRecord, scanConfig *domain.ScanConfig, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, relativePath string) ([]byte, string, string, string, error) {
|
||
if len(payloadEntries) == 0 {
|
||
payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems)
|
||
}
|
||
entry := sourceCollectionEntryPayload(collection, payloadEntries, relativePath)
|
||
if entry == nil {
|
||
return nil, "", "", "", os.ErrNotExist
|
||
}
|
||
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
|
||
if entryRelativePath == "" {
|
||
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
|
||
}
|
||
fileName := path.Base(entryRelativePath)
|
||
source := datasetIndexEntrySource(entry, archiveSources)
|
||
mimeType := stringValue(source["mimeType"])
|
||
if mimeType == "" {
|
||
mimeType = stringValue(entry["mimeType"])
|
||
}
|
||
if mimeType == "" {
|
||
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
|
||
}
|
||
if value, ok := entry["data"]; ok {
|
||
payload, err := json.MarshalIndent(normalizeStructuredValue(value), "", " ")
|
||
if err != nil {
|
||
return nil, "", "", "", err
|
||
}
|
||
if mimeType == "" {
|
||
mimeType = "application/json"
|
||
}
|
||
return payload, mimeType, fileName, "", nil
|
||
}
|
||
if sourcePath := stringValue(source["path"]); sourcePath != "" && !strings.Contains(sourcePath, "://") {
|
||
if payload, err := os.ReadFile(sourcePath); err == nil {
|
||
return payload, mimeType, fileName, "", nil
|
||
}
|
||
}
|
||
if sourcePath := stringValue(source["path"]); strings.HasPrefix(strings.ToLower(sourcePath), "bos://") && scanConfig != nil {
|
||
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
|
||
source["previewURL"] = previewURL
|
||
}
|
||
}
|
||
archiveKey := stringValue(source["archiveKey"])
|
||
archiveMemberPath := stringValue(source["archiveMemberPath"])
|
||
if archiveKey != "" && archiveMemberPath != "" {
|
||
archiveSource := mapValue(archiveSources[archiveKey])
|
||
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
|
||
payload, err := contentBytesForArchiveMember(stringValue(archiveSource["name"]), previewURL, archiveMemberPath)
|
||
if err == nil {
|
||
return payload, mimeType, fileName, "", nil
|
||
}
|
||
}
|
||
}
|
||
if previewURL := stringValue(source["previewURL"]); previewURL != "" {
|
||
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
|
||
return nil, "", fileName, previewURL, nil
|
||
}
|
||
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
|
||
if err == nil {
|
||
if mimeType == "" {
|
||
mimeType = decodedMimeType
|
||
}
|
||
return payload, mimeType, fileName, "", nil
|
||
}
|
||
}
|
||
return nil, "", "", "", os.ErrNotExist
|
||
}
|
||
|
||
func (server *server) handleSourceCollectionsList(writer http.ResponseWriter, request *http.Request) {
|
||
workflowByID, latestRuns, err := server.latestCompletedScanRuns(request.Context())
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
items := make([]domain.SourceCollectionRecord, 0)
|
||
for _, run := range latestRuns {
|
||
if !server.filterByTenant(request, run.OrganizationID) {
|
||
continue
|
||
}
|
||
artifact := artifactByName(run, "dataset-index.json")
|
||
payload := artifactStructuredData(artifact)
|
||
if artifact == nil || payload == nil {
|
||
continue
|
||
}
|
||
for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") {
|
||
collection := mapValue(raw)
|
||
if collection == nil {
|
||
continue
|
||
}
|
||
workflow := workflowByID[run.WorkflowID]
|
||
items = append(items, server.buildSourceCollectionRecord(request.Context(), run, &workflow, collection))
|
||
}
|
||
}
|
||
sort.Slice(items, func(left int, right int) bool {
|
||
if items[left].CreatedAt == items[right].CreatedAt {
|
||
if items[left].WorkflowName == items[right].WorkflowName {
|
||
return items[left].CollectionID < items[right].CollectionID
|
||
}
|
||
return items[left].WorkflowName < items[right].WorkflowName
|
||
}
|
||
return items[left].CreatedAt > items[right].CreatedAt
|
||
})
|
||
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
|
||
}
|
||
|
||
func (server *server) handleSourceCollectionDetail(writer http.ResponseWriter, request *http.Request) {
|
||
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if payload.Run == nil || payload.Collection == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
|
||
return
|
||
}
|
||
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
|
||
detail := domain.SourceCollectionDetail{
|
||
Collection: record,
|
||
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, ""),
|
||
}
|
||
renderJSON(writer, http.StatusOK, detail)
|
||
}
|
||
|
||
func (server *server) handleSourceCollectionEntries(writer http.ResponseWriter, request *http.Request) {
|
||
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if payload.Run == nil || payload.Collection == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
|
||
return
|
||
}
|
||
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
|
||
prefix := request.URL.Query().Get("prefix")
|
||
detail := domain.SourceCollectionDetail{
|
||
Collection: record,
|
||
Prefix: prefix,
|
||
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, prefix),
|
||
}
|
||
renderJSON(writer, http.StatusOK, detail)
|
||
}
|
||
|
||
func (server *server) handleSourceCollectionContent(writer http.ResponseWriter, request *http.Request) {
|
||
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if payload.Run == nil || payload.Collection == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
|
||
return
|
||
}
|
||
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
|
||
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
|
||
if relativePath == "" {
|
||
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
|
||
return
|
||
}
|
||
var effectiveScanConfig *domain.ScanConfig
|
||
if payload.Workflow != nil {
|
||
effectiveScanConfig, err = server.resolveScanConfig(request.Context(), payload.Workflow.ScanConfig)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
}
|
||
content, mimeType, fileName, redirectURL, err := sourceCollectionContentPayload(record, effectiveScanConfig, payload.Entries, payload.Items, payload.ArchiveSource, relativePath)
|
||
if err != nil {
|
||
if os.IsNotExist(err) {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source file not found"})
|
||
return
|
||
}
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
|
||
previewRequest := sourceBrowserPreviewRequest{
|
||
FileName: fileName,
|
||
Payload: content,
|
||
RedirectURL: redirectURL,
|
||
}
|
||
writeBrowserVideoPreviewResponse(writer, request, previewRequest, redirectURL)
|
||
return
|
||
}
|
||
if redirectURL != "" {
|
||
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
|
||
return
|
||
}
|
||
writeSourceContent(writer, request, fileName, mimeType, content)
|
||
}
|
||
|
||
func packageArtifactCandidatesByRelativePath(run domain.RunRecord, packageName string, relativePath string) []domain.ExecutionArtifact {
|
||
target := strings.Trim(filepath.ToSlash(relativePath), "/")
|
||
if target == "" {
|
||
return nil
|
||
}
|
||
if _, payload := wilorPackageManifestArtifact(run, packageName); payload != nil {
|
||
items := make([]domain.ExecutionArtifact, 0)
|
||
for _, artifact := range wilorPackageArtifacts(run, packageName) {
|
||
if strings.Trim(filepath.ToSlash(artifact.Name), "/") == target {
|
||
items = append(items, artifact)
|
||
}
|
||
}
|
||
sort.SliceStable(items, func(left int, right int) bool {
|
||
return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right])
|
||
})
|
||
return items
|
||
}
|
||
items := make([]domain.ExecutionArtifact, 0)
|
||
for _, artifact := range run.Artifacts {
|
||
if packageArtifactResolvedRelativePath(artifact, packageName) == target {
|
||
items = append(items, artifact)
|
||
}
|
||
}
|
||
sort.SliceStable(items, func(left int, right int) bool {
|
||
return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right])
|
||
})
|
||
return items
|
||
}
|
||
|
||
func workflowNodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
|
||
values := make(map[string]string, len(node.Data.Hyperparams))
|
||
for _, item := range node.Data.Hyperparams {
|
||
values[item.Key] = item.Value
|
||
}
|
||
return values
|
||
}
|
||
|
||
func (server *server) packageArtifactScanConfig(ctx context.Context, workflow *domain.WorkflowRecord, artifact domain.ExecutionArtifact) (*domain.ScanConfig, error) {
|
||
node := workflowNodeByID(workflow, artifact.NodeID)
|
||
if node == nil {
|
||
return nil, nil
|
||
}
|
||
params := workflowNodeHyperparamsMap(*node)
|
||
dataSource := chooseDataSource(params["DATA_SOURCE"])
|
||
sourcePath := strings.TrimSpace(params["SOURCE_PATH"])
|
||
if dataSource != "BOS" && strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
|
||
dataSource = "BOS"
|
||
}
|
||
if dataSource != "BOS" && strings.TrimSpace(node.Data.ConnectionID) != "" {
|
||
dataSource = "BOS"
|
||
}
|
||
if dataSource != "BOS" {
|
||
return nil, nil
|
||
}
|
||
scanConfig := &domain.ScanConfig{
|
||
DataSource: "BOS",
|
||
SourcePath: sourcePath,
|
||
ConnectionID: strings.TrimSpace(node.Data.ConnectionID),
|
||
Bucket: strings.TrimSpace(node.Data.Bucket),
|
||
Prefix: cleanBOSPrefix(node.Data.Prefix),
|
||
Endpoint: strings.TrimSpace(params["BOS_ENDPOINT"]),
|
||
AccessKeyID: strings.TrimSpace(params["BOS_ACCESS_KEY_ID"]),
|
||
SecretAccessKey: strings.TrimSpace(params["BOS_SECRET_ACCESS_KEY"]),
|
||
}
|
||
return server.resolveScanConfig(ctx, scanConfig)
|
||
}
|
||
|
||
func (server *server) packageContentPayload(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, packageName string, relativePath string) ([]byte, string, string, string, string, error) {
|
||
artifacts := packageArtifactCandidatesByRelativePath(run, packageName, relativePath)
|
||
if len(artifacts) == 0 {
|
||
return nil, "", "", "", "", os.ErrNotExist
|
||
}
|
||
fileName := path.Base(strings.Trim(filepath.ToSlash(relativePath), "/"))
|
||
for _, artifact := range artifacts {
|
||
mimeType := ""
|
||
if artifact.Metadata != nil {
|
||
mimeType = strings.TrimSpace(artifact.Metadata.MimeType)
|
||
}
|
||
if mimeType == "" {
|
||
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
|
||
}
|
||
filePath := strings.TrimSpace(artifact.Path)
|
||
if filePath != "" && !strings.Contains(filePath, "://") {
|
||
if _, err := os.Stat(filePath); err == nil {
|
||
return nil, mimeType, fileName, filePath, "", nil
|
||
}
|
||
}
|
||
structuredData := artifactStructuredData(&artifact)
|
||
if artifact.Type == "json" && structuredData != nil {
|
||
payload, err := json.MarshalIndent(normalizeStructuredValue(structuredData), "", " ")
|
||
if err != nil {
|
||
return nil, "", "", "", "", err
|
||
}
|
||
if mimeType == "" {
|
||
mimeType = "application/json"
|
||
}
|
||
return payload, mimeType, fileName, "", "", nil
|
||
}
|
||
source := mapValue(structuredData["source"])
|
||
sourcePath := stringValue(source["path"])
|
||
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
|
||
scanConfig, err := server.packageArtifactScanConfig(ctx, workflow, artifact)
|
||
if err != nil {
|
||
return nil, "", "", "", "", err
|
||
}
|
||
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
|
||
return nil, mimeType, fileName, "", previewURL, nil
|
||
}
|
||
}
|
||
if previewURL := packageArtifactPreviewURL(artifact); previewURL != "" {
|
||
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
|
||
return nil, mimeType, fileName, "", previewURL, nil
|
||
}
|
||
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
|
||
if err == nil {
|
||
if mimeType == "" {
|
||
mimeType = decodedMimeType
|
||
}
|
||
return payload, mimeType, fileName, "", "", nil
|
||
}
|
||
}
|
||
}
|
||
return nil, "", "", "", "", os.ErrNotExist
|
||
}
|
||
|
||
func (server *server) handlePackageContent(writer http.ResponseWriter, request *http.Request) {
|
||
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if run == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Run") {
|
||
return
|
||
}
|
||
packageName := chi.URLParam(request, "packageName")
|
||
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
|
||
if relativePath == "" {
|
||
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
|
||
return
|
||
}
|
||
workflow, err := server.store.GetWorkflow(request.Context(), run.WorkflowID)
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
content, mimeType, fileName, filePath, redirectURL, err := server.packageContentPayload(request.Context(), *run, workflow, packageName, relativePath)
|
||
if err != nil {
|
||
if os.IsNotExist(err) {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package file not found"})
|
||
return
|
||
}
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
|
||
previewRequest := sourceBrowserPreviewRequest{
|
||
FileName: fileName,
|
||
Payload: content,
|
||
}
|
||
directURL := ""
|
||
if filePath != "" {
|
||
previewRequest.RedirectURL = filePath
|
||
} else {
|
||
previewRequest.RedirectURL = redirectURL
|
||
directURL = redirectURL
|
||
}
|
||
writeBrowserVideoPreviewResponse(writer, request, previewRequest, directURL)
|
||
return
|
||
}
|
||
if filePath != "" {
|
||
if err := writeSourceContentFile(writer, request, fileName, mimeType, filePath); err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
}
|
||
return
|
||
}
|
||
if redirectURL != "" {
|
||
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
|
||
return
|
||
}
|
||
writeSourceContent(writer, request, fileName, mimeType, content)
|
||
}
|
||
|
||
func packageRecordFromManifest(run domain.RunRecord, item map[string]any, files []domain.ExecutionArtifact, rawFiles []domain.ExecutionArtifact) domain.PackageRecord {
|
||
pipelineTags := make([]string, 0)
|
||
for _, value := range listValue(item["pipelineTags"]) {
|
||
pipelineTags = append(pipelineTags, stringValue(value))
|
||
}
|
||
storagePath := stringValue(item["storagePath"])
|
||
for _, artifact := range rawFiles {
|
||
if candidate := packageArtifactStoragePath(artifact, stringValue(item["packageName"])); candidate != "" {
|
||
storagePath = candidate
|
||
break
|
||
}
|
||
}
|
||
fileCount := intValue(item["fileCount"])
|
||
if len(files) > 0 {
|
||
fileCount = len(files)
|
||
}
|
||
sourceVideo := stringValue(item["sourceVideo"])
|
||
if sourceVideo == "" {
|
||
sourceVideo = stringValue(mapValue(item["annotation"])["sourceVideo"])
|
||
}
|
||
if sourceVideo == "" {
|
||
sourceVideo = stringValue(mapValue(item["source"])["name"])
|
||
}
|
||
clipDuration := floatValue(item["clipDuration"])
|
||
if clipDuration <= 0 {
|
||
clipDuration = floatValue(item["duration"])
|
||
}
|
||
quality := map[string]any{}
|
||
for key, value := range mapValue(item["quality"]) {
|
||
quality[key] = value
|
||
}
|
||
if stringValue(quality["source_video"]) == "" && sourceVideo != "" {
|
||
quality["source_video"] = sourceVideo
|
||
}
|
||
return domain.PackageRecord{
|
||
RunID: run.ID,
|
||
WorkflowID: run.WorkflowID,
|
||
WorkflowName: run.WorkflowName,
|
||
ProjectID: run.ProjectID,
|
||
ProjectName: run.ProjectName,
|
||
PackageName: stringValue(item["packageName"]),
|
||
ClipID: stringValue(item["clipId"]),
|
||
SourceVideo: sourceVideo,
|
||
ClipDuration: clipDuration,
|
||
FileCount: fileCount,
|
||
PipelineTags: pipelineTags,
|
||
StoragePath: storagePath,
|
||
Quality: quality,
|
||
Annotation: mapValue(item["annotation"]),
|
||
CreatedAt: run.CreatedAt,
|
||
PreviewArtifact: choosePackagePreview(files),
|
||
Storage: assetStorageLocationFromPaths("", storagePath, ""),
|
||
}
|
||
}
|
||
|
||
func (server *server) packageDetail(ctx context.Context, runID string, packageName string) (*domain.PackageDetail, error) {
|
||
run, err := server.store.GetRun(ctx, runID)
|
||
if err != nil || run == nil {
|
||
return nil, err
|
||
}
|
||
manifest := artifactByName(*run, "packages-manifest.json")
|
||
manifestPayload := artifactStructuredData(manifest)
|
||
if manifest != nil && manifestPayload != nil {
|
||
rawFiles := dedupePackageArtifacts(*run, packageName)
|
||
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
|
||
for _, artifact := range rawFiles {
|
||
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
|
||
}
|
||
for _, raw := range listValue(manifestPayload["items"]) {
|
||
item := mapValue(raw)
|
||
if stringValue(item["packageName"]) != packageName {
|
||
continue
|
||
}
|
||
record := packageRecordFromManifest(*run, item, files, rawFiles)
|
||
return &domain.PackageDetail{Package: record, Files: files}, nil
|
||
}
|
||
}
|
||
if manifest, payload := wilorPackageManifestArtifact(*run, packageName); manifest != nil && payload != nil {
|
||
files, rawFiles := wilorPackageFilesForResponse(*run, packageName)
|
||
item := wilorPackageItem(*run, *manifest, payload, rawFiles)
|
||
record := packageRecordFromManifest(*run, item, files, rawFiles)
|
||
return &domain.PackageDetail{Package: record, Files: files}, nil
|
||
}
|
||
return nil, nil
|
||
}
|
||
|
||
func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) {
|
||
runs, err := server.listRunArtifactsForRequest(request.Context(), request, string(domain.RunStatusCompleted))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
items := make([]domain.PackageRecord, 0)
|
||
latestByWorkflowPackage := make(map[string]domain.PackageRecord)
|
||
for _, run := range runs {
|
||
if !server.filterByTenant(request, run.OrganizationID) {
|
||
continue
|
||
}
|
||
manifest := artifactByName(run, "packages-manifest.json")
|
||
manifestPayload := artifactStructuredData(manifest)
|
||
if manifest != nil && manifestPayload != nil {
|
||
for _, raw := range listValue(manifestPayload["items"]) {
|
||
item := mapValue(raw)
|
||
if item == nil {
|
||
continue
|
||
}
|
||
packageName := stringValue(item["packageName"])
|
||
rawFiles := dedupePackageArtifacts(run, packageName)
|
||
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
|
||
for _, artifact := range rawFiles {
|
||
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
|
||
}
|
||
record := packageRecordFromManifest(run, item, files, rawFiles)
|
||
key := run.WorkflowID + ":" + record.PackageName
|
||
current, exists := latestByWorkflowPackage[key]
|
||
if !exists || record.CreatedAt > current.CreatedAt {
|
||
latestByWorkflowPackage[key] = record
|
||
}
|
||
}
|
||
}
|
||
for _, wilorManifest := range wilorResultArtifacts(run) {
|
||
payload := wilorResultPayload(wilorManifest)
|
||
if payload == nil {
|
||
continue
|
||
}
|
||
packageName := wilorPackageName(wilorManifest, payload)
|
||
files, rawFiles := wilorPackageFilesForResponse(run, packageName)
|
||
item := wilorPackageItem(run, wilorManifest, payload, rawFiles)
|
||
record := packageRecordFromManifest(run, item, files, rawFiles)
|
||
key := run.WorkflowID + ":" + record.PackageName
|
||
current, exists := latestByWorkflowPackage[key]
|
||
if !exists || record.CreatedAt > current.CreatedAt {
|
||
latestByWorkflowPackage[key] = record
|
||
}
|
||
}
|
||
}
|
||
for _, item := range latestByWorkflowPackage {
|
||
items = append(items, item)
|
||
}
|
||
sort.Slice(items, func(left int, right int) bool {
|
||
if items[left].CreatedAt == items[right].CreatedAt {
|
||
return items[left].PackageName < items[right].PackageName
|
||
}
|
||
return items[left].CreatedAt > items[right].CreatedAt
|
||
})
|
||
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
|
||
}
|
||
|
||
func (server *server) handlePackageDetail(writer http.ResponseWriter, request *http.Request) {
|
||
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if run == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Package") {
|
||
return
|
||
}
|
||
item, err := server.packageDetail(request.Context(), run.ID, chi.URLParam(request, "packageName"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if item == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusOK, item)
|
||
}
|
||
|
||
var supportedSamplingRuleOperators = map[string]struct{}{
|
||
"contains": {},
|
||
"eq": {},
|
||
"exists": {},
|
||
"gt": {},
|
||
"gte": {},
|
||
"in": {},
|
||
"lt": {},
|
||
"lte": {},
|
||
"missing": {},
|
||
"ne": {},
|
||
}
|
||
|
||
func samplingRulesFromManifest(manifest map[string]any) ([]domain.SamplingRuleSummary, bool) {
|
||
rules := make([]domain.SamplingRuleSummary, 0)
|
||
ruleIDs := make(map[string]struct{})
|
||
for _, raw := range listValue(manifest["rules"]) {
|
||
item := mapValue(raw)
|
||
if item == nil {
|
||
return nil, false
|
||
}
|
||
id := strings.TrimSpace(stringValue(item["id"]))
|
||
field := strings.TrimSpace(stringValue(item["field"]))
|
||
operator := strings.ToLower(strings.TrimSpace(stringValue(item["operator"])))
|
||
samplePercent, ok := samplePercentValue(item["samplePercent"])
|
||
if id == "" || field == "" || !ok {
|
||
return nil, false
|
||
}
|
||
if _, exists := ruleIDs[id]; exists {
|
||
return nil, false
|
||
}
|
||
if _, supported := supportedSamplingRuleOperators[operator]; !supported {
|
||
return nil, false
|
||
}
|
||
ruleIDs[id] = struct{}{}
|
||
rules = append(rules, domain.SamplingRuleSummary{
|
||
ID: id,
|
||
Field: field,
|
||
Operator: operator,
|
||
Value: item["value"],
|
||
SamplePercent: samplePercent,
|
||
MatchedCount: intValue(item["matchedCount"]),
|
||
SampledCount: intValue(item["sampledCount"]),
|
||
})
|
||
}
|
||
return rules, len(rules) > 0
|
||
}
|
||
|
||
func samplePercentValue(value any) (float64, bool) {
|
||
switch typed := value.(type) {
|
||
case int:
|
||
return boundedSamplePercent(float64(typed))
|
||
case int8:
|
||
return boundedSamplePercent(float64(typed))
|
||
case int16:
|
||
return boundedSamplePercent(float64(typed))
|
||
case int32:
|
||
return boundedSamplePercent(float64(typed))
|
||
case int64:
|
||
return boundedSamplePercent(float64(typed))
|
||
case uint:
|
||
return boundedSamplePercent(float64(typed))
|
||
case uint8:
|
||
return boundedSamplePercent(float64(typed))
|
||
case uint16:
|
||
return boundedSamplePercent(float64(typed))
|
||
case uint32:
|
||
return boundedSamplePercent(float64(typed))
|
||
case uint64:
|
||
return boundedSamplePercent(float64(typed))
|
||
case float32:
|
||
return boundedSamplePercent(float64(typed))
|
||
case float64:
|
||
return boundedSamplePercent(typed)
|
||
case json.Number:
|
||
parsed, err := typed.Float64()
|
||
if err != nil {
|
||
return 0, false
|
||
}
|
||
return boundedSamplePercent(parsed)
|
||
default:
|
||
return 0, false
|
||
}
|
||
}
|
||
|
||
func boundedSamplePercent(value float64) (float64, bool) {
|
||
if math.IsNaN(value) || math.IsInf(value, 0) || value <= 0 || value > 100 {
|
||
return 0, false
|
||
}
|
||
return value, true
|
||
}
|
||
|
||
func buildSamplingItems(run domain.RunRecord, manifest map[string]any) []domain.SamplingItemRecord {
|
||
items := make([]domain.SamplingItemRecord, 0)
|
||
previewByPackage := make(map[string]*domain.ExecutionArtifact)
|
||
for _, raw := range listValue(manifest["items"]) {
|
||
item := mapValue(raw)
|
||
packageName := stringValue(item["packageName"])
|
||
if _, ok := previewByPackage[packageName]; !ok {
|
||
previewByPackage[packageName] = choosePackagePreview(packageArtifacts(run, packageName))
|
||
}
|
||
preview := previewByPackage[packageName]
|
||
items = append(items, domain.SamplingItemRecord{
|
||
ID: domain.NewID("sampling-item"),
|
||
PackageName: packageName,
|
||
ClipID: stringValue(item["clipId"]),
|
||
SourceVideo: stringValue(item["sourceVideo"]),
|
||
SampleRuleID: stringValue(item["sampleRuleId"]),
|
||
StoragePath: stringValue(item["storagePath"]),
|
||
Quality: mapValue(item["quality"]),
|
||
Annotation: mapValue(item["annotation"]),
|
||
PreviewArtifact: preview,
|
||
Status: domain.SamplingItemStatusPending,
|
||
})
|
||
}
|
||
return items
|
||
}
|
||
|
||
func (server *server) syncSamplingTasks(ctx context.Context) error {
|
||
existing, err := server.listSamplingTasksForContext(ctx, "")
|
||
if err != nil {
|
||
return err
|
||
}
|
||
existingByRun := make(map[string]domain.SamplingTaskRecord, len(existing))
|
||
for _, task := range existing {
|
||
existingByRun[task.RunID] = task
|
||
}
|
||
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
|
||
if err != nil {
|
||
return err
|
||
}
|
||
for _, run := range runs {
|
||
if !server.filterByTenantContext(ctx, run.OrganizationID) {
|
||
continue
|
||
}
|
||
if _, exists := existingByRun[run.ID]; exists {
|
||
continue
|
||
}
|
||
manifest := artifactByName(run, "audit-samples.json")
|
||
manifestPayload := artifactStructuredData(manifest)
|
||
if manifest == nil || manifestPayload == nil || len(listValue(manifestPayload["items"])) == 0 {
|
||
continue
|
||
}
|
||
rules, ok := samplingRulesFromManifest(manifestPayload)
|
||
if !ok {
|
||
continue
|
||
}
|
||
now := nowISO()
|
||
items := buildSamplingItems(run, manifestPayload)
|
||
task := domain.SamplingTaskRecord{
|
||
ID: domain.NewID("sampling"),
|
||
OrganizationID: normalizedTenantID(run.OrganizationID),
|
||
RunID: run.ID,
|
||
WorkflowID: run.WorkflowID,
|
||
WorkflowName: run.WorkflowName,
|
||
ProjectID: run.ProjectID,
|
||
ProjectName: run.ProjectName,
|
||
Name: fmt.Sprintf("%s 抽检任务", run.WorkflowName),
|
||
Description: fmt.Sprintf("来源于运行 %s 的真实抽检采样结果", run.ID),
|
||
Status: domain.SamplingTaskStatusPending,
|
||
Rules: rules,
|
||
Items: items,
|
||
CreatedAt: now,
|
||
UpdatedAt: now,
|
||
}
|
||
for index := range task.Items {
|
||
task.Items[index].TaskID = task.ID
|
||
}
|
||
if _, err := server.store.CreateSamplingTask(ctx, task); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (server *server) handleSamplingTasksList(writer http.ResponseWriter, request *http.Request) {
|
||
if err := server.syncSamplingTasks(request.Context()); err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
items, err := server.listSamplingTasksForRequest(request.Context(), request, request.URL.Query().Get("status"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
items = server.filterSamplingTasksForRequest(request, items)
|
||
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
|
||
}
|
||
|
||
func (server *server) handleSamplingTaskGet(writer http.ResponseWriter, request *http.Request) {
|
||
if err := server.syncSamplingTasks(request.Context()); err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
item, err := server.store.GetSamplingTask(request.Context(), chi.URLParam(request, "id"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
if item == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling task not found"})
|
||
return
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Sampling task") {
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusOK, item)
|
||
}
|
||
|
||
func validateSamplingReviewPayload(payload samplingReviewRequest) []validationIssue {
|
||
issues := make([]validationIssue, 0, 4)
|
||
if strings.TrimSpace(payload.VideoQuality) == "" {
|
||
issues = append(issues, validationIssue{Field: "videoQuality", Message: "videoQuality is required"})
|
||
}
|
||
if strings.TrimSpace(payload.AnnotationAccuracy) == "" {
|
||
issues = append(issues, validationIssue{Field: "annotationAccuracy", Message: "annotationAccuracy is required"})
|
||
}
|
||
if strings.TrimSpace(payload.AnnotationCompleteness) == "" {
|
||
issues = append(issues, validationIssue{Field: "annotationCompleteness", Message: "annotationCompleteness is required"})
|
||
}
|
||
if strings.TrimSpace(payload.OverallGrade) == "" {
|
||
issues = append(issues, validationIssue{Field: "overallGrade", Message: "overallGrade is required"})
|
||
}
|
||
return issues
|
||
}
|
||
|
||
func (server *server) handleSamplingItemReview(writer http.ResponseWriter, request *http.Request) {
|
||
if err := server.syncSamplingTasks(request.Context()); err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
var payload samplingReviewRequest
|
||
if !decodeJSON(writer, request, &payload) {
|
||
return
|
||
}
|
||
if issues := validateSamplingReviewPayload(payload); len(issues) > 0 {
|
||
renderValidationError(writer, issues...)
|
||
return
|
||
}
|
||
tasks, err := server.listSamplingTasksForRequest(request.Context(), request, "")
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
itemID := chi.URLParam(request, "id")
|
||
for _, task := range tasks {
|
||
if !server.filterByTenant(request, task.OrganizationID) {
|
||
continue
|
||
}
|
||
updated := false
|
||
for index := range task.Items {
|
||
if task.Items[index].ID != itemID {
|
||
continue
|
||
}
|
||
task.Items[index].Status = domain.SamplingItemStatusCompleted
|
||
task.Items[index].Review = &domain.SamplingReview{
|
||
VideoQuality: strings.TrimSpace(payload.VideoQuality),
|
||
AnnotationAccuracy: strings.TrimSpace(payload.AnnotationAccuracy),
|
||
AnnotationCompleteness: strings.TrimSpace(payload.AnnotationCompleteness),
|
||
OverallGrade: strings.TrimSpace(payload.OverallGrade),
|
||
Notes: strings.TrimSpace(payload.Notes),
|
||
ReviewedAt: nowISO(),
|
||
}
|
||
updated = true
|
||
break
|
||
}
|
||
if !updated {
|
||
continue
|
||
}
|
||
task.Status = domain.SamplingTaskStatusCompleted
|
||
for _, item := range task.Items {
|
||
if item.Status != domain.SamplingItemStatusCompleted {
|
||
task.Status = domain.SamplingTaskStatusPending
|
||
break
|
||
}
|
||
}
|
||
task.UpdatedAt = nowISO()
|
||
if err := server.store.UpsertSamplingTask(request.Context(), task); err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusOK, task)
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling item not found"})
|
||
}
|