droboticflow/internal/api/data_platform.go
2026-06-29 00:49:58 +08:00

3023 lines
99 KiB
Go

package api
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"math"
"mime"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
"github.com/baidubce/bce-sdk-go/services/bos"
"github.com/go-chi/chi/v5"
)
type sourceBrowserPreviewRequest struct {
FileName string
Payload []byte
RedirectURL string
}
type flushResponseWriter struct {
writer http.ResponseWriter
flusher http.Flusher
}
func (target flushResponseWriter) Write(payload []byte) (int, error) {
count, err := target.writer.Write(payload)
if count > 0 {
target.flusher.Flush()
}
return count, err
}
var sourceBrowserVideoPreviewPath = locateSourceBrowserVideoPreviewPath
var sourceBrowserVideoPreviewWriter = streamSourceBrowserVideoPreview
var sourceCollectionSignedPreviewURL = buildSourceCollectionSignedPreviewURL
type scanConfigRequest struct {
Name string `json:"name"`
Description string `json:"description"`
SourcePath string `json:"sourcePath"`
ConnectionID string `json:"connectionId,omitempty"`
Bucket string `json:"bucket,omitempty"`
Prefix string `json:"prefix,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
AccessKeyID string `json:"accessKeyId,omitempty"`
SecretAccessKey string `json:"secretAccessKey,omitempty"`
ScanMode string `json:"scanMode,omitempty"`
DataSourceIssue *validationIssue `json:"-"`
ScanModeIssue *validationIssue `json:"-"`
IncrementalUnit string `json:"incrementalUnit,omitempty"`
IncrementalUnitIssue *validationIssue `json:"-"`
DirectoryDepth int `json:"directoryDepth,omitempty"`
DirectoryDepthIssue *validationIssue `json:"-"`
Pattern string `json:"pattern,omitempty"`
Interval int `json:"interval"`
Enabled bool `json:"enabled"`
DataSource string `json:"dataSource,omitempty"`
Notifications domain.ScanNotificationSettings `json:"notifications,omitempty"`
}
type samplingReviewRequest struct {
VideoQuality string `json:"videoQuality"`
AnnotationAccuracy string `json:"annotationAccuracy"`
AnnotationCompleteness string `json:"annotationCompleteness"`
OverallGrade string `json:"overallGrade"`
Notes string `json:"notes,omitempty"`
}
func isScanWorkflow(item domain.WorkflowRecord) bool {
return item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil
}
func chooseWorkflowKind(value domain.WorkflowKind) domain.WorkflowKind {
if value == "" {
return domain.WorkflowKindStandard
}
return value
}
func chooseDataSource(value string) string {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "BOS"
}
return trimmed
}
func parseWorkflowDataSourceHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "BOS" {
return trimmed, nil
}
return "", &validationIssue{
Field: "dataSource",
Message: "DATA_SOURCE must be BOS for scan workflows",
}
}
func validateScanWorkflowDataSource(value string) *validationIssue {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" || trimmed == "BOS" {
return nil
}
return &validationIssue{
Field: "dataSource",
Message: "dataSource must be BOS for scan workflows",
}
}
func cleanBOSPrefix(value string) string {
return strings.Trim(strings.TrimSpace(value), "/")
}
func chooseBOSEndpointWithDefault(value string, fallback string) string {
if trimmedValue := strings.TrimSpace(value); trimmedValue != "" {
return trimmedValue
}
return strings.TrimSpace(fallback)
}
func chooseBOSEndpoint(value string) string {
return strings.TrimSpace(value)
}
func (server *server) chooseBOSEndpoint(value string) string {
return chooseBOSEndpointWithDefault(value, server.globalBOSEndpoint)
}
func chooseWorkflowScanMode(value string) string {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "FULL":
return "FULL"
default:
return "INCREMENTAL"
}
}
func parseWorkflowScanModeHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "FULL" || trimmed == "INCREMENTAL" {
return trimmed, nil
}
return "", &validationIssue{
Field: "scanMode",
Message: "SCAN_MODE must be FULL or INCREMENTAL",
}
}
func chooseWorkflowIncrementalUnit(value string) string {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "DIRECTORY":
return "DIRECTORY"
default:
return "OBJECT"
}
}
func parseWorkflowIncrementalUnitHyperparam(value string) (string, *validationIssue) {
trimmed := strings.ToUpper(strings.TrimSpace(value))
if trimmed == "" {
return "", nil
}
if trimmed == "OBJECT" || trimmed == "DIRECTORY" {
return trimmed, nil
}
return "", &validationIssue{
Field: "incrementalUnit",
Message: "INCREMENTAL_UNIT must be OBJECT or DIRECTORY",
}
}
func chooseWorkflowDirectoryDepth(value int) int {
if value > 0 {
return value
}
return 1
}
func parseWorkflowDirectoryDepthHyperparam(value string) (int, *validationIssue) {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return 0, nil
}
parsed, err := strconv.Atoi(trimmed)
if err != nil || parsed <= 0 {
return 0, &validationIssue{
Field: "directoryDepth",
Message: "INCREMENTAL_DIRECTORY_DEPTH must be a positive integer",
}
}
return parsed, nil
}
func scanSourcePath(dataSource string, sourcePath string, bucket string, prefix string) string {
if chooseDataSource(dataSource) == "BOS" {
if explicit := strings.TrimSpace(sourcePath); explicit != "" {
return explicit
}
if strings.TrimSpace(bucket) == "" {
return ""
}
canonical := "bos://" + strings.TrimSpace(bucket)
if cleanedPrefix := cleanBOSPrefix(prefix); cleanedPrefix != "" {
canonical += "/" + cleanedPrefix
}
return canonical
}
return strings.TrimSpace(sourcePath)
}
func mergeScanSecrets(existing *domain.ScanConfig, payload scanConfigRequest) (string, string) {
accessKeyID := strings.TrimSpace(payload.AccessKeyID)
secretAccessKey := strings.TrimSpace(payload.SecretAccessKey)
if accessKeyID == "" && existing != nil {
accessKeyID = existing.AccessKeyID
}
if secretAccessKey == "" && existing != nil {
secretAccessKey = existing.SecretAccessKey
}
return accessKeyID, secretAccessKey
}
func mergeScanConnectionID(existing *domain.ScanConfig, payload scanConfigRequest) string {
connectionID := strings.TrimSpace(payload.ConnectionID)
if connectionID == "" &&
existing != nil &&
strings.TrimSpace(existing.ConnectionID) != "" &&
strings.TrimSpace(payload.AccessKeyID) == "" &&
strings.TrimSpace(payload.SecretAccessKey) == "" {
connectionID = strings.TrimSpace(existing.ConnectionID)
}
return connectionID
}
func (server *server) getStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) {
connectionID := strings.TrimSpace(id)
if connectionID == "" {
return nil, nil
}
item, err := server.store.GetStorageConnection(ctx, connectionID)
if err != nil {
return nil, err
}
if item == nil {
return nil, fmt.Errorf("storage connection %s not found", connectionID)
}
if !server.filterByTenantContext(ctx, item.OrganizationID) {
return nil, fmt.Errorf("storage connection %s not found", connectionID)
}
return item, nil
}
func (server *server) getMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) {
connectionID := strings.TrimSpace(id)
if connectionID == "" {
return nil, nil
}
item, err := server.store.GetMongoConnection(ctx, connectionID)
if err != nil {
return nil, err
}
if item == nil {
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
}
if !server.filterByTenantContext(ctx, item.OrganizationID) {
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
}
return item, nil
}
func (server *server) resolveScanConfig(ctx context.Context, scanConfig *domain.ScanConfig) (*domain.ScanConfig, error) {
if scanConfig == nil {
return nil, nil
}
copy := *scanConfig
connection, err := server.getStorageConnection(ctx, copy.ConnectionID)
if err != nil {
return nil, err
}
if connection == nil {
if chooseDataSource(copy.DataSource) == "BOS" {
copy.Endpoint = server.chooseBOSEndpoint(copy.Endpoint)
}
return &copy, nil
}
copy.Endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint)
if strings.TrimSpace(copy.AccessKeyID) == "" {
copy.AccessKeyID = connection.AccessKeyID
}
if strings.TrimSpace(copy.SecretAccessKey) == "" {
copy.SecretAccessKey = connection.SecretAccessKey
}
return &copy, nil
}
func cloneHyperparams(items []domain.HyperParam) []domain.HyperParam {
out := make([]domain.HyperParam, len(items))
copy(out, items)
return out
}
func clonePorts(items []domain.PortDefinition) []domain.PortDefinition {
out := make([]domain.PortDefinition, len(items))
copy(out, items)
return out
}
func cloneOperatorIntoNode(operator domain.OperatorRecord, nodeID string, x float64, y float64, overrides map[string]string) domain.WorkflowNode {
hyperparams := cloneHyperparams(operator.Hyperparams)
seen := make(map[string]int, len(hyperparams))
for index, item := range hyperparams {
seen[item.Key] = index
}
for key, value := range overrides {
if index, ok := seen[key]; ok {
hyperparams[index].Value = value
continue
}
hyperparams = append(hyperparams, domain.HyperParam{Key: key, Value: value})
}
return domain.WorkflowNode{
ID: nodeID,
Type: "operator",
Data: domain.WorkflowNodeData{
Label: operator.Name,
Category: operator.Category,
Description: operator.Description,
Inputs: clonePorts(operator.Inputs),
Outputs: clonePorts(operator.Outputs),
Image: operator.Image,
SkillDefinition: operator.SkillDefinition,
Hyperparams: hyperparams,
},
Position: domain.Position{X: x, Y: y},
}
}
func operatorByName(items []domain.OperatorRecord, name string) (*domain.OperatorRecord, bool) {
for _, item := range items {
if item.Name == name {
copy := item
return &copy, true
}
}
return nil, false
}
func (server *server) buildScanWorkflowInput(ctx context.Context, projectID string, payload scanConfigRequest, existing *domain.ScanConfig) (domain.CreateWorkflowInput, error) {
operators, err := server.listAvailableOperators(ctx)
if err != nil {
return domain.CreateWorkflowInput{}, err
}
dataSource := chooseDataSource(payload.DataSource)
sourceOperatorName := "BOS 增量扫描"
sourceOperator, ok := operatorByName(operators, sourceOperatorName)
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator %s not found", sourceOperatorName)
}
indexOperator, ok := operatorByName(operators, "X 平台数据集索引")
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator X 平台数据集索引 not found")
}
validateOperator, ok := operatorByName(operators, "采集目录校验")
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator 采集目录校验 not found")
}
sourcePath := scanSourcePath(dataSource, payload.SourcePath, payload.Bucket, payload.Prefix)
connectionID := mergeScanConnectionID(existing, payload)
var connection *domain.StorageConnectionRecord
if connectionID != "" {
connection, err = server.getStorageConnection(ctx, connectionID)
if err != nil {
return domain.CreateWorkflowInput{}, err
}
}
endpoint := ""
accessKeyID := ""
secretAccessKey := ""
if connection != nil {
endpoint = server.storageEndpointForProvider(connection.Provider, connection.Endpoint)
} else if dataSource == "BOS" {
endpoint = server.chooseBOSEndpoint(payload.Endpoint)
accessKeyID, secretAccessKey = mergeScanSecrets(existing, payload)
}
sourceOverrides := map[string]string{
"DATA_SOURCE": dataSource,
"SCAN_MODE": chooseWorkflowScanMode(payload.ScanMode),
"INCREMENTAL_UNIT": chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
"INCREMENTAL_DIRECTORY_DEPTH": strconv.Itoa(chooseWorkflowDirectoryDepth(payload.DirectoryDepth)),
}
if dataSource == "BOS" {
sourceOverrides["SOURCE_PATH"] = sourcePath
sourceOverrides["BOS_ENDPOINT"] = endpoint
} else {
sourceOverrides["SOURCE_PATH"] = sourcePath
}
sourceNode := cloneOperatorIntoNode(*sourceOperator, "scan-source", 0, 0, sourceOverrides)
sourceNode.Data.Bucket = strings.TrimSpace(payload.Bucket)
sourceNode.Data.Prefix = cleanBOSPrefix(payload.Prefix)
sourceNode.Data.Pattern = strings.TrimSpace(payload.Pattern)
indexNode := cloneOperatorIntoNode(*indexOperator, "scan-index", 260, 0, nil)
validateNode := cloneOperatorIntoNode(*validateOperator, "scan-validate", 520, 0, nil)
schedule := &domain.WorkflowSchedule{SourcePath: sourcePath}
workflowType := domain.WorkflowTypeBatch
if payload.Enabled && payload.Interval > 0 {
schedule.Interval = payload.Interval
workflowType = domain.WorkflowTypeStreaming
}
return domain.CreateWorkflowInput{
ProjectID: projectID,
Name: strings.TrimSpace(payload.Name),
Description: strings.TrimSpace(payload.Description),
Type: workflowType,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: dataSource,
SourcePath: sourcePath,
ConnectionID: connectionID,
Bucket: strings.TrimSpace(payload.Bucket),
Prefix: cleanBOSPrefix(payload.Prefix),
Endpoint: endpoint,
AccessKeyID: accessKeyID,
SecretAccessKey: secretAccessKey,
ScanMode: chooseWorkflowScanMode(payload.ScanMode),
IncrementalUnit: chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
DirectoryDepth: chooseWorkflowDirectoryDepth(payload.DirectoryDepth),
Pattern: strings.TrimSpace(payload.Pattern),
Enabled: payload.Enabled && payload.Interval > 0,
Notifications: payload.Notifications,
},
Schedule: schedule,
Nodes: []domain.WorkflowNode{
sourceNode,
indexNode,
validateNode,
},
Edges: []domain.WorkflowEdge{
{ID: "scan-source-index", Source: sourceNode.ID, SourceHandle: "out-1", Target: indexNode.ID, TargetHandle: "in-1"},
{ID: "scan-index-validate", Source: indexNode.ID, SourceHandle: "out-1", Target: validateNode.ID, TargetHandle: "in-1"},
},
}, nil
}
func validateScanConfigEndpoint(payload scanConfigRequest, existing *domain.ScanConfig) *validationIssue {
if chooseDataSource(payload.DataSource) != "BOS" {
return nil
}
if strings.TrimSpace(mergeScanConnectionID(existing, payload)) != "" {
return nil
}
return validateOptionalHTTPURL("endpoint", payload.Endpoint)
}
func validateScanConfigDraftPayload(payload scanConfigRequest, existing ...*domain.ScanConfig) []validationIssue {
var previous *domain.ScanConfig
if len(existing) > 0 {
previous = existing[0]
}
issues := make([]validationIssue, 0, 5)
if strings.TrimSpace(payload.Name) == "" {
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
}
if payload.Interval < 0 {
issues = append(issues, validationIssue{Field: "interval", Message: "interval must be zero or greater"})
}
if issue := validateScanWorkflowDataSource(payload.DataSource); issue != nil {
issues = append(issues, *issue)
}
if payload.DataSourceIssue != nil {
issues = append(issues, *payload.DataSourceIssue)
}
if scanMode := strings.ToUpper(strings.TrimSpace(payload.ScanMode)); scanMode != "" && scanMode != "FULL" && scanMode != "INCREMENTAL" {
issues = append(issues, validationIssue{Field: "scanMode", Message: "scanMode must be FULL or INCREMENTAL"})
}
if payload.ScanModeIssue != nil {
issues = append(issues, *payload.ScanModeIssue)
}
if incrementalUnit := strings.ToUpper(strings.TrimSpace(payload.IncrementalUnit)); incrementalUnit != "" && incrementalUnit != "OBJECT" && incrementalUnit != "DIRECTORY" {
issues = append(issues, validationIssue{Field: "incrementalUnit", Message: "incrementalUnit must be OBJECT or DIRECTORY"})
}
if payload.IncrementalUnitIssue != nil {
issues = append(issues, *payload.IncrementalUnitIssue)
}
if payload.DirectoryDepthIssue != nil {
issues = append(issues, *payload.DirectoryDepthIssue)
}
if payload.DirectoryDepth < 0 {
issues = append(issues, validationIssue{Field: "directoryDepth", Message: "directoryDepth must be zero or greater"})
}
if issue := validateScanConfigEndpoint(payload, previous); issue != nil {
issues = append(issues, *issue)
}
return issues
}
func scanConfigPayloadReady(payload scanConfigRequest, existing *domain.ScanConfig) bool {
switch chooseDataSource(payload.DataSource) {
case "BOS":
connectionID := mergeScanConnectionID(existing, payload)
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
hasTarget := strings.TrimSpace(payload.SourcePath) != "" ||
(strings.TrimSpace(payload.Bucket) != "" && cleanBOSPrefix(payload.Prefix) != "")
return hasTarget &&
(strings.TrimSpace(connectionID) != "" ||
(strings.TrimSpace(accessKeyID) != "" && strings.TrimSpace(secretAccessKey) != ""))
default:
return strings.TrimSpace(payload.SourcePath) != ""
}
}
func validateScanConfigPayload(payload scanConfigRequest, existing *domain.ScanConfig) []validationIssue {
issues := validateScanConfigDraftPayload(payload, existing)
switch chooseDataSource(payload.DataSource) {
case "BOS":
if strings.TrimSpace(payload.SourcePath) == "" {
if strings.TrimSpace(payload.Bucket) == "" {
issues = append(issues, validationIssue{Field: "bucket", Message: "bucket is required for BOS scan configs when sourcePath is empty"})
}
if cleanBOSPrefix(payload.Prefix) == "" {
issues = append(issues, validationIssue{Field: "prefix", Message: "prefix is required for BOS scan configs when sourcePath is empty"})
}
}
connectionID := mergeScanConnectionID(existing, payload)
if strings.TrimSpace(connectionID) == "" {
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
if strings.TrimSpace(accessKeyID) == "" {
issues = append(issues, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required for BOS scan configs"})
}
if strings.TrimSpace(secretAccessKey) == "" {
issues = append(issues, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required for BOS scan configs"})
}
}
default:
if strings.TrimSpace(payload.SourcePath) == "" {
issues = append(issues, validationIssue{Field: "sourcePath", Message: "sourcePath is required"})
}
}
return issues
}
func (server *server) scanWorkflows(ctx context.Context, request *http.Request) ([]domain.WorkflowRecord, error) {
items, err := server.listAllWorkflowsForRequest(ctx, request)
if err != nil {
return nil, err
}
filtered := make([]domain.WorkflowRecord, 0)
for _, item := range items {
if isScanWorkflow(item) {
item.Kind = chooseWorkflowKind(item.Kind)
filtered = append(filtered, item)
}
}
return filtered, nil
}
func (server *server) handleScanConfigsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.scanWorkflows(request.Context(), request)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items = server.filterWorkflowsForRequest(request, items)
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)})
}
func (server *server) handleScanConfigsCreate(writer http.ResponseWriter, request *http.Request) {
var payload scanConfigRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Interval > 0 && !scanConfigPayloadReady(payload, nil) {
renderValidationError(writer, validateScanConfigPayload(payload, nil)...)
return
}
if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(nil, payload)); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
input, err := server.buildScanWorkflowInput(request.Context(), "", payload, nil)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
server.assignTenant(request, &input.OrganizationID)
item, err := server.store.CreateWorkflow(request.Context(), input)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
item.Kind = chooseWorkflowKind(item.Kind)
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
var payload scanConfigRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateScanConfigDraftPayload(payload, item.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Interval > 0 && !scanConfigPayloadReady(payload, item.ScanConfig) {
renderValidationError(writer, validateScanConfigPayload(payload, item.ScanConfig)...)
return
}
if issues := server.validateScanConnectionReference(request.Context(), server.normalizedScanConfigFromRequest(item.ScanConfig, payload)); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
input, err := server.buildScanWorkflowInput(request.Context(), item.ProjectID, payload, item.ScanConfig)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item.Name = input.Name
item.Description = input.Description
item.Type = input.Type
item.Kind = input.Kind
item.ScanConfig = input.ScanConfig
item.Schedule = input.Schedule
item.Nodes = input.Nodes
item.Edges = input.Edges
item.UpdatedAt = nowISO()
if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsDelete(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
writer.WriteHeader(http.StatusNotFound)
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
deleted, err := server.store.DeleteWorkflow(request.Context(), item.ID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func (server *server) handleScanConfigsRun(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Scan config") {
return
}
if issues := server.validateScanConnectionReference(request.Context(), item.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
run, err := server.store.CreateRun(request.Context(), item.ID, domain.RunTriggerManual)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusAccepted, run)
}
func artifactByName(run domain.RunRecord, name string) *domain.ExecutionArtifact {
for index := range run.Artifacts {
if run.Artifacts[index].Name == name {
return &run.Artifacts[index]
}
}
return nil
}
func mapValue(value any) map[string]any {
item, _ := normalizeStructuredValue(value).(map[string]any)
return item
}
func listValue(value any) []any {
items, _ := normalizeStructuredValue(value).([]any)
return items
}
func normalizeStructuredValue(value any) any {
switch typed := value.(type) {
case nil:
return nil
case map[string]any:
items := make(map[string]any, len(typed))
for key, item := range typed {
items[key] = normalizeStructuredValue(item)
}
return items
case []any:
items := make([]any, len(typed))
for index, item := range typed {
items[index] = normalizeStructuredValue(item)
}
return items
}
rv := reflect.ValueOf(value)
switch rv.Kind() {
case reflect.Map:
if rv.Type().Key().Kind() != reflect.String {
return value
}
items := make(map[string]any, rv.Len())
iter := rv.MapRange()
for iter.Next() {
items[iter.Key().String()] = normalizeStructuredValue(iter.Value().Interface())
}
return items
case reflect.Slice, reflect.Array:
items := make([]any, rv.Len())
for index := 0; index < rv.Len(); index++ {
items[index] = normalizeStructuredValue(rv.Index(index).Interface())
}
return items
default:
return value
}
}
func stringValue(value any) string {
switch typed := value.(type) {
case nil:
return ""
case string:
return typed
default:
text := strings.TrimSpace(fmt.Sprintf("%v", typed))
if text == "<nil>" {
return ""
}
return text
}
}
func boolValue(value any) bool {
typed, ok := value.(bool)
return ok && typed
}
func floatValue(value any) float64 {
switch typed := value.(type) {
case float64:
if math.IsNaN(typed) || math.IsInf(typed, 0) {
return 0
}
return typed
case float32:
value := float64(typed)
if math.IsNaN(value) || math.IsInf(value, 0) {
return 0
}
return value
case int:
return float64(typed)
case int32:
return float64(typed)
case int64:
return float64(typed)
default:
return 0
}
}
func intValue(value any) int {
parsed, ok := exactIntegerValue(value)
if !ok {
return 0
}
if strconv.IntSize == 32 {
const maxInt32 = int64(1<<31 - 1)
const minInt32 = -1 << 31
if parsed < minInt32 || parsed > maxInt32 {
return 0
}
}
return int(parsed)
}
func int64Value(value any) int64 {
parsed, ok := exactIntegerValue(value)
if !ok {
return 0
}
return parsed
}
func exactIntegerValue(value any) (int64, bool) {
switch typed := value.(type) {
case int:
return int64(typed), true
case int8:
return int64(typed), true
case int16:
return int64(typed), true
case int32:
return int64(typed), true
case int64:
return typed, true
case uint:
if uint64(typed) > uint64(math.MaxInt64) {
return 0, false
}
return int64(typed), true
case uint8:
return int64(typed), true
case uint16:
return int64(typed), true
case uint32:
return int64(typed), true
case uint64:
if typed > uint64(math.MaxInt64) {
return 0, false
}
return int64(typed), true
case float32:
const maxExactFloat32Integer = 1<<24 - 1
return exactFloatIntegerValue(float64(typed), maxExactFloat32Integer)
case float64:
const maxExactFloat64Integer = 1<<53 - 1
return exactFloatIntegerValue(typed, maxExactFloat64Integer)
case json.Number:
parsed, err := typed.Int64()
return parsed, err == nil
default:
return 0, false
}
}
func exactFloatIntegerValue(value float64, maxExactAbs float64) (int64, bool) {
if math.IsNaN(value) || math.IsInf(value, 0) || math.Trunc(value) != value {
return 0, false
}
if math.Abs(value) > maxExactAbs {
return 0, false
}
return int64(value), true
}
func artifactDataRef(artifact *domain.ExecutionArtifact) map[string]any {
if artifact == nil || artifact.Data == nil {
return nil
}
return mapValue(artifact.Data["dataRef"])
}
func artifactStructuredData(artifact *domain.ExecutionArtifact) map[string]any {
if artifact == nil {
return nil
}
if dataRef := artifactDataRef(artifact); dataRef != nil {
payloadPath := strings.TrimSpace(stringValue(dataRef["payloadPath"]))
if payloadPath != "" && !strings.Contains(payloadPath, "://") {
payload, err := os.ReadFile(payloadPath)
if err != nil {
return nil
}
var decoded any
if err := json.Unmarshal(payload, &decoded); err != nil {
return nil
}
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
return structured
}
return nil
}
}
if artifact.Data != nil {
return artifact.Data
}
if artifact.Type == domain.PortTypeJSON {
filePath := strings.TrimSpace(artifact.Path)
if filePath != "" && !strings.Contains(filePath, "://") {
payload, err := os.ReadFile(filePath)
if err == nil {
var decoded any
if err := json.Unmarshal(payload, &decoded); err == nil {
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
return structured
}
}
}
}
}
return nil
}
func sourceArtifactRelativePath(path string) string {
normalized := filepath.ToSlash(path)
marker := "/_source_tree/"
index := strings.Index(normalized, marker)
if index < 0 {
return ""
}
return strings.TrimPrefix(normalized[index+len(marker):], "/")
}
func sourceArtifactPath(artifact domain.ExecutionArtifact) string {
if payload := artifactStructuredData(&artifact); payload != nil {
if relative := stringValue(payload["relativePath"]); relative != "" {
return filepath.ToSlash(strings.Trim(relative, "/"))
}
}
return sourceArtifactRelativePath(artifact.Path)
}
func packageArtifactRelativePath(path string, packageName string) string {
normalized := filepath.ToSlash(path)
marker := "/packages/" + packageName + "/"
index := strings.Index(normalized, marker)
if index < 0 {
return ""
}
return strings.TrimPrefix(normalized[index+len(marker):], "/")
}
func packageContentURL(runID string, packageName string, relativePath string) string {
return fmt.Sprintf(
"/api/packages/%s/%s/content?relativePath=%s",
url.PathEscape(runID),
url.PathEscape(packageName),
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
)
}
func packageArtifactPreviewURL(artifact domain.ExecutionArtifact) string {
payload := artifactStructuredData(&artifact)
preview := mapValue(payload["preview"])
if preview != nil {
if value := strings.TrimSpace(stringValue(preview["url"])); value != "" {
return value
}
}
source := mapValue(payload["source"])
if source != nil {
if value := strings.TrimSpace(stringValue(source["previewURL"])); value != "" {
return value
}
}
return ""
}
func packageArtifactResolvedRelativePath(artifact domain.ExecutionArtifact, packageName string) string {
if relativePath := packageArtifactRelativePath(artifact.Path, packageName); relativePath != "" {
return relativePath
}
payload := artifactStructuredData(&artifact)
if payload == nil {
return ""
}
relativePath := strings.Trim(filepath.ToSlash(stringValue(payload["relativePath"])), "/")
if relativePath == "" {
return ""
}
packagePrefix := strings.Trim(filepath.ToSlash(path.Join("packages", packageName)), "/")
if packagePrefix == "" {
return ""
}
if relativePath == packagePrefix {
return ""
}
if strings.HasPrefix(relativePath, packagePrefix+"/") {
return strings.TrimPrefix(relativePath, packagePrefix+"/")
}
return ""
}
func packageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
copy := artifact
if copy.Data == nil {
copy.Data = make(map[string]any)
}
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
if relativePath == "" {
relativePath = filepath.ToSlash(strings.Trim(copy.Path, "/"))
}
if relativePath != "" {
copy.Path = relativePath
}
preview := mapValue(copy.Data["preview"])
if preview == nil {
preview = make(map[string]any)
}
kind := strings.TrimSpace(stringValue(preview["kind"]))
mimeType := ""
if copy.Metadata != nil {
mimeType = strings.TrimSpace(copy.Metadata.MimeType)
}
if kind == "video" || copy.Type == "video" || strings.HasPrefix(mimeType, "video/") {
preview["kind"] = "video"
preview["url"] = packageContentURL(runID, packageName, copy.Path)
copy.Data["preview"] = preview
}
return copy
}
func canonicalBOSPath(bucket string, object string) string {
bucket = strings.TrimSpace(bucket)
object = strings.Trim(strings.TrimSpace(object), "/")
if bucket == "" {
return ""
}
if object == "" {
return "bos://" + bucket
}
return "bos://" + bucket + "/" + object
}
func packageArtifactPriority(artifact domain.ExecutionArtifact) int {
score := 0
if artifact.NodeID == "upload" {
score += 40
}
artifactPath := strings.TrimSpace(artifact.Path)
if strings.HasPrefix(strings.ToLower(artifactPath), "bos://") {
score += 30
} else if artifactPath != "" && !strings.Contains(artifactPath, "://") {
score += 10
}
payload := artifactStructuredData(&artifact)
source := mapValue(payload["source"])
sourcePath := strings.TrimSpace(stringValue(source["path"]))
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
score += 30
}
if strings.TrimSpace(packageArtifactPreviewURL(artifact)) != "" {
score += 5
}
return score
}
func packageArtifactStoragePath(artifact domain.ExecutionArtifact, packageName string) string {
payload := artifactStructuredData(&artifact)
source := mapValue(payload["source"])
if sourcePath := strings.TrimSpace(stringValue(source["path"])); sourcePath != "" {
if bucket, object, ok := parseBOSPath(sourcePath); ok {
return canonicalBOSPath(bucket, path.Dir(object))
}
directory := filepath.ToSlash(filepath.Dir(sourcePath))
if directory != "." {
return directory
}
}
artifactPath := strings.TrimSpace(artifact.Path)
if bucket, object, ok := parseBOSPath(artifactPath); ok {
return canonicalBOSPath(bucket, path.Dir(object))
}
if artifactPath == "" {
return ""
}
directory := filepath.ToSlash(filepath.Dir(artifactPath))
if directory == "." {
return ""
}
return directory
}
func dedupePackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
bestByRelativePath := make(map[string]domain.ExecutionArtifact)
for _, artifact := range run.Artifacts {
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
if relativePath == "" {
continue
}
current, exists := bestByRelativePath[relativePath]
if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) {
bestByRelativePath[relativePath] = artifact
}
}
items := make([]domain.ExecutionArtifact, 0, len(bestByRelativePath))
for _, artifact := range bestByRelativePath {
items = append(items, artifact)
}
sort.Slice(items, func(left int, right int) bool {
leftPath := packageArtifactResolvedRelativePath(items[left], packageName)
rightPath := packageArtifactResolvedRelativePath(items[right], packageName)
return leftPath < rightPath
})
return items
}
func packageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range dedupePackageArtifacts(run, packageName) {
items = append(items, packageArtifactForResponse(run.ID, packageName, artifact))
}
return items
}
func choosePackagePreview(files []domain.ExecutionArtifact) *domain.ExecutionArtifact {
for _, file := range files {
if strings.HasSuffix(strings.ToLower(file.Name), ".mp4") {
copy := file
return &copy
}
}
if len(files) == 0 {
return nil
}
copy := files[0]
return &copy
}
func sourceDatasetList(data map[string]any, catalogKey string, fallbackKey string) []any {
if _, ok := data[catalogKey]; ok {
return listValue(data[catalogKey])
}
return listValue(data[fallbackKey])
}
func sourceDatasetHasCatalog(data map[string]any) bool {
_, hasCollections := data["catalogCollections"]
_, hasItems := data["catalogItems"]
_, hasEntries := data["catalogEntries"]
return hasCollections || hasItems || hasEntries
}
func sourceDatasetIsDisplayable(data map[string]any) bool {
if len(sourceDatasetList(data, "catalogCollections", "collections")) > 0 {
return true
}
if len(sourceDatasetList(data, "catalogEntries", "entries")) > 0 {
return true
}
return intValue(data["catalogCollectionCount"]) > 0 || intValue(data["collectionCount"]) > 0
}
func runSourceManifest(run domain.RunRecord) *domain.ExecutionArtifact {
manifest := artifactByName(run, "source-manifest.json")
if manifest == nil || artifactStructuredData(manifest) == nil {
return nil
}
return manifest
}
func runDatasetIndex(run domain.RunRecord) *domain.ExecutionArtifact {
artifact := artifactByName(run, "dataset-index.json")
if artifact == nil || artifactStructuredData(artifact) == nil {
return nil
}
return artifact
}
func runLooksLikeSourceDataset(workflow domain.WorkflowRecord, run domain.RunRecord) bool {
if manifest := runSourceManifest(run); manifest != nil {
payload := artifactStructuredData(manifest)
if strings.TrimSpace(stringValue(payload["sourcePath"])) != "" || strings.TrimSpace(stringValue(payload["dataSource"])) != "" {
return true
}
}
index := runDatasetIndex(run)
if index == nil {
return false
}
if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil {
return true
}
indexPayload := artifactStructuredData(index)
sourcePath := strings.TrimSpace(stringValue(indexPayload["sourcePath"]))
rootPath := strings.TrimSpace(stringValue(indexPayload["rootPath"]))
dataSource := strings.TrimSpace(stringValue(indexPayload["dataSource"]))
if dataSource != "" || strings.HasPrefix(sourcePath, "bos://") || strings.HasPrefix(rootPath, "bos://") {
return true
}
if len(mapValue(indexPayload["archiveSources"])) > 0 && strings.Contains(rootPath, "_materialized_inputs") {
return false
}
return sourceDatasetHasCatalog(indexPayload)
}
func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]domain.WorkflowRecord, map[string]domain.RunRecord, error) {
workflows, err := server.listAllWorkflowsForContext(ctx)
if err != nil {
return nil, nil, err
}
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
if err != nil {
return nil, nil, err
}
latestByWorkflow := make(map[string]domain.RunRecord)
for _, run := range runs {
workflow, ok := workflowByID[run.WorkflowID]
if !ok {
continue
}
index := runDatasetIndex(run)
if index == nil {
continue
}
if !runLooksLikeSourceDataset(workflow, run) {
continue
}
if !sourceDatasetIsDisplayable(artifactStructuredData(index)) {
continue
}
current, exists := latestByWorkflow[run.WorkflowID]
if !exists || run.CreatedAt > current.CreatedAt {
latestByWorkflow[run.WorkflowID] = run
}
}
return workflowByID, latestByWorkflow, nil
}
func parseBOSPath(sourcePath string) (string, string, bool) {
normalized := strings.TrimSpace(sourcePath)
if !strings.HasPrefix(strings.ToLower(normalized), "bos://") {
return "", "", false
}
trimmed := strings.TrimPrefix(normalized, "bos://")
trimmed = strings.TrimPrefix(trimmed, "BOS://")
bucket, object, found := strings.Cut(trimmed, "/")
bucket = strings.TrimSpace(bucket)
object = strings.Trim(strings.TrimSpace(object), "/")
if !found {
object = ""
}
if bucket == "" {
return "", "", false
}
return bucket, object, true
}
func normalizeLocalDirectory(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
directory := filepath.ToSlash(filepath.Dir(trimmed))
if directory == "." {
return ""
}
return directory
}
func assetStorageLocationFromPaths(rootPath string, currentPath string, endpoint string) domain.AssetStorageLocation {
location := domain.AssetStorageLocation{
Path: strings.TrimSpace(currentPath),
}
currentBucket, currentObject, hasCurrentBOS := parseBOSPath(currentPath)
rootBucket, rootObject, hasRootBOS := parseBOSPath(rootPath)
switch {
case hasCurrentBOS:
location.Provider = "bos"
location.Endpoint = chooseBOSEndpoint(endpoint)
location.Bucket = currentBucket
if hasRootBOS && rootBucket == currentBucket {
location.Prefix = rootObject
} else {
location.Prefix = path.Dir(currentObject)
if location.Prefix == "." {
location.Prefix = ""
}
}
case hasRootBOS:
location.Provider = "bos"
location.Endpoint = chooseBOSEndpoint(endpoint)
location.Bucket = rootBucket
location.Prefix = rootObject
if location.Path == "" {
location.Path = strings.TrimSpace(rootPath)
}
case strings.TrimSpace(currentPath) != "":
location.Provider = "k8s-job"
location.Prefix = normalizeLocalDirectory(currentPath)
case strings.TrimSpace(rootPath) != "":
location.Provider = "k8s-job"
location.Path = strings.TrimSpace(rootPath)
location.Prefix = normalizeLocalDirectory(rootPath)
default:
location.Provider = "unknown"
}
if location.Prefix == "." {
location.Prefix = ""
}
return location
}
func sourceRootPathForRun(run domain.RunRecord, scanConfig *domain.ScanConfig) string {
if scanConfig != nil {
if value := scanSourcePath(scanConfig.DataSource, scanConfig.SourcePath, scanConfig.Bucket, scanConfig.Prefix); value != "" {
return value
}
}
if manifest := runSourceManifest(run); manifest != nil {
payload := artifactStructuredData(manifest)
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
return value
}
}
if index := runDatasetIndex(run); index != nil {
payload := artifactStructuredData(index)
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
return value
}
if value := strings.TrimSpace(stringValue(payload["rootPath"])); value != "" {
return value
}
}
return ""
}
func (server *server) buildSourceCollectionRecord(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, collection map[string]any) domain.SourceCollectionRecord {
meta := mapValue(collection["meta"])
requiredFiles := make([]string, 0)
for _, item := range listValue(collection["requiredFiles"]) {
requiredFiles = append(requiredFiles, stringValue(item))
}
missingFiles := make([]string, 0)
for _, item := range listValue(collection["missingRequiredFiles"]) {
missingFiles = append(missingFiles, stringValue(item))
}
collectionName := stringValue(collection["name"])
if collectionName == "" {
if projectID := stringValue(meta["project_id"]); projectID != "" {
collectionName = projectID
} else if relativePath := strings.Trim(stringValue(collection["relativePath"]), "/"); relativePath != "" && relativePath != "." {
collectionName = path.Base(relativePath)
} else {
collectionName = stringValue(collection["id"])
}
}
var resolvedScanConfig *domain.ScanConfig
if workflow != nil {
if scanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig); err == nil {
resolvedScanConfig = scanConfig
}
}
storage := assetStorageLocationFromPaths(
sourceRootPathForRun(run, resolvedScanConfig),
stringValue(collection["path"]),
func() string {
if resolvedScanConfig == nil {
return ""
}
return resolvedScanConfig.Endpoint
}(),
)
return domain.SourceCollectionRecord{
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
CollectionID: stringValue(collection["id"]),
CollectionName: collectionName,
Path: stringValue(collection["path"]),
RelativePath: stringValue(collection["relativePath"]),
Meta: meta,
HasIntrinsics: boolValue(collection["hasIntrinsics"]),
HasVideoMeta: boolValue(collection["hasVideoMeta"]),
RequiredFiles: requiredFiles,
MissingFiles: missingFiles,
VideoCount: intValue(collection["videoCount"]),
FileCount: intValue(collection["fileCount"]),
TotalBytes: int64Value(collection["totalBytes"]),
Valid: boolValue(collection["valid"]),
CreatedAt: run.CreatedAt,
Storage: storage,
}
}
type sourceCollectionPayloadRecord struct {
Run *domain.RunRecord
Workflow *domain.WorkflowRecord
Collection map[string]any
Items []map[string]any
Entries []map[string]any
ArchiveSource map[string]any
}
func (server *server) sourceCollectionPayload(ctx context.Context, runID string, collectionID string) (*sourceCollectionPayloadRecord, error) {
run, err := server.store.GetRun(ctx, runID)
if err != nil || run == nil {
return &sourceCollectionPayloadRecord{Run: run}, err
}
workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
return &sourceCollectionPayloadRecord{Run: run}, err
}
artifact := artifactByName(*run, "dataset-index.json")
payload := artifactStructuredData(artifact)
if artifact == nil || payload == nil {
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
}
collections := sourceDatasetList(payload, "catalogCollections", "collections")
items := sourceDatasetList(payload, "catalogItems", "items")
entries := sourceDatasetList(payload, "catalogEntries", "entries")
var matchedCollection map[string]any
for _, item := range collections {
collection := mapValue(item)
if stringValue(collection["id"]) == collectionID {
matchedCollection = collection
break
}
}
if matchedCollection == nil {
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
}
matchedItems := make([]map[string]any, 0)
for _, item := range items {
row := mapValue(item)
if stringValue(row["collectionId"]) == collectionID {
matchedItems = append(matchedItems, row)
}
}
matchedEntries := make([]map[string]any, 0)
for _, item := range entries {
row := mapValue(item)
relativePath := strings.Trim(filepath.ToSlash(stringValue(row["relativePath"])), "/")
collectionRelativePath := strings.Trim(filepath.ToSlash(stringValue(matchedCollection["relativePath"])), "/")
if collectionRelativePath == "" || collectionRelativePath == "." {
if relativePath == "" {
continue
}
} else if relativePath != collectionRelativePath && !strings.HasPrefix(relativePath, collectionRelativePath+"/") {
continue
}
matchedEntries = append(matchedEntries, row)
}
return &sourceCollectionPayloadRecord{
Run: run,
Workflow: workflow,
Collection: matchedCollection,
Items: matchedItems,
Entries: matchedEntries,
ArchiveSource: mapValue(payload["archiveSources"]),
}, nil
}
func sourceEntryPayloadKind(relativePath string, mimeType string) string {
extension := strings.ToLower(path.Ext(relativePath))
switch {
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
return "video"
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
return "image"
case strings.HasPrefix(mimeType, "audio/") || extension == ".mp3" || extension == ".wav" || extension == ".m4a":
return "audio"
default:
return "binary"
}
}
func sourceEntryArtifactType(relativePath string, mimeType string) domain.PortType {
extension := strings.ToLower(path.Ext(relativePath))
switch {
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
return domain.PortTypeImage
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
return domain.PortTypeVideo
case extension == ".json":
return domain.PortTypeJSON
default:
return domain.PortTypeFile
}
}
func formatArtifactSize(sizeBytes int64) string {
switch {
case sizeBytes <= 0:
return "0 B"
case sizeBytes < 1024:
return fmt.Sprintf("%d B", sizeBytes)
case sizeBytes < 1024*1024:
return fmt.Sprintf("%.1f KiB", float64(sizeBytes)/1024)
case sizeBytes < 1024*1024*1024:
return fmt.Sprintf("%.1f MiB", float64(sizeBytes)/(1024*1024))
default:
return fmt.Sprintf("%.1f GiB", float64(sizeBytes)/(1024*1024*1024))
}
}
func sourceCollectionContentURL(runID string, collectionID string, relativePath string) string {
return fmt.Sprintf(
"/api/source-data/collections/%s/%s/content?relativePath=%s",
url.PathEscape(runID),
url.PathEscape(collectionID),
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
)
}
func datasetIndexEntrySource(entry map[string]any, archiveSources map[string]any) map[string]any {
source := mapValue(entry["source"])
if len(source) == 0 {
source = make(map[string]any)
}
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if relativePath != "" && stringValue(source["relativePath"]) == "" {
source["relativePath"] = relativePath
}
mimeType := stringValue(entry["mimeType"])
if mimeType == "" {
mimeType = stringValue(source["mimeType"])
}
if mimeType == "" && relativePath != "" {
mimeType = mime.TypeByExtension(path.Ext(relativePath))
}
if mimeType != "" {
source["mimeType"] = mimeType
}
if stringValue(source["previewURL"]) == "" {
archiveKey := stringValue(source["archiveKey"])
archiveSource := mapValue(archiveSources[archiveKey])
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
source["previewURL"] = previewURL
}
if archiveName := stringValue(archiveSource["name"]); archiveName != "" && stringValue(source["archiveName"]) == "" {
source["archiveName"] = archiveName
}
}
if pathValue := stringValue(entry["path"]); pathValue != "" && stringValue(source["path"]) == "" {
source["path"] = pathValue
}
return source
}
func sourceEntryArtifact(run domain.RunRecord, collectionID string, entry map[string]any, archiveSources map[string]any) *domain.ExecutionArtifact {
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if relativePath == "" {
relativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
if relativePath == "" {
return nil
}
source := datasetIndexEntrySource(entry, archiveSources)
mimeType := stringValue(source["mimeType"])
sizeBytes := int64Value(entry["sizeBytes"])
artifactType := sourceEntryArtifactType(relativePath, mimeType)
payload := make(map[string]any)
if value, ok := entry["data"]; ok {
payload["preview"] = map[string]any{
"kind": "json",
"value": normalizeStructuredValue(value),
}
}
if previewURL := stringValue(source["previewURL"]); previewURL != "" && len(payload) == 0 {
payload["preview"] = map[string]any{
"kind": sourceEntryPayloadKind(relativePath, mimeType),
"format": strings.TrimPrefix(strings.ToLower(path.Ext(relativePath)), "."),
"url": sourceCollectionContentURL(run.ID, collectionID, relativePath),
}
}
payload["relativePath"] = relativePath
if len(source) > 0 {
if previewURL := sourceCollectionContentURL(run.ID, collectionID, relativePath); previewURL != "" {
source["previewURL"] = previewURL
}
payload["source"] = source
}
return &domain.ExecutionArtifact{
ID: fmt.Sprintf("%s-source-%s", run.ID, strings.ReplaceAll(relativePath, "/", "-")),
RunID: run.ID,
NodeID: "dataset-index",
Name: path.Base(relativePath),
Type: artifactType,
Size: formatArtifactSize(sizeBytes),
Path: func() string {
if sourcePath := stringValue(source["path"]); sourcePath != "" {
return sourcePath
}
return relativePath
}(),
Metadata: &domain.ArtifactMetadata{MimeType: mimeType},
Data: payload,
}
}
func contentBytesForDataURL(previewURL string) ([]byte, string, error) {
if !strings.HasPrefix(previewURL, "data:") {
return nil, "", fmt.Errorf("not a data url")
}
header, encoded, found := strings.Cut(previewURL, ",")
if !found || !strings.Contains(header, ";base64") {
return nil, "", fmt.Errorf("invalid data url")
}
mimeType := strings.TrimPrefix(strings.SplitN(header, ";", 2)[0], "data:")
payload, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, "", err
}
return payload, mimeType, nil
}
func contentBytesForArchiveMember(archiveName string, archivePreviewURL string, memberPath string) ([]byte, error) {
payload, _, err := contentBytesForDataURL(archivePreviewURL)
if err != nil {
return nil, err
}
memberPath = strings.Trim(filepath.ToSlash(memberPath), "/")
if memberPath == "" {
return nil, fmt.Errorf("missing archive member path")
}
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
if err != nil {
return nil, err
}
for _, file := range archive.File {
if strings.Trim(filepath.ToSlash(file.Name), "/") != memberPath {
continue
}
handle, err := file.Open()
if err != nil {
return nil, err
}
defer handle.Close()
return io.ReadAll(handle)
}
return nil, os.ErrNotExist
}
reader := bytes.NewReader(payload)
var tarReader *tar.Reader
if strings.HasSuffix(strings.ToLower(archiveName), ".tgz") || strings.HasSuffix(strings.ToLower(archiveName), ".tar.gz") || (len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b) {
gzipReader, err := gzip.NewReader(reader)
if err == nil {
defer gzipReader.Close()
tarReader = tar.NewReader(gzipReader)
} else {
reader = bytes.NewReader(payload)
tarReader = tar.NewReader(reader)
}
} else {
tarReader = tar.NewReader(reader)
}
for {
header, err := tarReader.Next()
if err == io.EOF {
return nil, os.ErrNotExist
}
if err != nil {
return nil, err
}
if header.FileInfo().IsDir() {
continue
}
if strings.Trim(filepath.ToSlash(header.Name), "/") != memberPath {
continue
}
return io.ReadAll(tarReader)
}
}
func sourceCollectionEntryPayload(collection domain.SourceCollectionRecord, payloadEntries []map[string]any, relativePath string) map[string]any {
target := strings.Trim(filepath.ToSlash(relativePath), "/")
if target == "" {
return nil
}
collectionPrefix := strings.Trim(filepath.ToSlash(collection.RelativePath), "/")
fullTarget := target
if collectionPrefix != "" && target != collectionPrefix && !strings.HasPrefix(target, collectionPrefix+"/") {
fullTarget = collectionPrefix + "/" + target
}
for _, entry := range payloadEntries {
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if entryRelativePath == "" {
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
if entryRelativePath == fullTarget || entryRelativePath == target {
return entry
}
}
return nil
}
func writeSourceContent(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, payload []byte) {
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if mimeType == "" && len(payload) > 0 {
mimeType = http.DetectContentType(payload)
}
if mimeType != "" {
writer.Header().Set("Content-Type", mimeType)
}
writeSourceContentDisposition(writer, request, fileName)
http.ServeContent(writer, request, fileName, time.Time{}, bytes.NewReader(payload))
}
func sourceContentDownloadRequested(request *http.Request) bool {
value := strings.ToLower(strings.TrimSpace(request.URL.Query().Get("download")))
return value == "1" || value == "true" || value == "yes"
}
func writeSourceContentDisposition(writer http.ResponseWriter, request *http.Request, fileName string) {
if !sourceContentDownloadRequested(request) {
return
}
name := strings.TrimSpace(path.Base(fileName))
if name == "" || name == "." || name == "/" {
name = "artifact"
}
writer.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": name}))
}
func writeSourceContentFile(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, filePath string) error {
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if mimeType == "" {
header := make([]byte, 512)
count, readErr := file.Read(header)
if readErr != nil && readErr != io.EOF {
return readErr
}
mimeType = http.DetectContentType(header[:count])
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
}
if mimeType != "" {
writer.Header().Set("Content-Type", mimeType)
}
writeSourceContentDisposition(writer, request, fileName)
http.ServeContent(writer, request, fileName, fileInfo.ModTime(), file)
return nil
}
func sourceBrowserPreviewCacheKey(request sourceBrowserPreviewRequest) string {
hash := sha256.New()
hash.Write([]byte("browser-preview-v2"))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.FileName)))
hash.Write([]byte{0})
if request.RedirectURL != "" {
hash.Write([]byte("redirect"))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.RedirectURL)))
}
if len(request.Payload) > 0 {
payloadHash := sha256.Sum256(request.Payload)
hash.Write([]byte("payload"))
hash.Write([]byte{0})
hash.Write(payloadHash[:])
}
return hex.EncodeToString(hash.Sum(nil))
}
func sourceBrowserPreviewCacheDir() (string, error) {
cacheDir := filepath.Join(os.TempDir(), "droboticflow", "source-video-previews")
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
return "", err
}
return cacheDir, nil
}
func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string, error) {
cacheDir, err := sourceBrowserPreviewCacheDir()
if err != nil {
return "", err
}
cacheKey := sourceBrowserPreviewCacheKey(request)
return filepath.Join(cacheDir, cacheKey+".mp4"), nil
}
func parseBOSObjectPath(sourcePath string) (string, string, bool) {
bucket, object, ok := parseBOSPath(sourcePath)
if !ok || object == "" {
return "", "", false
}
return bucket, object, true
}
func buildSourceCollectionSignedPreviewURL(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil {
return "", fmt.Errorf("scan config is unavailable")
}
if chooseDataSource(scanConfig.DataSource) != "BOS" {
return "", fmt.Errorf("scan config is not BOS")
}
bucket, object, ok := parseBOSObjectPath(sourcePath)
if !ok {
return "", fmt.Errorf("invalid BOS source path: %s", sourcePath)
}
accessKeyID := strings.TrimSpace(scanConfig.AccessKeyID)
secretAccessKey := strings.TrimSpace(scanConfig.SecretAccessKey)
if accessKeyID == "" || secretAccessKey == "" {
return "", fmt.Errorf("BOS credentials are unavailable")
}
endpoint := chooseBOSEndpoint(scanConfig.Endpoint)
if endpoint == "" {
return "", fmt.Errorf("BOS endpoint is unavailable")
}
client, err := bos.NewClient(accessKeyID, secretAccessKey, endpoint)
if err != nil {
return "", err
}
previewURL := strings.TrimSpace(client.BasicGeneratePresignedUrl(bucket, object, 3600))
if previewURL == "" {
return "", fmt.Errorf("failed to generate BOS preview url")
}
return previewURL, nil
}
func locateSourceBrowserVideoPreviewPath(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
finalPath, err := sourceBrowserPreviewCachePath(request)
if err != nil {
return "", err
}
if fileInfo, err := os.Stat(finalPath); err == nil && fileInfo.Size() > 0 {
return finalPath, nil
}
return "", os.ErrNotExist
}
func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWriter, request sourceBrowserPreviewRequest) error {
if strings.TrimSpace(request.RedirectURL) == "" && len(request.Payload) == 0 {
return fmt.Errorf("source video content is unavailable")
}
finalPath, err := sourceBrowserPreviewCachePath(request)
if err != nil {
return err
}
inputRef := strings.TrimSpace(request.RedirectURL)
var cleanupInput func()
if inputRef == "" {
extension := path.Ext(request.FileName)
if extension == "" {
extension = ".bin"
}
cacheDir := filepath.Dir(finalPath)
inputFile, err := os.CreateTemp(cacheDir, "source-video-*"+extension)
if err != nil {
return err
}
if _, err := inputFile.Write(request.Payload); err != nil {
inputFile.Close()
os.Remove(inputFile.Name())
return err
}
if err := inputFile.Close(); err != nil {
os.Remove(inputFile.Name())
return err
}
inputRef = inputFile.Name()
cleanupInput = func() {
_ = os.Remove(inputFile.Name())
}
}
if cleanupInput != nil {
defer cleanupInput()
}
cacheDir := filepath.Dir(finalPath)
outputFile, err := os.CreateTemp(cacheDir, sourceBrowserPreviewCacheKey(request)+"-*.mp4")
if err != nil {
return err
}
tempOutputPath := outputFile.Name()
stderr := &bytes.Buffer{}
command := exec.CommandContext(
ctx,
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel", "error",
"-i", inputRef,
"-map", "0:v:0",
"-map", "0:a?",
"-threads", "1",
"-probesize", "32M",
"-analyzeduration", "0",
"-c:v", "libx264",
"-preset", "ultrafast",
"-tune", "zerolatency",
"-crf", "30",
"-vf", "fps=12,scale=min(960\\,iw):-2",
"-g", "12",
"-keyint_min", "12",
"-pix_fmt", "yuv420p",
"-movflags", "frag_keyframe+empty_moov+default_base_moof",
"-c:a", "aac",
"-b:a", "96k",
"-ac", "1",
"-f", "mp4",
"pipe:1",
)
command.Stderr = stderr
stdout, err := command.StdoutPipe()
if err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
return err
}
if err := command.Start(); err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
message := strings.TrimSpace(stderr.String())
if message != "" {
return fmt.Errorf("generate browser video preview: %s", message)
}
return err
}
streamTarget := io.Writer(writer)
if flusher, ok := writer.(http.Flusher); ok {
streamTarget = flushResponseWriter{writer: writer, flusher: flusher}
}
if _, err := io.Copy(io.MultiWriter(streamTarget, outputFile), stdout); err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
_ = command.Wait()
return err
}
if err := outputFile.Close(); err != nil {
_ = os.Remove(tempOutputPath)
_ = command.Wait()
return err
}
if err := command.Wait(); err != nil {
message := strings.TrimSpace(stderr.String())
_ = os.Remove(tempOutputPath)
if message != "" {
return fmt.Errorf("generate browser video preview: %s", message)
}
return fmt.Errorf("generate browser video preview: %w", err)
}
if err := os.Rename(tempOutputPath, finalPath); err != nil {
if fileInfo, statErr := os.Stat(finalPath); statErr == nil && fileInfo.Size() > 0 {
_ = os.Remove(tempOutputPath)
return nil
}
_ = os.Remove(tempOutputPath)
return err
}
return nil
}
func derivedSourceEntriesFromIndex(collection domain.SourceCollectionRecord, payloadItems []map[string]any) []map[string]any {
entries := make([]map[string]any, 0, len(payloadItems)+2)
if len(collection.Meta) > 0 {
entries = append(entries, map[string]any{
"collectionId": collection.CollectionID,
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/meta.json", "/"),
"sizeBytes": len(fmt.Sprintf("%v", collection.Meta)),
"mimeType": "application/json",
"data": collection.Meta,
"derivedFromIndex": true,
})
}
if collection.HasVideoMeta && len(payloadItems) > 0 {
videoRows := make([]map[string]any, 0, len(payloadItems))
for _, item := range payloadItems {
videoRows = append(videoRows, map[string]any{
"filename": stringValue(item["filename"]),
"codec": stringValue(item["codec"]),
"resolution": stringValue(item["resolution"]),
"frameRate": floatValue(item["frameRate"]),
"duration": floatValue(item["duration"]),
"startTime": item["startTime"],
})
}
entries = append(entries, map[string]any{
"collectionId": collection.CollectionID,
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/video_meta.json", "/"),
"mimeType": "application/json",
"data": videoRows,
"derivedFromIndex": true,
})
}
for _, item := range payloadItems {
entry := map[string]any{
"collectionId": collection.CollectionID,
"relativePath": stringValue(item["relativePath"]),
"path": stringValue(item["path"]),
"sizeBytes": int64Value(item["sizeBytes"]),
"mimeType": stringValue(mapValue(item["source"])["mimeType"]),
"source": mapValue(item["source"]),
"codec": stringValue(item["codec"]),
"resolution": stringValue(item["resolution"]),
"frameRate": floatValue(item["frameRate"]),
"duration": floatValue(item["duration"]),
"derivedFromIndex": true,
}
entries = append(entries, entry)
}
return entries
}
func buildSourceEntries(run domain.RunRecord, collection domain.SourceCollectionRecord, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, prefix string) []domain.SourceCollectionEntry {
if len(payloadEntries) == 0 {
payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems)
}
videoMetaByRelative := make(map[string]map[string]any, len(payloadItems))
for _, item := range payloadItems {
videoMetaByRelative[stringValue(item["relativePath"])] = item
}
normalizedPrefix := strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/")
prefixSlash := normalizedPrefix
if prefixSlash != "" {
prefixSlash += "/"
}
directories := make(map[string]domain.SourceCollectionEntry)
files := make([]domain.SourceCollectionEntry, 0)
collectionPrefix := strings.Trim(collection.RelativePath, "/")
if collectionPrefix != "" {
collectionPrefix += "/"
}
for _, entryPayload := range payloadEntries {
relativePath := strings.Trim(filepath.ToSlash(stringValue(entryPayload["relativePath"])), "/")
if relativePath == "" {
relativePath = strings.Trim(filepath.ToSlash(stringValue(entryPayload["path"])), "/")
}
if relativePath == "" {
continue
}
if collectionPrefix != "" && relativePath != strings.TrimSuffix(collectionPrefix, "/") && !strings.HasPrefix(relativePath, collectionPrefix) {
continue
}
entryPath := relativePath
if collectionPrefix != "" {
entryPath = strings.TrimPrefix(relativePath, collectionPrefix)
}
if entryPath == "" {
continue
}
if normalizedPrefix != "" {
if entryPath == normalizedPrefix {
// exact file match
} else if !strings.HasPrefix(entryPath, prefixSlash) {
continue
} else {
entryPath = strings.TrimPrefix(entryPath, prefixSlash)
}
}
if entryPath == "" {
continue
}
parts := strings.Split(entryPath, "/")
if len(parts) > 1 {
dirPath := parts[0]
displayPath := dirPath
if normalizedPrefix != "" {
displayPath = normalizedPrefix + "/" + dirPath
}
if _, exists := directories[displayPath]; !exists {
directories[displayPath] = domain.SourceCollectionEntry{
Path: displayPath,
Name: dirPath,
Kind: "directory",
CollectionName: collection.CollectionName,
DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]),
}
} else if boolValue(entryPayload["derivedFromIndex"]) {
entry := directories[displayPath]
entry.DerivedFromIndex = true
directories[displayPath] = entry
}
continue
}
displayPath := parts[0]
if normalizedPrefix != "" && displayPath != normalizedPrefix {
displayPath = normalizedPrefix + "/" + displayPath
}
artifact := sourceEntryArtifact(run, collection.CollectionID, entryPayload, archiveSources)
videoMeta := videoMetaByRelative[filepath.ToSlash(strings.Trim(collection.RelativePath, "/")+"/"+displayPath)]
entry := domain.SourceCollectionEntry{
Path: displayPath,
Name: parts[0],
Kind: "file",
CollectionName: collection.CollectionName,
SizeBytes: int64Value(entryPayload["sizeBytes"]),
Artifact: artifact,
DerivedFromIndex: boolValue(entryPayload["derivedFromIndex"]),
}
if videoMeta != nil {
entry.Kind = "video"
entry.Codec = stringValue(videoMeta["codec"])
entry.Resolution = stringValue(videoMeta["resolution"])
entry.FrameRate = floatValue(videoMeta["frameRate"])
entry.Duration = floatValue(videoMeta["duration"])
entry.Metadata = videoMeta
if entry.SizeBytes == 0 {
entry.SizeBytes = int64Value(videoMeta["sizeBytes"])
}
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".json") {
entry.Kind = "json"
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".mp4") {
entry.Kind = "video"
}
if entry.SizeBytes == 0 {
entry.SizeBytes = int64Value(entryPayload["sizeBytes"])
}
files = append(files, entry)
}
items := make([]domain.SourceCollectionEntry, 0, len(directories)+len(files))
for _, item := range directories {
items = append(items, item)
}
sort.Slice(items, func(left int, right int) bool { return items[left].Path < items[right].Path })
sort.Slice(files, func(left int, right int) bool { return files[left].Path < files[right].Path })
items = append(items, files...)
return items
}
func sourceCollectionContentPayload(collection domain.SourceCollectionRecord, scanConfig *domain.ScanConfig, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, relativePath string) ([]byte, string, string, string, error) {
if len(payloadEntries) == 0 {
payloadEntries = derivedSourceEntriesFromIndex(collection, payloadItems)
}
entry := sourceCollectionEntryPayload(collection, payloadEntries, relativePath)
if entry == nil {
return nil, "", "", "", os.ErrNotExist
}
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if entryRelativePath == "" {
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
fileName := path.Base(entryRelativePath)
source := datasetIndexEntrySource(entry, archiveSources)
mimeType := stringValue(source["mimeType"])
if mimeType == "" {
mimeType = stringValue(entry["mimeType"])
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if value, ok := entry["data"]; ok {
payload, err := json.MarshalIndent(normalizeStructuredValue(value), "", " ")
if err != nil {
return nil, "", "", "", err
}
if mimeType == "" {
mimeType = "application/json"
}
return payload, mimeType, fileName, "", nil
}
if sourcePath := stringValue(source["path"]); sourcePath != "" && !strings.Contains(sourcePath, "://") {
if payload, err := os.ReadFile(sourcePath); err == nil {
return payload, mimeType, fileName, "", nil
}
}
if sourcePath := stringValue(source["path"]); strings.HasPrefix(strings.ToLower(sourcePath), "bos://") && scanConfig != nil {
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
source["previewURL"] = previewURL
}
}
archiveKey := stringValue(source["archiveKey"])
archiveMemberPath := stringValue(source["archiveMemberPath"])
if archiveKey != "" && archiveMemberPath != "" {
archiveSource := mapValue(archiveSources[archiveKey])
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
payload, err := contentBytesForArchiveMember(stringValue(archiveSource["name"]), previewURL, archiveMemberPath)
if err == nil {
return payload, mimeType, fileName, "", nil
}
}
}
if previewURL := stringValue(source["previewURL"]); previewURL != "" {
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
return nil, "", fileName, previewURL, nil
}
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
if err == nil {
if mimeType == "" {
mimeType = decodedMimeType
}
return payload, mimeType, fileName, "", nil
}
}
return nil, "", "", "", os.ErrNotExist
}
func (server *server) handleSourceCollectionsList(writer http.ResponseWriter, request *http.Request) {
workflowByID, latestRuns, err := server.latestCompletedScanRuns(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items := make([]domain.SourceCollectionRecord, 0)
for _, run := range latestRuns {
if !server.filterByTenant(request, run.OrganizationID) {
continue
}
artifact := artifactByName(run, "dataset-index.json")
payload := artifactStructuredData(artifact)
if artifact == nil || payload == nil {
continue
}
for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") {
collection := mapValue(raw)
if collection == nil {
continue
}
workflow := workflowByID[run.WorkflowID]
items = append(items, server.buildSourceCollectionRecord(request.Context(), run, &workflow, collection))
}
}
sort.Slice(items, func(left int, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
if items[left].WorkflowName == items[right].WorkflowName {
return items[left].CollectionID < items[right].CollectionID
}
return items[left].WorkflowName < items[right].WorkflowName
}
return items[left].CreatedAt > items[right].CreatedAt
})
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleSourceCollectionDetail(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
detail := domain.SourceCollectionDetail{
Collection: record,
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, ""),
}
renderJSON(writer, http.StatusOK, detail)
}
func (server *server) handleSourceCollectionEntries(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
prefix := request.URL.Query().Get("prefix")
detail := domain.SourceCollectionDetail{
Collection: record,
Prefix: prefix,
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, prefix),
}
renderJSON(writer, http.StatusOK, detail)
}
func (server *server) handleSourceCollectionContent(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
if !server.ensureTenantAccess(writer, request, payload.Run.OrganizationID, "Source collection") {
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
if relativePath == "" {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
return
}
var effectiveScanConfig *domain.ScanConfig
if payload.Workflow != nil {
effectiveScanConfig, err = server.resolveScanConfig(request.Context(), payload.Workflow.ScanConfig)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
content, mimeType, fileName, redirectURL, err := sourceCollectionContentPayload(record, effectiveScanConfig, payload.Entries, payload.Items, payload.ArchiveSource, relativePath)
if err != nil {
if os.IsNotExist(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source file not found"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
Payload: content,
RedirectURL: redirectURL,
}
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] source browser preview stream failed: %v\n", err)
}
return
}
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
writeSourceContent(writer, request, fileName, mimeType, content)
}
func packageArtifactCandidatesByRelativePath(run domain.RunRecord, packageName string, relativePath string) []domain.ExecutionArtifact {
target := strings.Trim(filepath.ToSlash(relativePath), "/")
if target == "" {
return nil
}
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range run.Artifacts {
if packageArtifactResolvedRelativePath(artifact, packageName) == target {
items = append(items, artifact)
}
}
sort.SliceStable(items, func(left int, right int) bool {
return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right])
})
return items
}
func workflowNodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
values := make(map[string]string, len(node.Data.Hyperparams))
for _, item := range node.Data.Hyperparams {
values[item.Key] = item.Value
}
return values
}
func (server *server) packageArtifactScanConfig(ctx context.Context, workflow *domain.WorkflowRecord, artifact domain.ExecutionArtifact) (*domain.ScanConfig, error) {
node := workflowNodeByID(workflow, artifact.NodeID)
if node == nil {
return nil, nil
}
params := workflowNodeHyperparamsMap(*node)
dataSource := chooseDataSource(params["DATA_SOURCE"])
sourcePath := strings.TrimSpace(params["SOURCE_PATH"])
if dataSource != "BOS" && strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
dataSource = "BOS"
}
if dataSource != "BOS" && strings.TrimSpace(node.Data.ConnectionID) != "" {
dataSource = "BOS"
}
if dataSource != "BOS" {
return nil, nil
}
scanConfig := &domain.ScanConfig{
DataSource: "BOS",
SourcePath: sourcePath,
ConnectionID: strings.TrimSpace(node.Data.ConnectionID),
Bucket: strings.TrimSpace(node.Data.Bucket),
Prefix: cleanBOSPrefix(node.Data.Prefix),
Endpoint: strings.TrimSpace(params["BOS_ENDPOINT"]),
AccessKeyID: strings.TrimSpace(params["BOS_ACCESS_KEY_ID"]),
SecretAccessKey: strings.TrimSpace(params["BOS_SECRET_ACCESS_KEY"]),
}
return server.resolveScanConfig(ctx, scanConfig)
}
func (server *server) packageContentPayload(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, packageName string, relativePath string) ([]byte, string, string, string, string, error) {
artifacts := packageArtifactCandidatesByRelativePath(run, packageName, relativePath)
if len(artifacts) == 0 {
return nil, "", "", "", "", os.ErrNotExist
}
fileName := path.Base(strings.Trim(filepath.ToSlash(relativePath), "/"))
for _, artifact := range artifacts {
mimeType := ""
if artifact.Metadata != nil {
mimeType = strings.TrimSpace(artifact.Metadata.MimeType)
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
filePath := strings.TrimSpace(artifact.Path)
if filePath != "" && !strings.Contains(filePath, "://") {
if _, err := os.Stat(filePath); err == nil {
return nil, mimeType, fileName, filePath, "", nil
}
}
structuredData := artifactStructuredData(&artifact)
if artifact.Type == "json" && structuredData != nil {
payload, err := json.MarshalIndent(normalizeStructuredValue(structuredData), "", " ")
if err != nil {
return nil, "", "", "", "", err
}
if mimeType == "" {
mimeType = "application/json"
}
return payload, mimeType, fileName, "", "", nil
}
source := mapValue(structuredData["source"])
sourcePath := stringValue(source["path"])
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
scanConfig, err := server.packageArtifactScanConfig(ctx, workflow, artifact)
if err != nil {
return nil, "", "", "", "", err
}
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
return nil, mimeType, fileName, "", previewURL, nil
}
}
if previewURL := packageArtifactPreviewURL(artifact); previewURL != "" {
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
return nil, mimeType, fileName, "", previewURL, nil
}
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
if err == nil {
if mimeType == "" {
mimeType = decodedMimeType
}
return payload, mimeType, fileName, "", "", nil
}
}
}
return nil, "", "", "", "", os.ErrNotExist
}
func (server *server) handlePackageContent(writer http.ResponseWriter, request *http.Request) {
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if run == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Run") {
return
}
packageName := chi.URLParam(request, "packageName")
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
if relativePath == "" {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
return
}
workflow, err := server.store.GetWorkflow(request.Context(), run.WorkflowID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
content, mimeType, fileName, filePath, redirectURL, err := server.packageContentPayload(request.Context(), *run, workflow, packageName, relativePath)
if err != nil {
if os.IsNotExist(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package file not found"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
Payload: content,
}
if filePath != "" {
previewRequest.RedirectURL = filePath
} else {
previewRequest.RedirectURL = redirectURL
}
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] package browser preview stream failed: %v\n", err)
}
return
}
if filePath != "" {
if err := writeSourceContentFile(writer, request, fileName, mimeType, filePath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
writeSourceContent(writer, request, fileName, mimeType, content)
}
func packageRecordFromManifest(run domain.RunRecord, item map[string]any, files []domain.ExecutionArtifact, rawFiles []domain.ExecutionArtifact) domain.PackageRecord {
pipelineTags := make([]string, 0)
for _, value := range listValue(item["pipelineTags"]) {
pipelineTags = append(pipelineTags, stringValue(value))
}
storagePath := stringValue(item["storagePath"])
for _, artifact := range rawFiles {
if candidate := packageArtifactStoragePath(artifact, stringValue(item["packageName"])); candidate != "" {
storagePath = candidate
break
}
}
fileCount := intValue(item["fileCount"])
if len(files) > 0 {
fileCount = len(files)
}
return domain.PackageRecord{
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
PackageName: stringValue(item["packageName"]),
ClipID: stringValue(item["clipId"]),
SourceVideo: stringValue(item["sourceVideo"]),
ClipDuration: floatValue(item["clipDuration"]),
FileCount: fileCount,
PipelineTags: pipelineTags,
StoragePath: storagePath,
Quality: mapValue(item["quality"]),
Annotation: mapValue(item["annotation"]),
CreatedAt: run.CreatedAt,
PreviewArtifact: choosePackagePreview(files),
Storage: assetStorageLocationFromPaths("", storagePath, ""),
}
}
func (server *server) packageDetail(ctx context.Context, runID string, packageName string) (*domain.PackageDetail, error) {
run, err := server.store.GetRun(ctx, runID)
if err != nil || run == nil {
return nil, err
}
manifest := artifactByName(*run, "packages-manifest.json")
manifestPayload := artifactStructuredData(manifest)
if manifest == nil || manifestPayload == nil {
return nil, nil
}
rawFiles := dedupePackageArtifacts(*run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
}
for _, raw := range listValue(manifestPayload["items"]) {
item := mapValue(raw)
if stringValue(item["packageName"]) != packageName {
continue
}
record := packageRecordFromManifest(*run, item, files, rawFiles)
return &domain.PackageDetail{Package: record, Files: files}, nil
}
return nil, nil
}
func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) {
runs, err := server.listRunArtifactsForRequest(request.Context(), request, string(domain.RunStatusCompleted))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items := make([]domain.PackageRecord, 0)
latestByWorkflowPackage := make(map[string]domain.PackageRecord)
for _, run := range runs {
if !server.filterByTenant(request, run.OrganizationID) {
continue
}
manifest := artifactByName(run, "packages-manifest.json")
manifestPayload := artifactStructuredData(manifest)
if manifest == nil || manifestPayload == nil {
continue
}
for _, raw := range listValue(manifestPayload["items"]) {
item := mapValue(raw)
if item == nil {
continue
}
packageName := stringValue(item["packageName"])
rawFiles := dedupePackageArtifacts(run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
}
record := packageRecordFromManifest(run, item, files, rawFiles)
key := run.WorkflowID + ":" + record.PackageName
current, exists := latestByWorkflowPackage[key]
if !exists || record.CreatedAt > current.CreatedAt {
latestByWorkflowPackage[key] = record
}
}
}
for _, item := range latestByWorkflowPackage {
items = append(items, item)
}
sort.Slice(items, func(left int, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
return items[left].PackageName < items[right].PackageName
}
return items[left].CreatedAt > items[right].CreatedAt
})
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handlePackageDetail(writer http.ResponseWriter, request *http.Request) {
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if run == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
return
}
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Package") {
return
}
item, err := server.packageDetail(request.Context(), run.ID, chi.URLParam(request, "packageName"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
return
}
renderJSON(writer, http.StatusOK, item)
}
var supportedSamplingRuleOperators = map[string]struct{}{
"contains": {},
"eq": {},
"exists": {},
"gt": {},
"gte": {},
"in": {},
"lt": {},
"lte": {},
"missing": {},
"ne": {},
}
func samplingRulesFromManifest(manifest map[string]any) ([]domain.SamplingRuleSummary, bool) {
rules := make([]domain.SamplingRuleSummary, 0)
ruleIDs := make(map[string]struct{})
for _, raw := range listValue(manifest["rules"]) {
item := mapValue(raw)
if item == nil {
return nil, false
}
id := strings.TrimSpace(stringValue(item["id"]))
field := strings.TrimSpace(stringValue(item["field"]))
operator := strings.ToLower(strings.TrimSpace(stringValue(item["operator"])))
samplePercent, ok := samplePercentValue(item["samplePercent"])
if id == "" || field == "" || !ok {
return nil, false
}
if _, exists := ruleIDs[id]; exists {
return nil, false
}
if _, supported := supportedSamplingRuleOperators[operator]; !supported {
return nil, false
}
ruleIDs[id] = struct{}{}
rules = append(rules, domain.SamplingRuleSummary{
ID: id,
Field: field,
Operator: operator,
Value: item["value"],
SamplePercent: samplePercent,
MatchedCount: intValue(item["matchedCount"]),
SampledCount: intValue(item["sampledCount"]),
})
}
return rules, len(rules) > 0
}
func samplePercentValue(value any) (float64, bool) {
switch typed := value.(type) {
case int:
return boundedSamplePercent(float64(typed))
case int8:
return boundedSamplePercent(float64(typed))
case int16:
return boundedSamplePercent(float64(typed))
case int32:
return boundedSamplePercent(float64(typed))
case int64:
return boundedSamplePercent(float64(typed))
case uint:
return boundedSamplePercent(float64(typed))
case uint8:
return boundedSamplePercent(float64(typed))
case uint16:
return boundedSamplePercent(float64(typed))
case uint32:
return boundedSamplePercent(float64(typed))
case uint64:
return boundedSamplePercent(float64(typed))
case float32:
return boundedSamplePercent(float64(typed))
case float64:
return boundedSamplePercent(typed)
case json.Number:
parsed, err := typed.Float64()
if err != nil {
return 0, false
}
return boundedSamplePercent(parsed)
default:
return 0, false
}
}
func boundedSamplePercent(value float64) (float64, bool) {
if math.IsNaN(value) || math.IsInf(value, 0) || value <= 0 || value > 100 {
return 0, false
}
return value, true
}
func buildSamplingItems(run domain.RunRecord, manifest map[string]any) []domain.SamplingItemRecord {
items := make([]domain.SamplingItemRecord, 0)
previewByPackage := make(map[string]*domain.ExecutionArtifact)
for _, raw := range listValue(manifest["items"]) {
item := mapValue(raw)
packageName := stringValue(item["packageName"])
if _, ok := previewByPackage[packageName]; !ok {
previewByPackage[packageName] = choosePackagePreview(packageArtifacts(run, packageName))
}
preview := previewByPackage[packageName]
items = append(items, domain.SamplingItemRecord{
ID: domain.NewID("sampling-item"),
PackageName: packageName,
ClipID: stringValue(item["clipId"]),
SourceVideo: stringValue(item["sourceVideo"]),
SampleRuleID: stringValue(item["sampleRuleId"]),
StoragePath: stringValue(item["storagePath"]),
Quality: mapValue(item["quality"]),
Annotation: mapValue(item["annotation"]),
PreviewArtifact: preview,
Status: domain.SamplingItemStatusPending,
})
}
return items
}
func (server *server) syncSamplingTasks(ctx context.Context) error {
existing, err := server.listSamplingTasksForContext(ctx, "")
if err != nil {
return err
}
existingByRun := make(map[string]domain.SamplingTaskRecord, len(existing))
for _, task := range existing {
existingByRun[task.RunID] = task
}
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
if err != nil {
return err
}
for _, run := range runs {
if !server.filterByTenantContext(ctx, run.OrganizationID) {
continue
}
if _, exists := existingByRun[run.ID]; exists {
continue
}
manifest := artifactByName(run, "audit-samples.json")
manifestPayload := artifactStructuredData(manifest)
if manifest == nil || manifestPayload == nil || len(listValue(manifestPayload["items"])) == 0 {
continue
}
rules, ok := samplingRulesFromManifest(manifestPayload)
if !ok {
continue
}
now := nowISO()
items := buildSamplingItems(run, manifestPayload)
task := domain.SamplingTaskRecord{
ID: domain.NewID("sampling"),
OrganizationID: normalizedTenantID(run.OrganizationID),
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
Name: fmt.Sprintf("%s 抽检任务", run.WorkflowName),
Description: fmt.Sprintf("来源于运行 %s 的真实抽检采样结果", run.ID),
Status: domain.SamplingTaskStatusPending,
Rules: rules,
Items: items,
CreatedAt: now,
UpdatedAt: now,
}
for index := range task.Items {
task.Items[index].TaskID = task.ID
}
if _, err := server.store.CreateSamplingTask(ctx, task); err != nil {
return err
}
}
return nil
}
func (server *server) handleSamplingTasksList(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items, err := server.listSamplingTasksForRequest(request.Context(), request, request.URL.Query().Get("status"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items = server.filterSamplingTasksForRequest(request, items)
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleSamplingTaskGet(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item, err := server.store.GetSamplingTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling task not found"})
return
}
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Sampling task") {
return
}
renderJSON(writer, http.StatusOK, item)
}
func validateSamplingReviewPayload(payload samplingReviewRequest) []validationIssue {
issues := make([]validationIssue, 0, 4)
if strings.TrimSpace(payload.VideoQuality) == "" {
issues = append(issues, validationIssue{Field: "videoQuality", Message: "videoQuality is required"})
}
if strings.TrimSpace(payload.AnnotationAccuracy) == "" {
issues = append(issues, validationIssue{Field: "annotationAccuracy", Message: "annotationAccuracy is required"})
}
if strings.TrimSpace(payload.AnnotationCompleteness) == "" {
issues = append(issues, validationIssue{Field: "annotationCompleteness", Message: "annotationCompleteness is required"})
}
if strings.TrimSpace(payload.OverallGrade) == "" {
issues = append(issues, validationIssue{Field: "overallGrade", Message: "overallGrade is required"})
}
return issues
}
func (server *server) handleSamplingItemReview(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
var payload samplingReviewRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateSamplingReviewPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
tasks, err := server.listSamplingTasksForRequest(request.Context(), request, "")
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
itemID := chi.URLParam(request, "id")
for _, task := range tasks {
if !server.filterByTenant(request, task.OrganizationID) {
continue
}
updated := false
for index := range task.Items {
if task.Items[index].ID != itemID {
continue
}
task.Items[index].Status = domain.SamplingItemStatusCompleted
task.Items[index].Review = &domain.SamplingReview{
VideoQuality: strings.TrimSpace(payload.VideoQuality),
AnnotationAccuracy: strings.TrimSpace(payload.AnnotationAccuracy),
AnnotationCompleteness: strings.TrimSpace(payload.AnnotationCompleteness),
OverallGrade: strings.TrimSpace(payload.OverallGrade),
Notes: strings.TrimSpace(payload.Notes),
ReviewedAt: nowISO(),
}
updated = true
break
}
if !updated {
continue
}
task.Status = domain.SamplingTaskStatusCompleted
for _, item := range task.Items {
if item.Status != domain.SamplingItemStatusCompleted {
task.Status = domain.SamplingTaskStatusPending
break
}
}
task.UpdatedAt = nowISO()
if err := server.store.UpsertSamplingTask(request.Context(), task); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, task)
return
}
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling item not found"})
}