droboticflow/internal/api/data_platform.go
eust-w 43326d6a5e feat(platform): land workflow, data tooling, and docs updates
Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
2026-04-24 12:29:46 +08:00

2661 lines
88 KiB
Go

package api
import (
"archive/tar"
"archive/zip"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"mime"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
"github.com/baidubce/bce-sdk-go/services/bos"
"github.com/go-chi/chi/v5"
)
const (
defaultBOSEndpoint = "https://bj.bcebos.com"
)
type sourceBrowserPreviewRequest struct {
FileName string
Payload []byte
RedirectURL string
}
type flushResponseWriter struct {
writer http.ResponseWriter
flusher http.Flusher
}
func (target flushResponseWriter) Write(payload []byte) (int, error) {
count, err := target.writer.Write(payload)
if count > 0 {
target.flusher.Flush()
}
return count, err
}
var sourceBrowserVideoPreviewPath = locateSourceBrowserVideoPreviewPath
var sourceBrowserVideoPreviewWriter = streamSourceBrowserVideoPreview
var sourceCollectionSignedPreviewURL = buildSourceCollectionSignedPreviewURL
type scanConfigRequest struct {
Name string `json:"name"`
Description string `json:"description"`
SourcePath string `json:"sourcePath"`
ConnectionID string `json:"connectionId,omitempty"`
Bucket string `json:"bucket,omitempty"`
Prefix string `json:"prefix,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
AccessKeyID string `json:"accessKeyId,omitempty"`
SecretAccessKey string `json:"secretAccessKey,omitempty"`
ScanMode string `json:"scanMode,omitempty"`
IncrementalUnit string `json:"incrementalUnit,omitempty"`
DirectoryDepth int `json:"directoryDepth,omitempty"`
Pattern string `json:"pattern,omitempty"`
Interval int `json:"interval"`
Enabled bool `json:"enabled"`
DataSource string `json:"dataSource,omitempty"`
Notifications domain.ScanNotificationSettings `json:"notifications,omitempty"`
}
type samplingReviewRequest struct {
VideoQuality string `json:"videoQuality"`
AnnotationAccuracy string `json:"annotationAccuracy"`
AnnotationCompleteness string `json:"annotationCompleteness"`
OverallGrade string `json:"overallGrade"`
Notes string `json:"notes,omitempty"`
}
func isScanWorkflow(item domain.WorkflowRecord) bool {
return item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil
}
func chooseWorkflowKind(value domain.WorkflowKind) domain.WorkflowKind {
if value == "" {
return domain.WorkflowKindStandard
}
return value
}
func chooseDataSource(value string) string {
_ = value
return "BOS"
}
func cleanBOSPrefix(value string) string {
return strings.Trim(strings.TrimSpace(value), "/")
}
func chooseBOSEndpoint(value string) string {
if strings.TrimSpace(value) == "" {
return defaultBOSEndpoint
}
return strings.TrimSpace(value)
}
func chooseWorkflowScanMode(value string) string {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "FULL":
return "FULL"
default:
return "INCREMENTAL"
}
}
func chooseWorkflowIncrementalUnit(value string) string {
switch strings.ToUpper(strings.TrimSpace(value)) {
case "DIRECTORY":
return "DIRECTORY"
default:
return "OBJECT"
}
}
func chooseWorkflowDirectoryDepth(value int) int {
if value > 0 {
return value
}
return 1
}
func scanSourcePath(dataSource string, sourcePath string, bucket string, prefix string) string {
if chooseDataSource(dataSource) == "BOS" {
if explicit := strings.TrimSpace(sourcePath); explicit != "" {
return explicit
}
if strings.TrimSpace(bucket) == "" {
return ""
}
canonical := "bos://" + strings.TrimSpace(bucket)
if cleanedPrefix := cleanBOSPrefix(prefix); cleanedPrefix != "" {
canonical += "/" + cleanedPrefix
}
return canonical
}
return strings.TrimSpace(sourcePath)
}
func mergeScanSecrets(existing *domain.ScanConfig, payload scanConfigRequest) (string, string) {
accessKeyID := strings.TrimSpace(payload.AccessKeyID)
secretAccessKey := strings.TrimSpace(payload.SecretAccessKey)
if accessKeyID == "" && existing != nil {
accessKeyID = existing.AccessKeyID
}
if secretAccessKey == "" && existing != nil {
secretAccessKey = existing.SecretAccessKey
}
return accessKeyID, secretAccessKey
}
func mergeScanConnectionID(existing *domain.ScanConfig, payload scanConfigRequest) string {
connectionID := strings.TrimSpace(payload.ConnectionID)
if connectionID == "" &&
existing != nil &&
strings.TrimSpace(existing.ConnectionID) != "" &&
strings.TrimSpace(payload.AccessKeyID) == "" &&
strings.TrimSpace(payload.SecretAccessKey) == "" {
connectionID = strings.TrimSpace(existing.ConnectionID)
}
return connectionID
}
func (server *server) getStorageConnection(ctx context.Context, id string) (*domain.StorageConnectionRecord, error) {
connectionID := strings.TrimSpace(id)
if connectionID == "" {
return nil, nil
}
item, err := server.store.GetStorageConnection(ctx, connectionID)
if err != nil {
return nil, err
}
if item == nil {
return nil, fmt.Errorf("storage connection %s not found", connectionID)
}
return item, nil
}
func (server *server) getMongoConnection(ctx context.Context, id string) (*domain.MongoConnectionRecord, error) {
connectionID := strings.TrimSpace(id)
if connectionID == "" {
return nil, nil
}
item, err := server.store.GetMongoConnection(ctx, connectionID)
if err != nil {
return nil, err
}
if item == nil {
return nil, fmt.Errorf("mongo connection %s not found", connectionID)
}
return item, nil
}
func (server *server) resolveScanConfig(ctx context.Context, scanConfig *domain.ScanConfig) (*domain.ScanConfig, error) {
if scanConfig == nil {
return nil, nil
}
copy := *scanConfig
connection, err := server.getStorageConnection(ctx, copy.ConnectionID)
if err != nil {
return nil, err
}
if connection == nil {
return &copy, nil
}
copy.Endpoint = storageEndpointForProvider(connection.Provider, connection.Endpoint)
if strings.TrimSpace(copy.AccessKeyID) == "" {
copy.AccessKeyID = connection.AccessKeyID
}
if strings.TrimSpace(copy.SecretAccessKey) == "" {
copy.SecretAccessKey = connection.SecretAccessKey
}
return &copy, nil
}
func cloneHyperparams(items []domain.HyperParam) []domain.HyperParam {
out := make([]domain.HyperParam, len(items))
copy(out, items)
return out
}
func clonePorts(items []domain.PortDefinition) []domain.PortDefinition {
out := make([]domain.PortDefinition, len(items))
copy(out, items)
return out
}
func cloneOperatorIntoNode(operator domain.OperatorRecord, nodeID string, x float64, y float64, overrides map[string]string) domain.WorkflowNode {
hyperparams := cloneHyperparams(operator.Hyperparams)
seen := make(map[string]int, len(hyperparams))
for index, item := range hyperparams {
seen[item.Key] = index
}
for key, value := range overrides {
if index, ok := seen[key]; ok {
hyperparams[index].Value = value
continue
}
hyperparams = append(hyperparams, domain.HyperParam{Key: key, Value: value})
}
return domain.WorkflowNode{
ID: nodeID,
Type: "operator",
Data: domain.WorkflowNodeData{
Label: operator.Name,
Category: operator.Category,
Description: operator.Description,
Inputs: clonePorts(operator.Inputs),
Outputs: clonePorts(operator.Outputs),
Image: operator.Image,
SkillDefinition: operator.SkillDefinition,
Hyperparams: hyperparams,
},
Position: domain.Position{X: x, Y: y},
}
}
func operatorByName(items []domain.OperatorRecord, name string) (*domain.OperatorRecord, bool) {
for _, item := range items {
if item.Name == name {
copy := item
return &copy, true
}
}
return nil, false
}
func (server *server) buildScanWorkflowInput(ctx context.Context, projectID string, payload scanConfigRequest, existing *domain.ScanConfig) (domain.CreateWorkflowInput, error) {
operators, err := server.listAvailableOperators(ctx)
if err != nil {
return domain.CreateWorkflowInput{}, err
}
dataSource := chooseDataSource(payload.DataSource)
sourceOperatorName := "BOS 增量扫描"
sourceOperator, ok := operatorByName(operators, sourceOperatorName)
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator %s not found", sourceOperatorName)
}
indexOperator, ok := operatorByName(operators, "X 平台数据集索引")
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator X 平台数据集索引 not found")
}
validateOperator, ok := operatorByName(operators, "采集目录校验")
if !ok {
return domain.CreateWorkflowInput{}, fmt.Errorf("operator 采集目录校验 not found")
}
sourcePath := scanSourcePath(dataSource, payload.SourcePath, payload.Bucket, payload.Prefix)
connectionID := mergeScanConnectionID(existing, payload)
var connection *domain.StorageConnectionRecord
if connectionID != "" {
connection, err = server.getStorageConnection(ctx, connectionID)
if err != nil {
return domain.CreateWorkflowInput{}, err
}
}
endpoint := ""
accessKeyID := ""
secretAccessKey := ""
if connection != nil {
endpoint = storageEndpointForProvider(connection.Provider, connection.Endpoint)
} else if dataSource == "BOS" {
endpoint = chooseBOSEndpoint(payload.Endpoint)
accessKeyID, secretAccessKey = mergeScanSecrets(existing, payload)
}
sourceOverrides := map[string]string{
"DATA_SOURCE": dataSource,
"SCAN_MODE": chooseWorkflowScanMode(payload.ScanMode),
"INCREMENTAL_UNIT": chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
"INCREMENTAL_DIRECTORY_DEPTH": strconv.Itoa(chooseWorkflowDirectoryDepth(payload.DirectoryDepth)),
}
if dataSource == "BOS" {
sourceOverrides["SOURCE_PATH"] = sourcePath
sourceOverrides["BOS_ENDPOINT"] = endpoint
} else {
sourceOverrides["SOURCE_PATH"] = sourcePath
}
sourceNode := cloneOperatorIntoNode(*sourceOperator, "scan-source", 0, 0, sourceOverrides)
sourceNode.Data.Bucket = strings.TrimSpace(payload.Bucket)
sourceNode.Data.Prefix = cleanBOSPrefix(payload.Prefix)
sourceNode.Data.Pattern = strings.TrimSpace(payload.Pattern)
indexNode := cloneOperatorIntoNode(*indexOperator, "scan-index", 260, 0, nil)
validateNode := cloneOperatorIntoNode(*validateOperator, "scan-validate", 520, 0, nil)
schedule := &domain.WorkflowSchedule{SourcePath: sourcePath}
workflowType := domain.WorkflowTypeBatch
if payload.Enabled && payload.Interval > 0 {
schedule.Interval = payload.Interval
workflowType = domain.WorkflowTypeStreaming
}
return domain.CreateWorkflowInput{
ProjectID: projectID,
Name: strings.TrimSpace(payload.Name),
Description: strings.TrimSpace(payload.Description),
Type: workflowType,
Kind: domain.WorkflowKindScan,
ScanConfig: &domain.ScanConfig{
DataSource: dataSource,
SourcePath: sourcePath,
ConnectionID: connectionID,
Bucket: strings.TrimSpace(payload.Bucket),
Prefix: cleanBOSPrefix(payload.Prefix),
Endpoint: endpoint,
AccessKeyID: accessKeyID,
SecretAccessKey: secretAccessKey,
ScanMode: chooseWorkflowScanMode(payload.ScanMode),
IncrementalUnit: chooseWorkflowIncrementalUnit(payload.IncrementalUnit),
DirectoryDepth: chooseWorkflowDirectoryDepth(payload.DirectoryDepth),
Pattern: strings.TrimSpace(payload.Pattern),
Enabled: payload.Enabled && payload.Interval > 0,
Notifications: payload.Notifications,
},
Schedule: schedule,
Nodes: []domain.WorkflowNode{
sourceNode,
indexNode,
validateNode,
},
Edges: []domain.WorkflowEdge{
{ID: "scan-source-index", Source: sourceNode.ID, SourceHandle: "out-1", Target: indexNode.ID, TargetHandle: "in-1"},
{ID: "scan-index-validate", Source: indexNode.ID, SourceHandle: "out-1", Target: validateNode.ID, TargetHandle: "in-1"},
},
}, nil
}
func validateScanConfigDraftPayload(payload scanConfigRequest) []validationIssue {
issues := make([]validationIssue, 0, 4)
if strings.TrimSpace(payload.Name) == "" {
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
}
if payload.Interval < 0 {
issues = append(issues, validationIssue{Field: "interval", Message: "interval must be zero or greater"})
}
if scanMode := strings.ToUpper(strings.TrimSpace(payload.ScanMode)); scanMode != "" && scanMode != "FULL" && scanMode != "INCREMENTAL" {
issues = append(issues, validationIssue{Field: "scanMode", Message: "scanMode must be FULL or INCREMENTAL"})
}
if incrementalUnit := strings.ToUpper(strings.TrimSpace(payload.IncrementalUnit)); incrementalUnit != "" && incrementalUnit != "OBJECT" && incrementalUnit != "DIRECTORY" {
issues = append(issues, validationIssue{Field: "incrementalUnit", Message: "incrementalUnit must be OBJECT or DIRECTORY"})
}
if payload.DirectoryDepth < 0 {
issues = append(issues, validationIssue{Field: "directoryDepth", Message: "directoryDepth must be zero or greater"})
}
return issues
}
func scanConfigPayloadReady(payload scanConfigRequest, existing *domain.ScanConfig) bool {
switch chooseDataSource(payload.DataSource) {
case "BOS":
connectionID := mergeScanConnectionID(existing, payload)
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
hasTarget := strings.TrimSpace(payload.SourcePath) != "" ||
(strings.TrimSpace(payload.Bucket) != "" && cleanBOSPrefix(payload.Prefix) != "")
return hasTarget &&
(strings.TrimSpace(connectionID) != "" ||
(strings.TrimSpace(accessKeyID) != "" && strings.TrimSpace(secretAccessKey) != ""))
default:
return strings.TrimSpace(payload.SourcePath) != ""
}
}
func validateScanConfigPayload(payload scanConfigRequest, existing *domain.ScanConfig) []validationIssue {
issues := validateScanConfigDraftPayload(payload)
switch chooseDataSource(payload.DataSource) {
case "BOS":
if strings.TrimSpace(payload.SourcePath) == "" {
if strings.TrimSpace(payload.Bucket) == "" {
issues = append(issues, validationIssue{Field: "bucket", Message: "bucket is required for BOS scan configs when sourcePath is empty"})
}
if cleanBOSPrefix(payload.Prefix) == "" {
issues = append(issues, validationIssue{Field: "prefix", Message: "prefix is required for BOS scan configs when sourcePath is empty"})
}
}
connectionID := mergeScanConnectionID(existing, payload)
if strings.TrimSpace(connectionID) == "" {
accessKeyID, secretAccessKey := mergeScanSecrets(existing, payload)
if strings.TrimSpace(accessKeyID) == "" {
issues = append(issues, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required for BOS scan configs"})
}
if strings.TrimSpace(secretAccessKey) == "" {
issues = append(issues, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required for BOS scan configs"})
}
}
default:
if strings.TrimSpace(payload.SourcePath) == "" {
issues = append(issues, validationIssue{Field: "sourcePath", Message: "sourcePath is required"})
}
}
return issues
}
func (server *server) scanWorkflows(ctx context.Context) ([]domain.WorkflowRecord, error) {
items, err := server.store.ListAllWorkflows(ctx)
if err != nil {
return nil, err
}
filtered := make([]domain.WorkflowRecord, 0)
for _, item := range items {
if isScanWorkflow(item) {
item.Kind = chooseWorkflowKind(item.Kind)
filtered = append(filtered, item)
}
}
return filtered, nil
}
func (server *server) handleScanConfigsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.scanWorkflows(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)})
}
func (server *server) handleScanConfigsCreate(writer http.ResponseWriter, request *http.Request) {
var payload scanConfigRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Interval > 0 && !scanConfigPayloadReady(payload, nil) {
renderValidationError(writer, validateScanConfigPayload(payload, nil)...)
return
}
if issues := server.validateScanConnectionReference(request.Context(), normalizedScanConfigFromRequest(nil, payload)); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
input, err := server.buildScanWorkflowInput(request.Context(), "", payload, nil)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item, err := server.store.CreateWorkflow(request.Context(), input)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
item.Kind = chooseWorkflowKind(item.Kind)
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
var payload scanConfigRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateScanConfigDraftPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Interval > 0 && !scanConfigPayloadReady(payload, item.ScanConfig) {
renderValidationError(writer, validateScanConfigPayload(payload, item.ScanConfig)...)
return
}
if issues := server.validateScanConnectionReference(request.Context(), normalizedScanConfigFromRequest(item.ScanConfig, payload)); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
input, err := server.buildScanWorkflowInput(request.Context(), item.ProjectID, payload, item.ScanConfig)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item.Name = input.Name
item.Description = input.Description
item.Type = input.Type
item.Kind = input.Kind
item.ScanConfig = input.ScanConfig
item.Schedule = input.Schedule
item.Nodes = input.Nodes
item.Edges = input.Edges
item.UpdatedAt = nowISO()
if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleScanConfigsDelete(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
writer.WriteHeader(http.StatusNotFound)
return
}
deleted, err := server.store.DeleteWorkflow(request.Context(), item.ID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func (server *server) handleScanConfigsRun(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !isScanWorkflow(*item) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Scan config not found"})
return
}
if issues := server.validateScanConnectionReference(request.Context(), item.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
run, err := server.store.CreateRun(request.Context(), item.ID, domain.RunTriggerManual)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusAccepted, run)
}
func artifactByName(run domain.RunRecord, name string) *domain.ExecutionArtifact {
for index := range run.Artifacts {
if run.Artifacts[index].Name == name {
return &run.Artifacts[index]
}
}
return nil
}
func mapValue(value any) map[string]any {
item, _ := normalizeStructuredValue(value).(map[string]any)
return item
}
func listValue(value any) []any {
items, _ := normalizeStructuredValue(value).([]any)
return items
}
func normalizeStructuredValue(value any) any {
switch typed := value.(type) {
case nil:
return nil
case map[string]any:
items := make(map[string]any, len(typed))
for key, item := range typed {
items[key] = normalizeStructuredValue(item)
}
return items
case []any:
items := make([]any, len(typed))
for index, item := range typed {
items[index] = normalizeStructuredValue(item)
}
return items
}
rv := reflect.ValueOf(value)
switch rv.Kind() {
case reflect.Map:
if rv.Type().Key().Kind() != reflect.String {
return value
}
items := make(map[string]any, rv.Len())
iter := rv.MapRange()
for iter.Next() {
items[iter.Key().String()] = normalizeStructuredValue(iter.Value().Interface())
}
return items
case reflect.Slice, reflect.Array:
items := make([]any, rv.Len())
for index := 0; index < rv.Len(); index++ {
items[index] = normalizeStructuredValue(rv.Index(index).Interface())
}
return items
default:
return value
}
}
func stringValue(value any) string {
switch typed := value.(type) {
case nil:
return ""
case string:
return typed
default:
text := strings.TrimSpace(fmt.Sprintf("%v", typed))
if text == "<nil>" {
return ""
}
return text
}
}
func boolValue(value any) bool {
typed, ok := value.(bool)
return ok && typed
}
func floatValue(value any) float64 {
switch typed := value.(type) {
case float64:
return typed
case float32:
return float64(typed)
case int:
return float64(typed)
case int64:
return float64(typed)
default:
return 0
}
}
func intValue(value any) int {
return int(floatValue(value))
}
func int64Value(value any) int64 {
return int64(floatValue(value))
}
func artifactDataRef(artifact *domain.ExecutionArtifact) map[string]any {
if artifact == nil || artifact.Data == nil {
return nil
}
return mapValue(artifact.Data["dataRef"])
}
func artifactStructuredData(artifact *domain.ExecutionArtifact) map[string]any {
if artifact == nil {
return nil
}
if dataRef := artifactDataRef(artifact); dataRef != nil {
payloadPath := strings.TrimSpace(stringValue(dataRef["payloadPath"]))
if payloadPath != "" && !strings.Contains(payloadPath, "://") {
payload, err := os.ReadFile(payloadPath)
if err == nil {
var decoded any
if err := json.Unmarshal(payload, &decoded); err == nil {
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
return structured
}
}
}
}
}
if artifact.Data != nil {
return artifact.Data
}
if artifact.Type == domain.PortTypeJSON {
filePath := strings.TrimSpace(artifact.Path)
if filePath != "" && !strings.Contains(filePath, "://") {
payload, err := os.ReadFile(filePath)
if err == nil {
var decoded any
if err := json.Unmarshal(payload, &decoded); err == nil {
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
return structured
}
}
}
}
}
return nil
}
func sourceArtifactRelativePath(path string) string {
normalized := filepath.ToSlash(path)
marker := "/_source_tree/"
index := strings.Index(normalized, marker)
if index < 0 {
return ""
}
return strings.TrimPrefix(normalized[index+len(marker):], "/")
}
func sourceArtifactPath(artifact domain.ExecutionArtifact) string {
if payload := artifactStructuredData(&artifact); payload != nil {
if relative := stringValue(payload["relativePath"]); relative != "" {
return filepath.ToSlash(strings.Trim(relative, "/"))
}
}
return sourceArtifactRelativePath(artifact.Path)
}
func packageArtifactRelativePath(path string, packageName string) string {
normalized := filepath.ToSlash(path)
marker := "/packages/" + packageName + "/"
index := strings.Index(normalized, marker)
if index < 0 {
return ""
}
return strings.TrimPrefix(normalized[index+len(marker):], "/")
}
func packageContentURL(runID string, packageName string, relativePath string) string {
return fmt.Sprintf(
"/api/packages/%s/%s/content?relativePath=%s",
url.PathEscape(runID),
url.PathEscape(packageName),
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
)
}
func packageArtifactPreviewURL(artifact domain.ExecutionArtifact) string {
payload := artifactStructuredData(&artifact)
preview := mapValue(payload["preview"])
if preview != nil {
if value := strings.TrimSpace(stringValue(preview["url"])); value != "" {
return value
}
}
source := mapValue(payload["source"])
if source != nil {
if value := strings.TrimSpace(stringValue(source["previewURL"])); value != "" {
return value
}
}
return ""
}
func packageArtifactResolvedRelativePath(artifact domain.ExecutionArtifact, packageName string) string {
if relativePath := packageArtifactRelativePath(artifact.Path, packageName); relativePath != "" {
return relativePath
}
payload := artifactStructuredData(&artifact)
if payload == nil {
return ""
}
relativePath := strings.Trim(filepath.ToSlash(stringValue(payload["relativePath"])), "/")
if relativePath == "" {
return ""
}
packagePrefix := strings.Trim(filepath.ToSlash(path.Join("packages", packageName)), "/")
if packagePrefix == "" {
return ""
}
if relativePath == packagePrefix {
return ""
}
if strings.HasPrefix(relativePath, packagePrefix+"/") {
return strings.TrimPrefix(relativePath, packagePrefix+"/")
}
return ""
}
func packageArtifactForResponse(runID string, packageName string, artifact domain.ExecutionArtifact) domain.ExecutionArtifact {
copy := artifact
if copy.Data == nil {
copy.Data = make(map[string]any)
}
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
if relativePath == "" {
relativePath = filepath.ToSlash(strings.Trim(copy.Path, "/"))
}
if relativePath != "" {
copy.Path = relativePath
}
preview := mapValue(copy.Data["preview"])
if preview == nil {
preview = make(map[string]any)
}
kind := strings.TrimSpace(stringValue(preview["kind"]))
mimeType := ""
if copy.Metadata != nil {
mimeType = strings.TrimSpace(copy.Metadata.MimeType)
}
if kind == "video" || copy.Type == "video" || strings.HasPrefix(mimeType, "video/") {
preview["kind"] = "video"
preview["url"] = packageContentURL(runID, packageName, copy.Path)
copy.Data["preview"] = preview
}
return copy
}
func canonicalBOSPath(bucket string, object string) string {
bucket = strings.TrimSpace(bucket)
object = strings.Trim(strings.TrimSpace(object), "/")
if bucket == "" {
return ""
}
if object == "" {
return "bos://" + bucket
}
return "bos://" + bucket + "/" + object
}
func packageArtifactPriority(artifact domain.ExecutionArtifact) int {
score := 0
if artifact.NodeID == "upload" {
score += 40
}
artifactPath := strings.TrimSpace(artifact.Path)
if strings.HasPrefix(strings.ToLower(artifactPath), "bos://") {
score += 30
} else if artifactPath != "" && !strings.Contains(artifactPath, "://") {
score += 10
}
payload := artifactStructuredData(&artifact)
source := mapValue(payload["source"])
sourcePath := strings.TrimSpace(stringValue(source["path"]))
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
score += 30
}
if strings.TrimSpace(packageArtifactPreviewURL(artifact)) != "" {
score += 5
}
return score
}
func packageArtifactStoragePath(artifact domain.ExecutionArtifact, packageName string) string {
payload := artifactStructuredData(&artifact)
source := mapValue(payload["source"])
if sourcePath := strings.TrimSpace(stringValue(source["path"])); sourcePath != "" {
if bucket, object, ok := parseBOSPath(sourcePath); ok {
return canonicalBOSPath(bucket, path.Dir(object))
}
directory := filepath.ToSlash(filepath.Dir(sourcePath))
if directory != "." {
return directory
}
}
artifactPath := strings.TrimSpace(artifact.Path)
if bucket, object, ok := parseBOSPath(artifactPath); ok {
return canonicalBOSPath(bucket, path.Dir(object))
}
if artifactPath == "" {
return ""
}
directory := filepath.ToSlash(filepath.Dir(artifactPath))
if directory == "." {
return ""
}
return directory
}
func dedupePackageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
bestByRelativePath := make(map[string]domain.ExecutionArtifact)
for _, artifact := range run.Artifacts {
relativePath := packageArtifactResolvedRelativePath(artifact, packageName)
if relativePath == "" {
continue
}
current, exists := bestByRelativePath[relativePath]
if !exists || packageArtifactPriority(artifact) > packageArtifactPriority(current) {
bestByRelativePath[relativePath] = artifact
}
}
items := make([]domain.ExecutionArtifact, 0, len(bestByRelativePath))
for _, artifact := range bestByRelativePath {
items = append(items, artifact)
}
sort.Slice(items, func(left int, right int) bool {
leftPath := packageArtifactResolvedRelativePath(items[left], packageName)
rightPath := packageArtifactResolvedRelativePath(items[right], packageName)
return leftPath < rightPath
})
return items
}
func packageArtifacts(run domain.RunRecord, packageName string) []domain.ExecutionArtifact {
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range dedupePackageArtifacts(run, packageName) {
items = append(items, packageArtifactForResponse(run.ID, packageName, artifact))
}
return items
}
func choosePackagePreview(files []domain.ExecutionArtifact) *domain.ExecutionArtifact {
for _, file := range files {
if strings.HasSuffix(strings.ToLower(file.Name), ".mp4") {
copy := file
return &copy
}
}
if len(files) == 0 {
return nil
}
copy := files[0]
return &copy
}
func sourceDatasetList(data map[string]any, catalogKey string, fallbackKey string) []any {
if _, ok := data[catalogKey]; ok {
return listValue(data[catalogKey])
}
return listValue(data[fallbackKey])
}
func sourceDatasetHasCatalog(data map[string]any) bool {
_, hasCollections := data["catalogCollections"]
_, hasItems := data["catalogItems"]
_, hasEntries := data["catalogEntries"]
return hasCollections || hasItems || hasEntries
}
func sourceDatasetIsDisplayable(data map[string]any) bool {
if len(sourceDatasetList(data, "catalogCollections", "collections")) > 0 {
return true
}
if len(sourceDatasetList(data, "catalogEntries", "entries")) > 0 {
return true
}
return intValue(data["catalogCollectionCount"]) > 0 || intValue(data["collectionCount"]) > 0
}
func runSourceManifest(run domain.RunRecord) *domain.ExecutionArtifact {
manifest := artifactByName(run, "source-manifest.json")
if manifest == nil || artifactStructuredData(manifest) == nil {
return nil
}
return manifest
}
func runDatasetIndex(run domain.RunRecord) *domain.ExecutionArtifact {
artifact := artifactByName(run, "dataset-index.json")
if artifact == nil || artifactStructuredData(artifact) == nil {
return nil
}
return artifact
}
func runLooksLikeSourceDataset(workflow domain.WorkflowRecord, run domain.RunRecord) bool {
if manifest := runSourceManifest(run); manifest != nil {
payload := artifactStructuredData(manifest)
if strings.TrimSpace(stringValue(payload["sourcePath"])) != "" || strings.TrimSpace(stringValue(payload["dataSource"])) != "" {
return true
}
}
index := runDatasetIndex(run)
if index == nil {
return false
}
if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil {
return true
}
indexPayload := artifactStructuredData(index)
sourcePath := strings.TrimSpace(stringValue(indexPayload["sourcePath"]))
rootPath := strings.TrimSpace(stringValue(indexPayload["rootPath"]))
dataSource := strings.TrimSpace(stringValue(indexPayload["dataSource"]))
if dataSource != "" || strings.HasPrefix(sourcePath, "bos://") || strings.HasPrefix(rootPath, "bos://") {
return true
}
if len(mapValue(indexPayload["archiveSources"])) > 0 && strings.Contains(rootPath, "_materialized_inputs") {
return false
}
return sourceDatasetHasCatalog(indexPayload)
}
func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]domain.WorkflowRecord, map[string]domain.RunRecord, error) {
workflows, err := server.store.ListAllWorkflows(ctx)
if err != nil {
return nil, nil, err
}
workflowByID := make(map[string]domain.WorkflowRecord, len(workflows))
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
runs, err := server.store.ListRuns(ctx, string(domain.RunStatusCompleted))
if err != nil {
return nil, nil, err
}
latestByWorkflow := make(map[string]domain.RunRecord)
for _, run := range runs {
workflow, ok := workflowByID[run.WorkflowID]
if !ok {
continue
}
index := runDatasetIndex(run)
if index == nil {
continue
}
if !runLooksLikeSourceDataset(workflow, run) {
continue
}
if !sourceDatasetIsDisplayable(artifactStructuredData(index)) {
continue
}
current, exists := latestByWorkflow[run.WorkflowID]
if !exists || run.CreatedAt > current.CreatedAt {
latestByWorkflow[run.WorkflowID] = run
}
}
return workflowByID, latestByWorkflow, nil
}
func parseBOSPath(sourcePath string) (string, string, bool) {
normalized := strings.TrimSpace(sourcePath)
if !strings.HasPrefix(strings.ToLower(normalized), "bos://") {
return "", "", false
}
trimmed := strings.TrimPrefix(normalized, "bos://")
trimmed = strings.TrimPrefix(trimmed, "BOS://")
bucket, object, found := strings.Cut(trimmed, "/")
bucket = strings.TrimSpace(bucket)
object = strings.Trim(strings.TrimSpace(object), "/")
if !found {
object = ""
}
if bucket == "" {
return "", "", false
}
return bucket, object, true
}
func normalizeLocalDirectory(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return ""
}
directory := filepath.ToSlash(filepath.Dir(trimmed))
if directory == "." {
return ""
}
return directory
}
func assetStorageLocationFromPaths(rootPath string, currentPath string, endpoint string) domain.AssetStorageLocation {
location := domain.AssetStorageLocation{
Path: strings.TrimSpace(currentPath),
}
currentBucket, currentObject, hasCurrentBOS := parseBOSPath(currentPath)
rootBucket, rootObject, hasRootBOS := parseBOSPath(rootPath)
switch {
case hasCurrentBOS:
location.Provider = "bos"
location.Endpoint = chooseBOSEndpoint(endpoint)
location.Bucket = currentBucket
if hasRootBOS && rootBucket == currentBucket {
location.Prefix = rootObject
} else {
location.Prefix = path.Dir(currentObject)
if location.Prefix == "." {
location.Prefix = ""
}
}
case hasRootBOS:
location.Provider = "bos"
location.Endpoint = chooseBOSEndpoint(endpoint)
location.Bucket = rootBucket
location.Prefix = rootObject
if location.Path == "" {
location.Path = strings.TrimSpace(rootPath)
}
case strings.TrimSpace(currentPath) != "":
location.Provider = "k8s-job"
location.Prefix = normalizeLocalDirectory(currentPath)
case strings.TrimSpace(rootPath) != "":
location.Provider = "k8s-job"
location.Path = strings.TrimSpace(rootPath)
location.Prefix = normalizeLocalDirectory(rootPath)
default:
location.Provider = "unknown"
}
if location.Prefix == "." {
location.Prefix = ""
}
return location
}
func sourceRootPathForRun(run domain.RunRecord, scanConfig *domain.ScanConfig) string {
if scanConfig != nil {
if value := scanSourcePath(scanConfig.DataSource, scanConfig.SourcePath, scanConfig.Bucket, scanConfig.Prefix); value != "" {
return value
}
}
if manifest := runSourceManifest(run); manifest != nil {
payload := artifactStructuredData(manifest)
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
return value
}
}
if index := runDatasetIndex(run); index != nil {
payload := artifactStructuredData(index)
if value := strings.TrimSpace(stringValue(payload["sourcePath"])); value != "" {
return value
}
if value := strings.TrimSpace(stringValue(payload["rootPath"])); value != "" {
return value
}
}
return ""
}
func (server *server) buildSourceCollectionRecord(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, collection map[string]any) domain.SourceCollectionRecord {
meta := mapValue(collection["meta"])
requiredFiles := make([]string, 0)
for _, item := range listValue(collection["requiredFiles"]) {
requiredFiles = append(requiredFiles, stringValue(item))
}
missingFiles := make([]string, 0)
for _, item := range listValue(collection["missingRequiredFiles"]) {
missingFiles = append(missingFiles, stringValue(item))
}
collectionName := stringValue(collection["name"])
if collectionName == "" {
if projectID := stringValue(meta["project_id"]); projectID != "" {
collectionName = projectID
} else if relativePath := strings.Trim(stringValue(collection["relativePath"]), "/"); relativePath != "" && relativePath != "." {
collectionName = path.Base(relativePath)
} else {
collectionName = stringValue(collection["id"])
}
}
var resolvedScanConfig *domain.ScanConfig
if workflow != nil {
if scanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig); err == nil {
resolvedScanConfig = scanConfig
}
}
storage := assetStorageLocationFromPaths(
sourceRootPathForRun(run, resolvedScanConfig),
stringValue(collection["path"]),
func() string {
if resolvedScanConfig == nil {
return ""
}
return resolvedScanConfig.Endpoint
}(),
)
return domain.SourceCollectionRecord{
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
CollectionID: stringValue(collection["id"]),
CollectionName: collectionName,
Path: stringValue(collection["path"]),
RelativePath: stringValue(collection["relativePath"]),
Meta: meta,
HasIntrinsics: boolValue(collection["hasIntrinsics"]),
HasVideoMeta: boolValue(collection["hasVideoMeta"]),
RequiredFiles: requiredFiles,
MissingFiles: missingFiles,
VideoCount: intValue(collection["videoCount"]),
FileCount: intValue(collection["fileCount"]),
TotalBytes: int64Value(collection["totalBytes"]),
Valid: boolValue(collection["valid"]),
CreatedAt: run.CreatedAt,
Storage: storage,
}
}
type sourceCollectionPayloadRecord struct {
Run *domain.RunRecord
Workflow *domain.WorkflowRecord
Collection map[string]any
Items []map[string]any
Entries []map[string]any
ArchiveSource map[string]any
}
func (server *server) sourceCollectionPayload(ctx context.Context, runID string, collectionID string) (*sourceCollectionPayloadRecord, error) {
run, err := server.store.GetRun(ctx, runID)
if err != nil || run == nil {
return &sourceCollectionPayloadRecord{Run: run}, err
}
workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil {
return &sourceCollectionPayloadRecord{Run: run}, err
}
artifact := artifactByName(*run, "dataset-index.json")
payload := artifactStructuredData(artifact)
if artifact == nil || payload == nil {
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
}
collections := sourceDatasetList(payload, "catalogCollections", "collections")
items := sourceDatasetList(payload, "catalogItems", "items")
entries := sourceDatasetList(payload, "catalogEntries", "entries")
var matchedCollection map[string]any
for _, item := range collections {
collection := mapValue(item)
if stringValue(collection["id"]) == collectionID {
matchedCollection = collection
break
}
}
if matchedCollection == nil {
return &sourceCollectionPayloadRecord{Run: run, Workflow: workflow}, nil
}
matchedItems := make([]map[string]any, 0)
for _, item := range items {
row := mapValue(item)
if stringValue(row["collectionId"]) == collectionID {
matchedItems = append(matchedItems, row)
}
}
matchedEntries := make([]map[string]any, 0)
for _, item := range entries {
row := mapValue(item)
relativePath := strings.Trim(filepath.ToSlash(stringValue(row["relativePath"])), "/")
collectionRelativePath := strings.Trim(filepath.ToSlash(stringValue(matchedCollection["relativePath"])), "/")
if collectionRelativePath == "" || collectionRelativePath == "." {
if relativePath == "" {
continue
}
} else if relativePath != collectionRelativePath && !strings.HasPrefix(relativePath, collectionRelativePath+"/") {
continue
}
matchedEntries = append(matchedEntries, row)
}
return &sourceCollectionPayloadRecord{
Run: run,
Workflow: workflow,
Collection: matchedCollection,
Items: matchedItems,
Entries: matchedEntries,
ArchiveSource: mapValue(payload["archiveSources"]),
}, nil
}
func sourceEntryPayloadKind(relativePath string, mimeType string) string {
extension := strings.ToLower(path.Ext(relativePath))
switch {
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
return "video"
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
return "image"
case strings.HasPrefix(mimeType, "audio/") || extension == ".mp3" || extension == ".wav" || extension == ".m4a":
return "audio"
default:
return "binary"
}
}
func sourceEntryArtifactType(relativePath string, mimeType string) domain.PortType {
extension := strings.ToLower(path.Ext(relativePath))
switch {
case strings.HasPrefix(mimeType, "image/") || extension == ".jpg" || extension == ".jpeg" || extension == ".png" || extension == ".webp":
return domain.PortTypeImage
case strings.HasPrefix(mimeType, "video/") || extension == ".mp4" || extension == ".mov" || extension == ".mkv" || extension == ".avi":
return domain.PortTypeVideo
case extension == ".json":
return domain.PortTypeJSON
default:
return domain.PortTypeFile
}
}
func formatArtifactSize(sizeBytes int64) string {
switch {
case sizeBytes <= 0:
return "0 B"
case sizeBytes < 1024:
return fmt.Sprintf("%d B", sizeBytes)
case sizeBytes < 1024*1024:
return fmt.Sprintf("%.1f KiB", float64(sizeBytes)/1024)
case sizeBytes < 1024*1024*1024:
return fmt.Sprintf("%.1f MiB", float64(sizeBytes)/(1024*1024))
default:
return fmt.Sprintf("%.1f GiB", float64(sizeBytes)/(1024*1024*1024))
}
}
func sourceCollectionContentURL(runID string, collectionID string, relativePath string) string {
return fmt.Sprintf(
"/api/source-data/collections/%s/%s/content?relativePath=%s",
url.PathEscape(runID),
url.PathEscape(collectionID),
url.QueryEscape(strings.Trim(filepath.ToSlash(relativePath), "/")),
)
}
func datasetIndexEntrySource(entry map[string]any, archiveSources map[string]any) map[string]any {
source := mapValue(entry["source"])
if len(source) == 0 {
source = make(map[string]any)
}
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if relativePath != "" && stringValue(source["relativePath"]) == "" {
source["relativePath"] = relativePath
}
mimeType := stringValue(entry["mimeType"])
if mimeType == "" {
mimeType = stringValue(source["mimeType"])
}
if mimeType == "" && relativePath != "" {
mimeType = mime.TypeByExtension(path.Ext(relativePath))
}
if mimeType != "" {
source["mimeType"] = mimeType
}
if stringValue(source["previewURL"]) == "" {
archiveKey := stringValue(source["archiveKey"])
archiveSource := mapValue(archiveSources[archiveKey])
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
source["previewURL"] = previewURL
}
if archiveName := stringValue(archiveSource["name"]); archiveName != "" && stringValue(source["archiveName"]) == "" {
source["archiveName"] = archiveName
}
}
if pathValue := stringValue(entry["path"]); pathValue != "" && stringValue(source["path"]) == "" {
source["path"] = pathValue
}
return source
}
func sourceEntryArtifact(run domain.RunRecord, collectionID string, entry map[string]any, archiveSources map[string]any) *domain.ExecutionArtifact {
relativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if relativePath == "" {
relativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
if relativePath == "" {
return nil
}
source := datasetIndexEntrySource(entry, archiveSources)
mimeType := stringValue(source["mimeType"])
sizeBytes := int64Value(entry["sizeBytes"])
artifactType := sourceEntryArtifactType(relativePath, mimeType)
payload := make(map[string]any)
if value, ok := entry["data"]; ok {
payload["preview"] = map[string]any{
"kind": "json",
"value": normalizeStructuredValue(value),
}
}
if previewURL := stringValue(source["previewURL"]); previewURL != "" && len(payload) == 0 {
payload["preview"] = map[string]any{
"kind": sourceEntryPayloadKind(relativePath, mimeType),
"format": strings.TrimPrefix(strings.ToLower(path.Ext(relativePath)), "."),
"url": sourceCollectionContentURL(run.ID, collectionID, relativePath),
}
}
payload["relativePath"] = relativePath
if len(source) > 0 {
if previewURL := sourceCollectionContentURL(run.ID, collectionID, relativePath); previewURL != "" {
source["previewURL"] = previewURL
}
payload["source"] = source
}
return &domain.ExecutionArtifact{
ID: fmt.Sprintf("%s-source-%s", run.ID, strings.ReplaceAll(relativePath, "/", "-")),
RunID: run.ID,
NodeID: "dataset-index",
Name: path.Base(relativePath),
Type: artifactType,
Size: formatArtifactSize(sizeBytes),
Path: func() string {
if sourcePath := stringValue(source["path"]); sourcePath != "" {
return sourcePath
}
return relativePath
}(),
Metadata: &domain.ArtifactMetadata{MimeType: mimeType},
Data: payload,
}
}
func contentBytesForDataURL(previewURL string) ([]byte, string, error) {
if !strings.HasPrefix(previewURL, "data:") {
return nil, "", fmt.Errorf("not a data url")
}
header, encoded, found := strings.Cut(previewURL, ",")
if !found || !strings.Contains(header, ";base64") {
return nil, "", fmt.Errorf("invalid data url")
}
mimeType := strings.TrimPrefix(strings.SplitN(header, ";", 2)[0], "data:")
payload, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, "", err
}
return payload, mimeType, nil
}
func contentBytesForArchiveMember(archiveName string, archivePreviewURL string, memberPath string) ([]byte, error) {
payload, _, err := contentBytesForDataURL(archivePreviewURL)
if err != nil {
return nil, err
}
memberPath = strings.Trim(filepath.ToSlash(memberPath), "/")
if memberPath == "" {
return nil, fmt.Errorf("missing archive member path")
}
if strings.HasSuffix(strings.ToLower(archiveName), ".zip") {
archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
if err != nil {
return nil, err
}
for _, file := range archive.File {
if strings.Trim(filepath.ToSlash(file.Name), "/") != memberPath {
continue
}
handle, err := file.Open()
if err != nil {
return nil, err
}
defer handle.Close()
return io.ReadAll(handle)
}
return nil, os.ErrNotExist
}
reader := bytes.NewReader(payload)
var tarReader *tar.Reader
if strings.HasSuffix(strings.ToLower(archiveName), ".tgz") || strings.HasSuffix(strings.ToLower(archiveName), ".tar.gz") || (len(payload) >= 2 && payload[0] == 0x1f && payload[1] == 0x8b) {
gzipReader, err := gzip.NewReader(reader)
if err == nil {
defer gzipReader.Close()
tarReader = tar.NewReader(gzipReader)
} else {
reader = bytes.NewReader(payload)
tarReader = tar.NewReader(reader)
}
} else {
tarReader = tar.NewReader(reader)
}
for {
header, err := tarReader.Next()
if err == io.EOF {
return nil, os.ErrNotExist
}
if err != nil {
return nil, err
}
if header.FileInfo().IsDir() {
continue
}
if strings.Trim(filepath.ToSlash(header.Name), "/") != memberPath {
continue
}
return io.ReadAll(tarReader)
}
}
func sourceCollectionEntryPayload(collection domain.SourceCollectionRecord, payloadEntries []map[string]any, relativePath string) map[string]any {
target := strings.Trim(filepath.ToSlash(relativePath), "/")
if target == "" {
return nil
}
collectionPrefix := strings.Trim(filepath.ToSlash(collection.RelativePath), "/")
fullTarget := target
if collectionPrefix != "" && target != collectionPrefix && !strings.HasPrefix(target, collectionPrefix+"/") {
fullTarget = collectionPrefix + "/" + target
}
for _, entry := range payloadEntries {
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if entryRelativePath == "" {
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
if entryRelativePath == fullTarget || entryRelativePath == target {
return entry
}
}
return nil
}
func writeSourceContent(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, payload []byte) {
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if mimeType == "" && len(payload) > 0 {
mimeType = http.DetectContentType(payload)
}
if mimeType != "" {
writer.Header().Set("Content-Type", mimeType)
}
http.ServeContent(writer, request, fileName, time.Time{}, bytes.NewReader(payload))
}
func writeSourceContentFile(writer http.ResponseWriter, request *http.Request, fileName string, mimeType string, filePath string) error {
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
fileInfo, err := file.Stat()
if err != nil {
return err
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if mimeType == "" {
header := make([]byte, 512)
count, readErr := file.Read(header)
if readErr != nil && readErr != io.EOF {
return readErr
}
mimeType = http.DetectContentType(header[:count])
if _, err := file.Seek(0, io.SeekStart); err != nil {
return err
}
}
if mimeType != "" {
writer.Header().Set("Content-Type", mimeType)
}
http.ServeContent(writer, request, fileName, fileInfo.ModTime(), file)
return nil
}
func sourceBrowserPreviewCacheKey(request sourceBrowserPreviewRequest) string {
hash := sha256.New()
hash.Write([]byte("browser-preview-v2"))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.FileName)))
hash.Write([]byte{0})
if request.RedirectURL != "" {
hash.Write([]byte("redirect"))
hash.Write([]byte{0})
hash.Write([]byte(strings.TrimSpace(request.RedirectURL)))
}
if len(request.Payload) > 0 {
payloadHash := sha256.Sum256(request.Payload)
hash.Write([]byte("payload"))
hash.Write([]byte{0})
hash.Write(payloadHash[:])
}
return hex.EncodeToString(hash.Sum(nil))
}
func sourceBrowserPreviewCacheDir() (string, error) {
cacheDir := filepath.Join(os.TempDir(), "droboticflow", "source-video-previews")
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
return "", err
}
return cacheDir, nil
}
func sourceBrowserPreviewCachePath(request sourceBrowserPreviewRequest) (string, error) {
cacheDir, err := sourceBrowserPreviewCacheDir()
if err != nil {
return "", err
}
cacheKey := sourceBrowserPreviewCacheKey(request)
return filepath.Join(cacheDir, cacheKey+".mp4"), nil
}
func parseBOSObjectPath(sourcePath string) (string, string, bool) {
bucket, object, ok := parseBOSPath(sourcePath)
if !ok || object == "" {
return "", "", false
}
return bucket, object, true
}
func buildSourceCollectionSignedPreviewURL(scanConfig *domain.ScanConfig, sourcePath string) (string, error) {
if scanConfig == nil {
return "", fmt.Errorf("scan config is unavailable")
}
if chooseDataSource(scanConfig.DataSource) != "BOS" {
return "", fmt.Errorf("scan config is not BOS")
}
bucket, object, ok := parseBOSObjectPath(sourcePath)
if !ok {
return "", fmt.Errorf("invalid BOS source path: %s", sourcePath)
}
accessKeyID := strings.TrimSpace(scanConfig.AccessKeyID)
secretAccessKey := strings.TrimSpace(scanConfig.SecretAccessKey)
if accessKeyID == "" || secretAccessKey == "" {
return "", fmt.Errorf("BOS credentials are unavailable")
}
client, err := bos.NewClient(accessKeyID, secretAccessKey, chooseBOSEndpoint(scanConfig.Endpoint))
if err != nil {
return "", err
}
previewURL := strings.TrimSpace(client.BasicGeneratePresignedUrl(bucket, object, 3600))
if previewURL == "" {
return "", fmt.Errorf("failed to generate BOS preview url")
}
return previewURL, nil
}
func locateSourceBrowserVideoPreviewPath(ctx context.Context, request sourceBrowserPreviewRequest) (string, error) {
finalPath, err := sourceBrowserPreviewCachePath(request)
if err != nil {
return "", err
}
if fileInfo, err := os.Stat(finalPath); err == nil && fileInfo.Size() > 0 {
return finalPath, nil
}
return "", os.ErrNotExist
}
func streamSourceBrowserVideoPreview(ctx context.Context, writer http.ResponseWriter, request sourceBrowserPreviewRequest) error {
if strings.TrimSpace(request.RedirectURL) == "" && len(request.Payload) == 0 {
return fmt.Errorf("source video content is unavailable")
}
finalPath, err := sourceBrowserPreviewCachePath(request)
if err != nil {
return err
}
inputRef := strings.TrimSpace(request.RedirectURL)
var cleanupInput func()
if inputRef == "" {
extension := path.Ext(request.FileName)
if extension == "" {
extension = ".bin"
}
cacheDir := filepath.Dir(finalPath)
inputFile, err := os.CreateTemp(cacheDir, "source-video-*"+extension)
if err != nil {
return err
}
if _, err := inputFile.Write(request.Payload); err != nil {
inputFile.Close()
os.Remove(inputFile.Name())
return err
}
if err := inputFile.Close(); err != nil {
os.Remove(inputFile.Name())
return err
}
inputRef = inputFile.Name()
cleanupInput = func() {
_ = os.Remove(inputFile.Name())
}
}
if cleanupInput != nil {
defer cleanupInput()
}
cacheDir := filepath.Dir(finalPath)
outputFile, err := os.CreateTemp(cacheDir, sourceBrowserPreviewCacheKey(request)+"-*.mp4")
if err != nil {
return err
}
tempOutputPath := outputFile.Name()
stderr := &bytes.Buffer{}
command := exec.CommandContext(
ctx,
"ffmpeg",
"-y",
"-hide_banner",
"-loglevel", "error",
"-i", inputRef,
"-map", "0:v:0",
"-map", "0:a?",
"-threads", "1",
"-probesize", "32M",
"-analyzeduration", "0",
"-c:v", "libx264",
"-preset", "ultrafast",
"-tune", "zerolatency",
"-crf", "30",
"-vf", "fps=12,scale=min(960\\,iw):-2",
"-g", "12",
"-keyint_min", "12",
"-pix_fmt", "yuv420p",
"-movflags", "frag_keyframe+empty_moov+default_base_moof",
"-c:a", "aac",
"-b:a", "96k",
"-ac", "1",
"-f", "mp4",
"pipe:1",
)
command.Stderr = stderr
stdout, err := command.StdoutPipe()
if err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
return err
}
if err := command.Start(); err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
message := strings.TrimSpace(stderr.String())
if message != "" {
return fmt.Errorf("generate browser video preview: %s", message)
}
return err
}
streamTarget := io.Writer(writer)
if flusher, ok := writer.(http.Flusher); ok {
streamTarget = flushResponseWriter{writer: writer, flusher: flusher}
}
if _, err := io.Copy(io.MultiWriter(streamTarget, outputFile), stdout); err != nil {
outputFile.Close()
_ = os.Remove(tempOutputPath)
_ = command.Wait()
return err
}
if err := outputFile.Close(); err != nil {
_ = os.Remove(tempOutputPath)
_ = command.Wait()
return err
}
if err := command.Wait(); err != nil {
message := strings.TrimSpace(stderr.String())
_ = os.Remove(tempOutputPath)
if message != "" {
return fmt.Errorf("generate browser video preview: %s", message)
}
return fmt.Errorf("generate browser video preview: %w", err)
}
if err := os.Rename(tempOutputPath, finalPath); err != nil {
if fileInfo, statErr := os.Stat(finalPath); statErr == nil && fileInfo.Size() > 0 {
_ = os.Remove(tempOutputPath)
return nil
}
_ = os.Remove(tempOutputPath)
return err
}
return nil
}
func syntheticSourceEntries(collection domain.SourceCollectionRecord, payloadItems []map[string]any) []map[string]any {
entries := make([]map[string]any, 0, len(payloadItems)+2)
if len(collection.Meta) > 0 {
entries = append(entries, map[string]any{
"collectionId": collection.CollectionID,
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/meta.json", "/"),
"sizeBytes": len(fmt.Sprintf("%v", collection.Meta)),
"mimeType": "application/json",
"data": collection.Meta,
})
}
if collection.HasVideoMeta && len(payloadItems) > 0 {
videoRows := make([]map[string]any, 0, len(payloadItems))
for _, item := range payloadItems {
videoRows = append(videoRows, map[string]any{
"filename": stringValue(item["filename"]),
"codec": stringValue(item["codec"]),
"resolution": stringValue(item["resolution"]),
"frameRate": floatValue(item["frameRate"]),
"duration": floatValue(item["duration"]),
"startTime": item["startTime"],
})
}
entries = append(entries, map[string]any{
"collectionId": collection.CollectionID,
"relativePath": strings.Trim(strings.Trim(collection.RelativePath, "/")+"/video_meta.json", "/"),
"mimeType": "application/json",
"data": videoRows,
})
}
for _, item := range payloadItems {
entry := map[string]any{
"collectionId": collection.CollectionID,
"relativePath": stringValue(item["relativePath"]),
"path": stringValue(item["path"]),
"sizeBytes": int64Value(item["sizeBytes"]),
"mimeType": stringValue(mapValue(item["source"])["mimeType"]),
"source": mapValue(item["source"]),
"codec": stringValue(item["codec"]),
"resolution": stringValue(item["resolution"]),
"frameRate": floatValue(item["frameRate"]),
"duration": floatValue(item["duration"]),
}
entries = append(entries, entry)
}
return entries
}
func buildSourceEntries(run domain.RunRecord, collection domain.SourceCollectionRecord, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, prefix string) []domain.SourceCollectionEntry {
if len(payloadEntries) == 0 {
payloadEntries = syntheticSourceEntries(collection, payloadItems)
}
videoMetaByRelative := make(map[string]map[string]any, len(payloadItems))
for _, item := range payloadItems {
videoMetaByRelative[stringValue(item["relativePath"])] = item
}
normalizedPrefix := strings.Trim(strings.TrimSpace(filepath.ToSlash(prefix)), "/")
prefixSlash := normalizedPrefix
if prefixSlash != "" {
prefixSlash += "/"
}
directories := make(map[string]domain.SourceCollectionEntry)
files := make([]domain.SourceCollectionEntry, 0)
collectionPrefix := strings.Trim(collection.RelativePath, "/")
if collectionPrefix != "" {
collectionPrefix += "/"
}
for _, entryPayload := range payloadEntries {
relativePath := strings.Trim(filepath.ToSlash(stringValue(entryPayload["relativePath"])), "/")
if relativePath == "" {
relativePath = strings.Trim(filepath.ToSlash(stringValue(entryPayload["path"])), "/")
}
if relativePath == "" {
continue
}
if collectionPrefix != "" && relativePath != strings.TrimSuffix(collectionPrefix, "/") && !strings.HasPrefix(relativePath, collectionPrefix) {
continue
}
entryPath := relativePath
if collectionPrefix != "" {
entryPath = strings.TrimPrefix(relativePath, collectionPrefix)
}
if entryPath == "" {
continue
}
if normalizedPrefix != "" {
if entryPath == normalizedPrefix {
// exact file match
} else if !strings.HasPrefix(entryPath, prefixSlash) {
continue
} else {
entryPath = strings.TrimPrefix(entryPath, prefixSlash)
}
}
if entryPath == "" {
continue
}
parts := strings.Split(entryPath, "/")
if len(parts) > 1 {
dirPath := parts[0]
displayPath := dirPath
if normalizedPrefix != "" {
displayPath = normalizedPrefix + "/" + dirPath
}
if _, exists := directories[displayPath]; !exists {
directories[displayPath] = domain.SourceCollectionEntry{
Path: displayPath,
Name: dirPath,
Kind: "directory",
CollectionName: collection.CollectionName,
}
}
continue
}
displayPath := parts[0]
if normalizedPrefix != "" && displayPath != normalizedPrefix {
displayPath = normalizedPrefix + "/" + displayPath
}
artifact := sourceEntryArtifact(run, collection.CollectionID, entryPayload, archiveSources)
videoMeta := videoMetaByRelative[filepath.ToSlash(strings.Trim(collection.RelativePath, "/")+"/"+displayPath)]
entry := domain.SourceCollectionEntry{
Path: displayPath,
Name: parts[0],
Kind: "file",
CollectionName: collection.CollectionName,
SizeBytes: int64Value(entryPayload["sizeBytes"]),
Artifact: artifact,
}
if videoMeta != nil {
entry.Kind = "video"
entry.Codec = stringValue(videoMeta["codec"])
entry.Resolution = stringValue(videoMeta["resolution"])
entry.FrameRate = floatValue(videoMeta["frameRate"])
entry.Duration = floatValue(videoMeta["duration"])
entry.Metadata = videoMeta
if entry.SizeBytes == 0 {
entry.SizeBytes = int64Value(videoMeta["sizeBytes"])
}
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".json") {
entry.Kind = "json"
} else if strings.HasSuffix(strings.ToLower(entry.Name), ".mp4") {
entry.Kind = "video"
}
if entry.SizeBytes == 0 {
entry.SizeBytes = int64Value(entryPayload["sizeBytes"])
}
files = append(files, entry)
}
items := make([]domain.SourceCollectionEntry, 0, len(directories)+len(files))
for _, item := range directories {
items = append(items, item)
}
sort.Slice(items, func(left int, right int) bool { return items[left].Path < items[right].Path })
sort.Slice(files, func(left int, right int) bool { return files[left].Path < files[right].Path })
items = append(items, files...)
return items
}
func sourceCollectionContentPayload(collection domain.SourceCollectionRecord, scanConfig *domain.ScanConfig, payloadEntries []map[string]any, payloadItems []map[string]any, archiveSources map[string]any, relativePath string) ([]byte, string, string, string, error) {
if len(payloadEntries) == 0 {
payloadEntries = syntheticSourceEntries(collection, payloadItems)
}
entry := sourceCollectionEntryPayload(collection, payloadEntries, relativePath)
if entry == nil {
return nil, "", "", "", os.ErrNotExist
}
entryRelativePath := strings.Trim(filepath.ToSlash(stringValue(entry["relativePath"])), "/")
if entryRelativePath == "" {
entryRelativePath = strings.Trim(filepath.ToSlash(stringValue(entry["path"])), "/")
}
fileName := path.Base(entryRelativePath)
source := datasetIndexEntrySource(entry, archiveSources)
mimeType := stringValue(source["mimeType"])
if mimeType == "" {
mimeType = stringValue(entry["mimeType"])
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
if value, ok := entry["data"]; ok {
payload, err := json.MarshalIndent(normalizeStructuredValue(value), "", " ")
if err != nil {
return nil, "", "", "", err
}
if mimeType == "" {
mimeType = "application/json"
}
return payload, mimeType, fileName, "", nil
}
if sourcePath := stringValue(source["path"]); sourcePath != "" && !strings.Contains(sourcePath, "://") {
if payload, err := os.ReadFile(sourcePath); err == nil {
return payload, mimeType, fileName, "", nil
}
}
if sourcePath := stringValue(source["path"]); strings.HasPrefix(strings.ToLower(sourcePath), "bos://") && scanConfig != nil {
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
source["previewURL"] = previewURL
}
}
archiveKey := stringValue(source["archiveKey"])
archiveMemberPath := stringValue(source["archiveMemberPath"])
if archiveKey != "" && archiveMemberPath != "" {
archiveSource := mapValue(archiveSources[archiveKey])
if previewURL := stringValue(archiveSource["previewURL"]); previewURL != "" {
payload, err := contentBytesForArchiveMember(stringValue(archiveSource["name"]), previewURL, archiveMemberPath)
if err == nil {
return payload, mimeType, fileName, "", nil
}
}
}
if previewURL := stringValue(source["previewURL"]); previewURL != "" {
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
return nil, "", fileName, previewURL, nil
}
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
if err == nil {
if mimeType == "" {
mimeType = decodedMimeType
}
return payload, mimeType, fileName, "", nil
}
}
return nil, "", "", "", os.ErrNotExist
}
func (server *server) handleSourceCollectionsList(writer http.ResponseWriter, request *http.Request) {
workflowByID, latestRuns, err := server.latestCompletedScanRuns(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items := make([]domain.SourceCollectionRecord, 0)
for _, run := range latestRuns {
artifact := artifactByName(run, "dataset-index.json")
payload := artifactStructuredData(artifact)
if artifact == nil || payload == nil {
continue
}
for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") {
collection := mapValue(raw)
if collection == nil {
continue
}
workflow := workflowByID[run.WorkflowID]
items = append(items, server.buildSourceCollectionRecord(request.Context(), run, &workflow, collection))
}
}
sort.Slice(items, func(left int, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
if items[left].WorkflowName == items[right].WorkflowName {
return items[left].CollectionID < items[right].CollectionID
}
return items[left].WorkflowName < items[right].WorkflowName
}
return items[left].CreatedAt > items[right].CreatedAt
})
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleSourceCollectionDetail(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
detail := domain.SourceCollectionDetail{
Collection: record,
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, ""),
}
renderJSON(writer, http.StatusOK, detail)
}
func (server *server) handleSourceCollectionEntries(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
prefix := request.URL.Query().Get("prefix")
detail := domain.SourceCollectionDetail{
Collection: record,
Prefix: prefix,
Entries: buildSourceEntries(*payload.Run, record, payload.Entries, payload.Items, payload.ArchiveSource, prefix),
}
renderJSON(writer, http.StatusOK, detail)
}
func (server *server) handleSourceCollectionContent(writer http.ResponseWriter, request *http.Request) {
payload, err := server.sourceCollectionPayload(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "collectionId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if payload.Run == nil || payload.Collection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source collection not found"})
return
}
record := server.buildSourceCollectionRecord(request.Context(), *payload.Run, payload.Workflow, payload.Collection)
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
if relativePath == "" {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
return
}
var effectiveScanConfig *domain.ScanConfig
if payload.Workflow != nil {
effectiveScanConfig, err = server.resolveScanConfig(request.Context(), payload.Workflow.ScanConfig)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
content, mimeType, fileName, redirectURL, err := sourceCollectionContentPayload(record, effectiveScanConfig, payload.Entries, payload.Items, payload.ArchiveSource, relativePath)
if err != nil {
if os.IsNotExist(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Source file not found"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
Payload: content,
RedirectURL: redirectURL,
}
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] source browser preview stream failed: %v\n", err)
}
return
}
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
writeSourceContent(writer, request, fileName, mimeType, content)
}
func packageArtifactCandidatesByRelativePath(run domain.RunRecord, packageName string, relativePath string) []domain.ExecutionArtifact {
target := strings.Trim(filepath.ToSlash(relativePath), "/")
if target == "" {
return nil
}
items := make([]domain.ExecutionArtifact, 0)
for _, artifact := range run.Artifacts {
if packageArtifactResolvedRelativePath(artifact, packageName) == target {
items = append(items, artifact)
}
}
sort.SliceStable(items, func(left int, right int) bool {
return packageArtifactPriority(items[left]) > packageArtifactPriority(items[right])
})
return items
}
func workflowNodeHyperparamsMap(node domain.WorkflowNode) map[string]string {
values := make(map[string]string, len(node.Data.Hyperparams))
for _, item := range node.Data.Hyperparams {
values[item.Key] = item.Value
}
return values
}
func (server *server) packageArtifactScanConfig(ctx context.Context, workflow *domain.WorkflowRecord, artifact domain.ExecutionArtifact) (*domain.ScanConfig, error) {
node := workflowNodeByID(workflow, artifact.NodeID)
if node == nil {
return nil, nil
}
params := workflowNodeHyperparamsMap(*node)
dataSource := chooseDataSource(params["DATA_SOURCE"])
sourcePath := strings.TrimSpace(params["SOURCE_PATH"])
if dataSource != "BOS" && strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
dataSource = "BOS"
}
if dataSource != "BOS" && strings.TrimSpace(node.Data.ConnectionID) != "" {
dataSource = "BOS"
}
if dataSource != "BOS" {
return nil, nil
}
scanConfig := &domain.ScanConfig{
DataSource: "BOS",
SourcePath: sourcePath,
ConnectionID: strings.TrimSpace(node.Data.ConnectionID),
Bucket: strings.TrimSpace(node.Data.Bucket),
Prefix: cleanBOSPrefix(node.Data.Prefix),
Endpoint: strings.TrimSpace(params["BOS_ENDPOINT"]),
AccessKeyID: strings.TrimSpace(params["BOS_ACCESS_KEY_ID"]),
SecretAccessKey: strings.TrimSpace(params["BOS_SECRET_ACCESS_KEY"]),
}
return server.resolveScanConfig(ctx, scanConfig)
}
func (server *server) packageContentPayload(ctx context.Context, run domain.RunRecord, workflow *domain.WorkflowRecord, packageName string, relativePath string) ([]byte, string, string, string, string, error) {
artifacts := packageArtifactCandidatesByRelativePath(run, packageName, relativePath)
if len(artifacts) == 0 {
return nil, "", "", "", "", os.ErrNotExist
}
fileName := path.Base(strings.Trim(filepath.ToSlash(relativePath), "/"))
for _, artifact := range artifacts {
mimeType := ""
if artifact.Metadata != nil {
mimeType = strings.TrimSpace(artifact.Metadata.MimeType)
}
if mimeType == "" {
mimeType = mime.TypeByExtension(strings.ToLower(path.Ext(fileName)))
}
filePath := strings.TrimSpace(artifact.Path)
if filePath != "" && !strings.Contains(filePath, "://") {
if _, err := os.Stat(filePath); err == nil {
return nil, mimeType, fileName, filePath, "", nil
}
}
structuredData := artifactStructuredData(&artifact)
if artifact.Type == "json" && structuredData != nil {
payload, err := json.MarshalIndent(normalizeStructuredValue(structuredData), "", " ")
if err != nil {
return nil, "", "", "", "", err
}
if mimeType == "" {
mimeType = "application/json"
}
return payload, mimeType, fileName, "", "", nil
}
source := mapValue(structuredData["source"])
sourcePath := stringValue(source["path"])
if strings.HasPrefix(strings.ToLower(sourcePath), "bos://") {
scanConfig, err := server.packageArtifactScanConfig(ctx, workflow, artifact)
if err != nil {
return nil, "", "", "", "", err
}
if previewURL, err := sourceCollectionSignedPreviewURL(scanConfig, sourcePath); err == nil && previewURL != "" {
return nil, mimeType, fileName, "", previewURL, nil
}
}
if previewURL := packageArtifactPreviewURL(artifact); previewURL != "" {
if strings.HasPrefix(previewURL, "http://") || strings.HasPrefix(previewURL, "https://") {
return nil, mimeType, fileName, "", previewURL, nil
}
payload, decodedMimeType, err := contentBytesForDataURL(previewURL)
if err == nil {
if mimeType == "" {
mimeType = decodedMimeType
}
return payload, mimeType, fileName, "", "", nil
}
}
}
return nil, "", "", "", "", os.ErrNotExist
}
func (server *server) handlePackageContent(writer http.ResponseWriter, request *http.Request) {
run, err := server.store.GetRun(request.Context(), chi.URLParam(request, "runId"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if run == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
packageName := chi.URLParam(request, "packageName")
relativePath := strings.TrimSpace(request.URL.Query().Get("relativePath"))
if relativePath == "" {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "relativePath is required"})
return
}
workflow, err := server.store.GetWorkflow(request.Context(), run.WorkflowID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
content, mimeType, fileName, filePath, redirectURL, err := server.packageContentPayload(request.Context(), *run, workflow, packageName, relativePath)
if err != nil {
if os.IsNotExist(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package file not found"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
if strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("transcode")), "browser") && sourceEntryPayloadKind(relativePath, mimeType) == "video" {
previewRequest := sourceBrowserPreviewRequest{
FileName: fileName,
Payload: content,
}
if filePath != "" {
previewRequest.RedirectURL = filePath
} else {
previewRequest.RedirectURL = redirectURL
}
if previewPath, err := sourceBrowserVideoPreviewPath(request.Context(), previewRequest); err == nil && previewPath != "" {
if err := writeSourceContentFile(writer, request, strings.TrimSuffix(fileName, path.Ext(fileName))+"-browser.mp4", "video/mp4", previewPath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
writer.Header().Set("Content-Type", "video/mp4")
writer.Header().Set("Cache-Control", "no-store")
writer.WriteHeader(http.StatusOK)
if flusher, ok := writer.(http.Flusher); ok {
flusher.Flush()
}
if err := sourceBrowserVideoPreviewWriter(request.Context(), writer, previewRequest); err != nil {
fmt.Printf("[api] package browser preview stream failed: %v\n", err)
}
return
}
if filePath != "" {
if err := writeSourceContentFile(writer, request, fileName, mimeType, filePath); err != nil {
renderError(writer, http.StatusInternalServerError, err)
}
return
}
if redirectURL != "" {
http.Redirect(writer, request, redirectURL, http.StatusTemporaryRedirect)
return
}
writeSourceContent(writer, request, fileName, mimeType, content)
}
func packageRecordFromManifest(run domain.RunRecord, item map[string]any, files []domain.ExecutionArtifact, rawFiles []domain.ExecutionArtifact) domain.PackageRecord {
pipelineTags := make([]string, 0)
for _, value := range listValue(item["pipelineTags"]) {
pipelineTags = append(pipelineTags, stringValue(value))
}
storagePath := stringValue(item["storagePath"])
for _, artifact := range rawFiles {
if candidate := packageArtifactStoragePath(artifact, stringValue(item["packageName"])); candidate != "" {
storagePath = candidate
break
}
}
fileCount := intValue(item["fileCount"])
if len(files) > 0 {
fileCount = len(files)
}
return domain.PackageRecord{
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
PackageName: stringValue(item["packageName"]),
ClipID: stringValue(item["clipId"]),
SourceVideo: stringValue(item["sourceVideo"]),
ClipDuration: floatValue(item["clipDuration"]),
FileCount: fileCount,
PipelineTags: pipelineTags,
StoragePath: storagePath,
Quality: mapValue(item["quality"]),
Annotation: mapValue(item["annotation"]),
CreatedAt: run.CreatedAt,
PreviewArtifact: choosePackagePreview(files),
Storage: assetStorageLocationFromPaths("", storagePath, ""),
}
}
func (server *server) packageDetail(ctx context.Context, runID string, packageName string) (*domain.PackageDetail, error) {
run, err := server.store.GetRun(ctx, runID)
if err != nil || run == nil {
return nil, err
}
manifest := artifactByName(*run, "packages-manifest.json")
manifestPayload := artifactStructuredData(manifest)
if manifest == nil || manifestPayload == nil {
return nil, nil
}
rawFiles := dedupePackageArtifacts(*run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
}
for _, raw := range listValue(manifestPayload["items"]) {
item := mapValue(raw)
if stringValue(item["packageName"]) != packageName {
continue
}
record := packageRecordFromManifest(*run, item, files, rawFiles)
return &domain.PackageDetail{Package: record, Files: files}, nil
}
return nil, nil
}
func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) {
runs, err := server.store.ListRuns(request.Context(), string(domain.RunStatusCompleted))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items := make([]domain.PackageRecord, 0)
latestByWorkflowPackage := make(map[string]domain.PackageRecord)
for _, run := range runs {
manifest := artifactByName(run, "packages-manifest.json")
manifestPayload := artifactStructuredData(manifest)
if manifest == nil || manifestPayload == nil {
continue
}
for _, raw := range listValue(manifestPayload["items"]) {
item := mapValue(raw)
if item == nil {
continue
}
packageName := stringValue(item["packageName"])
rawFiles := dedupePackageArtifacts(run, packageName)
files := make([]domain.ExecutionArtifact, 0, len(rawFiles))
for _, artifact := range rawFiles {
files = append(files, packageArtifactForResponse(run.ID, packageName, artifact))
}
record := packageRecordFromManifest(run, item, files, rawFiles)
key := run.WorkflowID + ":" + record.PackageName
current, exists := latestByWorkflowPackage[key]
if !exists || record.CreatedAt > current.CreatedAt {
latestByWorkflowPackage[key] = record
}
}
}
for _, item := range latestByWorkflowPackage {
items = append(items, item)
}
sort.Slice(items, func(left int, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
return items[left].PackageName < items[right].PackageName
}
return items[left].CreatedAt > items[right].CreatedAt
})
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handlePackageDetail(writer http.ResponseWriter, request *http.Request) {
item, err := server.packageDetail(request.Context(), chi.URLParam(request, "runId"), chi.URLParam(request, "packageName"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Package not found"})
return
}
renderJSON(writer, http.StatusOK, item)
}
func samplingRulesFromManifest(manifest map[string]any) []domain.SamplingRuleSummary {
rules := make([]domain.SamplingRuleSummary, 0)
for _, raw := range listValue(manifest["rules"]) {
item := mapValue(raw)
if item == nil {
continue
}
rules = append(rules, domain.SamplingRuleSummary{
ID: stringValue(item["id"]),
Field: stringValue(item["field"]),
Operator: stringValue(item["operator"]),
Value: item["value"],
SamplePercent: floatValue(item["samplePercent"]),
MatchedCount: intValue(item["matchedCount"]),
SampledCount: intValue(item["sampledCount"]),
})
}
return rules
}
func buildSamplingItems(run domain.RunRecord, manifest map[string]any) []domain.SamplingItemRecord {
items := make([]domain.SamplingItemRecord, 0)
previewByPackage := make(map[string]*domain.ExecutionArtifact)
for _, raw := range listValue(manifest["items"]) {
item := mapValue(raw)
packageName := stringValue(item["packageName"])
if _, ok := previewByPackage[packageName]; !ok {
previewByPackage[packageName] = choosePackagePreview(packageArtifacts(run, packageName))
}
preview := previewByPackage[packageName]
items = append(items, domain.SamplingItemRecord{
ID: domain.NewID("sampling-item"),
PackageName: packageName,
ClipID: stringValue(item["clipId"]),
SourceVideo: stringValue(item["sourceVideo"]),
SampleRuleID: stringValue(item["sampleRuleId"]),
StoragePath: stringValue(item["storagePath"]),
Quality: mapValue(item["quality"]),
Annotation: mapValue(item["annotation"]),
PreviewArtifact: preview,
Status: domain.SamplingItemStatusPending,
})
}
return items
}
func (server *server) syncSamplingTasks(ctx context.Context) error {
existing, err := server.store.ListSamplingTasks(ctx, "")
if err != nil {
return err
}
existingByRun := make(map[string]domain.SamplingTaskRecord, len(existing))
for _, task := range existing {
existingByRun[task.RunID] = task
}
runs, err := server.store.ListRuns(ctx, string(domain.RunStatusCompleted))
if err != nil {
return err
}
for _, run := range runs {
if _, exists := existingByRun[run.ID]; exists {
continue
}
manifest := artifactByName(run, "audit-samples.json")
manifestPayload := artifactStructuredData(manifest)
if manifest == nil || manifestPayload == nil || len(listValue(manifestPayload["items"])) == 0 {
continue
}
now := nowISO()
items := buildSamplingItems(run, manifestPayload)
task := domain.SamplingTaskRecord{
ID: domain.NewID("sampling"),
RunID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
Name: fmt.Sprintf("%s 抽检任务", run.WorkflowName),
Description: fmt.Sprintf("来源于运行 %s 的真实抽检采样结果", run.ID),
Status: domain.SamplingTaskStatusPending,
Rules: samplingRulesFromManifest(manifestPayload),
Items: items,
CreatedAt: now,
UpdatedAt: now,
}
for index := range task.Items {
task.Items[index].TaskID = task.ID
}
if _, err := server.store.CreateSamplingTask(ctx, task); err != nil {
return err
}
}
return nil
}
func (server *server) handleSamplingTasksList(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items, err := server.store.ListSamplingTasks(request.Context(), request.URL.Query().Get("status"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleSamplingTaskGet(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item, err := server.store.GetSamplingTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling task not found"})
return
}
renderJSON(writer, http.StatusOK, item)
}
func validateSamplingReviewPayload(payload samplingReviewRequest) []validationIssue {
issues := make([]validationIssue, 0, 4)
if strings.TrimSpace(payload.VideoQuality) == "" {
issues = append(issues, validationIssue{Field: "videoQuality", Message: "videoQuality is required"})
}
if strings.TrimSpace(payload.AnnotationAccuracy) == "" {
issues = append(issues, validationIssue{Field: "annotationAccuracy", Message: "annotationAccuracy is required"})
}
if strings.TrimSpace(payload.AnnotationCompleteness) == "" {
issues = append(issues, validationIssue{Field: "annotationCompleteness", Message: "annotationCompleteness is required"})
}
if strings.TrimSpace(payload.OverallGrade) == "" {
issues = append(issues, validationIssue{Field: "overallGrade", Message: "overallGrade is required"})
}
return issues
}
func (server *server) handleSamplingItemReview(writer http.ResponseWriter, request *http.Request) {
if err := server.syncSamplingTasks(request.Context()); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
var payload samplingReviewRequest
if !decodeJSON(writer, request, &payload) {
return
}
if issues := validateSamplingReviewPayload(payload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
tasks, err := server.store.ListSamplingTasks(request.Context(), "")
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
itemID := chi.URLParam(request, "id")
for _, task := range tasks {
updated := false
for index := range task.Items {
if task.Items[index].ID != itemID {
continue
}
task.Items[index].Status = domain.SamplingItemStatusCompleted
task.Items[index].Review = &domain.SamplingReview{
VideoQuality: strings.TrimSpace(payload.VideoQuality),
AnnotationAccuracy: strings.TrimSpace(payload.AnnotationAccuracy),
AnnotationCompleteness: strings.TrimSpace(payload.AnnotationCompleteness),
OverallGrade: strings.TrimSpace(payload.OverallGrade),
Notes: strings.TrimSpace(payload.Notes),
ReviewedAt: nowISO(),
}
updated = true
break
}
if !updated {
continue
}
task.Status = domain.SamplingTaskStatusCompleted
for _, item := range task.Items {
if item.Status != domain.SamplingItemStatusCompleted {
task.Status = domain.SamplingTaskStatusPending
break
}
}
task.UpdatedAt = nowISO()
if err := server.store.UpsertSamplingTask(request.Context(), task); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, task)
return
}
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Sampling item not found"})
}