forked from longtao.wu/droboticflow
1443 lines
55 KiB
Go
1443 lines
55 KiB
Go
package api
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"os"
|
||
"path"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"unicode/utf8"
|
||
|
||
"droboticflow/internal/domain"
|
||
|
||
"github.com/go-chi/chi/v5"
|
||
)
|
||
|
||
// Manifest members are intentionally a projection of a top-level artifact, not
|
||
// additional run artifacts. A manifest can describe thousands of files, but
|
||
// those descriptions must never silently become lineage records or grant access
|
||
// to an arbitrary local path / pre-signed URL supplied by an operator payload.
|
||
const (
|
||
maxRunArtifactManifestMembers = 5000
|
||
maxRunArtifactInlineMemberBytes = 4 << 20
|
||
maxRunArtifactManifestBytes = 4 << 20
|
||
)
|
||
|
||
type runArtifactManifestMember struct {
|
||
ID string `json:"id"`
|
||
Name string `json:"name"`
|
||
Kind string `json:"kind"`
|
||
Group string `json:"group,omitempty"`
|
||
MimeType string `json:"mimeType,omitempty"`
|
||
SizeBytes int64 `json:"sizeBytes,omitempty"`
|
||
ContentAvailable bool `json:"contentAvailable"`
|
||
Details map[string]any `json:"details,omitempty"`
|
||
|
||
contentArtifact *domain.ExecutionArtifact `json:"-"`
|
||
inlineContent []byte `json:"-"`
|
||
bosObject *runArtifactMemberBOSLocation `json:"-"`
|
||
pinnedSource *annotationPinnedSource `json:"-"`
|
||
}
|
||
|
||
// runArtifactMemberBOSLocation lets directory-index members stream content
|
||
// straight from object storage on demand. The locator is resolved server-side
|
||
// from the stored connection and is never serialized to the public response.
|
||
type runArtifactMemberBOSLocation struct {
|
||
bucket string
|
||
key string
|
||
sizeBytes int64
|
||
connection *domain.StorageConnectionRecord
|
||
}
|
||
|
||
type runArtifactManifestDocument struct {
|
||
SchemaVersion string `json:"schemaVersion,omitempty"`
|
||
Recognized bool `json:"recognized"`
|
||
Message string `json:"message,omitempty"`
|
||
Summary map[string]any `json:"summary"`
|
||
Members []runArtifactManifestMember
|
||
}
|
||
|
||
func (server *server) handleRunArtifactMembers(writer http.ResponseWriter, request *http.Request) {
|
||
run, manifest, document, ok := server.loadAccessibleRunArtifactManifest(writer, request)
|
||
if !ok {
|
||
return
|
||
}
|
||
response, issues := pagedResponse(request, document.Members, 30, 200, false)
|
||
if len(issues) > 0 {
|
||
renderValidationError(writer, issues...)
|
||
return
|
||
}
|
||
response["runId"] = run.ID
|
||
response["artifactId"] = artifactResponseID(run.ID, manifest.NodeID, *manifest)
|
||
response["schemaVersion"] = document.SchemaVersion
|
||
response["recognized"] = document.Recognized
|
||
response["message"] = document.Message
|
||
response["summary"] = document.Summary
|
||
renderJSON(writer, http.StatusOK, response)
|
||
}
|
||
|
||
func (server *server) handleRunArtifactMemberContent(writer http.ResponseWriter, request *http.Request) {
|
||
run, _, document, ok := server.loadAccessibleRunArtifactManifest(writer, request)
|
||
if !ok {
|
||
return
|
||
}
|
||
server.serveRunArtifactManifestMemberContent(writer, request, run, document)
|
||
}
|
||
|
||
func (server *server) serveRunArtifactManifestMemberContent(writer http.ResponseWriter, request *http.Request, run *domain.RunRecord, document runArtifactManifestDocument) {
|
||
memberID := strings.TrimSpace(chi.URLParam(request, "memberId"))
|
||
if !validRunArtifactMemberID(memberID) {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Manifest member not found"})
|
||
return
|
||
}
|
||
for index := range document.Members {
|
||
member := &document.Members[index]
|
||
if member.ID != memberID {
|
||
continue
|
||
}
|
||
if member.pinnedSource != nil {
|
||
version, err := server.annotationFileVersion(request.Context(), run.OrganizationID, member.pinnedSource.Path)
|
||
if err != nil || version != member.pinnedSource.Version {
|
||
renderJSON(writer, http.StatusConflict, map[string]string{"error": "固定源文件版本不可用,已停止播放以避免时间错位"})
|
||
return
|
||
}
|
||
}
|
||
if len(member.inlineContent) > 0 {
|
||
writeSourceContent(writer, request, member.Name, member.MimeType, member.inlineContent)
|
||
return
|
||
}
|
||
if member.bosObject != nil {
|
||
server.serveRunArtifactMemberBOSObject(writer, request, *member.bosObject)
|
||
return
|
||
}
|
||
if member.contentArtifact != nil {
|
||
server.serveResolvedRunArtifactContent(writer, request, run, *member.contentArtifact)
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Manifest member content is not available"})
|
||
return
|
||
}
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Manifest member not found"})
|
||
}
|
||
|
||
func (server *server) loadAccessibleRunArtifactManifest(
|
||
writer http.ResponseWriter,
|
||
request *http.Request,
|
||
) (*domain.RunRecord, *domain.ExecutionArtifact, runArtifactManifestDocument, bool) {
|
||
run, err := server.loadRun(request.Context(), chi.URLParam(request, "id"))
|
||
if err != nil {
|
||
renderError(writer, http.StatusInternalServerError, err)
|
||
return nil, nil, runArtifactManifestDocument{}, false
|
||
}
|
||
if run == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
|
||
return nil, nil, runArtifactManifestDocument{}, false
|
||
}
|
||
if !server.ensureTenantAccess(writer, request, run.OrganizationID, "Run") ||
|
||
!server.ensureProjectVisibility(writer, request, run.OrganizationID, run.ProjectID) {
|
||
return nil, nil, runArtifactManifestDocument{}, false
|
||
}
|
||
manifest := runArtifactByID(run, chi.URLParam(request, "artifactId"))
|
||
if manifest == nil {
|
||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Artifact not found"})
|
||
return nil, nil, runArtifactManifestDocument{}, false
|
||
}
|
||
return run, manifest, server.runArtifactManifestDocumentWithContext(request.Context(), run, manifest), true
|
||
}
|
||
|
||
func validRunArtifactMemberID(value string) bool {
|
||
if len(value) != len("member-")+64 || !strings.HasPrefix(value, "member-") {
|
||
return false
|
||
}
|
||
for _, character := range value[len("member-"):] {
|
||
if !(character >= '0' && character <= '9') && !(character >= 'a' && character <= 'f') {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
func (server *server) runArtifactManifestDocument(run *domain.RunRecord, manifest *domain.ExecutionArtifact) runArtifactManifestDocument {
|
||
return server.runArtifactManifestDocumentWithContext(context.Background(), run, manifest)
|
||
}
|
||
|
||
func (server *server) runArtifactManifestDocumentWithContext(ctx context.Context, run *domain.RunRecord, manifest *domain.ExecutionArtifact) runArtifactManifestDocument {
|
||
if isSourceIndexArtifact(manifest) {
|
||
return server.sourceIndexManifestDocument(ctx, run, manifest)
|
||
}
|
||
transferMode, presentation := artifactResponsePresentation(*manifest)
|
||
document := runArtifactManifestDocument{
|
||
Summary: map[string]any{
|
||
"manifestName": strings.TrimSpace(manifest.Name),
|
||
"contentCount": 0,
|
||
"transferMode": string(transferMode),
|
||
"presentation": presentation,
|
||
},
|
||
Members: []runArtifactManifestMember{},
|
||
}
|
||
payload := server.runArtifactManifestPayload(ctx, run, manifest)
|
||
if payload == nil {
|
||
document.Message = "暂不支持预览,可下载受保护的引用内容。"
|
||
return document
|
||
}
|
||
if directoryIndex := server.directoryIndexDocument(ctx, run, manifest, payload); directoryIndex != nil {
|
||
return *directoryIndex
|
||
}
|
||
schemaVersion := strings.TrimSpace(stringValue(payload["schemaVersion"]))
|
||
document.SchemaVersion = safeRunArtifactMemberText(schemaVersion, 160)
|
||
var members []runArtifactManifestMember
|
||
switch schemaVersion {
|
||
case "droboticflow.directory-read/v1":
|
||
members = server.directoryManifestMembers(run, manifest, schemaVersion, payload)
|
||
case "droboticflow.orbbec.normalized/v1":
|
||
members = server.orbbecNormalizedManifestMembers(run, manifest, schemaVersion, payload)
|
||
case "droboticflow.video_face_blur.manifest/v1", "droboticflow.video_face_blur.manifest/v2":
|
||
members = server.blurredManifestMembers(run, manifest, schemaVersion, payload)
|
||
case "droboticflow.qwen3vl.behavior-rough-cut/v1":
|
||
members = server.roughCutManifestMembers(run, manifest, schemaVersion, payload)
|
||
case "droboticflow.annotation.accepted-partition/v1":
|
||
members = server.annotationManifestMembers(run, manifest, schemaVersion, payload)
|
||
case "droboticflow.delivery.manifest/v2", "droboticflow.delivery.manifest/v3", "droboticflow.delivery.manifest/v4", "droboticflow.delivery.manifest/v5":
|
||
members = server.packageManifestMembers(run, manifest, schemaVersion, payload)
|
||
case "droboticflow.delivery.part-index/v1", "droboticflow.delivery.part-index/v2":
|
||
return server.partIndexManifestDocument(run, manifest, document, payload)
|
||
default:
|
||
document.Message = "暂不支持预览,可下载受保护的引用内容。"
|
||
return document
|
||
}
|
||
document.Recognized = true
|
||
if len(members) > maxRunArtifactManifestMembers {
|
||
members = members[:maxRunArtifactManifestMembers]
|
||
document.Summary["memberLimitReached"] = true
|
||
}
|
||
document.Members = members
|
||
document.Summary["contentCount"] = len(members)
|
||
document.Summary["contentAvailableCount"] = countRunArtifactManifestMembersWithContent(members)
|
||
document.Summary["groups"] = runArtifactManifestMemberGroups(members)
|
||
return document
|
||
}
|
||
|
||
// runArtifactManifestPayload reads a new object-ref manifest server-side. The
|
||
// public response never contains its bucket/key/connection or raw JSON, while
|
||
// historical inline records remain readable through their stored Data payload.
|
||
func (server *server) runArtifactManifestPayload(ctx context.Context, run *domain.RunRecord, manifest *domain.ExecutionArtifact) map[string]any {
|
||
if run == nil || manifest == nil {
|
||
return nil
|
||
}
|
||
if manifest.Payload != nil && manifest.Payload.Mode == domain.ArtifactTransferModeRef && manifest.Payload.Ref != nil && manifest.Payload.Ref.Kind == domain.ArtifactRefKindObject {
|
||
if bucket, key, ok := artifactBOSLocation(*manifest); ok {
|
||
connection, err := server.resolveArtifactStorageConnection(ctx, run, *manifest, bucket)
|
||
if err == nil && connection != nil {
|
||
connection = server.storageConnectionWithResolvedEndpoint(connection)
|
||
browser, cleanup, browserErr := storageConnectionBrowserResolver(ctx, connection)
|
||
if browserErr == nil && browser != nil {
|
||
if cleanup != nil {
|
||
defer cleanup()
|
||
}
|
||
if summary, summaryErr := browser.GetObjectSummary(ctx, bucket, key); summaryErr == nil && (summary.SizeBytes <= 0 || summary.SizeBytes <= maxRunArtifactManifestBytes) {
|
||
if raw, _, readErr := browser.ReadObject(ctx, bucket, key); readErr == nil && len(raw) <= maxRunArtifactManifestBytes {
|
||
var decoded any
|
||
if json.Unmarshal(raw, &decoded) == nil {
|
||
if structured, ok := normalizeStructuredValue(decoded).(map[string]any); ok {
|
||
return structured
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return artifactStructuredData(manifest)
|
||
}
|
||
|
||
func countRunArtifactManifestMembersWithContent(members []runArtifactManifestMember) int {
|
||
count := 0
|
||
for _, member := range members {
|
||
if member.ContentAvailable {
|
||
count++
|
||
}
|
||
}
|
||
return count
|
||
}
|
||
|
||
func runArtifactManifestMemberGroups(members []runArtifactManifestMember) []map[string]any {
|
||
counts := make(map[string]int)
|
||
for _, member := range members {
|
||
group := strings.TrimSpace(member.Group)
|
||
if group == "" {
|
||
group = "内容"
|
||
}
|
||
counts[group]++
|
||
}
|
||
keys := make([]string, 0, len(counts))
|
||
for key := range counts {
|
||
keys = append(keys, key)
|
||
}
|
||
sort.Strings(keys)
|
||
groups := make([]map[string]any, 0, len(keys))
|
||
for _, key := range keys {
|
||
groups = append(groups, map[string]any{"name": key, "count": counts[key]})
|
||
}
|
||
return groups
|
||
}
|
||
|
||
func (server *server) directoryManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
|
||
entries := listValue(payload["entries"])
|
||
members := make([]runArtifactManifestMember, 0, len(entries))
|
||
for index, raw := range entries {
|
||
entry := mapValue(raw)
|
||
if entry == nil {
|
||
continue
|
||
}
|
||
name := firstNonEmptyText(stringValue(entry["path"]), stringValue(entry["relativePath"]), fmt.Sprintf("entry-%d", index+1))
|
||
candidate := manifestMemberCandidate(entry)
|
||
member := server.newRunArtifactManifestMember(run, manifest, schema, "directory", "目录文件", name, name, candidate, stringValue(entry["mimeType"]), int64Value(entry["sizeBytes"]), map[string]any{
|
||
"lastModified": safeRunArtifactMemberText(stringValue(entry["lastModified"]), 96),
|
||
})
|
||
if inlineText := stringValue(entry["inlineText"]); inlineText != "" && len(inlineText) <= maxRunArtifactInlineMemberBytes {
|
||
member.inlineContent = []byte(inlineText)
|
||
member.ContentAvailable = true
|
||
}
|
||
members = append(members, member)
|
||
}
|
||
return members
|
||
}
|
||
|
||
// ---- 目录索引产物(droboticflow.directory-index/v1)----
|
||
//
|
||
// directory.index 类产物是"目录引用"而非普通文件。成员 API 在此分支里按引用
|
||
// 实时只读列举对象存储(或工作区内已存储产物),让前端以文件管理器方式浏览,
|
||
// 且绝不内联目录全量清单、绝不开任意本地路径的内容读取。
|
||
|
||
func isDirectoryIndexManifest(manifest *domain.ExecutionArtifact, payload map[string]any) bool {
|
||
if manifest == nil || payload == nil {
|
||
return false
|
||
}
|
||
schemaVersion := strings.TrimSpace(stringValue(payload["schemaVersion"]))
|
||
datasetType := strings.TrimSpace(stringValue(payload["datasetType"]))
|
||
presentation := strings.TrimSpace(stringValue(payload["presentation"]))
|
||
if schemaVersion == "droboticflow.directory-index/v1" {
|
||
return true
|
||
}
|
||
if datasetType == "directory" && presentation == "collection" && schemaVersion != "droboticflow.directory-read/v1" {
|
||
return true
|
||
}
|
||
return manifest.Payload != nil && manifest.Payload.Ref != nil &&
|
||
manifest.Payload.Ref.Kind == domain.ArtifactRefKindDirectory && datasetType == "directory"
|
||
}
|
||
|
||
func (server *server) directoryIndexDocument(
|
||
ctx context.Context,
|
||
run *domain.RunRecord,
|
||
manifest *domain.ExecutionArtifact,
|
||
payload map[string]any,
|
||
) *runArtifactManifestDocument {
|
||
if !isDirectoryIndexManifest(manifest, payload) {
|
||
return nil
|
||
}
|
||
transferMode, presentation := artifactResponsePresentation(*manifest)
|
||
document := &runArtifactManifestDocument{
|
||
SchemaVersion: "droboticflow.directory-index/v1",
|
||
Recognized: true,
|
||
Summary: map[string]any{
|
||
"manifestName": strings.TrimSpace(manifest.Name),
|
||
"contentCount": 0,
|
||
"transferMode": string(transferMode),
|
||
"presentation": presentation,
|
||
},
|
||
Members: []runArtifactManifestMember{},
|
||
}
|
||
ref := mapValue(payload["ref"])
|
||
bucket := strings.TrimSpace(stringValue(ref["bucket"]))
|
||
prefix := strings.Trim(strings.TrimSpace(stringValue(ref["prefix"])), "/")
|
||
if bucket == "" && manifest.Payload != nil && manifest.Payload.Ref != nil &&
|
||
manifest.Payload.Ref.Kind == domain.ArtifactRefKindDirectory {
|
||
bucket = strings.TrimSpace(manifest.Payload.Ref.Bucket)
|
||
prefix = strings.Trim(strings.TrimSpace(manifest.Payload.Ref.Prefix), "/")
|
||
}
|
||
rootPath := strings.TrimSpace(stringValue(payload["rootPath"]))
|
||
var members []runArtifactManifestMember
|
||
switch {
|
||
case bucket != "":
|
||
listed, truncated, listErr := server.directoryIndexBOSMembers(ctx, run, manifest, bucket, prefix)
|
||
if listErr != nil {
|
||
document.Message = "目录内容实时列举失败:" + safeRunArtifactMemberText(listErr.Error(), 300)
|
||
return document
|
||
}
|
||
members = listed
|
||
if truncated {
|
||
document.Summary["memberLimitReached"] = true
|
||
}
|
||
case rootPath != "":
|
||
listed, listErr := server.directoryIndexFilesystemMembers(ctx, run, manifest, rootPath)
|
||
if listErr != nil {
|
||
document.Message = "目录内容实时列举失败,请重试"
|
||
return document
|
||
}
|
||
members = listed
|
||
default:
|
||
document.Message = "目录引用缺少可列举的存储位置,仅元数据可见。"
|
||
return document
|
||
}
|
||
if len(members) > maxRunArtifactManifestMembers {
|
||
members = members[:maxRunArtifactManifestMembers]
|
||
document.Summary["memberLimitReached"] = true
|
||
}
|
||
document.Members = members
|
||
document.Summary["contentCount"] = len(members)
|
||
document.Summary["contentAvailableCount"] = countRunArtifactManifestMembersWithContent(members)
|
||
document.Summary["groups"] = runArtifactManifestMemberGroups(members)
|
||
document.Summary["dataSource"] = "BOS"
|
||
if bucket == "" {
|
||
document.Summary["dataSource"] = "FILESYSTEM"
|
||
}
|
||
return document
|
||
}
|
||
|
||
func (server *server) directoryIndexBOSMembers(
|
||
ctx context.Context,
|
||
run *domain.RunRecord,
|
||
manifest *domain.ExecutionArtifact,
|
||
bucket string,
|
||
prefix string,
|
||
) ([]runArtifactManifestMember, bool, error) {
|
||
connection, err := server.resolveArtifactStorageConnection(ctx, run, *manifest, bucket)
|
||
if err != nil {
|
||
return nil, false, err
|
||
}
|
||
if connection == nil {
|
||
return nil, false, fmt.Errorf("未找到 bucket %s 对应的存储连接", bucket)
|
||
}
|
||
connection = server.storageConnectionWithResolvedEndpoint(connection)
|
||
browser, cleanup, browserErr := storageConnectionBrowserResolver(ctx, connection)
|
||
if browserErr != nil {
|
||
return nil, false, browserErr
|
||
}
|
||
if cleanup != nil {
|
||
defer cleanup()
|
||
}
|
||
members := make([]runArtifactManifestMember, 0, 64)
|
||
truncated := false
|
||
appendItems := func(items []domain.ObjectStorageObjectSummary) bool {
|
||
for _, item := range items {
|
||
key := strings.Trim(strings.TrimSpace(item.Key), "/")
|
||
if key == "" || key == prefix {
|
||
continue
|
||
}
|
||
if len(members) >= maxRunArtifactManifestMembers {
|
||
truncated = true
|
||
return false
|
||
}
|
||
relative := key
|
||
if prefix != "" && strings.HasPrefix(key, prefix+"/") {
|
||
relative = key[len(prefix)+1:]
|
||
}
|
||
name := relative
|
||
group := ""
|
||
if idx := strings.LastIndex(relative, "/"); idx >= 0 {
|
||
name = relative[idx+1:]
|
||
group = relative[:idx]
|
||
}
|
||
if name == "" {
|
||
continue
|
||
}
|
||
if group == "" {
|
||
group = "根目录"
|
||
}
|
||
member := server.newRunArtifactManifestMember(
|
||
run,
|
||
manifest,
|
||
"droboticflow.directory-index/v1",
|
||
manifestMemberKind(name, item.ContentType),
|
||
group,
|
||
name,
|
||
key,
|
||
"",
|
||
item.ContentType,
|
||
item.SizeBytes,
|
||
map[string]any{"lastModified": safeRunArtifactMemberText(item.LastModified, 96)},
|
||
)
|
||
member.bosObject = &runArtifactMemberBOSLocation{bucket: bucket, key: key, sizeBytes: item.SizeBytes, connection: connection}
|
||
member.ContentAvailable = true
|
||
members = append(members, member)
|
||
}
|
||
return true
|
||
}
|
||
if pageBrowser, ok := browser.(pagedObjectStorageBrowser); ok {
|
||
marker := ""
|
||
for {
|
||
page, pageErr := pageBrowser.ListObjectsPage(ctx, bucket, prefix, marker, 1000)
|
||
if pageErr != nil {
|
||
return nil, false, pageErr
|
||
}
|
||
if !appendItems(page.Items) {
|
||
break
|
||
}
|
||
if !page.HasMore || page.NextMarker == "" || len(members) > maxRunArtifactManifestMembers {
|
||
break
|
||
}
|
||
marker = page.NextMarker
|
||
}
|
||
} else {
|
||
items, listErr := browser.ListObjects(ctx, bucket, prefix)
|
||
if listErr != nil {
|
||
return nil, false, listErr
|
||
}
|
||
appendItems(items)
|
||
}
|
||
return members, truncated, nil
|
||
}
|
||
|
||
// directoryIndexFilesystemMembers 只列举元数据;内容读取仍走既有安全边界——
|
||
// 仅当候选路径命中本 run 已持久化的产物时才允许回源。
|
||
func (server *server) directoryIndexFilesystemMembers(
|
||
ctx context.Context,
|
||
run *domain.RunRecord,
|
||
manifest *domain.ExecutionArtifact,
|
||
rootPath string,
|
||
) ([]runArtifactManifestMember, error) {
|
||
root := filepath.Clean(rootPath)
|
||
contentRun := run
|
||
if run != nil && manifest != nil && strings.TrimSpace(manifest.RunID) != "" && manifest.RunID != run.ID && manifest.RunID == run.ParentRunID {
|
||
copy := *run
|
||
copy.ID = manifest.RunID
|
||
contentRun = ©
|
||
}
|
||
members := make([]runArtifactManifestMember, 0, 64)
|
||
walkErr := filepath.WalkDir(root, func(current string, entry os.DirEntry, walkErr error) error {
|
||
if ctx.Err() != nil {
|
||
return ctx.Err()
|
||
}
|
||
if walkErr != nil {
|
||
return walkErr
|
||
}
|
||
if current == root {
|
||
return nil
|
||
}
|
||
if len(members) >= maxRunArtifactManifestMembers {
|
||
return filepath.SkipAll
|
||
}
|
||
if entry.IsDir() {
|
||
return nil
|
||
}
|
||
info, infoErr := entry.Info()
|
||
if infoErr != nil {
|
||
return infoErr
|
||
}
|
||
relative, relativeErr := filepath.Rel(root, current)
|
||
if relativeErr != nil {
|
||
return relativeErr
|
||
}
|
||
relative = filepath.ToSlash(relative)
|
||
name := path.Base(relative)
|
||
group := ""
|
||
if idx := strings.LastIndex(relative, "/"); idx >= 0 {
|
||
group = relative[:idx]
|
||
}
|
||
if group == "" {
|
||
group = "根目录"
|
||
}
|
||
member := server.newRunArtifactManifestMember(
|
||
run,
|
||
manifest,
|
||
"droboticflow.directory-index/v1",
|
||
manifestMemberKind(name, ""),
|
||
group,
|
||
name,
|
||
relative,
|
||
current,
|
||
"",
|
||
info.Size(),
|
||
nil,
|
||
)
|
||
if contentRun != nil {
|
||
if contentArtifact := server.safeRunArtifactManifestContentArtifact(contentRun, current, name, "", info.Size()); contentArtifact != nil {
|
||
member.contentArtifact = contentArtifact
|
||
member.ContentAvailable = true
|
||
}
|
||
}
|
||
members = append(members, member)
|
||
return nil
|
||
})
|
||
return members, walkErr
|
||
}
|
||
|
||
func (server *server) serveRunArtifactMemberBOSObject(
|
||
writer http.ResponseWriter,
|
||
request *http.Request,
|
||
location runArtifactMemberBOSLocation,
|
||
) {
|
||
connection := server.storageConnectionWithResolvedEndpoint(location.connection)
|
||
browser, cleanup, err := storageConnectionBrowserResolver(request.Context(), connection)
|
||
if err != nil || browser == nil {
|
||
renderError(writer, http.StatusInternalServerError, fmt.Errorf("目录成员内容服务不可用"))
|
||
return
|
||
}
|
||
if cleanup != nil {
|
||
defer cleanup()
|
||
}
|
||
summary, summaryErr := browser.GetObjectSummary(request.Context(), location.bucket, location.key)
|
||
if summaryErr != nil {
|
||
renderObjectStorageWriteError(writer, summaryErr)
|
||
return
|
||
}
|
||
requestedRange, hasRange, rangeErr := parseObjectStorageByteRange(request.Header.Get("Range"), summary.SizeBytes)
|
||
if rangeErr != nil {
|
||
if summary.SizeBytes > 0 {
|
||
writer.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", summary.SizeBytes))
|
||
}
|
||
writer.Header().Set("Accept-Ranges", "bytes")
|
||
renderError(writer, http.StatusRequestedRangeNotSatisfiable, rangeErr)
|
||
return
|
||
}
|
||
contentType := strings.TrimSpace(summary.ContentType)
|
||
if contentType == "" {
|
||
contentType = "application/octet-stream"
|
||
}
|
||
statusCode := http.StatusOK
|
||
if hasRange {
|
||
statusCode = http.StatusPartialContent
|
||
} else if summary.SizeBytes > 0 {
|
||
requestedRange = objectStorageByteRange{start: 0, end: summary.SizeBytes - 1}
|
||
}
|
||
writeObjectStorageRangeHeaders(writer, summary, contentType, requestedRange, statusCode)
|
||
writer.WriteHeader(statusCode)
|
||
reader, _, readErr := browser.ReadObjectRange(request.Context(), location.bucket, location.key, requestedRange.start, requestedRange.end)
|
||
if readErr != nil {
|
||
renderObjectStorageWriteError(writer, readErr)
|
||
return
|
||
}
|
||
defer reader.Close()
|
||
if _, copyErr := io.Copy(writer, reader); copyErr != nil {
|
||
fmt.Printf("[api] directory index member stream failed: %v\n", copyErr)
|
||
}
|
||
}
|
||
|
||
func (server *server) orbbecNormalizedManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
|
||
return server.videoManifestMembers(run, manifest, schema, payload, "normalized")
|
||
}
|
||
|
||
func (server *server) blurredManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
|
||
return server.videoManifestMembers(run, manifest, schema, payload, "blurred")
|
||
}
|
||
|
||
func (server *server) videoManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any, mode string) []runArtifactManifestMember {
|
||
items := listValue(payload["items"])
|
||
members := make([]runArtifactManifestMember, 0, len(items)*2)
|
||
for index, raw := range items {
|
||
item := mapValue(raw)
|
||
if item == nil {
|
||
continue
|
||
}
|
||
name := firstNonEmptyText(stringValue(item["filename"]), stringValue(item["name"]), fmt.Sprintf("video-%d.mp4", index+1))
|
||
candidate := manifestMemberCandidate(item)
|
||
group := firstNonEmptyText(stringValue(item["collectionName"]), stringValue(item["collectionId"]), "视频")
|
||
details := map[string]any{
|
||
"cameraSide": safeRunArtifactMemberText(stringValue(item["cameraSide"]), 32),
|
||
"collection": safeRunArtifactMemberText(group, 160),
|
||
}
|
||
if metrics := mapValue(item["faceBlurMetrics"]); metrics != nil {
|
||
details["framesProcessed"] = int64Value(metrics["framesProcessed"])
|
||
details["facesDetectedCount"] = int64Value(metrics["facesDetectedCount"])
|
||
details["durationSeconds"] = floatValue(metrics["duration"])
|
||
}
|
||
member := server.newRunArtifactManifestMember(run, manifest, schema, mode+"-video", group, name, firstNonEmptyText(stringValue(item["relativePath"]), name), candidate, firstNonEmptyText(stringValue(item["mimeType"]), "video/mp4"), int64Value(item["sizeBytes"]), details)
|
||
members = append(members, member)
|
||
|
||
pairedFiles := mapValue(item["pairedFiles"])
|
||
ptsName := safeManifestMemberLeafName(stringValue(pairedFiles["pts"]))
|
||
if ptsName != "" {
|
||
ptsCandidate, ok := safeManifestMemberSiblingCandidate(candidate, ptsName)
|
||
if ok {
|
||
members = append(members, server.newRunArtifactManifestMember(run, manifest, schema, mode+"-pts", group, ptsName, firstNonEmptyText(stringValue(item["relativePath"]), name)+"#pts", ptsCandidate, "text/csv", 0, map[string]any{
|
||
"pairedWith": safeRunArtifactMemberText(name, 240),
|
||
"kind": "PTS",
|
||
}))
|
||
}
|
||
}
|
||
}
|
||
return members
|
||
}
|
||
|
||
func (server *server) roughCutManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
|
||
items := listValue(payload["items"])
|
||
members := make([]runArtifactManifestMember, 0, len(items))
|
||
for index, raw := range items {
|
||
item := mapValue(raw)
|
||
if item == nil {
|
||
continue
|
||
}
|
||
clipID := firstNonEmptyText(stringValue(item["clipId"]), fmt.Sprintf("rough-cut-%d", index+1))
|
||
name := firstNonEmptyText(stringValue(item["clipName"]), clipID+".mp4")
|
||
candidate := manifestMemberCandidate(mapValue(item["source"]))
|
||
if candidate == "" {
|
||
candidate = manifestMemberCandidate(item)
|
||
}
|
||
details := map[string]any{
|
||
"clipId": safeRunArtifactMemberText(clipID, 300),
|
||
"startSecond": floatValue(item["startSecond"]),
|
||
"endSecond": floatValue(item["endSecond"]),
|
||
"duration": floatValue(item["duration"]),
|
||
"behavior": safeRunArtifactMemberText(stringValue(item["behavior"]), 512),
|
||
"objectChange": safeRunArtifactMemberText(stringValue(item["objectChange"]), 512),
|
||
"labels": manifestMemberActionLabels(item),
|
||
}
|
||
if rawPTS := mapValue(item["rawPTS"]); rawPTS != nil {
|
||
details["ptsStart"] = safeRunArtifactMemberText(stringValue(rawPTS["startTimestamp"]), 96)
|
||
details["ptsEnd"] = safeRunArtifactMemberText(stringValue(rawPTS["endTimestamp"]), 96)
|
||
}
|
||
member := server.newRunArtifactManifestMember(run, manifest, schema, "rough-cut-video", "粗切片段", name, clipID, candidate, firstNonEmptyText(stringValue(mapValue(item["source"])["mimeType"]), "video/mp4"), int64Value(mapValue(item["ref"])["sizeBytes"]), details)
|
||
if provenance := mapValue(item["provenance"]); provenance != nil {
|
||
if sourceID := strings.TrimSpace(stringValue(provenance["sourceVideoArtifactId"])); sourceID != "" {
|
||
if sourceArtifact := runArtifactByID(run, sourceID); sourceArtifact != nil {
|
||
member.contentArtifact = cloneRunArtifactMemberContentArtifact(*sourceArtifact, name, "video/mp4")
|
||
member.ContentAvailable = true
|
||
}
|
||
}
|
||
}
|
||
members = append(members, member)
|
||
}
|
||
return members
|
||
}
|
||
|
||
func (server *server) annotationManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
|
||
items := listValue(payload["items"])
|
||
members := make([]runArtifactManifestMember, 0, len(items))
|
||
for index, raw := range items {
|
||
item := mapValue(raw)
|
||
if item == nil {
|
||
continue
|
||
}
|
||
annotation := mapValue(item["annotation"])
|
||
clipID := firstNonEmptyText(stringValue(item["clipId"]), stringValue(annotation["clipId"]), fmt.Sprintf("annotation-%d", index+1))
|
||
name := firstNonEmptyText(stringValue(annotation["sourceVideo"]), stringValue(item["sourceVideo"]), clipID+".json")
|
||
member := server.newRunArtifactManifestMember(run, manifest, schema, "annotation", "人工标注", name, clipID, "", "application/json", 0, map[string]any{
|
||
"clipId": safeRunArtifactMemberText(clipID, 300),
|
||
"duration": floatValue(annotation["duration"]),
|
||
"revision": int64Value(annotation["revision"]),
|
||
"labels": manifestMemberActionLabels(annotation),
|
||
})
|
||
if sourceArtifact := runArtifactForManifestClip(run, clipID); sourceArtifact != nil {
|
||
member.contentArtifact = cloneRunArtifactMemberContentArtifact(*sourceArtifact, name, "video/mp4")
|
||
member.Kind = "video"
|
||
member.MimeType = "video/mp4"
|
||
member.ContentAvailable = true
|
||
}
|
||
members = append(members, member)
|
||
}
|
||
return members
|
||
}
|
||
|
||
// partIndexManifestDocument presents a single-file part delivery. The index is
|
||
// the deliverable itself, so the listing exposes the JSON plus the split
|
||
// classification instead of per-slice media members.
|
||
func (server *server) partIndexManifestDocument(
|
||
run *domain.RunRecord,
|
||
manifest *domain.ExecutionArtifact,
|
||
document runArtifactManifestDocument,
|
||
payload map[string]any,
|
||
) runArtifactManifestDocument {
|
||
document.Recognized = true
|
||
document.Members = []runArtifactManifestMember{}
|
||
classify := func(entries []any, disposition string) []map[string]any {
|
||
classified := make([]map[string]any, 0, len(entries))
|
||
for index, entry := range entries {
|
||
slice := mapValue(entry)
|
||
if slice == nil {
|
||
continue
|
||
}
|
||
split := mapValue(slice["taskSplit"])
|
||
classification := mapValue(slice["classification"])
|
||
segments := listValue(slice["segments"])
|
||
detail := map[string]any{
|
||
"sliceId": safeRunArtifactMemberText(firstNonEmptyText(stringValue(slice["sliceId"]), fmt.Sprintf("slice-%06d", index+1)), 300),
|
||
"sliceName": safeRunArtifactMemberText(stringValue(slice["sliceName"]), 300),
|
||
"deliveryDisposition": firstNonEmptyText(stringValue(classification["deliveryDisposition"]), disposition),
|
||
"descriptionStatus": safeRunArtifactMemberText(stringValue(classification["descriptionStatus"]), 80),
|
||
"segmentCount": len(segments),
|
||
"startTimestampUs": int64Value(split["startTimestampUs"]),
|
||
"endTimestampUs": int64Value(split["endTimestampUs"]),
|
||
"startSecond": floatValue(split["startSecond"]),
|
||
"endSecond": floatValue(split["endSecond"]),
|
||
}
|
||
if evidence := mapValue(classification["discardEvidence"]); evidence != nil {
|
||
detail["discardReason"] = safeRunArtifactMemberText(stringValue(evidence["reason"]), 1000)
|
||
}
|
||
classified = append(classified, detail)
|
||
}
|
||
return classified
|
||
}
|
||
slices := classify(listValue(payload["slices"]), "keep")
|
||
discarded := classify(listValue(payload["discardedSlices"]), "discard")
|
||
counts := mapValue(payload["deliveryClassification"])
|
||
document.Summary["datasetType"] = safeRunArtifactMemberText(stringValue(payload["datasetType"]), 160)
|
||
document.Summary["keptSliceCount"] = len(slices)
|
||
document.Summary["discardedSliceCount"] = len(discarded)
|
||
document.Summary["keptDurationSeconds"] = floatValue(counts["keptDurationSeconds"])
|
||
document.Summary["discardedDurationSeconds"] = floatValue(counts["discardedDurationSeconds"])
|
||
document.Summary["slices"] = slices
|
||
if len(discarded) > 0 {
|
||
document.Summary["discardedSlices"] = discarded
|
||
}
|
||
if coverage := mapValue(payload["coverage"]); coverage != nil {
|
||
document.Summary["coverage"] = map[string]any{
|
||
"mode": safeRunArtifactMemberText(stringValue(coverage["mode"]), 120),
|
||
"sliceCount": int64Value(coverage["sliceCount"]),
|
||
"durationSeconds": floatValue(coverage["durationSeconds"]),
|
||
}
|
||
}
|
||
if delivery := mapValue(payload["delivery"]); delivery != nil {
|
||
document.Summary["deliveryKind"] = safeRunArtifactMemberText(stringValue(delivery["kind"]), 80)
|
||
document.Summary["fileName"] = safeRunArtifactMemberText(stringValue(delivery["fileName"]), 200)
|
||
}
|
||
// The JSON document itself is the single downloadable member. Its bytes are
|
||
// served through the regular artifact content endpoint, never inlined here.
|
||
// A delivered index normally carries a durable object ref; a locally
|
||
// resolvable payload (replica, or a payloadPath-only record) must still be
|
||
// downloadable, otherwise the only deliverable of this mode is unreachable.
|
||
fileName := manifest.Name
|
||
sizeBytes := manifest.SizeBytes
|
||
hasContent := false
|
||
if manifest.Payload != nil && manifest.Payload.Mode == domain.ArtifactTransferModeRef && manifest.Payload.Ref != nil && manifest.Payload.Ref.Kind == domain.ArtifactRefKindObject {
|
||
fileName = firstNonEmptyText(filepath.Base(manifest.Payload.Ref.Key), manifest.Name)
|
||
sizeBytes = manifest.Payload.Ref.SizeBytes
|
||
hasContent = true
|
||
} else if dataRef := artifactDataRef(manifest); dataRef != nil {
|
||
if payloadPath := strings.TrimSpace(stringValue(dataRef["payloadPath"])); payloadPath != "" && !strings.Contains(payloadPath, "://") {
|
||
fileName = firstNonEmptyText(filepath.Base(payloadPath), manifest.Name)
|
||
hasContent = true
|
||
}
|
||
}
|
||
if hasContent {
|
||
member := server.newRunArtifactManifestMember(run, manifest, document.SchemaVersion, "part-index-document", "part-index", fileName, "part-index#"+fileName, "", "application/json", sizeBytes, nil)
|
||
member.contentArtifact = cloneRunArtifactMemberContentArtifact(*manifest, fileName, "application/json")
|
||
member.ContentAvailable = true
|
||
document.Members = []runArtifactManifestMember{member}
|
||
}
|
||
document.Summary["contentCount"] = len(document.Members)
|
||
document.Summary["contentAvailableCount"] = countRunArtifactManifestMembersWithContent(document.Members)
|
||
document.Summary["groups"] = runArtifactManifestMemberGroups(document.Members)
|
||
document.Message = "该 part 以单份 JSON 索引交付:切分与分类在索引内,媒体由后续流程按索引打包。"
|
||
return document
|
||
}
|
||
|
||
func (server *server) packageManifestMembers(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
|
||
if schema == "droboticflow.delivery.manifest/v5" {
|
||
return server.packageManifestV5Members(run, manifest, schema, payload)
|
||
}
|
||
type packageFile struct {
|
||
name string
|
||
mime string
|
||
kind string
|
||
size int64
|
||
}
|
||
items := listValue(payload["items"])
|
||
members := make([]runArtifactManifestMember, 0, len(items)*10+len(listValue(payload["calibration"])))
|
||
for index, raw := range items {
|
||
item := mapValue(raw)
|
||
if item == nil {
|
||
continue
|
||
}
|
||
packageName := firstNonEmptyText(stringValue(item["packageName"]), fmt.Sprintf("package-%d", index+1))
|
||
clipID := firstNonEmptyText(stringValue(item["clipId"]), packageName)
|
||
videoName := firstNonEmptyText(stringValue(item["clipName"]), clipID+".mp4")
|
||
baseCandidate := strings.TrimSpace(stringValue(item["storagePath"]))
|
||
details := map[string]any{
|
||
"clipId": safeRunArtifactMemberText(clipID, 300),
|
||
"startSecond": floatValue(item["startSecond"]),
|
||
"endSecond": floatValue(item["endSecond"]),
|
||
"duration": floatValue(item["duration"]),
|
||
"labels": manifestMemberActionLabels(mapValue(item["annotation"])),
|
||
}
|
||
segmentManifest := mapValue(item["segmentManifest"])
|
||
segmentVideo := mapValue(segmentManifest["video"])
|
||
rightVideoName := strings.TrimSpace(stringValue(item["rightClipName"]))
|
||
files := []packageFile{
|
||
{name: videoName, mime: "video/mp4", kind: "package-left-video", size: int64Value(segmentVideo["bytes"])},
|
||
}
|
||
if schema == "droboticflow.delivery.manifest/v4" {
|
||
if rightVideoName != "" {
|
||
rightVideo := mapValue(mapValue(segmentManifest["videos"])["right"])
|
||
files = append(files, packageFile{name: rightVideoName, mime: "video/mp4", kind: "package-right-video", size: int64Value(rightVideo["bytes"])})
|
||
}
|
||
files = append(files,
|
||
packageFile{name: clipID + "_left_pts.csv", mime: "text/csv", kind: "package-left-pts"},
|
||
packageFile{name: clipID + "_right_pts.csv", mime: "text/csv", kind: "package-right-pts"},
|
||
packageFile{name: clipID + "_imu.csv", mime: "text/csv", kind: "package-imu"},
|
||
packageFile{name: "semantic.json", mime: "application/json", kind: "package-timestamped-annotation"},
|
||
packageFile{name: "synchronized-slice.json", mime: "application/json", kind: "package-synchronization"},
|
||
packageFile{name: "segment.json", mime: "application/json", kind: "package-segment"},
|
||
)
|
||
if mapValue(segmentManifest["headpose"]) != nil {
|
||
files = append(files, packageFile{name: "headpose.json", mime: "application/json", kind: "package-headpose"})
|
||
}
|
||
if mapValue(segmentManifest["handpose"]) != nil {
|
||
files = append(files, packageFile{name: "handpose.json", mime: "application/json", kind: "package-handpose"})
|
||
}
|
||
} else {
|
||
files = append(files,
|
||
packageFile{name: "semantic.json", mime: "application/json", kind: "package-semantic"},
|
||
packageFile{name: "segment.json", mime: "application/json", kind: "package-segment"},
|
||
)
|
||
}
|
||
for _, file := range files {
|
||
member := server.newRunArtifactManifestMember(run, manifest, schema, file.kind, packageName, file.name, packageName+"#"+file.name, safeManifestMemberJoinedCandidate(baseCandidate, file.name), file.mime, file.size, details)
|
||
if sourceArtifact := runArtifactForManifestPackageFile(run, packageName, file.name, clipID); sourceArtifact != nil {
|
||
member.contentArtifact = cloneRunArtifactMemberContentArtifact(*sourceArtifact, file.name, file.mime)
|
||
member.ContentAvailable = true
|
||
}
|
||
members = append(members, member)
|
||
}
|
||
}
|
||
if schema != "droboticflow.delivery.manifest/v4" {
|
||
return members
|
||
}
|
||
for index, raw := range listValue(payload["calibration"]) {
|
||
calibration := mapValue(raw)
|
||
if calibration == nil {
|
||
continue
|
||
}
|
||
name := firstNonEmptyText(stringValue(calibration["name"]), fmt.Sprintf("calibration-%d", index+1))
|
||
member := server.newRunArtifactManifestMember(run, manifest, schema, "package-calibration", "calibration", name, "calibration#"+name, "", "application/x-yaml", int64Value(calibration["bytes"]), nil)
|
||
if sourceArtifact := runArtifactForManifestPackageFile(run, "calibration", name, ""); sourceArtifact != nil {
|
||
member.contentArtifact = cloneRunArtifactMemberContentArtifact(*sourceArtifact, name, "application/x-yaml")
|
||
member.ContentAvailable = true
|
||
}
|
||
members = append(members, member)
|
||
}
|
||
return members
|
||
}
|
||
|
||
func (server *server) packageManifestV5Members(run *domain.RunRecord, manifest *domain.ExecutionArtifact, schema string, payload map[string]any) []runArtifactManifestMember {
|
||
members := make([]runArtifactManifestMember, 0, len(listValue(payload["items"]))*8+len(listValue(payload["calibration"])))
|
||
appendMember := func(group string, relative string, file map[string]any, details map[string]any) {
|
||
name := safeManifestMemberName(stringValue(file["name"]))
|
||
if name == "member" {
|
||
return
|
||
}
|
||
relative = strings.Trim(strings.TrimSpace(relative), "/")
|
||
if relative == "" || !safeRunArtifactManifestBOSKey(relative) {
|
||
return
|
||
}
|
||
mimeType := packageManifestMemberMimeType(name)
|
||
member := server.newRunArtifactManifestMember(run, manifest, schema, "package-v5-file", group, name, relative, "", mimeType, int64Value(file["bytes"]), details)
|
||
if contentArtifact := runArtifactForDeliveryDirectoryMember(run, relative, name, mimeType, int64Value(file["bytes"])); contentArtifact != nil {
|
||
member.contentArtifact = contentArtifact
|
||
member.ContentAvailable = true
|
||
}
|
||
members = append(members, member)
|
||
}
|
||
for index, raw := range listValue(payload["items"]) {
|
||
item := mapValue(raw)
|
||
if item == nil {
|
||
continue
|
||
}
|
||
sliceID := firstNonEmptyText(stringValue(item["sliceId"]), fmt.Sprintf("slice-%06d", index+1))
|
||
directory := strings.Trim(strings.TrimSpace(stringValue(item["directory"])), "/")
|
||
if directory == "" || !safeRunArtifactManifestBOSKey(directory) {
|
||
continue
|
||
}
|
||
details := map[string]any{
|
||
"sliceId": safeRunArtifactMemberText(sliceID, 300),
|
||
"startSecond": floatValue(item["startSecond"]),
|
||
"endSecond": floatValue(item["endSecond"]),
|
||
"startTimestampUs": int64Value(item["startTimestampUs"]),
|
||
"endTimestampUs": int64Value(item["endTimestampUs"]),
|
||
}
|
||
for _, rawFile := range listValue(item["files"]) {
|
||
file := mapValue(rawFile)
|
||
if file == nil {
|
||
continue
|
||
}
|
||
name := safeManifestMemberName(stringValue(file["name"]))
|
||
appendMember(directory, path.Join(directory, name), file, details)
|
||
}
|
||
}
|
||
for index, raw := range listValue(payload["calibration"]) {
|
||
file := mapValue(raw)
|
||
if file == nil {
|
||
continue
|
||
}
|
||
name := firstNonEmptyText(stringValue(file["name"]), fmt.Sprintf("calibration-%d", index+1))
|
||
relative := firstNonEmptyText(stringValue(file["path"]), path.Join("calibration", name))
|
||
appendMember("calibration", relative, file, nil)
|
||
}
|
||
return members
|
||
}
|
||
|
||
func packageManifestMemberMimeType(name string) string {
|
||
lower := strings.ToLower(strings.TrimSpace(name))
|
||
switch {
|
||
case strings.HasSuffix(lower, ".mp4"):
|
||
return "video/mp4"
|
||
case strings.HasSuffix(lower, ".json"):
|
||
return "application/json"
|
||
case strings.HasSuffix(lower, ".csv"):
|
||
return "text/csv"
|
||
case strings.HasSuffix(lower, ".yaml"), strings.HasSuffix(lower, ".yml"):
|
||
return "application/x-yaml"
|
||
default:
|
||
return "application/octet-stream"
|
||
}
|
||
}
|
||
|
||
func runArtifactForDeliveryDirectoryMember(run *domain.RunRecord, relative string, name string, mimeType string, sizeBytes int64) *domain.ExecutionArtifact {
|
||
if run == nil || !safeRunArtifactManifestBOSKey(relative) {
|
||
return nil
|
||
}
|
||
for _, artifact := range allRunArtifactMembers(run) {
|
||
if artifact.PortID != "deliveryDirectory" && artifact.PortName != "deliveryDirectory" {
|
||
continue
|
||
}
|
||
if artifact.Payload != nil && artifact.Payload.Ref != nil && artifact.Payload.Ref.Kind == domain.ArtifactRefKindDirectory {
|
||
ref := artifact.Payload.Ref
|
||
prefix := strings.Trim(strings.TrimSpace(ref.Prefix), "/")
|
||
if prefix == "" || !safeRunArtifactManifestBOSKey(prefix) {
|
||
continue
|
||
}
|
||
key := path.Join(prefix, relative)
|
||
objectRef := &domain.ArtifactReference{Provider: ref.Provider, ConnectionID: ref.ConnectionID, Bucket: ref.Bucket, Key: key, Kind: domain.ArtifactRefKindObject, SizeBytes: sizeBytes}
|
||
return &domain.ExecutionArtifact{RunID: run.ID, NodeID: artifact.NodeID, Name: safeManifestMemberName(name), Type: memberArtifactPortType(name, mimeType), Path: "bos://" + ref.Bucket + "/" + key, SizeBytes: sizeBytes, Metadata: &domain.ArtifactMetadata{MimeType: mimeType}, Payload: &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: memberArtifactPortType(name, mimeType), Ref: objectRef}}
|
||
}
|
||
root := filepath.Clean(strings.TrimSpace(artifact.Path))
|
||
if root == "." || root == "" || !filepath.IsAbs(root) {
|
||
continue
|
||
}
|
||
candidate := filepath.Join(root, filepath.FromSlash(relative))
|
||
if runArtifactMemberPathWithin(root, candidate) {
|
||
return &domain.ExecutionArtifact{RunID: run.ID, NodeID: artifact.NodeID, Name: safeManifestMemberName(name), Type: memberArtifactPortType(name, mimeType), Path: candidate, SizeBytes: sizeBytes, Metadata: &domain.ArtifactMetadata{MimeType: mimeType}}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (server *server) newRunArtifactManifestMember(
|
||
run *domain.RunRecord,
|
||
manifest *domain.ExecutionArtifact,
|
||
schema string,
|
||
kind string,
|
||
group string,
|
||
name string,
|
||
logicalIdentity string,
|
||
candidate string,
|
||
mimeType string,
|
||
sizeBytes int64,
|
||
details map[string]any,
|
||
) runArtifactManifestMember {
|
||
name = safeManifestMemberName(name)
|
||
member := runArtifactManifestMember{
|
||
ID: stableRunArtifactManifestMemberID(manifest.ID, schema, kind, logicalIdentity),
|
||
Name: name,
|
||
Kind: manifestMemberKind(name, mimeType),
|
||
Group: safeRunArtifactMemberText(group, 240),
|
||
MimeType: safeRunArtifactMemberText(mimeType, 160),
|
||
SizeBytes: sizeBytes,
|
||
Details: sanitizeRunArtifactMemberDetails(details),
|
||
}
|
||
if candidate == "" {
|
||
return member
|
||
}
|
||
if contentArtifact := server.safeRunArtifactManifestContentArtifact(run, candidate, name, mimeType, sizeBytes); contentArtifact != nil {
|
||
member.contentArtifact = contentArtifact
|
||
member.ContentAvailable = true
|
||
}
|
||
return member
|
||
}
|
||
|
||
func stableRunArtifactManifestMemberID(manifestID string, schema string, kind string, logicalIdentity string) string {
|
||
hash := sha256.Sum256([]byte(strings.Join([]string{strings.TrimSpace(manifestID), strings.TrimSpace(schema), strings.TrimSpace(kind), strings.TrimSpace(logicalIdentity)}, "\x00")))
|
||
return "member-" + hex.EncodeToString(hash[:])
|
||
}
|
||
|
||
func safeManifestMemberName(value string) string {
|
||
name := path.Base(strings.TrimSpace(value))
|
||
if name == "" || name == "." || name == "/" || name == ".." || !utf8.ValidString(name) || hasRunArtifactMemberControlCharacter(name) {
|
||
return "member"
|
||
}
|
||
return safeRunArtifactMemberText(name, 512)
|
||
}
|
||
|
||
func safeRunArtifactMemberText(value string, maximum int) string {
|
||
value = strings.TrimSpace(value)
|
||
if value == "" || !utf8.ValidString(value) || hasRunArtifactMemberControlCharacter(value) {
|
||
return ""
|
||
}
|
||
if maximum > 0 && len(value) > maximum {
|
||
return value[:maximum]
|
||
}
|
||
return value
|
||
}
|
||
|
||
func hasRunArtifactMemberControlCharacter(value string) bool {
|
||
for _, character := range value {
|
||
if character < 0x20 || character == 0x7f {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func sanitizeRunArtifactMemberDetails(details map[string]any) map[string]any {
|
||
if len(details) == 0 {
|
||
return nil
|
||
}
|
||
result := make(map[string]any, len(details))
|
||
for key, value := range details {
|
||
if key == "" || hasRunArtifactMemberControlCharacter(key) {
|
||
continue
|
||
}
|
||
switch typed := value.(type) {
|
||
case string:
|
||
if safe := safeRunArtifactMemberText(typed, 512); safe != "" {
|
||
result[key] = safe
|
||
}
|
||
case []string:
|
||
values := make([]string, 0, len(typed))
|
||
for _, item := range typed {
|
||
if safe := safeRunArtifactMemberText(item, 256); safe != "" {
|
||
values = append(values, safe)
|
||
}
|
||
}
|
||
if len(values) > 0 {
|
||
result[key] = values
|
||
}
|
||
case int64, int, float64, float32, bool:
|
||
result[key] = typed
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func manifestMemberKind(name string, mimeType string) string {
|
||
mimeType = strings.ToLower(strings.TrimSpace(mimeType))
|
||
if strings.HasPrefix(mimeType, "video/") || strings.HasSuffix(strings.ToLower(name), ".mp4") || strings.HasSuffix(strings.ToLower(name), ".webm") {
|
||
return "video"
|
||
}
|
||
if strings.HasPrefix(mimeType, "image/") {
|
||
return "image"
|
||
}
|
||
if strings.HasPrefix(mimeType, "audio/") {
|
||
return "audio"
|
||
}
|
||
if strings.HasPrefix(mimeType, "text/") || strings.HasSuffix(strings.ToLower(name), ".csv") || strings.HasSuffix(strings.ToLower(name), ".json") || strings.HasSuffix(strings.ToLower(name), ".yaml") || strings.HasSuffix(strings.ToLower(name), ".yml") {
|
||
return "text"
|
||
}
|
||
return "file"
|
||
}
|
||
|
||
func manifestMemberCandidate(item map[string]any) string {
|
||
if item == nil {
|
||
return ""
|
||
}
|
||
for _, key := range []string{"storagePath", "path"} {
|
||
if value := strings.TrimSpace(stringValue(item[key])); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
if source := mapValue(item["source"]); source != nil {
|
||
if value := strings.TrimSpace(stringValue(source["path"])); value != "" {
|
||
return value
|
||
}
|
||
}
|
||
if ref := mapValue(item["ref"]); ref != nil && strings.EqualFold(strings.TrimSpace(stringValue(ref["provider"])), "BOS") {
|
||
bucket := strings.Trim(strings.TrimSpace(stringValue(ref["bucket"])), "/")
|
||
key := strings.Trim(strings.TrimSpace(stringValue(ref["key"])), "/")
|
||
if bucket != "" && key != "" {
|
||
return "bos://" + bucket + "/" + key
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func safeManifestMemberLeafName(value string) string {
|
||
value = strings.TrimSpace(value)
|
||
if value == "" || strings.Contains(value, "/") || strings.Contains(value, "\\") || value == "." || value == ".." || hasRunArtifactMemberControlCharacter(value) {
|
||
return ""
|
||
}
|
||
return value
|
||
}
|
||
|
||
func safeManifestMemberSiblingCandidate(base string, leaf string) (string, bool) {
|
||
leaf = safeManifestMemberLeafName(leaf)
|
||
if leaf == "" {
|
||
return "", false
|
||
}
|
||
if bucket, key, ok := safeRunArtifactManifestBOSPath(base); ok {
|
||
return "bos://" + bucket + "/" + path.Join(path.Dir(key), leaf), true
|
||
}
|
||
if strings.TrimSpace(base) == "" || !filepath.IsAbs(base) {
|
||
return "", false
|
||
}
|
||
return filepath.Join(filepath.Dir(base), leaf), true
|
||
}
|
||
|
||
func safeManifestMemberJoinedCandidate(base string, leaf string) string {
|
||
leaf = safeManifestMemberLeafName(leaf)
|
||
if leaf == "" {
|
||
return ""
|
||
}
|
||
if bucket, key, ok := safeRunArtifactManifestBOSPath(base); ok {
|
||
return "bos://" + bucket + "/" + path.Join(key, leaf)
|
||
}
|
||
if strings.TrimSpace(base) == "" || !filepath.IsAbs(base) {
|
||
return ""
|
||
}
|
||
return filepath.Join(base, leaf)
|
||
}
|
||
|
||
func (server *server) safeRunArtifactManifestContentArtifact(run *domain.RunRecord, candidate string, name string, mimeType string, sizeBytes int64) *domain.ExecutionArtifact {
|
||
if bucket, key, ok := safeRunArtifactManifestBOSPath(candidate); ok {
|
||
return &domain.ExecutionArtifact{
|
||
RunID: run.ID,
|
||
Name: safeManifestMemberName(name),
|
||
Type: memberArtifactPortType(name, mimeType),
|
||
Path: "bos://" + bucket + "/" + key,
|
||
SizeBytes: sizeBytes,
|
||
Metadata: &domain.ArtifactMetadata{MimeType: safeRunArtifactMemberText(mimeType, 160)},
|
||
}
|
||
}
|
||
if filePath, ok := server.safeRunArtifactManifestLocalFile(run, candidate); ok {
|
||
return &domain.ExecutionArtifact{
|
||
RunID: run.ID,
|
||
Name: safeManifestMemberName(name),
|
||
Type: memberArtifactPortType(name, mimeType),
|
||
Path: filePath,
|
||
SizeBytes: sizeBytes,
|
||
Metadata: &domain.ArtifactMetadata{MimeType: safeRunArtifactMemberText(mimeType, 160)},
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func memberArtifactPortType(name string, mimeType string) domain.PortType {
|
||
switch manifestMemberKind(name, mimeType) {
|
||
case "video":
|
||
return domain.PortTypeVideo
|
||
case "image":
|
||
return domain.PortTypeImage
|
||
case "text":
|
||
if strings.HasSuffix(strings.ToLower(name), ".json") {
|
||
return domain.PortTypeJSON
|
||
}
|
||
}
|
||
return domain.PortTypeFile
|
||
}
|
||
|
||
func safeRunArtifactManifestBOSPath(candidate string) (string, string, bool) {
|
||
bucket, key, ok := parseBOSPath(candidate)
|
||
if !ok || !safeRunArtifactManifestBOSBucket(bucket) || !safeRunArtifactManifestBOSKey(key) {
|
||
return "", "", false
|
||
}
|
||
return bucket, key, true
|
||
}
|
||
|
||
func safeRunArtifactManifestBOSBucket(bucket string) bool {
|
||
bucket = strings.TrimSpace(bucket)
|
||
if bucket == "" || len(bucket) > 63 || hasRunArtifactMemberControlCharacter(bucket) {
|
||
return false
|
||
}
|
||
for _, character := range bucket {
|
||
if !(character == '.' || character == '-' || (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9')) {
|
||
return false
|
||
}
|
||
}
|
||
return bucket[0] != '.' && bucket[0] != '-' && bucket[len(bucket)-1] != '.' && bucket[len(bucket)-1] != '-'
|
||
}
|
||
|
||
func safeRunArtifactManifestBOSKey(key string) bool {
|
||
key = strings.Trim(strings.TrimSpace(key), "/")
|
||
if key == "" || len(key) > 2048 || strings.Contains(key, "\\") || hasRunArtifactMemberControlCharacter(key) || path.Clean(key) != key {
|
||
return false
|
||
}
|
||
for _, segment := range strings.Split(key, "/") {
|
||
if segment == "" || segment == "." || segment == ".." {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
func (server *server) safeRunArtifactManifestLocalFile(run *domain.RunRecord, candidate string) (string, bool) {
|
||
if run == nil || !safeRunArtifactManifestRunID(run.ID) {
|
||
return "", false
|
||
}
|
||
candidate = strings.TrimSpace(candidate)
|
||
if candidate == "" || !filepath.IsAbs(candidate) || hasRunArtifactMemberControlCharacter(candidate) {
|
||
return "", false
|
||
}
|
||
cleaned := filepath.Clean(candidate)
|
||
if cleaned != candidate {
|
||
return "", false
|
||
}
|
||
workspaceRoot := cleanWorkspaceMountPath(server.workspaceMountPath)
|
||
runRoot := filepath.Join(workspaceRoot, "system", "runs", run.ID)
|
||
if !runArtifactMemberPathWithin(runRoot, cleaned) {
|
||
return "", false
|
||
}
|
||
if runArtifactMemberPathHasSymlink(runRoot, cleaned) {
|
||
return "", false
|
||
}
|
||
realRoot, err := filepath.EvalSymlinks(runRoot)
|
||
if err != nil {
|
||
return "", false
|
||
}
|
||
realCandidate, err := filepath.EvalSymlinks(cleaned)
|
||
if err != nil || !runArtifactMemberPathWithin(realRoot, realCandidate) {
|
||
return "", false
|
||
}
|
||
info, err := os.Stat(realCandidate)
|
||
if err != nil || info.IsDir() {
|
||
return "", false
|
||
}
|
||
return realCandidate, true
|
||
}
|
||
|
||
func safeRunArtifactManifestRunID(value string) bool {
|
||
value = strings.TrimSpace(value)
|
||
return value != "" && value == path.Base(value) && !strings.Contains(value, "\\") && !hasRunArtifactMemberControlCharacter(value)
|
||
}
|
||
|
||
func runArtifactMemberPathHasSymlink(root string, candidate string) bool {
|
||
relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(candidate))
|
||
if err != nil || relative == "." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) || relative == ".." {
|
||
return true
|
||
}
|
||
current := filepath.Clean(root)
|
||
for _, segment := range strings.Split(relative, string(filepath.Separator)) {
|
||
current = filepath.Join(current, segment)
|
||
info, statErr := os.Lstat(current)
|
||
if statErr != nil || info.Mode()&os.ModeSymlink != 0 {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func runArtifactMemberPathWithin(root string, candidate string) bool {
|
||
relative, err := filepath.Rel(filepath.Clean(root), filepath.Clean(candidate))
|
||
if err != nil || relative == "." || relative == ".." {
|
||
return false
|
||
}
|
||
return !strings.HasPrefix(relative, ".."+string(filepath.Separator))
|
||
}
|
||
|
||
func cloneRunArtifactMemberContentArtifact(item domain.ExecutionArtifact, name string, mimeType string) *domain.ExecutionArtifact {
|
||
copy := item
|
||
copy.Name = safeManifestMemberName(name)
|
||
if copy.Metadata == nil {
|
||
copy.Metadata = &domain.ArtifactMetadata{}
|
||
} else {
|
||
metadata := *copy.Metadata
|
||
copy.Metadata = &metadata
|
||
}
|
||
if strings.TrimSpace(mimeType) != "" {
|
||
copy.Metadata.MimeType = safeRunArtifactMemberText(mimeType, 160)
|
||
}
|
||
return ©
|
||
}
|
||
|
||
func manifestMemberActionLabels(item map[string]any) []string {
|
||
if item == nil {
|
||
return nil
|
||
}
|
||
annotation := item
|
||
if nested := mapValue(item["annotation"]); nested != nil {
|
||
annotation = nested
|
||
}
|
||
labels := make([]string, 0)
|
||
for _, raw := range listValue(annotation["actions"]) {
|
||
action := mapValue(raw)
|
||
if label := safeRunArtifactMemberText(stringValue(action["label"]), 256); label != "" {
|
||
labels = append(labels, label)
|
||
continue
|
||
}
|
||
if phrases := listValue(action["phrases"]); len(phrases) > 0 {
|
||
if phrase := safeRunArtifactMemberText(stringValue(phrases[0]), 256); phrase != "" {
|
||
labels = append(labels, phrase)
|
||
}
|
||
}
|
||
}
|
||
return labels
|
||
}
|
||
|
||
func allRunArtifactMembers(run *domain.RunRecord) []domain.ExecutionArtifact {
|
||
if run == nil {
|
||
return nil
|
||
}
|
||
items := make([]domain.ExecutionArtifact, 0, len(run.Artifacts))
|
||
seen := make(map[string]struct{})
|
||
appendUnique := func(item domain.ExecutionArtifact) {
|
||
key := firstNonEmptyText(item.ID, item.Path, item.Name)
|
||
if key == "" {
|
||
return
|
||
}
|
||
if _, exists := seen[key]; exists {
|
||
return
|
||
}
|
||
seen[key] = struct{}{}
|
||
items = append(items, item)
|
||
}
|
||
for _, item := range run.Artifacts {
|
||
appendUnique(item)
|
||
}
|
||
for _, execution := range run.NodeExecutions {
|
||
for _, item := range execution.Artifacts {
|
||
appendUnique(item)
|
||
}
|
||
for _, item := range execution.InputArtifacts {
|
||
appendUnique(item)
|
||
}
|
||
for _, lineage := range execution.InputLineage {
|
||
appendUnique(lineage.Artifact)
|
||
}
|
||
}
|
||
return items
|
||
}
|
||
|
||
func runArtifactForManifestClip(run *domain.RunRecord, clipID string) *domain.ExecutionArtifact {
|
||
clipID = strings.TrimSpace(clipID)
|
||
if clipID == "" {
|
||
return nil
|
||
}
|
||
for _, item := range allRunArtifactMembers(run) {
|
||
payload := artifactStructuredData(&item)
|
||
if payload != nil && strings.TrimSpace(stringValue(payload["clipId"])) == clipID {
|
||
copy := item
|
||
return ©
|
||
}
|
||
if strings.Contains(strings.TrimSpace(item.Name), clipID) && manifestMemberKind(item.Name, artifactMimeType(item)) == "video" {
|
||
copy := item
|
||
return ©
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func runArtifactForManifestPackageFile(run *domain.RunRecord, packageName string, fileName string, clipID string) *domain.ExecutionArtifact {
|
||
packageName = strings.TrimSpace(packageName)
|
||
fileName = strings.TrimSpace(fileName)
|
||
for _, item := range allRunArtifactMembers(run) {
|
||
if strings.TrimSpace(item.Name) != fileName {
|
||
continue
|
||
}
|
||
itemPath := filepath.ToSlash(item.Path)
|
||
if packageName != "" && strings.Contains(itemPath, packageName) {
|
||
copy := item
|
||
return ©
|
||
}
|
||
if fileName != "" && strings.Contains(fileName, clipID) {
|
||
copy := item
|
||
return ©
|
||
}
|
||
}
|
||
return nil
|
||
}
|