✨ Publish Capture deliveries to configured project directories
This commit is contained in:
parent
cd5bfdb409
commit
169fb5d60c
24
docs/capture-delivery-directory.md
Normal file
24
docs/capture-delivery-directory.md
Normal file
@ -0,0 +1,24 @@
|
||||
# Configurable final delivery directory
|
||||
|
||||
The continuous-processing settings now include a required delivery root in the same connection/bucket as intake. The UI previews:
|
||||
|
||||
`<delivery root>/<project name>__<project ID>/<capture name>/`
|
||||
|
||||
`clips/`, `calibration/` and package/evidence manifests are direct children. No Run or part directory is added. Only flat legacy sources or a single `part0001` are supported with this delivery layout; multiple parts fail intake rather than being merged.
|
||||
|
||||
## Safety and persistence
|
||||
|
||||
- API validates disjoint input/output prefixes (including other workflow listeners), rejects reserved system prefixes and probes write/delete permission before enabling intake. The project directory name is server-owned and reused across subsequent configuration updates.
|
||||
- New admissions freeze connection, bucket and complete target prefix on the Capture record. Existing snapshots are not changed by settings edits or project renames.
|
||||
- An old task without a delivery snapshot fails publication explicitly. It is not silently migrated and cannot fall back to the Run directory. No existing output is moved automatically.
|
||||
- The package operator still prepares and validates files in its internal workspace. The worker publishes the `delivery.directory.index` artifact into the frozen destination. Other intermediate artifacts keep their existing storage policy.
|
||||
- `.delivery-owner.json` reserves the destination using conditional creation. Every member also uses conditional creation and metadata SHA-256/size verification. Retrying identical publication is idempotent; changed content or a different owner is rejected, not overwritten.
|
||||
- Files can be visible during publication, but `manifest.json` is written last and is the authoritative completion marker. The business task cannot complete until publication succeeds. This is not a claim of atomic BOS directory rename.
|
||||
- `packages-manifest.json` is published after data members. `manifest.json` indexes every published member with size and SHA-256. The output remains a directory, not a ZIP.
|
||||
- Temporary upload objects stay under `system/runs/<run>/delivery-staging/` and successful uploads remove them. Interrupted staging is covered by normal Run cleanup.
|
||||
- Project and workflow deletion protect configured/frozen final delivery destinations. Deleting source business records is not permission to delete delivered data.
|
||||
- Capture records show the frozen destination and an “open delivery directory” link after delivery.
|
||||
|
||||
## Release boundary
|
||||
|
||||
API and worker must be deployed together. The earlier part-layout change additionally needs both updated directory/normalizer operator images. Do not deploy only the frontend or start a new scanner with old operators that ignore its explicit file selection.
|
||||
@ -23,6 +23,7 @@ func normalizeCaptureAutomationConfig(config domain.CaptureAutomationConfig) dom
|
||||
config.ConnectionID = strings.TrimSpace(config.ConnectionID)
|
||||
config.Bucket = strings.TrimSpace(config.Bucket)
|
||||
config.Prefix = strings.Trim(strings.TrimSpace(config.Prefix), "/")
|
||||
config.DeliveryPrefix = strings.Trim(strings.TrimSpace(config.DeliveryPrefix), "/")
|
||||
config.Schema = strings.TrimSpace(config.Schema)
|
||||
if config.Schema == "" {
|
||||
config.Schema = "orbbec_ego_stereo_v1"
|
||||
@ -235,6 +236,11 @@ func validateCaptureAutomationConfig(workflow *domain.WorkflowRecord, config dom
|
||||
if config.Prefix == "" {
|
||||
issues = append(issues, validationIssue{Field: "captureAutomation.prefix", Message: "持续接入目录不能为空"})
|
||||
}
|
||||
if !domain.ValidDeliveryPrefix(config.DeliveryPrefix) {
|
||||
issues = append(issues, validationIssue{Field: "captureAutomation.deliveryPrefix", Message: "请选择有效交付保存目录,不能使用系统产物目录"})
|
||||
} else if domain.StoragePrefixesOverlap(config.Prefix, config.DeliveryPrefix) {
|
||||
issues = append(issues, validationIssue{Field: "captureAutomation.deliveryPrefix", Message: "交付目录与监听目录不能相同或互相包含"})
|
||||
}
|
||||
if config.PollIntervalSeconds < 5 || config.PollIntervalSeconds > 3600 {
|
||||
issues = append(issues, validationIssue{Field: "captureAutomation.pollIntervalSeconds", Message: "检查间隔必须在5到3600秒之间"})
|
||||
}
|
||||
@ -310,6 +316,66 @@ func (server *server) handleCaptureAutomationUpdate(writer http.ResponseWriter,
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
// The project path is server-owned and remains stable after a rename.
|
||||
config.DeliveryProjectDirectory = ""
|
||||
workflows, listErr := server.store.ListAllWorkflowsByOrganization(request.Context(), workflow.OrganizationID)
|
||||
if listErr != nil {
|
||||
renderError(writer, 500, listErr)
|
||||
return
|
||||
}
|
||||
for _, other := range workflows {
|
||||
if other.CaptureAutomation == nil {
|
||||
continue
|
||||
}
|
||||
old := other.CaptureAutomation
|
||||
if other.ProjectID == workflow.ProjectID && old.DeliveryProjectDirectory != "" {
|
||||
config.DeliveryProjectDirectory = old.DeliveryProjectDirectory
|
||||
}
|
||||
if other.ID != workflow.ID && old.Bucket == config.Bucket && config.Enabled {
|
||||
if (old.Enabled && config.DeliveryPrefix != "" && domain.StoragePrefixesOverlap(config.DeliveryPrefix, old.Prefix)) || (old.DeliveryPrefix != "" && domain.StoragePrefixesOverlap(config.Prefix, old.DeliveryPrefix)) {
|
||||
renderValidationError(writer, validationIssue{Field: "captureAutomation.deliveryPrefix", Message: "交付目录与其他工作流监听目录重叠"})
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if config.DeliveryProjectDirectory == "" && config.DeliveryPrefix != "" {
|
||||
project, e := server.store.GetProject(request.Context(), workflow.ProjectID)
|
||||
if e != nil || project == nil {
|
||||
renderJSON(writer, 400, map[string]string{"error": "项目不存在"})
|
||||
return
|
||||
}
|
||||
config.DeliveryProjectDirectory = domain.DeliveryProjectDirectory(project.Name, project.ID)
|
||||
}
|
||||
if config.Enabled {
|
||||
captures, e := server.store.ListCaptureRecords(request.Context(), workflow.OrganizationID, "", 0)
|
||||
if e != nil {
|
||||
renderError(writer, 500, e)
|
||||
return
|
||||
}
|
||||
for _, capture := range captures {
|
||||
if d := capture.Delivery; d != nil && d.Bucket == config.Bucket && domain.StoragePrefixesOverlap(config.Prefix, d.Prefix) {
|
||||
renderValidationError(writer, validationIssue{Field: "captureAutomation.prefix", Message: "监听目录与已有任务的交付位置重叠"})
|
||||
return
|
||||
}
|
||||
}
|
||||
browser, _, cleanup, e := server.resolveObjectStorageBrowser(request.Context(), config.ConnectionID)
|
||||
if cleanup != nil {
|
||||
defer cleanup()
|
||||
}
|
||||
if e != nil {
|
||||
renderError(writer, 400, e)
|
||||
return
|
||||
}
|
||||
probe := config.DeliveryPrefix + "/.write-probe-" + domain.NewID("delivery")
|
||||
if _, e = browser.PutObject(request.Context(), config.Bucket, probe, []byte("delivery-write-check"), "text/plain"); e != nil {
|
||||
renderJSON(writer, 400, map[string]string{"error": "交付目录不可写:" + e.Error()})
|
||||
return
|
||||
}
|
||||
if e = browser.DeleteObject(request.Context(), config.Bucket, probe); e != nil {
|
||||
renderJSON(writer, 400, map[string]string{"error": "交付目录校验文件无法清理:" + probe})
|
||||
return
|
||||
}
|
||||
}
|
||||
workflow.CaptureAutomation = &config
|
||||
workflow.MaxRetryAttempts = config.MaxRetryAttempts
|
||||
workflow.RetryBackoffSeconds = config.RetryBackoffSeconds
|
||||
|
||||
@ -28,6 +28,11 @@ func captureTestInput(connectionID, bucket, prefix, fingerprint string) map[stri
|
||||
}
|
||||
|
||||
func TestCaptureAutomationCanBindReleasedWorkflowWithoutEditingDefinition(t *testing.T) {
|
||||
original := storageConnectionBrowserResolver
|
||||
t.Cleanup(func() { storageConnectionBrowserResolver = original })
|
||||
storageConnectionBrowserResolver = func(context.Context, *domain.StorageConnectionRecord) (objectStorageBrowser, func(), error) {
|
||||
return deliveryProbeBrowser{}, nil, nil
|
||||
}
|
||||
ctx := t.Context()
|
||||
dataStore := store.NewMemoryStore()
|
||||
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{OrganizationID: domain.DefaultOrganizationID, Name: "Fanyu"})
|
||||
@ -57,7 +62,8 @@ func TestCaptureAutomationCanBindReleasedWorkflowWithoutEditingDefinition(t *tes
|
||||
t.Fatalf("release workflow: %v", err)
|
||||
}
|
||||
body, _ := json.Marshal(domain.CaptureAutomationConfig{
|
||||
Enabled: true, ConnectionID: connection.ID, Bucket: "drobodata-fanyu", Prefix: "incoming",
|
||||
DeliveryPrefix: "deliveries",
|
||||
Enabled: true, ConnectionID: connection.ID, Bucket: "drobodata-fanyu", Prefix: "incoming",
|
||||
Schema: "orbbec_ego_stereo_v1", CompletionMode: "required_files", ManifestName: "source-manifest.json",
|
||||
WorkflowInputID: "sourceDirectory", PollIntervalSeconds: 30, MaxRetryAttempts: 3, RetryBackoffSeconds: 45,
|
||||
})
|
||||
|
||||
47
internal/api/capture_delivery_test.go
Normal file
47
internal/api/capture_delivery_test.go
Normal file
@ -0,0 +1,47 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type deliveryProbeBrowser struct{ objectStorageBrowser }
|
||||
|
||||
func (deliveryProbeBrowser) PutObject(context.Context, string, string, []byte, string) (domain.ObjectStorageObjectSummary, error) {
|
||||
return domain.ObjectStorageObjectSummary{}, nil
|
||||
}
|
||||
func (deliveryProbeBrowser) DeleteObject(context.Context, string, string) error { return nil }
|
||||
|
||||
func TestCaptureDeliveryConfigRejectsOverlap(t *testing.T) {
|
||||
workflow := &domain.WorkflowRecord{Type: domain.WorkflowTypeStreaming, DefinitionState: domain.WorkflowDefinitionStateReleased, WorkflowInputs: []domain.WorkflowInputDefinition{{ID: "sourceDirectory"}}}
|
||||
for _, prefix := range []string{"raw", "raw/outputs", "system/runs", ""} {
|
||||
config := normalizeCaptureAutomationConfig(domain.CaptureAutomationConfig{Enabled: true, ConnectionID: "c", Bucket: "b", Prefix: "raw", DeliveryPrefix: prefix})
|
||||
issues := validateCaptureAutomationConfig(workflow, config)
|
||||
found := false
|
||||
for _, i := range issues {
|
||||
if strings.Contains(i.Field, "deliveryPrefix") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal(prefix, issues)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProjectDeletionProtectsFrozenDelivery(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db := store.NewMemoryStore()
|
||||
s := &server{store: db}
|
||||
d := &domain.CaptureDeliveryTarget{ConnectionID: "c", Bucket: "b", Prefix: "deliveries/project/capture"}
|
||||
if e := db.UpsertCaptureRecord(ctx, domain.CaptureRecord{ID: "capture", OrganizationID: "org", WorkflowID: "w", Delivery: d}); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
plan := domain.ProjectDeletionPlan{OrganizationID: "org", Targets: []domain.ProjectDeletionTarget{{Kind: "bos", ConnectionID: "c", Bucket: "b", Prefix: "deliveries/project"}}}
|
||||
if e := s.validateDeletionReferences(ctx, plan); e == nil || !strings.Contains(e.Error(), "交付") {
|
||||
t.Fatal(e)
|
||||
}
|
||||
}
|
||||
@ -213,6 +213,9 @@ func (s *server) buildProjectDeletionPlan(ctx context.Context, p *domain.Project
|
||||
plan.Counts["captures"] += preview.CaptureRecordCount
|
||||
if w.CaptureAutomation != nil && w.CaptureAutomation.Prefix != "" {
|
||||
plan.Preserved = append(plan.Preserved, "输入:bos://"+w.CaptureAutomation.Bucket+"/"+w.CaptureAutomation.Prefix)
|
||||
if w.CaptureAutomation.DeliveryPrefix != "" {
|
||||
plan.Preserved = append(plan.Preserved, "最终交付:bos://"+w.CaptureAutomation.Bucket+"/"+w.CaptureAutomation.DeliveryPrefix)
|
||||
}
|
||||
}
|
||||
policy, e := s.effectiveArtifactStorage(ctx, &w)
|
||||
if e != nil {
|
||||
@ -281,6 +284,15 @@ func (s *server) validateDeletionReferences(ctx context.Context, plan domain.Pro
|
||||
if t.Kind != "bos" {
|
||||
continue
|
||||
}
|
||||
captures, e := s.store.ListCaptureRecords(ctx, plan.OrganizationID, "", 0)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, capture := range captures {
|
||||
if d := capture.Delivery; d != nil && d.Bucket == t.Bucket && prefixOverlap(t.Prefix, d.Prefix) {
|
||||
return fmt.Errorf("最终交付成品受保护,不能级联删除:%s", d.Prefix)
|
||||
}
|
||||
}
|
||||
if !validDeletionPrefix(t.Prefix) {
|
||||
return fmt.Errorf("拒绝不安全路径 %s", t.Prefix)
|
||||
}
|
||||
@ -292,6 +304,9 @@ func (s *server) validateDeletionReferences(ctx context.Context, plan domain.Pro
|
||||
}
|
||||
for _, w := range workflows {
|
||||
cfg := w.CaptureAutomation
|
||||
if cfg != nil && cfg.Bucket == t.Bucket && cfg.DeliveryPrefix != "" && prefixOverlap(t.Prefix, cfg.DeliveryPrefix) {
|
||||
return fmt.Errorf("最终交付目录受保护:%s", cfg.DeliveryPrefix)
|
||||
}
|
||||
if cfg != nil && cfg.ConnectionID == t.ConnectionID && cfg.Bucket == t.Bucket && cfg.Prefix != "" && prefixOverlap(t.Prefix, cfg.Prefix) {
|
||||
return fmt.Errorf("目标与原始输入目录重叠,已保护:%s", t.Prefix)
|
||||
}
|
||||
|
||||
@ -286,6 +286,15 @@ func (server *server) deleteWorkflowGeneratedArtifacts(ctx context.Context, work
|
||||
return err
|
||||
}
|
||||
for _, prefix := range prefixes {
|
||||
captures, e := server.store.ListCaptureRecords(ctx, workflow.OrganizationID, "", 0)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
for _, capture := range captures {
|
||||
if d := capture.Delivery; d != nil && d.Bucket == bucket && prefixOverlap(prefix, d.Prefix) {
|
||||
return fmt.Errorf("最终交付成品受保护,不能删除:%s", d.Prefix)
|
||||
}
|
||||
}
|
||||
if err := browser.DeletePrefix(ctx, bucket, prefix); err != nil && !isBOSNoSuchKey(err) {
|
||||
return fmt.Errorf("清理生成产物 %s/%s: %w", bucket, prefix, err)
|
||||
}
|
||||
|
||||
@ -1,20 +1,22 @@
|
||||
package domain
|
||||
|
||||
type CaptureAutomationConfig struct {
|
||||
Enabled bool `json:"enabled" bson:"enabled"`
|
||||
HasStarted bool `json:"hasStarted,omitempty" bson:"hasStarted,omitempty"`
|
||||
ConnectionID string `json:"connectionId" bson:"connectionId"`
|
||||
Bucket string `json:"bucket" bson:"bucket"`
|
||||
Prefix string `json:"prefix" bson:"prefix"`
|
||||
Schema string `json:"schema,omitempty" bson:"schema,omitempty"`
|
||||
CompletionMode string `json:"completionMode,omitempty" bson:"completionMode,omitempty"`
|
||||
ManifestName string `json:"manifestName,omitempty" bson:"manifestName,omitempty"`
|
||||
WorkflowInputID string `json:"workflowInputId,omitempty" bson:"workflowInputId,omitempty"`
|
||||
PollIntervalSeconds int `json:"pollIntervalSeconds,omitempty" bson:"pollIntervalSeconds,omitempty"`
|
||||
StabilityWindowSeconds int `json:"stabilityWindowSeconds,omitempty" bson:"stabilityWindowSeconds,omitempty"`
|
||||
RequiredPatterns []string `json:"requiredPatterns,omitempty" bson:"requiredPatterns,omitempty"`
|
||||
MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"`
|
||||
RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"`
|
||||
DeliveryPrefix string `json:"deliveryPrefix,omitempty" bson:"deliveryPrefix,omitempty"`
|
||||
DeliveryProjectDirectory string `json:"deliveryProjectDirectory,omitempty" bson:"deliveryProjectDirectory,omitempty"`
|
||||
Enabled bool `json:"enabled" bson:"enabled"`
|
||||
HasStarted bool `json:"hasStarted,omitempty" bson:"hasStarted,omitempty"`
|
||||
ConnectionID string `json:"connectionId" bson:"connectionId"`
|
||||
Bucket string `json:"bucket" bson:"bucket"`
|
||||
Prefix string `json:"prefix" bson:"prefix"`
|
||||
Schema string `json:"schema,omitempty" bson:"schema,omitempty"`
|
||||
CompletionMode string `json:"completionMode,omitempty" bson:"completionMode,omitempty"`
|
||||
ManifestName string `json:"manifestName,omitempty" bson:"manifestName,omitempty"`
|
||||
WorkflowInputID string `json:"workflowInputId,omitempty" bson:"workflowInputId,omitempty"`
|
||||
PollIntervalSeconds int `json:"pollIntervalSeconds,omitempty" bson:"pollIntervalSeconds,omitempty"`
|
||||
StabilityWindowSeconds int `json:"stabilityWindowSeconds,omitempty" bson:"stabilityWindowSeconds,omitempty"`
|
||||
RequiredPatterns []string `json:"requiredPatterns,omitempty" bson:"requiredPatterns,omitempty"`
|
||||
MaxRetryAttempts int `json:"maxRetryAttempts,omitempty" bson:"maxRetryAttempts,omitempty"`
|
||||
RetryBackoffSeconds int `json:"retryBackoffSeconds,omitempty" bson:"retryBackoffSeconds,omitempty"`
|
||||
}
|
||||
|
||||
type CaptureRecordStatus string
|
||||
@ -57,6 +59,7 @@ type CaptureActionRecord struct {
|
||||
}
|
||||
|
||||
type CaptureRecord struct {
|
||||
Delivery *CaptureDeliveryTarget `json:"delivery,omitempty" bson:"delivery,omitempty"`
|
||||
SourcePrefix string `json:"sourcePrefix,omitempty" bson:"sourcePrefix,omitempty"`
|
||||
SourceKeys []string `json:"sourceKeys,omitempty" bson:"sourceKeys,omitempty"`
|
||||
ID string `json:"id" bson:"id"`
|
||||
@ -95,6 +98,12 @@ type CaptureRecord struct {
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
}
|
||||
|
||||
type CaptureDeliveryTarget struct {
|
||||
ConnectionID string `json:"connectionId" bson:"connectionId"`
|
||||
Bucket string `json:"bucket" bson:"bucket"`
|
||||
Prefix string `json:"prefix" bson:"prefix"`
|
||||
}
|
||||
|
||||
type CaptureRecordCounts struct {
|
||||
Total int64 `json:"total"`
|
||||
Incomplete int64 `json:"incomplete"`
|
||||
|
||||
62
internal/domain/capture_delivery.go
Normal file
62
internal/domain/capture_delivery.go
Normal file
@ -0,0 +1,62 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
func ValidDeliveryPrefix(prefix string) bool {
|
||||
if prefix == "" || strings.HasPrefix(prefix, "/") || strings.ContainsAny(prefix, "\\:") {
|
||||
return false
|
||||
}
|
||||
for _, part := range strings.Split(prefix, "/") {
|
||||
if part == "" || part == "." || part == ".." {
|
||||
return false
|
||||
}
|
||||
for _, r := range part {
|
||||
if unicode.IsControl(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, root := range []string{"system", "workflows", "artifacts", "models", ".droboticflow-trash"} {
|
||||
if prefix == root || strings.HasPrefix(prefix, root+"/") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func StoragePrefixesOverlap(a, b string) bool {
|
||||
a = strings.Trim(a, "/")
|
||||
b = strings.Trim(b, "/")
|
||||
return a == "" || b == "" || a == b || strings.HasPrefix(a, b+"/") || strings.HasPrefix(b, a+"/")
|
||||
}
|
||||
|
||||
func DeliveryProjectDirectory(name, id string) string {
|
||||
name = strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) || strings.ContainsRune(`/\:*?"<>|`, r) {
|
||||
return '-'
|
||||
}
|
||||
return r
|
||||
}, strings.TrimSpace(name))
|
||||
if name == "" || name == "." || name == ".." {
|
||||
name = "项目"
|
||||
}
|
||||
return name + "__" + id
|
||||
}
|
||||
|
||||
func CaptureDeliveryPrefix(config CaptureAutomationConfig, captureID string) (string, error) {
|
||||
if !ValidDeliveryPrefix(config.DeliveryPrefix) || config.DeliveryProjectDirectory == "" {
|
||||
return "", fmt.Errorf("请先配置交付保存目录")
|
||||
}
|
||||
parts := strings.Split(captureID, "/")
|
||||
if len(parts) > 2 || (len(parts) == 2 && !strings.EqualFold(parts[1], "part0001")) {
|
||||
return "", fmt.Errorf("当前交付结构仅支持单个 part0001,不支持多个 part")
|
||||
}
|
||||
if parts[0] == "" || parts[0] == "." || parts[0] == ".." || strings.ContainsAny(parts[0], `\`) {
|
||||
return "", fmt.Errorf("无效采集目录名")
|
||||
}
|
||||
return config.DeliveryPrefix + "/" + config.DeliveryProjectDirectory + "/" + parts[0], nil
|
||||
}
|
||||
26
internal/domain/capture_delivery_test.go
Normal file
26
internal/domain/capture_delivery_test.go
Normal file
@ -0,0 +1,26 @@
|
||||
package domain
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCaptureDeliveryPath(t *testing.T) {
|
||||
config := CaptureAutomationConfig{DeliveryPrefix: "deliveries/fanyu", DeliveryProjectDirectory: DeliveryProjectDirectory("测试项目", "project-1")}
|
||||
for _, id := range []string{"capture", "capture/part0001"} {
|
||||
got, e := CaptureDeliveryPrefix(config, id)
|
||||
if e != nil || got != "deliveries/fanyu/测试项目__project-1/capture" {
|
||||
t.Fatal(got, e)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"capture/part0002", "../part0001", "a/b/c"} {
|
||||
if _, e := CaptureDeliveryPrefix(config, id); e == nil {
|
||||
t.Fatal(id)
|
||||
}
|
||||
}
|
||||
for _, prefix := range []string{"", "system/runs", "workflows/a", "../a", "a/../b", "/root", "a\\b"} {
|
||||
if ValidDeliveryPrefix(prefix) {
|
||||
t.Fatal(prefix)
|
||||
}
|
||||
}
|
||||
if StoragePrefixesOverlap("raw/a", "raw/abc") || !StoragePrefixesOverlap("raw", "raw/a") {
|
||||
t.Fatal("prefix boundary check")
|
||||
}
|
||||
}
|
||||
@ -18,6 +18,7 @@ import (
|
||||
)
|
||||
|
||||
type ObjectMetadata struct {
|
||||
UserMeta map[string]string
|
||||
Exists bool
|
||||
ETag string
|
||||
VersionID string
|
||||
@ -71,6 +72,7 @@ func (publisher BOSPublisher) Head(ctx context.Context, connection domain.Storag
|
||||
return ObjectMetadata{}, err
|
||||
}
|
||||
return ObjectMetadata{
|
||||
UserMeta: result.UserMeta,
|
||||
Exists: true,
|
||||
ETag: strings.Trim(result.ETag, `"`),
|
||||
VersionID: strings.TrimSpace(result.VersionId),
|
||||
@ -79,6 +81,31 @@ func (publisher BOSPublisher) Head(ctx context.Context, connection domain.Storag
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (publisher BOSPublisher) ListPrefix(ctx context.Context, connection domain.StorageConnectionRecord, bucket, prefix string) ([]string, error) {
|
||||
client, err := publisher.client(connection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := []string{}
|
||||
marker := ""
|
||||
for {
|
||||
result, e := client.ListObjectsWithContext(ctx, bucket, &bosapi.ListObjectsArgs{Prefix: prefix, Marker: marker, MaxKeys: 1000})
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
for _, item := range result.Contents {
|
||||
keys = append(keys, item.Key)
|
||||
}
|
||||
if !result.IsTruncated {
|
||||
return keys, nil
|
||||
}
|
||||
if result.NextMarker == "" || result.NextMarker == marker {
|
||||
return nil, fmt.Errorf("incomplete listing")
|
||||
}
|
||||
marker = result.NextMarker
|
||||
}
|
||||
}
|
||||
|
||||
func multipartCopyPartSize(sizeBytes int64) int64 {
|
||||
const (
|
||||
minimum = int64(256 * 1024 * 1024)
|
||||
|
||||
@ -380,6 +380,14 @@ func (runtime *Runtime) normalizeOutputArtifacts(ctx context.Context, workflow *
|
||||
}
|
||||
}
|
||||
}
|
||||
if statErr == nil && info.IsDir() && artifact.Name == "delivery.directory.index" {
|
||||
delivered, e := runtime.publishCaptureDelivery(ctx, run, nodeID, artifact)
|
||||
if e != nil {
|
||||
return nil, e
|
||||
}
|
||||
result[index] = delivered
|
||||
continue
|
||||
}
|
||||
if policy == nil {
|
||||
continue // Draft/legacy workflows retain workspace-path compatibility.
|
||||
}
|
||||
|
||||
@ -120,7 +120,7 @@ func captureRecordID(workflowID, prefix string) string {
|
||||
}
|
||||
|
||||
func captureAutomationFingerprint(workflowID string, config domain.CaptureAutomationConfig) string {
|
||||
sum := sha256.Sum256([]byte(strings.Join([]string{workflowID, config.ConnectionID, config.Bucket, config.Prefix, config.CompletionMode, config.ManifestName, config.WorkflowInputID}, "|")))
|
||||
sum := sha256.Sum256([]byte(strings.Join([]string{workflowID, config.ConnectionID, config.Bucket, config.Prefix, config.CompletionMode, config.ManifestName, config.WorkflowInputID, config.DeliveryPrefix, config.DeliveryProjectDirectory}, "|")))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
@ -455,6 +455,19 @@ func (runtime *Runtime) launchCaptureRun(ctx context.Context, workflow domain.Wo
|
||||
record.UpdatedAt = nowISO()
|
||||
return runtime.store.UpsertCaptureRecord(ctx, *record)
|
||||
}
|
||||
if record.Delivery == nil {
|
||||
prefix, deliveryErr := domain.CaptureDeliveryPrefix(config, record.CaptureID)
|
||||
if deliveryErr != nil {
|
||||
record.Status = domain.CaptureRecordFailed
|
||||
record.Error = deliveryErr.Error()
|
||||
return runtime.store.UpsertCaptureRecord(ctx, *record)
|
||||
}
|
||||
record.Delivery = &domain.CaptureDeliveryTarget{ConnectionID: config.ConnectionID, Bucket: config.Bucket, Prefix: prefix}
|
||||
record.Status = domain.CaptureRecordLaunching
|
||||
if err := runtime.store.UpsertCaptureRecord(ctx, *record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
payload := domain.ArtifactPayload{
|
||||
Mode: domain.ArtifactTransferModeRef,
|
||||
Type: input.Type,
|
||||
@ -609,6 +622,7 @@ func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow d
|
||||
ContractRevision: domain.FanyuV58CaptureContractRevision,
|
||||
}
|
||||
if existing != nil {
|
||||
record.Delivery = existing.Delivery
|
||||
record.DiscoveredAt = existing.DiscoveredAt
|
||||
record.Actions = append([]domain.CaptureActionRecord(nil), existing.Actions...)
|
||||
record.StatusBeforeDiscard = existing.StatusBeforeDiscard
|
||||
|
||||
@ -278,6 +278,7 @@ func TestLaunchCaptureRunBindsOneCompleteDirectory(t *testing.T) {
|
||||
runtime := &Runtime{store: dataStore}
|
||||
config := normalizedCaptureAutomation(&domain.CaptureAutomationConfig{
|
||||
Enabled: true, ConnectionID: "bos-fanyu", Bucket: "drobodata-fanyu", Prefix: "incoming", CompletionMode: "required_files",
|
||||
DeliveryPrefix: "deliveries", DeliveryProjectDirectory: "Fanyu__project",
|
||||
})
|
||||
paused := config
|
||||
paused.Enabled = false
|
||||
@ -317,6 +318,9 @@ func TestLaunchCaptureRunBindsOneCompleteDirectory(t *testing.T) {
|
||||
t.Fatalf("capture was not queued: %#v err=%v", stored, err)
|
||||
}
|
||||
run, err := dataStore.GetRun(ctx, stored.RunID)
|
||||
if stored.Delivery == nil || stored.Delivery.Prefix != "deliveries/Fanyu__project/capture-000001" {
|
||||
t.Fatalf("delivery snapshot missing: %+v", stored.Delivery)
|
||||
}
|
||||
if err != nil || run == nil {
|
||||
t.Fatalf("get run: %#v err=%v", run, err)
|
||||
}
|
||||
|
||||
195
internal/worker/capture_delivery.go
Normal file
195
internal/worker/capture_delivery.go
Normal file
@ -0,0 +1,195 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/objectsync"
|
||||
)
|
||||
|
||||
type deliveryPrefixLister interface {
|
||||
ListPrefix(context.Context, domain.StorageConnectionRecord, string, string) ([]string, error)
|
||||
}
|
||||
|
||||
// Every final object uses conditional create. A different delivery can never
|
||||
// overwrite it, even when two workers race. Retries accept only identical bytes.
|
||||
func publishDeliveryObject(ctx context.Context, p objectsync.Publisher, c domain.StorageConnectionRecord, bucket, stage, key, owner, sha string, size int64) error {
|
||||
old, err := p.Head(ctx, c, bucket, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if old.Exists {
|
||||
if old.UserMeta["delivery-owner"] == owner && old.UserMeta["sha256"] == sha && old.SizeBytes == size {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("交付目录冲突,拒绝覆盖 %s", key)
|
||||
}
|
||||
src, err := p.Head(ctx, c, bucket, stage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = p.PublishConditional(ctx, c, bucket, stage, key, size, src.ETag, "", "*", map[string]string{"delivery-owner": owner, "sha256": sha})
|
||||
if err != nil {
|
||||
// An ambiguous network response or an identical concurrent retry is safe.
|
||||
got, e := p.Head(ctx, c, bucket, key)
|
||||
if e == nil && got.Exists && got.UserMeta["delivery-owner"] == owner && got.UserMeta["sha256"] == sha && got.SizeBytes == size {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
got, err := p.Head(ctx, c, bucket, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !got.Exists || got.SizeBytes != size || got.UserMeta["sha256"] != sha {
|
||||
return fmt.Errorf("交付文件校验失败 %s", key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runtime *Runtime) publishCaptureDelivery(ctx context.Context, run *domain.RunRecord, nodeID string, artifact domain.ExecutionArtifact) (domain.ExecutionArtifact, error) {
|
||||
record, err := runtime.store.GetCaptureRecordByRunID(ctx, run.ID)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
if record == nil || record.Delivery == nil {
|
||||
return artifact, fmt.Errorf("该任务未配置交付目录,请明确配置后重新接入;不会回退到 runs 目录")
|
||||
}
|
||||
target := record.Delivery
|
||||
if !domain.ValidDeliveryPrefix(target.Prefix) {
|
||||
return artifact, fmt.Errorf("无效交付目标目录")
|
||||
}
|
||||
listeners, e := runtime.store.ListEnabledCaptureAutomations(ctx, run.OrganizationID)
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
for _, workflow := range listeners {
|
||||
if c := workflow.CaptureAutomation; c != nil && c.Bucket == target.Bucket && domain.StoragePrefixesOverlap(c.Prefix, target.Prefix) {
|
||||
return artifact, fmt.Errorf("交付位置与已启用监听目录重叠,已停止发布")
|
||||
}
|
||||
}
|
||||
connection, err := runtime.artifactStorageConnection(ctx, target.ConnectionID, run.OrganizationID)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
publisher := runtime.outputArtifactPublisher()
|
||||
lister, ok := publisher.(deliveryPrefixLister)
|
||||
if !ok {
|
||||
return artifact, fmt.Errorf("交付存储不支持目录冲突检查")
|
||||
}
|
||||
owner := record.ID + ":" + record.RunID
|
||||
stageRoot := "system/runs/" + run.ID + "/delivery-staging/" + nodeID
|
||||
publishBytes := func(key string, body []byte) error {
|
||||
sum := sha256.Sum256(body)
|
||||
sha := hex.EncodeToString(sum[:])
|
||||
stage := stageRoot + "/" + sha
|
||||
if _, e := publisher.PutInternal(ctx, *connection, target.Bucket, stage, bytes.NewReader(body), int64(len(body)), "application/json", nil); e != nil {
|
||||
return e
|
||||
}
|
||||
if e := publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, int64(len(body))); e != nil {
|
||||
return e
|
||||
}
|
||||
return publisher.Delete(ctx, *connection, target.Bucket, stage)
|
||||
}
|
||||
marker := target.Prefix + "/.delivery-owner.json"
|
||||
known, err := publisher.Head(ctx, *connection, target.Bucket, marker)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
if !known.Exists {
|
||||
keys, e := lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
if len(keys) > 0 {
|
||||
return artifact, fmt.Errorf("目标目录已有文件,拒绝覆盖 %s", target.Prefix)
|
||||
}
|
||||
}
|
||||
markerBody, _ := json.Marshal(map[string]string{"owner": owner})
|
||||
if err = publishBytes(marker, markerBody); err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
paths := []string{}
|
||||
err = filepath.WalkDir(artifact.Path, func(path string, entry os.DirEntry, e error) error {
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if entry.Type()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("交付目录不允许符号链接")
|
||||
}
|
||||
if !entry.IsDir() {
|
||||
paths = append(paths, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
sort.Slice(paths, func(i, j int) bool {
|
||||
a, b := filepath.Base(paths[i]) == "packages-manifest.json", filepath.Base(paths[j]) == "packages-manifest.json"
|
||||
if a != b {
|
||||
return !a
|
||||
}
|
||||
return paths[i] < paths[j]
|
||||
})
|
||||
entries := []directoryManifestEntry{}
|
||||
var total int64
|
||||
for _, path := range paths {
|
||||
relative, e := filepath.Rel(artifact.Path, path)
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
relative = filepath.ToSlash(relative)
|
||||
if strings.HasPrefix(relative, "../") || relative == "manifest.json" || relative == ".delivery-owner.json" {
|
||||
return artifact, fmt.Errorf("无效交付成员 %s", relative)
|
||||
}
|
||||
stage := stageRoot + "/files/" + relative
|
||||
meta, sha, e := putArtifactFile(ctx, publisher, *connection, target.Bucket, stage, path)
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
key := target.Prefix + "/" + relative
|
||||
if e = publishDeliveryObject(ctx, publisher, *connection, target.Bucket, stage, key, owner, sha, meta.SizeBytes); e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
if e = publisher.Delete(ctx, *connection, target.Bucket, stage); e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
entries = append(entries, directoryManifestEntry{Path: relative, Key: key, SizeBytes: meta.SizeBytes, SHA256: sha})
|
||||
total += meta.SizeBytes
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return artifact, fmt.Errorf("交付目录为空")
|
||||
}
|
||||
expected := map[string]bool{marker: true, target.Prefix + "/manifest.json": true}
|
||||
for _, entry := range entries {
|
||||
expected[entry.Key] = true
|
||||
}
|
||||
keys, e := lister.ListPrefix(ctx, *connection, target.Bucket, target.Prefix+"/")
|
||||
if e != nil {
|
||||
return artifact, e
|
||||
}
|
||||
for _, key := range keys {
|
||||
if !expected[key] {
|
||||
return artifact, fmt.Errorf("目标目录包含非本次交付文件,拒绝完成 %s", key)
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(directoryManifest{SchemaVersion: 1, Kind: "directory", FileCount: len(entries), TotalBytes: total, Entries: entries})
|
||||
manifestKey := target.Prefix + "/manifest.json"
|
||||
if err = publishBytes(manifestKey, body); err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
artifact.Path = "bos://" + target.Bucket + "/" + target.Prefix
|
||||
artifact.SizeBytes = total
|
||||
artifact.Payload = &domain.ArtifactPayload{Mode: domain.ArtifactTransferModeRef, Type: artifact.Type, Ref: &domain.ArtifactReference{Provider: domain.StorageProviderBOS, ConnectionID: target.ConnectionID, Bucket: target.Bucket, Prefix: target.Prefix, ManifestKey: manifestKey, Kind: domain.ArtifactRefKindDirectory}}
|
||||
return artifact, nil
|
||||
}
|
||||
125
internal/worker/capture_delivery_test.go
Normal file
125
internal/worker/capture_delivery_test.go
Normal file
@ -0,0 +1,125 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/objectsync"
|
||||
"droboticflow/internal/store"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type deliveryPublisher struct {
|
||||
memoryArtifactPublisher
|
||||
metadata map[string]map[string]string
|
||||
published []string
|
||||
fail string
|
||||
}
|
||||
|
||||
func (p *deliveryPublisher) Head(ctx context.Context, c domain.StorageConnectionRecord, b, k string) (objectsync.ObjectMetadata, error) {
|
||||
m, e := p.memoryArtifactPublisher.Head(ctx, c, b, k)
|
||||
m.ETag = "test"
|
||||
m.UserMeta = p.metadata[b+"/"+k]
|
||||
return m, e
|
||||
}
|
||||
func (p *deliveryPublisher) ListPrefix(_ context.Context, _ domain.StorageConnectionRecord, b, k string) ([]string, error) {
|
||||
result := []string{}
|
||||
for key := range p.objects {
|
||||
if strings.HasPrefix(key, b+"/"+k) {
|
||||
result = append(result, strings.TrimPrefix(key, b+"/"))
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func (p *deliveryPublisher) PublishConditional(ctx context.Context, c domain.StorageConnectionRecord, b, src, target string, size int64, etag, match, none string, meta map[string]string) (objectsync.ObjectMetadata, error) {
|
||||
if none != "*" {
|
||||
return objectsync.ObjectMetadata{}, fmt.Errorf("non-conditional write")
|
||||
}
|
||||
if target == p.fail {
|
||||
return objectsync.ObjectMetadata{}, fmt.Errorf("injected interruption")
|
||||
}
|
||||
key := b + "/" + target
|
||||
if _, ok := p.objects[key]; ok {
|
||||
return objectsync.ObjectMetadata{}, fmt.Errorf("conflict")
|
||||
}
|
||||
p.objects[key] = append([]byte(nil), p.objects[b+"/"+src]...)
|
||||
if p.metadata == nil {
|
||||
p.metadata = map[string]map[string]string{}
|
||||
}
|
||||
p.metadata[key] = meta
|
||||
p.published = append(p.published, target)
|
||||
return p.Head(ctx, c, b, target)
|
||||
}
|
||||
|
||||
func TestCaptureDeliveryPublishesChosenDirectoryAndRetries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
db := store.NewMemoryStore()
|
||||
p := &deliveryPublisher{}
|
||||
runtime := &Runtime{store: db, artifactPublisher: p}
|
||||
c := domain.StorageConnectionRecord{ID: "c", OrganizationID: "org", Provider: domain.StorageProviderBOS, Bucket: "b"}
|
||||
if e := db.UpsertStorageConnection(ctx, c); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
prefix := "deliveries/测试__project/capture"
|
||||
record := domain.CaptureRecord{ID: "capture", OrganizationID: "org", RunID: "run-1", Delivery: &domain.CaptureDeliveryTarget{ConnectionID: "c", Bucket: "b", Prefix: prefix}}
|
||||
if e := db.UpsertCaptureRecord(ctx, record); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
root := t.TempDir()
|
||||
if e := os.Mkdir(filepath.Join(root, "clips"), 0755); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := os.WriteFile(filepath.Join(root, "clips", "left.mp4"), []byte("test-video"), 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := os.WriteFile(filepath.Join(root, "packages-manifest.json"), []byte(`{"items":[]}`), 0600); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
artifact := domain.ExecutionArtifact{Name: "delivery.directory.index", Path: root, Type: domain.PortTypeFile}
|
||||
run := &domain.RunRecord{ID: "run-1", OrganizationID: "org"}
|
||||
p.fail = prefix + "/clips/left.mp4"
|
||||
if _, e := runtime.publishCaptureDelivery(ctx, run, "package", artifact); e == nil {
|
||||
t.Fatal("failure swallowed")
|
||||
}
|
||||
if _, ok := p.objects["b/"+prefix+"/manifest.json"]; ok {
|
||||
t.Fatal("partial delivery marked complete")
|
||||
}
|
||||
p.fail = ""
|
||||
got, e := runtime.publishCaptureDelivery(ctx, run, "package", artifact)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if got.Path != "bos://b/"+prefix || got.Payload.Ref.Prefix != prefix {
|
||||
t.Fatal(got)
|
||||
}
|
||||
if p.published[len(p.published)-1] != prefix+"/manifest.json" {
|
||||
t.Fatal("completion manifest not last")
|
||||
}
|
||||
count := len(p.published)
|
||||
if _, e = runtime.publishCaptureDelivery(ctx, run, "package", artifact); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if len(p.published) != count {
|
||||
t.Fatal("retry rewrote completed objects")
|
||||
}
|
||||
record.RunID = "run-2"
|
||||
if e = db.UpsertCaptureRecord(ctx, record); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if _, e = runtime.publishCaptureDelivery(ctx, &domain.RunRecord{ID: "run-2", OrganizationID: "org"}, "package", artifact); e == nil {
|
||||
t.Fatal("new attempt overwrote prior delivery")
|
||||
}
|
||||
if len(p.published) != count {
|
||||
t.Fatal("conflict mutated output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureDeliveryMissingConfigurationFailsClosed(t *testing.T) {
|
||||
r := &Runtime{store: store.NewMemoryStore()}
|
||||
if _, e := r.publishCaptureDelivery(context.Background(), &domain.RunRecord{ID: "old"}, "package", domain.ExecutionArtifact{}); e == nil {
|
||||
t.Fatal("legacy task silently used current config")
|
||||
}
|
||||
}
|
||||
@ -91,6 +91,9 @@ func discoverCaptureGroups(objects []captureObject, config domain.CaptureAutomat
|
||||
sort.Strings(names)
|
||||
for _, part := range names {
|
||||
group := captureGroup{CaptureID: parent.CaptureID + "/" + part, Prefix: parent.Prefix + "/" + part, SourcePrefix: parent.Prefix}
|
||||
if config.DeliveryPrefix != "" && (len(names) != 1 || !strings.EqualFold(part, "part0001")) {
|
||||
group.Missing = append(group.Missing, "当前交付结构只支持单个 part0001;多个 part 不会自动合并或覆盖")
|
||||
}
|
||||
captureName := regexp.QuoteMeta(parent.CaptureID)
|
||||
partName := regexp.QuoteMeta(part)
|
||||
requirements := []captureRequirement{
|
||||
|
||||
@ -136,3 +136,17 @@ func TestPartCapturePrefersVideoSpecificPTS(t *testing.T) {
|
||||
t.Fatal("video-specific PTS missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartCaptureDeliveryRejectsMultipleParts(t *testing.T) {
|
||||
objects := append(calibrationObjects(), partObjects("part0001")...)
|
||||
objects = append(objects, partObjects("part0002")...)
|
||||
groups, e := discoverCaptureGroups(objects, normalizedCaptureAutomation(&domain.CaptureAutomationConfig{Prefix: "incoming", DeliveryPrefix: "deliveries"}))
|
||||
if e != nil || len(groups) != 2 {
|
||||
t.Fatal(groups, e)
|
||||
}
|
||||
for _, group := range groups {
|
||||
if !strings.Contains(strings.Join(group.Missing, ""), "多个 part") {
|
||||
t.Fatal(group)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
48
src/components/CaptureDeliveryField.tsx
Normal file
48
src/components/CaptureDeliveryField.tsx
Normal file
@ -0,0 +1,48 @@
|
||||
import { FolderOpen } from "lucide-react";
|
||||
|
||||
export function CaptureDeliveryField({
|
||||
bucket,
|
||||
prefix,
|
||||
projectDirectory,
|
||||
onChange,
|
||||
onBrowse,
|
||||
}: {
|
||||
bucket: string;
|
||||
prefix: string;
|
||||
projectDirectory: string;
|
||||
onChange: (value: string) => void;
|
||||
onBrowse: () => void;
|
||||
}) {
|
||||
return (
|
||||
<div className="md:col-span-2 space-y-2 rounded-lg border border-cyan-200 bg-cyan-50/40 p-3 dark:bg-cyan-950/10">
|
||||
<label className="block text-xs font-medium" htmlFor="capture-delivery-prefix">
|
||||
交付保存目录 *
|
||||
</label>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<input
|
||||
id="capture-delivery-prefix"
|
||||
value={prefix}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
placeholder="例如 deliveries/fanyu"
|
||||
className="min-w-0 flex-1 rounded-md border border-[color:var(--app-border)] bg-[color:var(--app-panel)] px-3 py-2 text-sm"
|
||||
/>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onBrowse}
|
||||
className="inline-flex items-center gap-2 rounded-md border border-[color:var(--app-border)] px-3 py-2 text-xs"
|
||||
>
|
||||
<FolderOpen className="h-4 w-4" />
|
||||
浏览目录
|
||||
</button>
|
||||
</div>
|
||||
<p className="text-xs text-slate-500">与监听目录分开,不能相同或互相包含。修改仅影响之后新接入的任务。</p>
|
||||
<p className="break-all text-xs font-mono">
|
||||
最终路径:bos://{bucket}/{prefix.replace(/^\/+|\/+$/g, "") || "{交付根目录}"}/{projectDirectory}/
|
||||
{"{采集目录名}"}/
|
||||
</p>
|
||||
<p className="text-xs text-slate-500">
|
||||
目录内直接包含 clips、calibration 和清单,不包含 part 或 Run ID;多个 part 将阻止接入。删除项目不删除最终成品。
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
import React, { useEffect, useState, useDeferredValue } from "react";
|
||||
import { continuousProcessingStatus } from "../lib/continuous-processing.js";
|
||||
import { useParams, useNavigate } from "react-router-dom";
|
||||
import { Link, useParams, useNavigate } from "react-router-dom";
|
||||
import { CaptureDeliveryField } from "../components/CaptureDeliveryField";
|
||||
import { useTimeZone } from "../context/TimeZoneContext";
|
||||
import { useUser } from "../context/UserContext";
|
||||
import { PageSubnav } from "../components/PageSubnav";
|
||||
@ -993,6 +994,8 @@ export default function ProjectDetail() {
|
||||
completionMode: "required_files",
|
||||
manifestName: response.config?.manifestName || "source-manifest.json",
|
||||
workflowInputId: response.config?.workflowInputId || directoryInput?.id || "sourceDirectory",
|
||||
deliveryPrefix: response.config?.deliveryPrefix || "",
|
||||
deliveryProjectDirectory: response.config?.deliveryProjectDirectory,
|
||||
pollIntervalSeconds: response.config?.pollIntervalSeconds || 30,
|
||||
stabilityWindowSeconds: response.config?.stabilityWindowSeconds || 30,
|
||||
requiredPatterns: response.config?.requiredPatterns,
|
||||
@ -1016,9 +1019,9 @@ export default function ProjectDetail() {
|
||||
setAutomationNotice("");
|
||||
if (
|
||||
enabled &&
|
||||
(!configToSave.connectionId || !configToSave.bucket.trim() || !configToSave.prefix.trim())
|
||||
(!configToSave.connectionId || !configToSave.bucket.trim() || !configToSave.prefix.trim() || !configToSave.deliveryPrefix?.trim())
|
||||
) {
|
||||
setAutomationError("请选择原始数据根目录及其存储连接、Bucket 后再启动正式处理。");
|
||||
setAutomationError("请选择原始数据根目录、交付保存目录及其存储连接、Bucket 后再启动正式处理。");
|
||||
return;
|
||||
}
|
||||
setIsAutomationSaving(true);
|
||||
@ -1097,6 +1100,15 @@ export default function ProjectDetail() {
|
||||
|
||||
const handleReferencePickerSelect = (selection: ObjectStorageReferenceSelection) => {
|
||||
if (!referencePickerTarget) return;
|
||||
if (referencePickerTarget.inputId === "__capture_delivery__") {
|
||||
if (selection.kind !== "directory" || selection.connectionId !== automationDraft.connectionId || selection.bucket !== automationDraft.bucket) {
|
||||
setAutomationError("交付目录必须是当前存储连接和 Bucket 下的目录。");
|
||||
} else {
|
||||
setAutomationDraft(current => ({...current, deliveryPrefix: selection.prefix || selection.key || ""}));
|
||||
}
|
||||
setReferencePickerTarget(null);
|
||||
return;
|
||||
}
|
||||
if (referencePickerTarget.inputId === "__capture_automation__") {
|
||||
if (selection.kind !== "directory") {
|
||||
setAutomationError("持续接入必须选择整个目录,不能选择单个文件。");
|
||||
@ -2512,6 +2524,7 @@ export default function ProjectDetail() {
|
||||
</button>
|
||||
</div>
|
||||
</label>
|
||||
<CaptureDeliveryField bucket={automationDraft.bucket} prefix={automationDraft.deliveryPrefix || ""} projectDirectory={automationDraft.deliveryProjectDirectory || `${project.name.replace(/[\\/:*?"<>|]/g, "-")}__${project.id}`} onChange={value=>setAutomationDraft(current=>({...current,deliveryPrefix:value}))} onBrowse={()=>setReferencePickerTarget({inputId:"__capture_delivery__",inputType:"file"})} />
|
||||
<details className="md:col-span-2 order-last">
|
||||
<summary className="cursor-pointer text-sm text-[color:var(--app-text)]">
|
||||
处理设置(格式与并发)
|
||||
@ -2741,6 +2754,10 @@ export default function ProjectDetail() {
|
||||
>
|
||||
<td className="px-4 py-2.5">
|
||||
<div className="font-medium text-[color:var(--app-text)]">{record.captureId}</div>
|
||||
{record.delivery && <div className="mt-1 max-w-[280px] space-y-1 break-all text-[10px]">
|
||||
<div>交付位置:bos://{record.delivery.bucket}/{record.delivery.prefix}</div>
|
||||
{record.businessStage === "delivered" && <Link className="text-blue-600 underline" to={`/object-storage?connectionId=${encodeURIComponent(record.delivery.connectionId)}&bucket=${encodeURIComponent(record.delivery.bucket)}&prefix=${encodeURIComponent(record.delivery.prefix)}`}>打开交付目录</Link>}
|
||||
</div>}
|
||||
<div className="mt-0.5 max-w-[240px] truncate font-mono text-[10px] text-slate-500 dark:text-gray-600">
|
||||
{record.prefix}
|
||||
</div>
|
||||
@ -3060,17 +3077,17 @@ export default function ProjectDetail() {
|
||||
connections={storageConnections}
|
||||
allowDirectory
|
||||
value={
|
||||
referencePickerTarget?.inputId === "__capture_automation__"
|
||||
(referencePickerTarget?.inputId === "__capture_automation__" || referencePickerTarget?.inputId === "__capture_delivery__")
|
||||
? {
|
||||
kind: "directory",
|
||||
connectionId: automationDraft.connectionId,
|
||||
bucket: automationDraft.bucket,
|
||||
prefix: automationDraft.prefix,
|
||||
prefix: referencePickerTarget?.inputId === "__capture_delivery__" ? automationDraft.deliveryPrefix : automationDraft.prefix,
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
title={
|
||||
referencePickerTarget?.inputId === "__capture_automation__" ? "选择持续接入目录" : "选择本次运行输入存储路径"
|
||||
referencePickerTarget?.inputId === "__capture_delivery__" ? "选择交付保存目录" : referencePickerTarget?.inputId === "__capture_automation__" ? "选择持续接入目录" : "选择本次运行输入存储路径"
|
||||
}
|
||||
onClose={() => setReferencePickerTarget(null)}
|
||||
onSelect={handleReferencePickerSelect}
|
||||
|
||||
@ -989,6 +989,8 @@ export interface ScanConfig {
|
||||
}
|
||||
|
||||
export interface CaptureAutomationConfig {
|
||||
deliveryPrefix?: string;
|
||||
deliveryProjectDirectory?: string;
|
||||
enabled: boolean;
|
||||
hasStarted?: boolean;
|
||||
connectionId: string;
|
||||
@ -1086,6 +1088,7 @@ export interface CaptureActionRecord {
|
||||
}
|
||||
|
||||
export interface CaptureRecord {
|
||||
delivery?: { connectionId: string; bucket: string; prefix: string };
|
||||
id: string;
|
||||
projectId: string;
|
||||
workflowId: string;
|
||||
|
||||
21
tests/frontend/capture-delivery-config.test.mjs
Normal file
21
tests/frontend/capture-delivery-config.test.mjs
Normal file
@ -0,0 +1,21 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
test("continuous settings expose delivery root and exact project/capture path", () => {
|
||||
const source =
|
||||
readFileSync("src/pages/ProjectDetail.tsx", "utf8") +
|
||||
readFileSync("src/components/CaptureDeliveryField.tsx", "utf8");
|
||||
for (const text of [
|
||||
"交付保存目录",
|
||||
"__capture_delivery__",
|
||||
"deliveryProjectDirectory",
|
||||
"record.delivery.prefix",
|
||||
"打开交付目录",
|
||||
])
|
||||
assert.ok(source.includes(text), text);
|
||||
assert.ok(source.includes("selection.connectionId !== automationDraft.connectionId"));
|
||||
assert.ok(source.includes("selection.bucket !== automationDraft.bucket"));
|
||||
assert.ok(source.includes("!configToSave.deliveryPrefix?.trim()"));
|
||||
assert.ok(source.includes('record.businessStage === "delivered"'));
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user