forked from longtao.wu/droboticflow
851 lines
31 KiB
Go
851 lines
31 KiB
Go
package worker
|
||
|
||
import (
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"regexp"
|
||
"sort"
|
||
"strings"
|
||
"time"
|
||
|
||
"droboticflow/internal/domain"
|
||
"droboticflow/internal/sourceidentity"
|
||
|
||
"github.com/baidubce/bce-sdk-go/services/bos"
|
||
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
|
||
)
|
||
|
||
const captureAutomationNodeID = "__capture_automation__"
|
||
|
||
const captureRequeueStaleAfter = 2 * time.Minute
|
||
|
||
type captureObject = sourceidentity.Object
|
||
|
||
type captureGroup struct {
|
||
SourcePrefix string
|
||
SourceKeys []string
|
||
CaptureID string
|
||
Prefix string
|
||
ManifestKey string
|
||
Fingerprint string
|
||
Missing []string
|
||
Objects []captureObject
|
||
TotalBytes int64
|
||
}
|
||
|
||
type captureRequirement struct {
|
||
Name string
|
||
Pattern string
|
||
}
|
||
|
||
var defaultOrbbecCaptureRequirements = []captureRequirement{
|
||
{Name: "左相机视频", Pattern: `(?i)(^|/).*camera_left.*\.(mp4|mov|mkv)$`},
|
||
{Name: "右相机视频", Pattern: `(?i)(^|/).*camera_right.*\.(mp4|mov|mkv)$`},
|
||
{Name: "左相机 PTS", Pattern: `(?i)(^|/).*camera_left.*pts\.csv$`},
|
||
{Name: "右相机 PTS", Pattern: `(?i)(^|/).*camera_right.*pts\.csv$`},
|
||
{Name: "IMU", Pattern: `(?i)(^|/)([^/]*_)?imu(?:_part[0-9]+)?\.csv$`},
|
||
{Name: "相机标定", Pattern: `(?i)(^|/).*calibration_camera(?:\.(yaml|yml|json))?$`},
|
||
{Name: "IMU 标定", Pattern: `(?i)(^|/).*calibration_imu(?:\.(yaml|yml|json))?$`},
|
||
}
|
||
|
||
func normalizedCaptureAutomation(config *domain.CaptureAutomationConfig) domain.CaptureAutomationConfig {
|
||
if config == nil {
|
||
return domain.CaptureAutomationConfig{}
|
||
}
|
||
next := *config
|
||
next.ConnectionID = strings.TrimSpace(next.ConnectionID)
|
||
next.Bucket = strings.TrimSpace(next.Bucket)
|
||
next.Prefix = strings.Trim(strings.TrimSpace(next.Prefix), "/")
|
||
next.Schema = strings.TrimSpace(next.Schema)
|
||
if next.Schema == "" {
|
||
next.Schema = "orbbec_ego_stereo_v1"
|
||
}
|
||
// One completion policy, including workflows saved before this change.
|
||
next.CompletionMode = "required_files"
|
||
next.ManifestName = strings.Trim(strings.TrimSpace(next.ManifestName), "/")
|
||
if next.ManifestName == "" {
|
||
next.ManifestName = "source-manifest.json"
|
||
}
|
||
next.WorkflowInputID = strings.TrimSpace(next.WorkflowInputID)
|
||
if next.WorkflowInputID == "" {
|
||
next.WorkflowInputID = "sourceDirectory"
|
||
}
|
||
if next.PollIntervalSeconds <= 0 {
|
||
next.PollIntervalSeconds = 30
|
||
}
|
||
if next.PollIntervalSeconds < 5 {
|
||
next.PollIntervalSeconds = 5
|
||
}
|
||
if next.StabilityWindowSeconds <= 0 {
|
||
next.StabilityWindowSeconds = 30
|
||
}
|
||
return next
|
||
}
|
||
|
||
func captureRequirements(config domain.CaptureAutomationConfig) []captureRequirement {
|
||
if len(config.RequiredPatterns) == 0 {
|
||
return append([]captureRequirement(nil), defaultOrbbecCaptureRequirements...)
|
||
}
|
||
items := make([]captureRequirement, 0, len(config.RequiredPatterns))
|
||
for index, pattern := range config.RequiredPatterns {
|
||
if strings.TrimSpace(pattern) == "" {
|
||
continue
|
||
}
|
||
items = append(items, captureRequirement{Name: fmt.Sprintf("必需数据 %d", index+1), Pattern: pattern})
|
||
}
|
||
return items
|
||
}
|
||
|
||
func captureFingerprint(objects []captureObject) string { return sourceidentity.Fingerprint(objects) }
|
||
|
||
func captureRecordID(workflowID, prefix string) string {
|
||
sum := sha256.Sum256([]byte(strings.TrimSpace(workflowID) + "|" + strings.Trim(strings.TrimSpace(prefix), "/")))
|
||
return "capture-" + hex.EncodeToString(sum[:12])
|
||
}
|
||
|
||
func captureAutomationFingerprint(workflowID string, config domain.CaptureAutomationConfig) string {
|
||
sum := sha256.Sum256([]byte(strings.Join([]string{workflowID, config.ConnectionID, config.Bucket, config.Prefix, config.CompletionMode, config.ManifestName, config.WorkflowInputID, config.DeliveryPrefix, config.DeliveryProjectDirectory}, "|")))
|
||
return hex.EncodeToString(sum[:])
|
||
}
|
||
|
||
func discoverFlatCaptureGroups(objects []captureObject, config domain.CaptureAutomationConfig) ([]captureGroup, error) {
|
||
requirements := captureRequirements(config)
|
||
compiledRequirements := make([]*regexp.Regexp, 0, len(requirements))
|
||
for _, requirement := range requirements {
|
||
compiled, err := regexp.Compile(requirement.Pattern)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("invalid required pattern %q: %w", requirement.Pattern, err)
|
||
}
|
||
compiledRequirements = append(compiledRequirements, compiled)
|
||
}
|
||
root := strings.Trim(config.Prefix, "/")
|
||
rootWithSlash := ""
|
||
if root != "" {
|
||
rootWithSlash = root + "/"
|
||
}
|
||
grouped := map[string][]captureObject{}
|
||
for _, object := range objects {
|
||
key := strings.Trim(strings.TrimSpace(object.Key), "/")
|
||
if key == "" || (rootWithSlash != "" && !strings.HasPrefix(key, rootWithSlash)) {
|
||
continue
|
||
}
|
||
relative := strings.TrimPrefix(key, rootWithSlash)
|
||
parts := strings.SplitN(relative, "/", 2)
|
||
if len(parts) != 2 || parts[0] == "_droboticflow" {
|
||
continue
|
||
}
|
||
if strings.SplitN(parts[1], "/", 2)[0] == "_droboticflow" {
|
||
continue
|
||
}
|
||
grouped[parts[0]] = append(grouped[parts[0]], captureObject{Key: key, ETag: object.ETag, Size: object.Size})
|
||
}
|
||
captureIDs := make([]string, 0, len(grouped))
|
||
for captureID := range grouped {
|
||
captureIDs = append(captureIDs, captureID)
|
||
}
|
||
sort.Strings(captureIDs)
|
||
items := make([]captureGroup, 0, len(captureIDs))
|
||
for _, captureID := range captureIDs {
|
||
capturePrefix := strings.Trim(strings.Join([]string{root, captureID}, "/"), "/")
|
||
captureObjects := grouped[captureID]
|
||
relativeKeys := make([]string, 0, len(captureObjects))
|
||
manifestKey := strings.Trim(strings.Join([]string{capturePrefix, config.ManifestName}, "/"), "/")
|
||
totalBytes := int64(0)
|
||
for _, object := range captureObjects {
|
||
relative := strings.TrimPrefix(object.Key, capturePrefix+"/")
|
||
relativeKeys = append(relativeKeys, relative)
|
||
totalBytes += object.Size
|
||
}
|
||
missing := make([]string, 0)
|
||
for index, requirement := range requirements {
|
||
matched := false
|
||
for objectIndex, relative := range relativeKeys {
|
||
if compiledRequirements[index].MatchString(relative) {
|
||
matched = true
|
||
if captureObjects[objectIndex].Size <= 0 {
|
||
missing = append(missing, "空文件 "+relative)
|
||
}
|
||
}
|
||
}
|
||
if !matched {
|
||
missing = append(missing, requirement.Name)
|
||
}
|
||
}
|
||
items = append(items, captureGroup{
|
||
CaptureID: captureID, Prefix: capturePrefix, ManifestKey: manifestKey,
|
||
Fingerprint: captureFingerprint(captureObjects), Missing: missing,
|
||
Objects: captureObjects, TotalBytes: totalBytes,
|
||
})
|
||
}
|
||
return items, nil
|
||
}
|
||
|
||
func (runtime *Runtime) captureBOSClient(connection domain.StorageConnectionRecord) (*bos.Client, error) {
|
||
endpoint := runtime.storageConnectionEndpoint(&connection)
|
||
if strings.TrimSpace(endpoint) == "" {
|
||
return nil, fmt.Errorf("BOS endpoint is required")
|
||
}
|
||
return bos.NewClient(strings.TrimSpace(connection.AccessKeyID), strings.TrimSpace(connection.SecretAccessKey), endpoint)
|
||
}
|
||
|
||
func listCaptureObjects(ctx context.Context, client *bos.Client, bucket, prefix string) ([]captureObject, error) {
|
||
items := make([]captureObject, 0)
|
||
marker := ""
|
||
requestPrefix := strings.Trim(strings.TrimSpace(prefix), "/")
|
||
if requestPrefix != "" {
|
||
requestPrefix += "/"
|
||
}
|
||
for {
|
||
result, err := client.ListObjectsWithContext(ctx, strings.TrimSpace(bucket), &bosapi.ListObjectsArgs{Prefix: requestPrefix, Marker: marker, MaxKeys: 1000})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if result == nil {
|
||
break
|
||
}
|
||
for _, object := range result.Contents {
|
||
items = append(items, captureObject{Key: strings.TrimSpace(object.Key), ETag: strings.Trim(object.ETag, `"`), Size: int64(object.Size)})
|
||
}
|
||
if !result.IsTruncated || strings.TrimSpace(result.NextMarker) == "" || result.NextMarker == marker {
|
||
break
|
||
}
|
||
marker = result.NextMarker
|
||
}
|
||
return items, nil
|
||
}
|
||
|
||
func validateCaptureManifest(ctx context.Context, client *bos.Client, bucket string, group captureGroup) []string {
|
||
if group.ManifestKey == "" {
|
||
return []string{"完成清单"}
|
||
}
|
||
result, err := client.GetObjectWithContext(ctx, bucket, group.ManifestKey, nil)
|
||
if err != nil {
|
||
return []string{"无法读取完成清单"}
|
||
}
|
||
defer result.Body.Close()
|
||
payload, err := io.ReadAll(io.LimitReader(result.Body, 5<<20))
|
||
if err != nil {
|
||
return []string{"无法读取完成清单"}
|
||
}
|
||
var manifest map[string]any
|
||
if err := json.Unmarshal(payload, &manifest); err != nil {
|
||
return []string{"完成清单不是有效 JSON"}
|
||
}
|
||
status := strings.ToLower(strings.TrimSpace(fmt.Sprint(manifest["status"])))
|
||
if status != "" && status != "complete" && status != "completed" && status != "ready" && status != "finalized" && status != "success" {
|
||
return []string{"完成清单尚未封存"}
|
||
}
|
||
known := make(map[string]struct{}, len(group.Objects))
|
||
for _, object := range group.Objects {
|
||
known[object.Key] = struct{}{}
|
||
}
|
||
missing := make([]string, 0)
|
||
if rawFiles, ok := manifest["files"].([]any); ok {
|
||
for _, raw := range rawFiles {
|
||
fileName := ""
|
||
switch typed := raw.(type) {
|
||
case string:
|
||
fileName = typed
|
||
case map[string]any:
|
||
for _, key := range []string{"path", "key", "relativePath", "name"} {
|
||
if value := strings.TrimSpace(fmt.Sprint(typed[key])); value != "" && value != "<nil>" {
|
||
fileName = value
|
||
break
|
||
}
|
||
}
|
||
}
|
||
fileName = strings.Trim(strings.TrimSpace(fileName), "/")
|
||
if fileName == "" {
|
||
continue
|
||
}
|
||
key := fileName
|
||
if !strings.HasPrefix(key, group.Prefix+"/") {
|
||
key = strings.Trim(strings.Join([]string{group.Prefix, fileName}, "/"), "/")
|
||
}
|
||
if _, exists := known[key]; !exists {
|
||
missing = append(missing, "清单文件 "+fileName)
|
||
}
|
||
}
|
||
}
|
||
return missing
|
||
}
|
||
|
||
func runStatusForCapture(run *domain.RunRecord) domain.CaptureRecordStatus {
|
||
if run == nil {
|
||
return domain.CaptureRecordQueued
|
||
}
|
||
switch run.Status {
|
||
case domain.RunStatusQueued:
|
||
return domain.CaptureRecordQueued
|
||
case domain.RunStatusRunning, domain.RunStatusStopping:
|
||
return domain.CaptureRecordRunning
|
||
case domain.RunStatusWaitingForAudit, domain.RunStatusWaitingForHuman:
|
||
return domain.CaptureRecordWaiting
|
||
case domain.RunStatusCompleted:
|
||
return domain.CaptureRecordCompleted
|
||
case domain.RunStatusFailed:
|
||
return domain.CaptureRecordFailed
|
||
case domain.RunStatusCancelled:
|
||
return domain.CaptureRecordCancelled
|
||
default:
|
||
return domain.CaptureRecordQueued
|
||
}
|
||
}
|
||
|
||
func captureCancelledByUser(record *domain.CaptureRecord) bool {
|
||
if record == nil || record.Status != domain.CaptureRecordCancelled {
|
||
return false
|
||
}
|
||
for index := len(record.Actions) - 1; index >= 0; index-- {
|
||
switch record.Actions[index].Action {
|
||
case domain.CaptureActionCancelled:
|
||
return true
|
||
case domain.CaptureActionReprocessRequested, domain.CaptureActionReprocessQueued:
|
||
return false
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func captureStatusBeforeReprocess(record *domain.CaptureRecord) domain.CaptureRecordStatus {
|
||
if record != nil {
|
||
for index := len(record.Actions) - 1; index >= 0; index-- {
|
||
if record.Actions[index].Action == domain.CaptureActionReprocessRequested {
|
||
status := record.Actions[index].FromStatus
|
||
if status == domain.CaptureRecordFailed || status == domain.CaptureRecordCancelled {
|
||
return status
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return domain.CaptureRecordCancelled
|
||
}
|
||
|
||
func recoverStaleCaptureRequeue(record *domain.CaptureRecord, now time.Time) bool {
|
||
if record == nil || record.Status != domain.CaptureRecordRequeueing {
|
||
return false
|
||
}
|
||
updatedAt, ok := parseRFC3339Nano(record.UpdatedAt)
|
||
if ok && now.Sub(updatedAt) < captureRequeueStaleAfter {
|
||
return false
|
||
}
|
||
previousStatus := captureStatusBeforeReprocess(record)
|
||
reason := "重新处理请求超过 2 分钟未创建新 Run,已恢复原状态"
|
||
record.Status = previousStatus
|
||
record.Error = reason
|
||
record.Actions = append(record.Actions, domain.CaptureActionRecord{
|
||
ID: domain.NewID("capture-action"), Action: domain.CaptureActionReprocessFailed,
|
||
FromStatus: domain.CaptureRecordRequeueing, ToStatus: previousStatus,
|
||
ActorUserID: "system", Reason: reason, RunID: record.RunID, CreatedAt: nowISO(),
|
||
})
|
||
if len(record.Actions) > 100 {
|
||
record.Actions = append([]domain.CaptureActionRecord(nil), record.Actions[len(record.Actions)-100:]...)
|
||
}
|
||
record.UpdatedAt = nowISO()
|
||
return true
|
||
}
|
||
|
||
func workflowInputDefinition(workflow domain.WorkflowRecord, inputID string) *domain.WorkflowInputDefinition {
|
||
for index := range workflow.WorkflowInputs {
|
||
if workflow.WorkflowInputs[index].ID == inputID {
|
||
return &workflow.WorkflowInputs[index]
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (runtime *Runtime) reconcileCaptureRun(ctx context.Context, record *domain.CaptureRecord) error {
|
||
if record == nil || (strings.TrimSpace(record.RunID) == "" && record.Status != domain.CaptureRecordLaunching) {
|
||
return nil
|
||
}
|
||
run, err := runtime.store.GetRunByWorkflowTaskID(ctx, record.WorkflowID, record.ID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if run == nil && record.RunID != "" {
|
||
run, err = runtime.store.GetRun(ctx, record.RunID)
|
||
}
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if run == nil {
|
||
if record.Status == domain.CaptureRecordLaunching && strings.TrimSpace(record.RunID) == "" {
|
||
return fmt.Errorf("capture_launch_unresolved: Capture %s under workflow %s has no observable Run; quiesce Workers and recover this claim under its original workflow before activation; do not reset or repeat admission", record.ID, record.WorkflowID)
|
||
}
|
||
return nil
|
||
}
|
||
if run.WorkflowID != record.WorkflowID || run.ProjectID != record.ProjectID ||
|
||
(run.OrganizationID != "" && record.OrganizationID != "" && run.OrganizationID != record.OrganizationID) {
|
||
return fmt.Errorf("capture_run_scope_mismatch: %s", record.ID)
|
||
}
|
||
before := *record
|
||
record.RunID = run.ID
|
||
if !captureCancelledByUser(record) {
|
||
record.Status = runStatusForCapture(run)
|
||
}
|
||
record.BusinessStage = domain.CaptureBusinessStageForStatus(record.Status)
|
||
if domain.RunHasNoDeliverables(run) {
|
||
record.BusinessStage = domain.CaptureStageNoDeliverables
|
||
}
|
||
if record.Status == domain.CaptureRecordWaiting && record.SamplingDecision != nil {
|
||
if record.SamplingDecision.Decision == domain.CaptureSamplingNotSelected {
|
||
record.BusinessStage = domain.CaptureStagePackaging
|
||
} else {
|
||
record.BusinessStage = domain.CaptureStageWaitingAcceptance
|
||
}
|
||
}
|
||
record.StartedAt = run.StartedAt
|
||
record.Error = run.Error
|
||
if run.Failure != nil && strings.TrimSpace(run.Failure.Message) != "" {
|
||
record.Error = run.Failure.Message
|
||
}
|
||
record.CompletedAt = ""
|
||
if run.Status == domain.RunStatusCompleted || run.Status == domain.RunStatusFailed || run.Status == domain.RunStatusCancelled {
|
||
record.CompletedAt = run.EndedAt
|
||
}
|
||
if record.RunID == before.RunID && record.Status == before.Status &&
|
||
record.BusinessStage == before.BusinessStage && record.StartedAt == before.StartedAt &&
|
||
record.Error == before.Error && record.CompletedAt == before.CompletedAt {
|
||
return nil
|
||
}
|
||
record.UpdatedAt = nowISO()
|
||
updated, err := runtime.store.UpdateCaptureRunStatus(ctx, before, *record)
|
||
if err != nil {
|
||
*record = before
|
||
return err
|
||
}
|
||
if !updated {
|
||
*record = before
|
||
return fmt.Errorf("capture_status_conflict: %s", record.ID)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (runtime *Runtime) launchCaptureRun(ctx context.Context, workflow domain.WorkflowRecord, config domain.CaptureAutomationConfig, record *domain.CaptureRecord) error {
|
||
if workflow.ProjectID != "" {
|
||
p, err := runtime.store.GetProject(ctx, workflow.ProjectID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if p == nil || p.DeletionTaskID != "" {
|
||
return nil
|
||
}
|
||
}
|
||
if record == nil || record.Status != domain.CaptureRecordReady || record.WorkflowID != workflow.ID || record.OrganizationID != workflow.OrganizationID || record.ProjectID != workflow.ProjectID {
|
||
return nil
|
||
}
|
||
existingRun, err := runtime.store.GetRunByWorkflowTaskID(ctx, workflow.ID, record.ID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if existingRun != nil {
|
||
record.RunID = existingRun.ID
|
||
record.Status = runStatusForCapture(existingRun)
|
||
record.UpdatedAt = nowISO()
|
||
return runtime.store.UpsertCaptureRecord(ctx, *record)
|
||
}
|
||
// A scan may have started before the administrator paused intake or
|
||
// changed its root. Re-read admission settings before claiming new work.
|
||
latest, err := runtime.store.GetWorkflow(ctx, workflow.ID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if latest == nil || latest.Hidden || latest.DefinitionState != domain.WorkflowDefinitionStateReleased || latest.Type != domain.WorkflowTypeStreaming || latest.CaptureAutomation == nil || !latest.CaptureAutomation.Enabled {
|
||
return nil
|
||
}
|
||
if captureAutomationFingerprint(workflow.ID, normalizedCaptureAutomation(latest.CaptureAutomation)) != captureAutomationFingerprint(workflow.ID, config) {
|
||
return nil
|
||
}
|
||
if err := validateV4CaptureSourceSelection(workflow, record); err != nil {
|
||
return err
|
||
}
|
||
if err := validateRawHumanCaptureMetadataSelection(workflow, record); err != nil {
|
||
return err
|
||
}
|
||
claimed, err := runtime.store.ClaimCaptureRecordForLaunch(ctx, record.ID, record.Fingerprint)
|
||
if err != nil || !claimed {
|
||
return err
|
||
}
|
||
persistedClaim, readErr := runtime.store.GetCaptureRecord(ctx, record.ID)
|
||
if readErr != nil {
|
||
return readErr
|
||
}
|
||
if persistedClaim == nil || persistedClaim.Status != domain.CaptureRecordLaunching || persistedClaim.WorkflowID != workflow.ID || persistedClaim.RunID != "" {
|
||
return fmt.Errorf("capture claim changed: %s", record.ID)
|
||
}
|
||
*record = *persistedClaim
|
||
input := workflowInputDefinition(workflow, config.WorkflowInputID)
|
||
if input == nil {
|
||
record.Status = domain.CaptureRecordFailed
|
||
record.Error = "工作流缺少目录输入 " + config.WorkflowInputID
|
||
record.UpdatedAt = nowISO()
|
||
return runtime.store.UpsertCaptureRecord(ctx, *record)
|
||
}
|
||
if record.Delivery == nil {
|
||
prefix, deliveryErr := domain.CaptureDeliveryPrefix(config, record.CaptureID)
|
||
if deliveryErr != nil {
|
||
record.Status = domain.CaptureRecordFailed
|
||
record.Error = deliveryErr.Error()
|
||
return runtime.store.UpsertCaptureRecord(ctx, *record)
|
||
}
|
||
record.Delivery = &domain.CaptureDeliveryTarget{ConnectionID: config.ConnectionID, Bucket: config.Bucket, Prefix: prefix}
|
||
record.Status = domain.CaptureRecordLaunching
|
||
if err := runtime.store.UpsertCaptureRecord(ctx, *record); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
payload := domain.ArtifactPayload{
|
||
Mode: domain.ArtifactTransferModeRef,
|
||
Type: input.Type,
|
||
Ref: &domain.ArtifactReference{
|
||
Provider: domain.StorageProviderBOS, ConnectionID: config.ConnectionID,
|
||
Bucket: config.Bucket, Prefix: record.Prefix,
|
||
VersionID: record.Fingerprint, Kind: domain.ArtifactRefKindDirectory,
|
||
},
|
||
}
|
||
if record.SourcePrefix != "" {
|
||
payload.Ref.Prefix = record.SourcePrefix
|
||
payload.Ref.ObjectKeys = append([]string(nil), record.SourceKeys...)
|
||
payload.Ref.CaptureUnit = record.CaptureID
|
||
}
|
||
run, err := runtime.store.CreateRunWithInputs(ctx, workflow.ID, domain.RunTriggerScheduled, map[string]domain.ArtifactPayload{config.WorkflowInputID: payload}, record.ID)
|
||
if err != nil {
|
||
// CreateRun may have committed before its response failed. Keep the
|
||
// persisted launch claim and adopt an observable Run; never reopen
|
||
// admission on an unknown result or infer failure from claim age.
|
||
persisted, readErr := runtime.store.GetCaptureRecord(ctx, record.ID)
|
||
if readErr != nil {
|
||
return readErr
|
||
}
|
||
if persisted == nil {
|
||
return fmt.Errorf("capture claim disappeared: %s", record.ID)
|
||
}
|
||
*record = *persisted
|
||
if readbackErr := runtime.reconcileCaptureRun(ctx, record); readbackErr != nil {
|
||
return fmt.Errorf("create Capture Run failed (%v): %w", err, readbackErr)
|
||
}
|
||
return nil
|
||
}
|
||
before := *record
|
||
record.RunID = run.ID
|
||
record.Status = domain.CaptureRecordQueued
|
||
record.BusinessStage = domain.CaptureStageQueued
|
||
record.ContractRevision = domain.FanyuV58CaptureContractRevision
|
||
record.Error = ""
|
||
record.UpdatedAt = nowISO()
|
||
updated, writeErr := runtime.store.UpdateCaptureRunStatus(ctx, before, *record)
|
||
if writeErr != nil {
|
||
*record = before
|
||
return writeErr
|
||
}
|
||
if !updated {
|
||
current, readErr := runtime.store.GetCaptureRecord(ctx, record.ID)
|
||
if readErr != nil {
|
||
return readErr
|
||
}
|
||
if current == nil || current.RunID != run.ID || current.WorkflowID != workflow.ID {
|
||
return fmt.Errorf("capture_status_conflict: %s", record.ID)
|
||
}
|
||
*record = *current
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (runtime *Runtime) captureAutomationDue(ctx context.Context, workflow domain.WorkflowRecord, config domain.CaptureAutomationConfig, fingerprint string, now time.Time) (bool, error) {
|
||
state, err := runtime.store.GetSourceState(ctx, workflow.ID, captureAutomationNodeID, fingerprint)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
if state == nil {
|
||
return true, nil
|
||
}
|
||
last, ok := parseRFC3339Nano(state.LastScanCompletedAt)
|
||
return !ok || now.Sub(last) >= time.Duration(config.PollIntervalSeconds)*time.Second, nil
|
||
}
|
||
|
||
// Intake moves between versions, but records with Runs remain with the original
|
||
// workflow. Read only this project's exact tenant/family history for deduplication
|
||
// and reconcile hidden in-flight owners without migrating their lineage.
|
||
func (runtime *Runtime) captureAutomationFamilyRecords(ctx context.Context, workflow domain.WorkflowRecord) ([]domain.CaptureRecord, error) {
|
||
domain.NormalizeWorkflowDefinitionMetadata(&workflow)
|
||
versions, err := runtime.store.ListWorkflowsByProject(ctx, workflow.ProjectID)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var records []domain.CaptureRecord
|
||
for _, version := range versions {
|
||
domain.NormalizeWorkflowDefinitionMetadata(&version)
|
||
if version.OrganizationID != workflow.OrganizationID || version.ProjectID != workflow.ProjectID || version.DefinitionFamilyID != workflow.DefinitionFamilyID || version.DefinitionState != domain.WorkflowDefinitionStateReleased {
|
||
continue
|
||
}
|
||
items, err := runtime.store.ListCaptureRecords(ctx, version.OrganizationID, version.ID, 0)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
for _, item := range items {
|
||
if item.OrganizationID != workflow.OrganizationID || item.ProjectID != workflow.ProjectID || item.WorkflowID != version.ID {
|
||
continue
|
||
}
|
||
// Reconcile claims for the current listener as well as hidden owners.
|
||
// Missing Runs surface an actionable error without unfenced retries.
|
||
if item.Status == domain.CaptureRecordLaunching {
|
||
if err := runtime.reconcileCaptureRun(ctx, &item); err != nil {
|
||
return nil, err
|
||
}
|
||
} else if version.ID != workflow.ID {
|
||
switch item.Status {
|
||
case domain.CaptureRecordQueued, domain.CaptureRecordRunning, domain.CaptureRecordWaiting:
|
||
if err := runtime.reconcileCaptureRun(ctx, &item); err != nil {
|
||
return nil, err
|
||
}
|
||
case domain.CaptureRecordRequeueing:
|
||
if recoverStaleCaptureRequeue(&item, time.Now().UTC()) {
|
||
if err := runtime.store.UpsertCaptureRecord(ctx, item); err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
}
|
||
}
|
||
records = append(records, item)
|
||
}
|
||
}
|
||
return records, nil
|
||
}
|
||
|
||
func selectCanonicalFamilyCaptureRecord(activeWorkflowID string, previous, candidate domain.CaptureRecord) (domain.CaptureRecord, error) {
|
||
if previous.ID == candidate.ID {
|
||
return candidate, nil
|
||
}
|
||
prevUnstarted := domain.CaptureAwaitingFirstRun(previous)
|
||
candUnstarted := domain.CaptureAwaitingFirstRun(candidate)
|
||
switch {
|
||
case !prevUnstarted && candUnstarted:
|
||
return previous, nil
|
||
case prevUnstarted && !candUnstarted:
|
||
return candidate, nil
|
||
case prevUnstarted && candUnstarted:
|
||
if previous.WorkflowID == activeWorkflowID && candidate.WorkflowID != activeWorkflowID {
|
||
return previous, nil
|
||
}
|
||
if candidate.WorkflowID == activeWorkflowID && previous.WorkflowID != activeWorkflowID {
|
||
return candidate, nil
|
||
}
|
||
if candidate.UpdatedAt > previous.UpdatedAt || (candidate.UpdatedAt == previous.UpdatedAt && candidate.ID < previous.ID) {
|
||
return candidate, nil
|
||
}
|
||
return previous, nil
|
||
default:
|
||
return domain.CaptureRecord{}, fmt.Errorf("capture workflow family has conflicting records for prefix %s", candidate.Prefix)
|
||
}
|
||
}
|
||
|
||
func (runtime *Runtime) processCaptureAutomation(ctx context.Context, workflow domain.WorkflowRecord) error {
|
||
if workflow.Type != domain.WorkflowTypeStreaming {
|
||
return nil
|
||
}
|
||
config := normalizedCaptureAutomation(workflow.CaptureAutomation)
|
||
if !config.Enabled {
|
||
return nil
|
||
}
|
||
if workflow.Hidden || workflow.DefinitionState != domain.WorkflowDefinitionStateReleased {
|
||
return nil
|
||
}
|
||
fingerprint := captureAutomationFingerprint(workflow.ID, config)
|
||
now := time.Now().UTC()
|
||
due, err := runtime.captureAutomationDue(ctx, workflow, config, fingerprint, now)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !due {
|
||
return nil
|
||
}
|
||
connection, err := runtime.store.GetStorageConnection(ctx, config.ConnectionID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if connection == nil {
|
||
return fmt.Errorf("capture automation storage connection %s not found", config.ConnectionID)
|
||
}
|
||
client, err := runtime.captureBOSClient(*connection)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
objects, err := listCaptureObjects(ctx, client, config.Bucket, config.Prefix)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
groups, err := discoverCaptureGroupsForWorkflow(objects, config, workflow)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
// Listing can outlive activation or an intake pause. Do not publish records
|
||
// from a captured old listener after the handoff. Deployment still quiesces
|
||
// scans because this check and the following store writes are not a transaction.
|
||
latest, err := runtime.store.GetWorkflow(ctx, workflow.ID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if latest == nil || latest.Hidden || latest.DefinitionState != domain.WorkflowDefinitionStateReleased || latest.CaptureAutomation == nil || !latest.CaptureAutomation.Enabled || captureAutomationFingerprint(workflow.ID, normalizedCaptureAutomation(latest.CaptureAutomation)) != fingerprint {
|
||
return nil
|
||
}
|
||
priorRecords, err := runtime.captureAutomationFamilyRecords(ctx, workflow)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
byPrefix := map[string]*domain.CaptureRecord{}
|
||
admittedPartParents := map[string]bool{}
|
||
for _, prior := range priorRecords {
|
||
if prior.ConnectionID != config.ConnectionID || prior.Bucket != config.Bucket {
|
||
continue
|
||
}
|
||
copy := prior
|
||
if previous := byPrefix[prior.Prefix]; previous != nil && previous.ID != prior.ID {
|
||
selected, selectErr := selectCanonicalFamilyCaptureRecord(workflow.ID, *previous, prior)
|
||
if selectErr != nil {
|
||
return selectErr
|
||
}
|
||
copy = selected
|
||
}
|
||
byPrefix[prior.Prefix] = ©
|
||
if prior.SourcePrefix != "" && prior.SourcePrefix != prior.Prefix && preserveCaptureParent(&prior) {
|
||
admittedPartParents[prior.SourcePrefix] = true
|
||
}
|
||
}
|
||
newCount := 0
|
||
historicalParents := map[string]bool{}
|
||
for _, group := range groups {
|
||
if admittedPartParents[group.Prefix] {
|
||
group.Missing = append(group.Missing, "该目录已有 part 接入记录,不能重新作为整目录接入")
|
||
}
|
||
if group.SourcePrefix != "" && group.SourcePrefix != group.Prefix {
|
||
parent := byPrefix[group.SourcePrefix]
|
||
if preserveCaptureParent(parent) {
|
||
if !historicalParents[parent.ID] && parent.RunID != "" && parent.WorkflowID == workflow.ID {
|
||
if e := runtime.reconcileCaptureRun(ctx, parent); e != nil {
|
||
return e
|
||
}
|
||
}
|
||
historicalParents[parent.ID] = true
|
||
continue
|
||
}
|
||
}
|
||
existing := byPrefix[group.Prefix]
|
||
if existing != nil && existing.WorkflowID != workflow.ID {
|
||
// Historical admissions (including a launch claim with no Run yet)
|
||
// never switch protocols. Unstarted transfers belong to activation.
|
||
continue
|
||
}
|
||
if existing != nil && existing.Status == domain.CaptureRecordDiscarded {
|
||
continue
|
||
}
|
||
if existing != nil && existing.RunID != "" {
|
||
if existing.Status == domain.CaptureRecordRequeueing {
|
||
if !recoverStaleCaptureRequeue(existing, now) {
|
||
continue
|
||
}
|
||
if err := runtime.store.UpsertCaptureRecord(ctx, *existing); err != nil {
|
||
return err
|
||
}
|
||
continue
|
||
}
|
||
if existing.Fingerprint != group.Fingerprint {
|
||
before := *existing
|
||
existing.Error = "源目录在创建运行后发生变化,请复制为新的 capture 目录后重新处理"
|
||
existing.UpdatedAt = nowISO()
|
||
updated, err := runtime.store.UpdateCaptureRunStatus(ctx, before, *existing)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if !updated {
|
||
return fmt.Errorf("capture_status_conflict: %s", existing.ID)
|
||
}
|
||
}
|
||
if err := runtime.reconcileCaptureRun(ctx, existing); err != nil {
|
||
return err
|
||
}
|
||
continue
|
||
}
|
||
if existing != nil && !domain.CaptureAwaitingFirstRun(*existing) {
|
||
continue
|
||
}
|
||
missing := append([]string(nil), group.Missing...)
|
||
nowText := nowISO()
|
||
record := domain.CaptureRecord{
|
||
SourcePrefix: group.SourcePrefix, SourceKeys: group.SourceKeys,
|
||
ID: captureRecordID(workflow.ID, group.Prefix), OrganizationID: workflow.OrganizationID,
|
||
ProjectID: workflow.ProjectID, WorkflowID: workflow.ID, ConnectionID: config.ConnectionID,
|
||
Bucket: config.Bucket, Prefix: group.Prefix, CaptureID: group.CaptureID,
|
||
ManifestKey: group.ManifestKey, Fingerprint: group.Fingerprint, MissingTypes: missing,
|
||
FileCount: len(group.Objects), TotalBytes: group.TotalBytes, UpdatedAt: nowText,
|
||
ContractRevision: domain.FanyuV58CaptureContractRevision,
|
||
}
|
||
if existing != nil {
|
||
record.ID = existing.ID
|
||
record.Delivery = existing.Delivery
|
||
record.DiscoveredAt = existing.DiscoveredAt
|
||
record.Actions = append([]domain.CaptureActionRecord(nil), existing.Actions...)
|
||
record.StatusBeforeDiscard = existing.StatusBeforeDiscard
|
||
} else {
|
||
record.DiscoveredAt = nowText
|
||
newCount++
|
||
}
|
||
if len(missing) == 0 {
|
||
if existing == nil || existing.Fingerprint != group.Fingerprint {
|
||
record.Status = domain.CaptureRecordStabilizing
|
||
record.StabilityChecks = 1
|
||
record.StabilityObservedAt = nowText
|
||
} else {
|
||
record.StabilityChecks = existing.StabilityChecks + 1
|
||
record.StabilityObservedAt = existing.StabilityObservedAt
|
||
if record.StabilityObservedAt == "" {
|
||
record.StabilityObservedAt = nowText
|
||
}
|
||
stableSince, stableSinceOK := parseRFC3339Nano(record.StabilityObservedAt)
|
||
stableEnough := stableSinceOK && now.Sub(stableSince) >= time.Duration(config.StabilityWindowSeconds)*time.Second
|
||
if record.StabilityChecks >= 2 && stableEnough {
|
||
record.Status = domain.CaptureRecordReady
|
||
record.ReadyAt = nowText
|
||
} else {
|
||
record.Status = domain.CaptureRecordStabilizing
|
||
}
|
||
}
|
||
} else {
|
||
record.Status = domain.CaptureRecordIncomplete
|
||
record.StabilityChecks = 0
|
||
record.StabilityObservedAt = ""
|
||
}
|
||
if err := runtime.store.UpsertCaptureRecord(ctx, record); err != nil {
|
||
return err
|
||
}
|
||
if record.Status == domain.CaptureRecordReady {
|
||
if err := runtime.launchCaptureRun(ctx, workflow, config, &record); err != nil {
|
||
fmt.Printf("[worker] launch capture %s for workflow %s failed: %v\n", record.CaptureID, workflow.ID, err)
|
||
}
|
||
}
|
||
}
|
||
return runtime.store.UpsertSourceState(ctx, domain.SourceStateRecord{
|
||
ID: workflow.ID + ":" + captureAutomationNodeID + ":" + fingerprint,
|
||
OrganizationID: workflow.OrganizationID, WorkflowID: workflow.ID, NodeID: captureAutomationNodeID,
|
||
SourceFingerprint: fingerprint, DataSource: "BOS", SourcePath: "bos://" + config.Bucket + "/" + config.Prefix,
|
||
Bucket: config.Bucket, Prefix: config.Prefix, Pattern: "*",
|
||
ScanMode: "INCREMENTAL", IncrementalUnit: "DIRECTORY", DirectoryDepth: 1,
|
||
LastListedCount: len(groups), LastNewCount: newCount,
|
||
LastScanStartedAt: now.Format(time.RFC3339Nano), LastScanCompletedAt: nowISO(), UpdatedAt: nowISO(),
|
||
})
|
||
}
|
||
|
||
func (runtime *Runtime) enqueueCaptureAutomations(ctx context.Context, workflows []domain.WorkflowRecord) {
|
||
for _, workflow := range workflows {
|
||
if workflow.CaptureAutomation == nil || !workflow.CaptureAutomation.Enabled {
|
||
continue
|
||
}
|
||
if err := runtime.processCaptureAutomation(ctx, workflow); err != nil {
|
||
fmt.Printf("[worker] capture automation for workflow %s failed: %v\n", workflow.ID, err)
|
||
}
|
||
}
|
||
}
|