droboticflow/internal/worker/human_release.go
2026-11-08 10:00:31 -08:00

595 lines
24 KiB
Go

package worker
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"sort"
"strings"
"time"
"droboticflow/internal/domain"
)
const humanReleaseDispatchLease = 5 * time.Minute
func humanReleaseChildRunID(uniqueKey string) string {
digest := sha256.Sum256([]byte(strings.TrimSpace(uniqueKey)))
return "run-human-release-" + hex.EncodeToString(digest[:16])
}
func humanReleaseChildForKey(runs []domain.RunRecord, uniqueKey string) *domain.RunRecord {
for index := range runs {
if runs[index].Metadata != nil && runs[index].Metadata.HumanReleaseUniqueKey == uniqueKey {
copy := runs[index]
return &copy
}
}
return nil
}
func humanReleaseTarget(workflow *domain.WorkflowRecord, gateNodeID string) (string, error) {
if workflow == nil {
return "", fmt.Errorf("human release workflow is missing")
}
edges := domain.GetDecisionOutgoingEdges(workflow.Edges, gateNodeID, domain.AuditStatusApproved)
targets := make([]string, 0, len(edges))
seen := map[string]struct{}{}
for _, edge := range edges {
if _, exists := seen[edge.Target]; exists {
continue
}
seen[edge.Target] = struct{}{}
targets = append(targets, edge.Target)
}
// A multiport human gate may provide approved videos and annotation JSON to
// the same package node. It is still one downstream execution target.
if len(targets) != 1 {
return "", fmt.Errorf("human release gate %s must have exactly one approved downstream target", gateNodeID)
}
return targets[0], nil
}
func humanReleaseBaseline(workflow domain.WorkflowRecord, parent domain.RunRecord, gateNodeID string, targetNodeID string, artifacts []domain.ExecutionArtifact) []domain.NodeExecutionRecord {
allowed := rerunAllowedNodeSet(workflow, &domain.RunRecord{Trigger: domain.RunTriggerHumanRelease, RestartFromNodeID: targetNodeID})
items := make([]domain.NodeExecutionRecord, 0)
for _, execution := range parent.NodeExecutions {
if nodeAllowedForExecution(allowed, execution.NodeID) {
continue
}
copy := execution
copy.Artifacts = cloneArtifacts(execution.Artifacts)
copy.InputArtifacts = cloneArtifacts(execution.InputArtifacts)
copy.InputLineage = cloneInputLineage(execution.InputLineage)
if copy.NodeID == gateNodeID {
copy.Status = domain.NodeExecutionStatusCompleted
copy.Artifacts = cloneArtifacts(artifacts)
copy.EndedAt = nowISO()
copy.Logs = append(copy.Logs, logEntry("INFO", "Released accepted human-work partition", copy.NodeLabel))
}
items = append(items, copy)
}
return items
}
func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch domain.HumanBatchRecord, partition domain.ReleasePartitionRecord) error {
if partition.Status == domain.ReleasePartitionDispatched {
return runtime.syncDeliveryPackage(ctx, batch, partition)
}
if partition.Status == domain.ReleasePartitionDispatching {
child, err := runtime.store.GetRun(ctx, humanReleaseChildRunID(partition.UniqueKey))
if err != nil {
return err
}
if child != nil {
partition.ChildRunID = child.ID
partition.Status = domain.ReleasePartitionDispatched
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
return err
}
return runtime.syncDeliveryPackage(ctx, batch, partition)
}
updatedAt, _ := time.Parse(time.RFC3339Nano, partition.UpdatedAt)
if updatedAt.IsZero() || time.Since(updatedAt) < humanReleaseDispatchLease {
return nil
}
partition.Status = domain.ReleasePartitionFailed
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
return err
}
}
if partition.Status != domain.ReleasePartitionPending && partition.Status != domain.ReleasePartitionFailed {
return nil
}
claimed, err := runtime.store.ClaimReleasePartition(ctx, batch.OrganizationID, partition.ID)
if err != nil {
return err
}
if !claimed {
return nil
}
partition.Status = domain.ReleasePartitionDispatching
partition.UpdatedAt = nowISO()
parent, err := runtime.store.GetRun(ctx, batch.RunID)
if err != nil || parent == nil {
return err
}
workflow, err := runtime.store.GetWorkflow(ctx, batch.WorkflowID)
if err != nil || workflow == nil {
return err
}
runs, err := runtime.store.ListRunsByOrganization(ctx, batch.OrganizationID, "")
if err != nil {
return err
}
if existing := humanReleaseChildForKey(runs, partition.UniqueKey); existing != nil {
partition.ChildRunID = existing.ID
partition.Status = domain.ReleasePartitionDispatched
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
return err
}
return runtime.syncDeliveryPackage(ctx, batch, partition)
}
target, err := humanReleaseTarget(workflow, batch.GateNodeID)
if err != nil {
return err
}
var gateNode *domain.WorkflowNode
for index := range workflow.Nodes {
if workflow.Nodes[index].ID == batch.GateNodeID {
copy := workflow.Nodes[index]
gateNode = &copy
break
}
}
if gateNode == nil {
return fmt.Errorf("human release gate node %s is missing", batch.GateNodeID)
}
var gateExecution *domain.NodeExecutionRecord
for index := range parent.NodeExecutions {
if parent.NodeExecutions[index].NodeID == batch.GateNodeID {
copy := parent.NodeExecutions[index]
gateExecution = &copy
break
}
}
if gateExecution == nil {
return fmt.Errorf("human release gate execution %s is missing", batch.GateNodeID)
}
frozenPackage, err := runtime.freezeLegacyDeliveryRuntime(ctx, workflow, parent, target)
if err != nil {
return err
}
artifacts, err := runtime.releasePartitionArtifacts(ctx, parent, *gateNode, *gateExecution, partition)
if err != nil {
return err
}
now := nowISO()
child := &domain.RunRecord{
ID: humanReleaseChildRunID(partition.UniqueKey), OrganizationID: parent.OrganizationID,
WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: parent.ProjectID, ProjectName: parent.ProjectName,
CaptureDirectories: append([]string(nil), parent.CaptureDirectories...),
Trigger: domain.RunTriggerHumanRelease, Attempt: max(parent.Attempt, 1), ParentRunID: parent.ID,
RestartFromNodeID: target, TaskID: parent.TaskID, AvailableAt: now, CurrentRunAt: now,
Mode: parent.Mode, ReleaseID: parent.ReleaseID, WorkflowVersion: parent.WorkflowVersion,
GitCommitSha: parent.GitCommitSha, ImageDigest: parent.ImageDigest,
Status: domain.RunStatusQueued, NodeExecutions: []domain.NodeExecutionRecord{}, Logs: []domain.LogEntryRecord{},
ResourceSeries: []domain.ResourceSample{}, Artifacts: []domain.ExecutionArtifact{}, K8sRefs: []domain.K8sRef{},
CreatedAt: now, UpdatedAt: now,
}
baseline := humanReleaseBaseline(*workflow, *parent, batch.GateNodeID, target, artifacts)
for _, execution := range baseline {
applyExecution(child, execution)
}
child.K8sRefs = []domain.K8sRef{}
child.Status = domain.RunStatusQueued
child.AvailableAt = nowISO()
child.StartedAt = ""
child.EndedAt = ""
child.Error = ""
child.Failure = nil
child.Progress = nil
if child.Metadata == nil {
child.Metadata = &domain.RunMetadata{}
}
child.Metadata.CaptureDeliveryPackage = frozenPackage
child.Metadata.HumanReleasePartitionID = partition.ID
child.Metadata.HumanReleaseUniqueKey = partition.UniqueKey
child.UpdatedAt = nowISO()
if err := runtime.store.UpsertRun(ctx, *child); err != nil {
return err
}
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusRunning, child.ID, now); err != nil {
return err
}
partition.ChildRunID = child.ID
partition.Status = domain.ReleasePartitionDispatched
partition.ManifestPath = "annotations-manifest.json"
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
return err
}
return runtime.syncDeliveryPackage(ctx, batch, partition)
}
func (runtime *Runtime) syncDeliveryPackage(ctx context.Context, batch domain.HumanBatchRecord, partition domain.ReleasePartitionRecord) error {
packages, err := runtime.store.ListDeliveryPackages(ctx, batch.OrganizationID, domain.DeliveryPackageQuery{ReleasePartitionID: partition.ID})
if err != nil {
return err
}
parentRun, _ := runtime.store.GetRun(ctx, batch.RunID)
mode := domain.RunModeProduction
if parentRun != nil && parentRun.Mode != "" {
mode = parentRun.Mode
}
var item domain.DeliveryPackageRecord
if len(packages) > 0 {
item = packages[0]
} else {
workgroupID, vendorID, supplierID := "", "", ""
var legacyAssignment map[string]string
if len(partition.ItemIDs) > 0 {
if humanItem, itemErr := runtime.store.GetHumanItem(ctx, partition.ItemIDs[0]); itemErr == nil && humanItem != nil {
workgroupID, vendorID, supplierID = humanItem.WorkgroupID, humanItem.VendorID, humanItem.SupplierID
legacyAssignment = humanItem.LegacyAssignment
}
}
item = domain.DeliveryPackageRecord{
ID: "delivery-package-" + partition.ID, OrganizationID: batch.OrganizationID, ProjectID: batch.ProjectID,
WorkflowID: batch.WorkflowID, SupplierID: supplierID, LegacyAssignment: legacyAssignment, WorkgroupID: workgroupID, VendorID: vendorID, ParentRunID: batch.RunID, ChildRunID: partition.ChildRunID,
ReleasePartitionID: partition.ID, ManifestDigest: partition.ManifestDigest, ItemIDs: partition.ItemIDs,
Status: domain.DeliveryPackageBuilding, Mode: mode, Temporary: mode == domain.RunModeDebug, Version: 1, CreatedAt: nowISO(), UpdatedAt: nowISO(),
}
}
if item.Mode == "" {
item.Mode = mode
item.Temporary = mode == domain.RunModeDebug
}
item.ChildRunID = partition.ChildRunID
child, err := runtime.store.GetRun(ctx, partition.ChildRunID)
if err != nil {
return err
}
if child != nil {
if child.Mode == domain.RunModeDebug {
item.Mode = domain.RunModeDebug
item.Temporary = true
}
artifactIDs := make([]string, 0, len(child.Artifacts))
finalArtifactIDs := make([]string, 0, len(child.Artifacts))
headposeIDs := make([]string, 0)
handposeIDs := make([]string, 0)
for _, artifact := range child.Artifacts {
artifactIDs = append(artifactIDs, artifact.ID)
if artifact.RunID == child.ID && !domain.IsInternalDeliveryAudit(artifact) {
finalArtifactIDs = append(finalArtifactIDs, artifact.ID)
}
identity := strings.ToLower(artifact.Name + " " + artifact.Path + " " + artifact.NodeID)
if strings.Contains(identity, "headpose") || strings.Contains(identity, "head-pose") {
headposeIDs = append(headposeIDs, artifact.ID)
} else if strings.Contains(identity, "handpose") || strings.Contains(identity, "wilor") {
handposeIDs = append(handposeIDs, artifact.ID)
}
}
item.ArtifactIDs = artifactIDs
item.HeadposeArtifactIDs = headposeIDs
item.HandposeArtifactIDs = handposeIDs
switch child.Status {
case domain.RunStatusCompleted:
if batch.RequirePrePackageAcceptance {
evidence, evidenceErr := fanyuV5PrePackageAcceptanceEvidence(ctx, runtime, &batch)
if evidenceErr != nil {
return fmt.Errorf("verify v5 pre-package acceptance before finalizing delivery: %w", evidenceErr)
}
item.Status = domain.DeliveryPackageAccepted
item.FinalExportArtifactIDs = append([]string(nil), finalArtifactIDs...)
if strings.TrimSpace(item.FinalExportedAt) == "" {
item.FinalExportedAt = nowISO()
}
item.QualitySummary = map[string]any{"prePackageAcceptance": evidence}
if evidence["sampled"] != false && len(item.AcceptanceReviews) == 0 {
item.AcceptanceReviews = append(item.AcceptanceReviews, domain.DeliveryPackageAcceptanceReview{
ID: domain.NewID("package-acceptance"), Decision: domain.HumanDecisionApproved,
ReviewerUserID: strings.TrimSpace(fmt.Sprint(evidence["confirmedByUserId"])),
Comment: "v5 pre-package aggregate acceptance confirmed before package execution",
CreatedAt: nowISO(),
})
}
} else if item.Status == domain.DeliveryPackageBuilding || item.Status == domain.DeliveryPackageFailed {
item.Status = domain.DeliveryPackagePendingAcceptance
}
case domain.RunStatusFailed, domain.RunStatusCancelled:
item.Status = domain.DeliveryPackageFailed
}
if domain.RunHasNoDeliverables(child) {
item.Status = domain.DeliveryPackageNoDeliverables
item.FinalExportArtifactIDs = nil
item.FinalExportedAt = ""
if item.QualitySummary == nil {
item.QualitySummary = map[string]any{}
}
item.QualitySummary["outcome"] = "no_deliverables"
item.QualitySummary["message"] = "无可交付片段"
}
}
item.UpdatedAt = nowISO()
if err := runtime.store.UpsertDeliveryPackage(ctx, item); err != nil {
return err
}
if batch.RequirePrePackageAcceptance {
// v5 was confirmed before package execution. Do not create the legacy
// post-package acceptance/reject work item a second time.
if (item.Status == domain.DeliveryPackageAccepted || item.Status == domain.DeliveryPackageNoDeliverables) && parentRun != nil && parentRun.Status == domain.RunStatusWaitingForHuman {
now := nowISO()
for index := range parentRun.NodeExecutions {
if parentRun.NodeExecutions[index].NodeID != batch.GateNodeID {
continue
}
parentRun.NodeExecutions[index].Status = domain.NodeExecutionStatusCompleted
parentRun.NodeExecutions[index].EndedAt = now
parentRun.NodeExecutions[index].Logs = append(parentRun.NodeExecutions[index].Logs, logEntry("INFO", "Aggregate acceptance confirmed and final package completed", parentRun.NodeExecutions[index].NodeLabel))
}
parentRun.Status = domain.RunStatusCompleted
parentRun.DeliveryOutcome = ""
if item.Status == domain.DeliveryPackageNoDeliverables {
parentRun.DeliveryOutcome = "no_deliverables"
}
parentRun.CurrentNodeID = ""
parentRun.EndedAt = now
parentRun.Error = ""
parentRun.Failure = nil
parentRun.UpdatedAt = now
if err := runtime.store.UpsertRun(ctx, *parentRun); err != nil {
return err
}
if err := runtime.store.UpdateWorkflowRunState(ctx, batch.WorkflowID, domain.WorkflowStatusCompleted, child.ID, now); err != nil {
return err
}
}
if (item.Status == domain.DeliveryPackageAccepted || item.Status == domain.DeliveryPackageNoDeliverables) && parentRun != nil && child != nil && parentRun.Status == domain.RunStatusCompleted && parentRun.TaskID != "" {
capture, e := runtime.store.GetCaptureRecord(ctx, parentRun.TaskID)
if e != nil {
return e
}
if capture != nil && capture.OrganizationID == parentRun.OrganizationID && capture.WorkflowID == parentRun.WorkflowID && (capture.RunID == parentRun.ID || capture.RunID == child.ID) {
if capture.Status != domain.CaptureRecordCompleted || (item.Status == domain.DeliveryPackageNoDeliverables && capture.BusinessStage != domain.CaptureStageNoDeliverables) {
if e := runtime.reconcileCaptureRun(ctx, capture); e != nil {
return e
}
}
}
}
return nil
}
if item.Status == domain.DeliveryPackagePendingAcceptance || item.Status == domain.DeliveryPackageAccepted || item.Status == domain.DeliveryPackageRejected {
return runtime.ensureDeliveryPackageAcceptanceItem(ctx, batch, &item)
}
return nil
}
func deliveryPackageAcceptanceBatchID(batchID string) string {
return "human-batch-delivery-acceptance-" + strings.TrimSpace(batchID)
}
func deliveryPackageAcceptanceItemID(packageID string) string {
return "human-delivery-acceptance-" + strings.TrimSpace(packageID)
}
func (runtime *Runtime) ensureDeliveryPackageAcceptanceItem(ctx context.Context, sourceBatch domain.HumanBatchRecord, item *domain.DeliveryPackageRecord) error {
if item == nil {
return nil
}
batchID := deliveryPackageAcceptanceBatchID(sourceBatch.ID)
acceptanceBatch, err := runtime.store.GetHumanBatch(ctx, batchID)
if err != nil {
return err
}
if acceptanceBatch == nil {
now := nowISO()
acceptanceBatch = &domain.HumanBatchRecord{
ID: batchID, OrganizationID: item.OrganizationID, PluginID: "annotation-qa", ProjectID: item.ProjectID,
WorkflowID: item.WorkflowID, RunID: item.ChildRunID, GateNodeID: "delivery-package-acceptance",
Status: domain.HumanBatchActive, Stages: []domain.HumanStageDefinition{{
ID: "acceptance", Label: "成品最终验收", Kind: domain.HumanStageAcceptance, RequiredRole: "quality_reviewer",
}}, MaxReworkRounds: sourceBatch.MaxReworkRounds,
ItemCount: 1, CreatedAt: now, UpdatedAt: now,
}
if err := runtime.store.UpsertHumanBatch(ctx, *acceptanceBatch); err != nil {
return err
}
}
humanItemID := deliveryPackageAcceptanceItemID(item.ID)
humanItem, err := runtime.store.GetHumanItem(ctx, humanItemID)
if err != nil {
return err
}
if humanItem == nil {
status := domain.HumanItemReady
if item.Status == domain.DeliveryPackageAccepted {
status = domain.HumanItemAccepted
} else if item.Status == domain.DeliveryPackageRejected {
status = domain.HumanItemRejected
}
artifactID := ""
if len(item.ArtifactIDs) > 0 {
artifactID = item.ArtifactIDs[0]
}
now := nowISO()
humanItem = &domain.HumanItemRecord{
ID: humanItemID, OrganizationID: item.OrganizationID, PluginID: "annotation-qa", BatchID: batchID,
ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.ChildRunID, GateNodeID: "delivery-package-acceptance",
Subject: domain.HumanSubjectRef{ArtifactID: artifactID, ItemKey: item.ID, MediaType: "application/json", Payload: map[string]any{
"deliveryPackageId": item.ID, "artifactIds": item.ArtifactIDs, "manifestDigest": item.ManifestDigest,
}}, CurrentStageID: "acceptance", Status: status, SupplierID: item.SupplierID, LegacyAssignment: item.LegacyAssignment, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID,
Version: 1, MaxReworkRounds: sourceBatch.MaxReworkRounds, CreatedAt: now, UpdatedAt: now,
}
if err := runtime.store.UpsertHumanItem(ctx, *humanItem); err != nil {
return err
}
_ = runtime.store.AppendHumanEvent(ctx, domain.HumanEventRecord{
ID: "human-event-created-" + humanItemID, OrganizationID: item.OrganizationID, BatchID: batchID,
ItemID: humanItemID, StageID: "acceptance", EventType: "created", ActorUserID: "system",
ItemVersion: humanItem.Version, Payload: map[string]any{"deliveryPackageId": item.ID}, CreatedAt: now,
})
}
if item.AcceptanceHumanItemID != humanItemID {
item.AcceptanceHumanItemID = humanItemID
item.UpdatedAt = nowISO()
return runtime.store.UpsertDeliveryPackage(ctx, *item)
}
return nil
}
func humanReleaseBatchTerminal(items []domain.HumanItemRecord) bool {
if len(items) == 0 {
return false
}
for _, item := range items {
if item.Status != domain.HumanItemAccepted && item.Status != domain.HumanItemRejected && item.Status != domain.HumanItemEscalated && item.Status != domain.HumanItemCancelled {
return false
}
}
return true
}
func aggregateAcceptanceBatch(batch *domain.HumanBatchRecord) bool {
// Both the legacy confirm-only aggregate item and the v5.8 independent
// acceptance item summarize an already releasable source batch. The latter
// intentionally allows rejection, so ConfirmOnly cannot decide whether the
// aggregate batch should create a release partition of its own.
return batch != nil && strings.TrimSpace(batch.SourceBatchID) != "" && len(batch.Stages) == 1 && batch.Stages[0].ID == "prepackage_acceptance"
}
func (runtime *Runtime) ensureDueHumanReleasePartitions(ctx context.Context, batch *domain.HumanBatchRecord) error {
if batch == nil || batch.ReleasePolicy == "" || aggregateAcceptanceBatch(batch) {
return nil
}
if batch.RequirePrePackageAcceptance {
acceptanceItemID := "human-item-fanyu-v5-prepackage-acceptance-" + strings.TrimSpace(batch.ID)
acceptanceItem, err := runtime.store.GetHumanItem(ctx, acceptanceItemID)
if err != nil {
return err
}
if acceptanceItem == nil || acceptanceItem.Status != domain.HumanItemAccepted {
return nil
}
}
items, err := runtime.store.ListHumanItems(ctx, batch.OrganizationID, domain.HumanItemQuery{BatchID: batch.ID})
if err != nil {
return err
}
accepted := make([]domain.HumanItemRecord, 0)
for _, item := range items {
if item.Status == domain.HumanItemAccepted && item.ReleasePartitionID == "" {
accepted = append(accepted, item)
}
}
if len(accepted) == 0 {
return nil
}
sort.Slice(accepted, func(i, j int) bool { return accepted[i].AcceptedAt < accepted[j].AcceptedAt })
terminal := humanReleaseBatchTerminal(items)
oldest, _ := time.Parse(time.RFC3339Nano, accepted[0].AcceptedAt)
waitExpired := !oldest.IsZero() && time.Since(oldest) >= time.Duration(batch.PartitionMaxWaitSeconds)*time.Second
if batch.ReleasePolicy == domain.HumanReleaseAllAccepted && !terminal {
return nil
}
if batch.ReleasePolicy == domain.HumanReleaseAcceptedPartition && len(accepted) < batch.PartitionMaxItems && !waitExpired && !terminal {
return nil
}
limit := batch.PartitionMaxItems
if limit <= 0 || batch.ReleasePolicy == domain.HumanReleaseAllAccepted {
limit = len(accepted)
}
for len(accepted) > 0 {
count := limit
if count > len(accepted) {
count = len(accepted)
}
chunk := accepted[:count]
accepted = accepted[count:]
itemIDs := make([]string, 0, len(chunk))
for _, item := range chunk {
itemIDs = append(itemIDs, item.ID)
}
sort.Strings(itemIDs)
sequence := batch.NextPartitionSequence + 1
digest := domain.HumanReleaseManifestDigest(itemIDs)
uniqueKey := strings.Join([]string{batch.GateNodeID, fmt.Sprint(sequence), digest}, ":")
partition := domain.ReleasePartitionRecord{
OrganizationID: batch.OrganizationID, BatchID: batch.ID, GateNodeID: batch.GateNodeID,
Sequence: sequence, UniqueKey: uniqueKey, ManifestDigest: digest,
ItemIDs: itemIDs, Status: domain.ReleasePartitionPending,
}
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
return err
}
stored, err := runtime.store.GetReleasePartitionByUniqueKey(ctx, batch.OrganizationID, uniqueKey)
if err != nil || stored == nil {
return err
}
for index := range chunk {
previousVersion := chunk[index].Version
chunk[index].ReleasePartitionID = stored.ID
chunk[index].Version++
if _, err := runtime.store.UpdateHumanItemVersioned(ctx, chunk[index], previousVersion); err != nil {
return err
}
}
batch.NextPartitionSequence = sequence
batch.ReleasedCount += len(chunk)
}
return runtime.store.UpsertHumanBatch(ctx, *batch)
}
func (runtime *Runtime) reconcileHumanReleasePartitions(ctx context.Context) error {
organizations, err := runtime.store.ListOrganizations(ctx)
if err != nil {
return err
}
for _, organization := range organizations {
for _, status := range []domain.HumanItemStatus{domain.HumanItemClaimed, domain.HumanItemInProgress} {
items, err := runtime.store.ListHumanItems(ctx, organization.ID, domain.HumanItemQuery{CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: status})
if err != nil {
return err
}
for _, item := range items {
if !domain.ExpireSliceProcessing(&item, time.Now()) {
continue
}
version := item.Version
item.Version++
if _, err := runtime.store.UpdateHumanItemVersioned(ctx, item, version); err != nil {
return err
}
}
}
batches, err := runtime.store.ListHumanBatches(ctx, organization.ID, domain.HumanBatchQuery{})
if err != nil {
return err
}
for index := range batches {
if aggregateAcceptanceBatch(&batches[index]) {
continue
}
if err := runtime.ensureDueHumanReleasePartitions(ctx, &batches[index]); err != nil {
fmt.Printf("[worker] reconcile human release batch %s in organization %s failed: %v\n", batches[index].ID, organization.ID, err)
continue
}
partitions, err := runtime.store.ListReleasePartitions(ctx, organization.ID, batches[index].ID)
if err != nil {
return err
}
for _, partition := range partitions {
if err := runtime.dispatchHumanReleasePartition(ctx, batches[index], partition); err != nil {
partition.Status = domain.ReleasePartitionFailed
_ = runtime.store.UpsertReleasePartition(ctx, partition)
fmt.Printf("[worker] reconcile human release partition %s for batch %s failed: %v\n", partition.ID, batches[index].ID, err)
continue
}
}
}
}
return nil
}