forked from longtao.wu/droboticflow
595 lines
24 KiB
Go
595 lines
24 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
const humanReleaseDispatchLease = 5 * time.Minute
|
|
|
|
func humanReleaseChildRunID(uniqueKey string) string {
|
|
digest := sha256.Sum256([]byte(strings.TrimSpace(uniqueKey)))
|
|
return "run-human-release-" + hex.EncodeToString(digest[:16])
|
|
}
|
|
|
|
func humanReleaseChildForKey(runs []domain.RunRecord, uniqueKey string) *domain.RunRecord {
|
|
for index := range runs {
|
|
if runs[index].Metadata != nil && runs[index].Metadata.HumanReleaseUniqueKey == uniqueKey {
|
|
copy := runs[index]
|
|
return ©
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func humanReleaseTarget(workflow *domain.WorkflowRecord, gateNodeID string) (string, error) {
|
|
if workflow == nil {
|
|
return "", fmt.Errorf("human release workflow is missing")
|
|
}
|
|
edges := domain.GetDecisionOutgoingEdges(workflow.Edges, gateNodeID, domain.AuditStatusApproved)
|
|
targets := make([]string, 0, len(edges))
|
|
seen := map[string]struct{}{}
|
|
for _, edge := range edges {
|
|
if _, exists := seen[edge.Target]; exists {
|
|
continue
|
|
}
|
|
seen[edge.Target] = struct{}{}
|
|
targets = append(targets, edge.Target)
|
|
}
|
|
// A multiport human gate may provide approved videos and annotation JSON to
|
|
// the same package node. It is still one downstream execution target.
|
|
if len(targets) != 1 {
|
|
return "", fmt.Errorf("human release gate %s must have exactly one approved downstream target", gateNodeID)
|
|
}
|
|
return targets[0], nil
|
|
}
|
|
|
|
func humanReleaseBaseline(workflow domain.WorkflowRecord, parent domain.RunRecord, gateNodeID string, targetNodeID string, artifacts []domain.ExecutionArtifact) []domain.NodeExecutionRecord {
|
|
allowed := rerunAllowedNodeSet(workflow, &domain.RunRecord{Trigger: domain.RunTriggerHumanRelease, RestartFromNodeID: targetNodeID})
|
|
items := make([]domain.NodeExecutionRecord, 0)
|
|
for _, execution := range parent.NodeExecutions {
|
|
if nodeAllowedForExecution(allowed, execution.NodeID) {
|
|
continue
|
|
}
|
|
copy := execution
|
|
copy.Artifacts = cloneArtifacts(execution.Artifacts)
|
|
copy.InputArtifacts = cloneArtifacts(execution.InputArtifacts)
|
|
copy.InputLineage = cloneInputLineage(execution.InputLineage)
|
|
if copy.NodeID == gateNodeID {
|
|
copy.Status = domain.NodeExecutionStatusCompleted
|
|
copy.Artifacts = cloneArtifacts(artifacts)
|
|
copy.EndedAt = nowISO()
|
|
copy.Logs = append(copy.Logs, logEntry("INFO", "Released accepted human-work partition", copy.NodeLabel))
|
|
}
|
|
items = append(items, copy)
|
|
}
|
|
return items
|
|
}
|
|
|
|
func (runtime *Runtime) dispatchHumanReleasePartition(ctx context.Context, batch domain.HumanBatchRecord, partition domain.ReleasePartitionRecord) error {
|
|
if partition.Status == domain.ReleasePartitionDispatched {
|
|
return runtime.syncDeliveryPackage(ctx, batch, partition)
|
|
}
|
|
if partition.Status == domain.ReleasePartitionDispatching {
|
|
child, err := runtime.store.GetRun(ctx, humanReleaseChildRunID(partition.UniqueKey))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if child != nil {
|
|
partition.ChildRunID = child.ID
|
|
partition.Status = domain.ReleasePartitionDispatched
|
|
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
|
|
return err
|
|
}
|
|
return runtime.syncDeliveryPackage(ctx, batch, partition)
|
|
}
|
|
updatedAt, _ := time.Parse(time.RFC3339Nano, partition.UpdatedAt)
|
|
if updatedAt.IsZero() || time.Since(updatedAt) < humanReleaseDispatchLease {
|
|
return nil
|
|
}
|
|
partition.Status = domain.ReleasePartitionFailed
|
|
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if partition.Status != domain.ReleasePartitionPending && partition.Status != domain.ReleasePartitionFailed {
|
|
return nil
|
|
}
|
|
claimed, err := runtime.store.ClaimReleasePartition(ctx, batch.OrganizationID, partition.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !claimed {
|
|
return nil
|
|
}
|
|
partition.Status = domain.ReleasePartitionDispatching
|
|
partition.UpdatedAt = nowISO()
|
|
parent, err := runtime.store.GetRun(ctx, batch.RunID)
|
|
if err != nil || parent == nil {
|
|
return err
|
|
}
|
|
workflow, err := runtime.store.GetWorkflow(ctx, batch.WorkflowID)
|
|
if err != nil || workflow == nil {
|
|
return err
|
|
}
|
|
runs, err := runtime.store.ListRunsByOrganization(ctx, batch.OrganizationID, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if existing := humanReleaseChildForKey(runs, partition.UniqueKey); existing != nil {
|
|
partition.ChildRunID = existing.ID
|
|
partition.Status = domain.ReleasePartitionDispatched
|
|
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
|
|
return err
|
|
}
|
|
return runtime.syncDeliveryPackage(ctx, batch, partition)
|
|
}
|
|
target, err := humanReleaseTarget(workflow, batch.GateNodeID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var gateNode *domain.WorkflowNode
|
|
for index := range workflow.Nodes {
|
|
if workflow.Nodes[index].ID == batch.GateNodeID {
|
|
copy := workflow.Nodes[index]
|
|
gateNode = ©
|
|
break
|
|
}
|
|
}
|
|
if gateNode == nil {
|
|
return fmt.Errorf("human release gate node %s is missing", batch.GateNodeID)
|
|
}
|
|
var gateExecution *domain.NodeExecutionRecord
|
|
for index := range parent.NodeExecutions {
|
|
if parent.NodeExecutions[index].NodeID == batch.GateNodeID {
|
|
copy := parent.NodeExecutions[index]
|
|
gateExecution = ©
|
|
break
|
|
}
|
|
}
|
|
if gateExecution == nil {
|
|
return fmt.Errorf("human release gate execution %s is missing", batch.GateNodeID)
|
|
}
|
|
frozenPackage, err := runtime.freezeLegacyDeliveryRuntime(ctx, workflow, parent, target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
artifacts, err := runtime.releasePartitionArtifacts(ctx, parent, *gateNode, *gateExecution, partition)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := nowISO()
|
|
child := &domain.RunRecord{
|
|
ID: humanReleaseChildRunID(partition.UniqueKey), OrganizationID: parent.OrganizationID,
|
|
WorkflowID: workflow.ID, WorkflowName: workflow.Name, ProjectID: parent.ProjectID, ProjectName: parent.ProjectName,
|
|
CaptureDirectories: append([]string(nil), parent.CaptureDirectories...),
|
|
Trigger: domain.RunTriggerHumanRelease, Attempt: max(parent.Attempt, 1), ParentRunID: parent.ID,
|
|
RestartFromNodeID: target, TaskID: parent.TaskID, AvailableAt: now, CurrentRunAt: now,
|
|
Mode: parent.Mode, ReleaseID: parent.ReleaseID, WorkflowVersion: parent.WorkflowVersion,
|
|
GitCommitSha: parent.GitCommitSha, ImageDigest: parent.ImageDigest,
|
|
Status: domain.RunStatusQueued, NodeExecutions: []domain.NodeExecutionRecord{}, Logs: []domain.LogEntryRecord{},
|
|
ResourceSeries: []domain.ResourceSample{}, Artifacts: []domain.ExecutionArtifact{}, K8sRefs: []domain.K8sRef{},
|
|
CreatedAt: now, UpdatedAt: now,
|
|
}
|
|
baseline := humanReleaseBaseline(*workflow, *parent, batch.GateNodeID, target, artifacts)
|
|
for _, execution := range baseline {
|
|
applyExecution(child, execution)
|
|
}
|
|
child.K8sRefs = []domain.K8sRef{}
|
|
child.Status = domain.RunStatusQueued
|
|
child.AvailableAt = nowISO()
|
|
child.StartedAt = ""
|
|
child.EndedAt = ""
|
|
child.Error = ""
|
|
child.Failure = nil
|
|
child.Progress = nil
|
|
if child.Metadata == nil {
|
|
child.Metadata = &domain.RunMetadata{}
|
|
}
|
|
child.Metadata.CaptureDeliveryPackage = frozenPackage
|
|
child.Metadata.HumanReleasePartitionID = partition.ID
|
|
child.Metadata.HumanReleaseUniqueKey = partition.UniqueKey
|
|
child.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertRun(ctx, *child); err != nil {
|
|
return err
|
|
}
|
|
if err := runtime.store.UpdateWorkflowRunState(ctx, workflow.ID, domain.WorkflowStatusRunning, child.ID, now); err != nil {
|
|
return err
|
|
}
|
|
partition.ChildRunID = child.ID
|
|
partition.Status = domain.ReleasePartitionDispatched
|
|
partition.ManifestPath = "annotations-manifest.json"
|
|
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
|
|
return err
|
|
}
|
|
return runtime.syncDeliveryPackage(ctx, batch, partition)
|
|
}
|
|
|
|
func (runtime *Runtime) syncDeliveryPackage(ctx context.Context, batch domain.HumanBatchRecord, partition domain.ReleasePartitionRecord) error {
|
|
packages, err := runtime.store.ListDeliveryPackages(ctx, batch.OrganizationID, domain.DeliveryPackageQuery{ReleasePartitionID: partition.ID})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
parentRun, _ := runtime.store.GetRun(ctx, batch.RunID)
|
|
mode := domain.RunModeProduction
|
|
if parentRun != nil && parentRun.Mode != "" {
|
|
mode = parentRun.Mode
|
|
}
|
|
var item domain.DeliveryPackageRecord
|
|
if len(packages) > 0 {
|
|
item = packages[0]
|
|
} else {
|
|
workgroupID, vendorID, supplierID := "", "", ""
|
|
var legacyAssignment map[string]string
|
|
if len(partition.ItemIDs) > 0 {
|
|
if humanItem, itemErr := runtime.store.GetHumanItem(ctx, partition.ItemIDs[0]); itemErr == nil && humanItem != nil {
|
|
workgroupID, vendorID, supplierID = humanItem.WorkgroupID, humanItem.VendorID, humanItem.SupplierID
|
|
legacyAssignment = humanItem.LegacyAssignment
|
|
}
|
|
}
|
|
item = domain.DeliveryPackageRecord{
|
|
ID: "delivery-package-" + partition.ID, OrganizationID: batch.OrganizationID, ProjectID: batch.ProjectID,
|
|
WorkflowID: batch.WorkflowID, SupplierID: supplierID, LegacyAssignment: legacyAssignment, WorkgroupID: workgroupID, VendorID: vendorID, ParentRunID: batch.RunID, ChildRunID: partition.ChildRunID,
|
|
ReleasePartitionID: partition.ID, ManifestDigest: partition.ManifestDigest, ItemIDs: partition.ItemIDs,
|
|
Status: domain.DeliveryPackageBuilding, Mode: mode, Temporary: mode == domain.RunModeDebug, Version: 1, CreatedAt: nowISO(), UpdatedAt: nowISO(),
|
|
}
|
|
}
|
|
if item.Mode == "" {
|
|
item.Mode = mode
|
|
item.Temporary = mode == domain.RunModeDebug
|
|
}
|
|
item.ChildRunID = partition.ChildRunID
|
|
child, err := runtime.store.GetRun(ctx, partition.ChildRunID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if child != nil {
|
|
if child.Mode == domain.RunModeDebug {
|
|
item.Mode = domain.RunModeDebug
|
|
item.Temporary = true
|
|
}
|
|
artifactIDs := make([]string, 0, len(child.Artifacts))
|
|
finalArtifactIDs := make([]string, 0, len(child.Artifacts))
|
|
headposeIDs := make([]string, 0)
|
|
handposeIDs := make([]string, 0)
|
|
for _, artifact := range child.Artifacts {
|
|
artifactIDs = append(artifactIDs, artifact.ID)
|
|
if artifact.RunID == child.ID && !domain.IsInternalDeliveryAudit(artifact) {
|
|
finalArtifactIDs = append(finalArtifactIDs, artifact.ID)
|
|
}
|
|
identity := strings.ToLower(artifact.Name + " " + artifact.Path + " " + artifact.NodeID)
|
|
if strings.Contains(identity, "headpose") || strings.Contains(identity, "head-pose") {
|
|
headposeIDs = append(headposeIDs, artifact.ID)
|
|
} else if strings.Contains(identity, "handpose") || strings.Contains(identity, "wilor") {
|
|
handposeIDs = append(handposeIDs, artifact.ID)
|
|
}
|
|
}
|
|
item.ArtifactIDs = artifactIDs
|
|
item.HeadposeArtifactIDs = headposeIDs
|
|
item.HandposeArtifactIDs = handposeIDs
|
|
switch child.Status {
|
|
case domain.RunStatusCompleted:
|
|
if batch.RequirePrePackageAcceptance {
|
|
evidence, evidenceErr := fanyuV5PrePackageAcceptanceEvidence(ctx, runtime, &batch)
|
|
if evidenceErr != nil {
|
|
return fmt.Errorf("verify v5 pre-package acceptance before finalizing delivery: %w", evidenceErr)
|
|
}
|
|
item.Status = domain.DeliveryPackageAccepted
|
|
item.FinalExportArtifactIDs = append([]string(nil), finalArtifactIDs...)
|
|
if strings.TrimSpace(item.FinalExportedAt) == "" {
|
|
item.FinalExportedAt = nowISO()
|
|
}
|
|
item.QualitySummary = map[string]any{"prePackageAcceptance": evidence}
|
|
if evidence["sampled"] != false && len(item.AcceptanceReviews) == 0 {
|
|
item.AcceptanceReviews = append(item.AcceptanceReviews, domain.DeliveryPackageAcceptanceReview{
|
|
ID: domain.NewID("package-acceptance"), Decision: domain.HumanDecisionApproved,
|
|
ReviewerUserID: strings.TrimSpace(fmt.Sprint(evidence["confirmedByUserId"])),
|
|
Comment: "v5 pre-package aggregate acceptance confirmed before package execution",
|
|
CreatedAt: nowISO(),
|
|
})
|
|
}
|
|
} else if item.Status == domain.DeliveryPackageBuilding || item.Status == domain.DeliveryPackageFailed {
|
|
item.Status = domain.DeliveryPackagePendingAcceptance
|
|
}
|
|
case domain.RunStatusFailed, domain.RunStatusCancelled:
|
|
item.Status = domain.DeliveryPackageFailed
|
|
}
|
|
if domain.RunHasNoDeliverables(child) {
|
|
item.Status = domain.DeliveryPackageNoDeliverables
|
|
item.FinalExportArtifactIDs = nil
|
|
item.FinalExportedAt = ""
|
|
if item.QualitySummary == nil {
|
|
item.QualitySummary = map[string]any{}
|
|
}
|
|
item.QualitySummary["outcome"] = "no_deliverables"
|
|
item.QualitySummary["message"] = "无可交付片段"
|
|
}
|
|
}
|
|
item.UpdatedAt = nowISO()
|
|
if err := runtime.store.UpsertDeliveryPackage(ctx, item); err != nil {
|
|
return err
|
|
}
|
|
if batch.RequirePrePackageAcceptance {
|
|
// v5 was confirmed before package execution. Do not create the legacy
|
|
// post-package acceptance/reject work item a second time.
|
|
if (item.Status == domain.DeliveryPackageAccepted || item.Status == domain.DeliveryPackageNoDeliverables) && parentRun != nil && parentRun.Status == domain.RunStatusWaitingForHuman {
|
|
now := nowISO()
|
|
for index := range parentRun.NodeExecutions {
|
|
if parentRun.NodeExecutions[index].NodeID != batch.GateNodeID {
|
|
continue
|
|
}
|
|
parentRun.NodeExecutions[index].Status = domain.NodeExecutionStatusCompleted
|
|
parentRun.NodeExecutions[index].EndedAt = now
|
|
parentRun.NodeExecutions[index].Logs = append(parentRun.NodeExecutions[index].Logs, logEntry("INFO", "Aggregate acceptance confirmed and final package completed", parentRun.NodeExecutions[index].NodeLabel))
|
|
}
|
|
parentRun.Status = domain.RunStatusCompleted
|
|
parentRun.DeliveryOutcome = ""
|
|
if item.Status == domain.DeliveryPackageNoDeliverables {
|
|
parentRun.DeliveryOutcome = "no_deliverables"
|
|
}
|
|
parentRun.CurrentNodeID = ""
|
|
parentRun.EndedAt = now
|
|
parentRun.Error = ""
|
|
parentRun.Failure = nil
|
|
parentRun.UpdatedAt = now
|
|
if err := runtime.store.UpsertRun(ctx, *parentRun); err != nil {
|
|
return err
|
|
}
|
|
if err := runtime.store.UpdateWorkflowRunState(ctx, batch.WorkflowID, domain.WorkflowStatusCompleted, child.ID, now); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if (item.Status == domain.DeliveryPackageAccepted || item.Status == domain.DeliveryPackageNoDeliverables) && parentRun != nil && child != nil && parentRun.Status == domain.RunStatusCompleted && parentRun.TaskID != "" {
|
|
capture, e := runtime.store.GetCaptureRecord(ctx, parentRun.TaskID)
|
|
if e != nil {
|
|
return e
|
|
}
|
|
if capture != nil && capture.OrganizationID == parentRun.OrganizationID && capture.WorkflowID == parentRun.WorkflowID && (capture.RunID == parentRun.ID || capture.RunID == child.ID) {
|
|
if capture.Status != domain.CaptureRecordCompleted || (item.Status == domain.DeliveryPackageNoDeliverables && capture.BusinessStage != domain.CaptureStageNoDeliverables) {
|
|
if e := runtime.reconcileCaptureRun(ctx, capture); e != nil {
|
|
return e
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
if item.Status == domain.DeliveryPackagePendingAcceptance || item.Status == domain.DeliveryPackageAccepted || item.Status == domain.DeliveryPackageRejected {
|
|
return runtime.ensureDeliveryPackageAcceptanceItem(ctx, batch, &item)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func deliveryPackageAcceptanceBatchID(batchID string) string {
|
|
return "human-batch-delivery-acceptance-" + strings.TrimSpace(batchID)
|
|
}
|
|
|
|
func deliveryPackageAcceptanceItemID(packageID string) string {
|
|
return "human-delivery-acceptance-" + strings.TrimSpace(packageID)
|
|
}
|
|
|
|
func (runtime *Runtime) ensureDeliveryPackageAcceptanceItem(ctx context.Context, sourceBatch domain.HumanBatchRecord, item *domain.DeliveryPackageRecord) error {
|
|
if item == nil {
|
|
return nil
|
|
}
|
|
batchID := deliveryPackageAcceptanceBatchID(sourceBatch.ID)
|
|
acceptanceBatch, err := runtime.store.GetHumanBatch(ctx, batchID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if acceptanceBatch == nil {
|
|
now := nowISO()
|
|
acceptanceBatch = &domain.HumanBatchRecord{
|
|
ID: batchID, OrganizationID: item.OrganizationID, PluginID: "annotation-qa", ProjectID: item.ProjectID,
|
|
WorkflowID: item.WorkflowID, RunID: item.ChildRunID, GateNodeID: "delivery-package-acceptance",
|
|
Status: domain.HumanBatchActive, Stages: []domain.HumanStageDefinition{{
|
|
ID: "acceptance", Label: "成品最终验收", Kind: domain.HumanStageAcceptance, RequiredRole: "quality_reviewer",
|
|
}}, MaxReworkRounds: sourceBatch.MaxReworkRounds,
|
|
ItemCount: 1, CreatedAt: now, UpdatedAt: now,
|
|
}
|
|
if err := runtime.store.UpsertHumanBatch(ctx, *acceptanceBatch); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
humanItemID := deliveryPackageAcceptanceItemID(item.ID)
|
|
humanItem, err := runtime.store.GetHumanItem(ctx, humanItemID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if humanItem == nil {
|
|
status := domain.HumanItemReady
|
|
if item.Status == domain.DeliveryPackageAccepted {
|
|
status = domain.HumanItemAccepted
|
|
} else if item.Status == domain.DeliveryPackageRejected {
|
|
status = domain.HumanItemRejected
|
|
}
|
|
artifactID := ""
|
|
if len(item.ArtifactIDs) > 0 {
|
|
artifactID = item.ArtifactIDs[0]
|
|
}
|
|
now := nowISO()
|
|
humanItem = &domain.HumanItemRecord{
|
|
ID: humanItemID, OrganizationID: item.OrganizationID, PluginID: "annotation-qa", BatchID: batchID,
|
|
ProjectID: item.ProjectID, WorkflowID: item.WorkflowID, RunID: item.ChildRunID, GateNodeID: "delivery-package-acceptance",
|
|
Subject: domain.HumanSubjectRef{ArtifactID: artifactID, ItemKey: item.ID, MediaType: "application/json", Payload: map[string]any{
|
|
"deliveryPackageId": item.ID, "artifactIds": item.ArtifactIDs, "manifestDigest": item.ManifestDigest,
|
|
}}, CurrentStageID: "acceptance", Status: status, SupplierID: item.SupplierID, LegacyAssignment: item.LegacyAssignment, WorkgroupID: item.WorkgroupID, VendorID: item.VendorID,
|
|
Version: 1, MaxReworkRounds: sourceBatch.MaxReworkRounds, CreatedAt: now, UpdatedAt: now,
|
|
}
|
|
if err := runtime.store.UpsertHumanItem(ctx, *humanItem); err != nil {
|
|
return err
|
|
}
|
|
_ = runtime.store.AppendHumanEvent(ctx, domain.HumanEventRecord{
|
|
ID: "human-event-created-" + humanItemID, OrganizationID: item.OrganizationID, BatchID: batchID,
|
|
ItemID: humanItemID, StageID: "acceptance", EventType: "created", ActorUserID: "system",
|
|
ItemVersion: humanItem.Version, Payload: map[string]any{"deliveryPackageId": item.ID}, CreatedAt: now,
|
|
})
|
|
}
|
|
if item.AcceptanceHumanItemID != humanItemID {
|
|
item.AcceptanceHumanItemID = humanItemID
|
|
item.UpdatedAt = nowISO()
|
|
return runtime.store.UpsertDeliveryPackage(ctx, *item)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func humanReleaseBatchTerminal(items []domain.HumanItemRecord) bool {
|
|
if len(items) == 0 {
|
|
return false
|
|
}
|
|
for _, item := range items {
|
|
if item.Status != domain.HumanItemAccepted && item.Status != domain.HumanItemRejected && item.Status != domain.HumanItemEscalated && item.Status != domain.HumanItemCancelled {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func aggregateAcceptanceBatch(batch *domain.HumanBatchRecord) bool {
|
|
// Both the legacy confirm-only aggregate item and the v5.8 independent
|
|
// acceptance item summarize an already releasable source batch. The latter
|
|
// intentionally allows rejection, so ConfirmOnly cannot decide whether the
|
|
// aggregate batch should create a release partition of its own.
|
|
return batch != nil && strings.TrimSpace(batch.SourceBatchID) != "" && len(batch.Stages) == 1 && batch.Stages[0].ID == "prepackage_acceptance"
|
|
}
|
|
|
|
func (runtime *Runtime) ensureDueHumanReleasePartitions(ctx context.Context, batch *domain.HumanBatchRecord) error {
|
|
if batch == nil || batch.ReleasePolicy == "" || aggregateAcceptanceBatch(batch) {
|
|
return nil
|
|
}
|
|
if batch.RequirePrePackageAcceptance {
|
|
acceptanceItemID := "human-item-fanyu-v5-prepackage-acceptance-" + strings.TrimSpace(batch.ID)
|
|
acceptanceItem, err := runtime.store.GetHumanItem(ctx, acceptanceItemID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if acceptanceItem == nil || acceptanceItem.Status != domain.HumanItemAccepted {
|
|
return nil
|
|
}
|
|
}
|
|
items, err := runtime.store.ListHumanItems(ctx, batch.OrganizationID, domain.HumanItemQuery{BatchID: batch.ID})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
accepted := make([]domain.HumanItemRecord, 0)
|
|
for _, item := range items {
|
|
if item.Status == domain.HumanItemAccepted && item.ReleasePartitionID == "" {
|
|
accepted = append(accepted, item)
|
|
}
|
|
}
|
|
if len(accepted) == 0 {
|
|
return nil
|
|
}
|
|
sort.Slice(accepted, func(i, j int) bool { return accepted[i].AcceptedAt < accepted[j].AcceptedAt })
|
|
terminal := humanReleaseBatchTerminal(items)
|
|
oldest, _ := time.Parse(time.RFC3339Nano, accepted[0].AcceptedAt)
|
|
waitExpired := !oldest.IsZero() && time.Since(oldest) >= time.Duration(batch.PartitionMaxWaitSeconds)*time.Second
|
|
if batch.ReleasePolicy == domain.HumanReleaseAllAccepted && !terminal {
|
|
return nil
|
|
}
|
|
if batch.ReleasePolicy == domain.HumanReleaseAcceptedPartition && len(accepted) < batch.PartitionMaxItems && !waitExpired && !terminal {
|
|
return nil
|
|
}
|
|
limit := batch.PartitionMaxItems
|
|
if limit <= 0 || batch.ReleasePolicy == domain.HumanReleaseAllAccepted {
|
|
limit = len(accepted)
|
|
}
|
|
for len(accepted) > 0 {
|
|
count := limit
|
|
if count > len(accepted) {
|
|
count = len(accepted)
|
|
}
|
|
chunk := accepted[:count]
|
|
accepted = accepted[count:]
|
|
itemIDs := make([]string, 0, len(chunk))
|
|
for _, item := range chunk {
|
|
itemIDs = append(itemIDs, item.ID)
|
|
}
|
|
sort.Strings(itemIDs)
|
|
sequence := batch.NextPartitionSequence + 1
|
|
digest := domain.HumanReleaseManifestDigest(itemIDs)
|
|
uniqueKey := strings.Join([]string{batch.GateNodeID, fmt.Sprint(sequence), digest}, ":")
|
|
partition := domain.ReleasePartitionRecord{
|
|
OrganizationID: batch.OrganizationID, BatchID: batch.ID, GateNodeID: batch.GateNodeID,
|
|
Sequence: sequence, UniqueKey: uniqueKey, ManifestDigest: digest,
|
|
ItemIDs: itemIDs, Status: domain.ReleasePartitionPending,
|
|
}
|
|
if err := runtime.store.UpsertReleasePartition(ctx, partition); err != nil {
|
|
return err
|
|
}
|
|
stored, err := runtime.store.GetReleasePartitionByUniqueKey(ctx, batch.OrganizationID, uniqueKey)
|
|
if err != nil || stored == nil {
|
|
return err
|
|
}
|
|
for index := range chunk {
|
|
previousVersion := chunk[index].Version
|
|
chunk[index].ReleasePartitionID = stored.ID
|
|
chunk[index].Version++
|
|
if _, err := runtime.store.UpdateHumanItemVersioned(ctx, chunk[index], previousVersion); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
batch.NextPartitionSequence = sequence
|
|
batch.ReleasedCount += len(chunk)
|
|
}
|
|
return runtime.store.UpsertHumanBatch(ctx, *batch)
|
|
}
|
|
|
|
func (runtime *Runtime) reconcileHumanReleasePartitions(ctx context.Context) error {
|
|
organizations, err := runtime.store.ListOrganizations(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, organization := range organizations {
|
|
for _, status := range []domain.HumanItemStatus{domain.HumanItemClaimed, domain.HumanItemInProgress} {
|
|
items, err := runtime.store.ListHumanItems(ctx, organization.ID, domain.HumanItemQuery{CurrentStageID: domain.FanyuV58StageAnnotationCombined, Status: status})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, item := range items {
|
|
if !domain.ExpireSliceProcessing(&item, time.Now()) {
|
|
continue
|
|
}
|
|
version := item.Version
|
|
item.Version++
|
|
if _, err := runtime.store.UpdateHumanItemVersioned(ctx, item, version); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
batches, err := runtime.store.ListHumanBatches(ctx, organization.ID, domain.HumanBatchQuery{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for index := range batches {
|
|
if aggregateAcceptanceBatch(&batches[index]) {
|
|
continue
|
|
}
|
|
if err := runtime.ensureDueHumanReleasePartitions(ctx, &batches[index]); err != nil {
|
|
fmt.Printf("[worker] reconcile human release batch %s in organization %s failed: %v\n", batches[index].ID, organization.ID, err)
|
|
continue
|
|
}
|
|
partitions, err := runtime.store.ListReleasePartitions(ctx, organization.ID, batches[index].ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, partition := range partitions {
|
|
if err := runtime.dispatchHumanReleasePartition(ctx, batches[index], partition); err != nil {
|
|
partition.Status = domain.ReleasePartitionFailed
|
|
_ = runtime.store.UpsertReleasePartition(ctx, partition)
|
|
fmt.Printf("[worker] reconcile human release partition %s for batch %s failed: %v\n", partition.ID, batches[index].ID, err)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|