droboticflow/internal/api/slice_recovery.go

87 lines
2.1 KiB
Go

package api
import (
"context"
"time"
"droboticflow/internal/domain"
)
const sliceLeaseDuration = domain.SliceProcessingLeaseDuration
// Persisted leases survive API restarts. A dead process must not leave the
// workbench spinning forever; retry remains explicit to avoid an OOM loop.
func (server *server) recoverInterruptedSlice(ctx context.Context, item *domain.HumanItemRecord, now time.Time) error {
if durableAnnotationActive(item) {
return nil
}
if !domain.ExpireSliceProcessing(item, now) {
return nil
}
version := item.Version
item.Version++
item.UpdatedAt = now.Format(time.RFC3339Nano)
updated, updateErr := server.store.UpdateHumanItemVersioned(ctx, *item, version)
if updateErr != nil {
return updateErr
}
if !updated {
fresh, err := server.store.GetHumanItem(ctx, item.ID)
if err != nil {
return err
}
if fresh != nil {
*item = *fresh
}
}
return nil
}
func (server *server) keepSliceLease(ctx context.Context, cancel context.CancelFunc, itemID, requestID string, done <-chan struct{}) {
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
deadline := time.Now().Add(sliceLeaseDuration)
for {
select {
case <-done:
return
case <-ctx.Done():
return
case now := <-ticker.C:
if now.After(deadline) {
cancel()
return
}
item, err := server.store.GetHumanItem(ctx, itemID)
if err != nil {
continue
}
if item == nil {
cancel()
return
}
w := mapValue(item.Subject.Payload["sliceWorkflow"])
if item.ProjectID != "" {
p, e := server.store.GetProject(ctx, item.ProjectID)
if e != nil || p == nil || p.DeletionTaskID != "" {
cancel()
return
}
}
if stringValue(w["requestId"]) != requestID || stringValue(w["status"]) != "processing" {
cancel()
return
}
w["leaseExpiresAt"] = now.Add(sliceLeaseDuration).Format(time.RFC3339Nano)
w["heartbeatAt"] = now.Format(time.RFC3339Nano)
item.Subject.Payload["sliceWorkflow"] = w
version := item.Version
item.Version++
ok, err := server.store.UpdateHumanItemVersioned(ctx, *item, version)
if err == nil && ok {
deadline = now.Add(sliceLeaseDuration)
}
}
}
}