forked from longtao.wu/droboticflow
87 lines
2.1 KiB
Go
87 lines
2.1 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"droboticflow/internal/domain"
|
|
)
|
|
|
|
const sliceLeaseDuration = domain.SliceProcessingLeaseDuration
|
|
|
|
// Persisted leases survive API restarts. A dead process must not leave the
|
|
// workbench spinning forever; retry remains explicit to avoid an OOM loop.
|
|
func (server *server) recoverInterruptedSlice(ctx context.Context, item *domain.HumanItemRecord, now time.Time) error {
|
|
if durableAnnotationActive(item) {
|
|
return nil
|
|
}
|
|
if !domain.ExpireSliceProcessing(item, now) {
|
|
return nil
|
|
}
|
|
version := item.Version
|
|
item.Version++
|
|
item.UpdatedAt = now.Format(time.RFC3339Nano)
|
|
updated, updateErr := server.store.UpdateHumanItemVersioned(ctx, *item, version)
|
|
if updateErr != nil {
|
|
return updateErr
|
|
}
|
|
if !updated {
|
|
fresh, err := server.store.GetHumanItem(ctx, item.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if fresh != nil {
|
|
*item = *fresh
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (server *server) keepSliceLease(ctx context.Context, cancel context.CancelFunc, itemID, requestID string, done <-chan struct{}) {
|
|
ticker := time.NewTicker(15 * time.Second)
|
|
defer ticker.Stop()
|
|
deadline := time.Now().Add(sliceLeaseDuration)
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-ctx.Done():
|
|
return
|
|
case now := <-ticker.C:
|
|
if now.After(deadline) {
|
|
cancel()
|
|
return
|
|
}
|
|
item, err := server.store.GetHumanItem(ctx, itemID)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if item == nil {
|
|
cancel()
|
|
return
|
|
}
|
|
w := mapValue(item.Subject.Payload["sliceWorkflow"])
|
|
if item.ProjectID != "" {
|
|
p, e := server.store.GetProject(ctx, item.ProjectID)
|
|
if e != nil || p == nil || p.DeletionTaskID != "" {
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
if stringValue(w["requestId"]) != requestID || stringValue(w["status"]) != "processing" {
|
|
cancel()
|
|
return
|
|
}
|
|
w["leaseExpiresAt"] = now.Add(sliceLeaseDuration).Format(time.RFC3339Nano)
|
|
w["heartbeatAt"] = now.Format(time.RFC3339Nano)
|
|
item.Subject.Payload["sliceWorkflow"] = w
|
|
version := item.Version
|
|
item.Version++
|
|
ok, err := server.store.UpdateHumanItemVersioned(ctx, *item, version)
|
|
if err == nil && ok {
|
|
deadline = now.Add(sliceLeaseDuration)
|
|
}
|
|
}
|
|
}
|
|
}
|