forked from longtao.wu/droboticflow
545 lines
18 KiB
Go
545 lines
18 KiB
Go
package api
|
||
|
||
import (
|
||
"context"
|
||
"droboticflow/internal/config"
|
||
"droboticflow/internal/domain"
|
||
"droboticflow/internal/store"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"strconv"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
type annotationJobLease struct{ ItemID, JobID, Owner string }
|
||
type annotationJobLeaseKey struct{}
|
||
|
||
func (s *server) newAnnotationProcessingJob(ctx context.Context, t domain.AnnotationTaskRecord, actor, id, kind string, input any, revision int) (*domain.AnnotationProcessingJob, error) {
|
||
run, e := s.store.GetRun(ctx, t.RunID)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
if run == nil {
|
||
return nil, fmt.Errorf("annotation source run unavailable")
|
||
}
|
||
fingerprint, e := s.annotationSourceFingerprint(ctx, t, *run)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
b, e := json.Marshal(input)
|
||
if e != nil {
|
||
return nil, e
|
||
}
|
||
var values map[string]any
|
||
if e = json.Unmarshal(b, &values); e != nil {
|
||
return nil, e
|
||
}
|
||
return &domain.AnnotationProcessingJob{ID: id, TaskID: t.ID, Kind: kind, ActorID: actor, Input: values, SourceFingerprint: fingerprint, SourceIndex: s.annotationSourceIndex, Revision: revision, Status: "queued", Stage: "queued", TotalSlices: len(listValue(values["sliceSegments"])), CreatedAt: nowISO(), UpdatedAt: nowISO()}, nil
|
||
}
|
||
|
||
func durableAnnotationActive(item *domain.HumanItemRecord) bool {
|
||
if item == nil || item.ProcessingJob == nil {
|
||
return false
|
||
}
|
||
return domain.HasActiveAnnotationJob(item, "submit", item.PendingCombinedSubmissionID) ||
|
||
domain.HasActiveAnnotationJob(item, "slice", stringValue(mapValue(item.Subject.Payload["sliceWorkflow"])["requestId"]))
|
||
}
|
||
|
||
func (s *server) assertAnnotationLease(ctx context.Context) error {
|
||
lease, ok := ctx.Value(annotationJobLeaseKey{}).(annotationJobLease)
|
||
if !ok {
|
||
return ctx.Err()
|
||
}
|
||
h, e := s.store.GetHumanItem(ctx, lease.ItemID)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != lease.JobID || h.ProcessingJob.Owner != lease.Owner || h.ProcessingJob.Status != "running" {
|
||
return fmt.Errorf("annotation processing lease lost")
|
||
}
|
||
if h.Status == domain.HumanItemCancelled {
|
||
return fmt.Errorf("annotation task cancelled")
|
||
}
|
||
project, e := s.store.GetProject(ctx, h.ProjectID)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if project == nil || project.DeletionTaskID != "" {
|
||
return fmt.Errorf("annotation project removed or deleting")
|
||
}
|
||
expires, e := time.Parse(time.RFC3339Nano, h.ProcessingJob.LeaseUntil)
|
||
if e != nil || time.Now().After(expires) {
|
||
return fmt.Errorf("annotation processing lease expired")
|
||
}
|
||
return ctx.Err()
|
||
}
|
||
|
||
// RunAnnotationProcessor has no public mutation endpoints. Authenticated API
|
||
// requests create fenced outboxes; this independent process performs the work.
|
||
func RunAnnotationProcessor(ctx context.Context, db store.Store, cfg config.AppConfig, parallel int, backfill bool) error {
|
||
queue, ok := db.(store.AnnotationProcessingStore)
|
||
if !ok {
|
||
return fmt.Errorf("annotation processing store unavailable")
|
||
}
|
||
sourceIndex := os.Getenv("ANNOTATION_SOURCE_INDEX") == "true"
|
||
role := os.Getenv("ANNOTATION_PROCESSOR_ROLE")
|
||
if role == "" {
|
||
role = "all"
|
||
}
|
||
if role != "all" && role != "revision" && role != "validation" {
|
||
return fmt.Errorf("invalid annotation processor role")
|
||
}
|
||
maximum := 4
|
||
if sourceIndex {
|
||
maximum = 32
|
||
}
|
||
if parallel < 1 || parallel > maximum {
|
||
return fmt.Errorf("annotation processor concurrency must be 1..%d", maximum)
|
||
}
|
||
indexParallel := 1
|
||
if sourceIndex {
|
||
indexParallel = 2
|
||
}
|
||
if raw := os.Getenv("ANNOTATION_INDEX_CONCURRENCY"); raw != "" {
|
||
n, err := strconv.Atoi(raw)
|
||
if err != nil || n < 1 || n > 4 {
|
||
return fmt.Errorf("annotation index concurrency must be 1..4")
|
||
}
|
||
indexParallel = n
|
||
}
|
||
if role == "revision" {
|
||
indexParallel = 0
|
||
}
|
||
s := &server{store: db, workspaceMountPath: cfg.WorkspaceMountPath, globalBOSEndpoint: cfg.GlobalBOSEndpoint, externalAnnotationProcessor: true, annotationSourceIndex: sourceIndex}
|
||
s.materializationSlotsOnce.Do(func() { s.materializationSlots = make(chan struct{}, parallel) })
|
||
if backfill {
|
||
orgs, e := db.ListOrganizations(ctx)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
for _, org := range orgs {
|
||
tasks, e := db.ListAnnotationTasks(ctx, org.ID, "")
|
||
if e != nil {
|
||
return e
|
||
}
|
||
for _, t := range tasks {
|
||
if t.SourceIsFullVideo && t.TemplateVersion >= 5 && t.Status == domain.AnnotationTaskStatusPendingAnnotation {
|
||
if e = queue.EnqueueAnnotationIndex(ctx, t); e != nil {
|
||
return e
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
var wg sync.WaitGroup
|
||
jobParallel := parallel
|
||
if role == "validation" {
|
||
jobParallel = 0
|
||
}
|
||
for n := 0; n < jobParallel; n++ {
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for ctx.Err() == nil {
|
||
now := time.Now().UTC()
|
||
item, e := queue.ClaimAnnotationProcessing(ctx, domain.NewID("annotation-lease"), now.Format(time.RFC3339Nano), now.Add(2*time.Minute).Format(time.RFC3339Nano))
|
||
if e != nil {
|
||
log.Printf("[annotation-processor] claim: %v", e)
|
||
} else if item != nil {
|
||
s.executeAnnotationJob(ctx, *item)
|
||
continue
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-time.After(time.Second):
|
||
}
|
||
}
|
||
}()
|
||
}
|
||
for indexWorker := 0; indexWorker < indexParallel; indexWorker++ {
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
for ctx.Err() == nil {
|
||
now := time.Now().UTC()
|
||
j, e := queue.ClaimAnnotationIndex(ctx, domain.NewID("index-lease"), now.Format(time.RFC3339Nano), now.Add(90*time.Second).Format(time.RFC3339Nano))
|
||
if e != nil {
|
||
log.Printf("[annotation-processor] index claim: %v", e)
|
||
} else if j != nil {
|
||
started := time.Now()
|
||
indexCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||
stopHeartbeat := keepAnnotationIndexLease(indexCtx, queue, *j, cancel)
|
||
taskID := j.TaskID
|
||
if taskID == "" {
|
||
taskID = j.ID
|
||
}
|
||
task, readErr := db.GetAnnotationTask(indexCtx, taskID)
|
||
var stopWatch func()
|
||
if task != nil {
|
||
stopWatch = s.watchSupplierDeletion(indexCtx, task.SupplierID, cancel)
|
||
}
|
||
if readErr == nil && task != nil {
|
||
_, readErr = s.annotationFrameTimelineForTask(indexCtx, *task, "left")
|
||
// Legacy timeline jobs may predate this release. They must
|
||
// not silently trigger full-media validation/source copies.
|
||
if readErr == nil && (s.annotationSourceIndex && j.TaskID != "" || j.RevalidateSource) {
|
||
run, err := db.GetRun(indexCtx, task.RunID)
|
||
if err != nil {
|
||
readErr = err
|
||
} else if run == nil {
|
||
readErr = fmt.Errorf("source run unavailable")
|
||
} else {
|
||
key, err := s.annotationSourceFingerprint(indexCtx, *task, *run)
|
||
if err != nil {
|
||
readErr = err
|
||
} else {
|
||
_, readErr = s.annotationSourceCertificate(indexCtx, *task, *run, key)
|
||
if readErr == nil {
|
||
j.SourceFingerprint = key
|
||
}
|
||
}
|
||
}
|
||
}
|
||
} else if task == nil {
|
||
readErr = fmt.Errorf("annotation task no longer exists")
|
||
}
|
||
stopHeartbeat()
|
||
if readErr == nil {
|
||
readErr = indexCtx.Err()
|
||
}
|
||
if stopWatch != nil {
|
||
stopWatch()
|
||
}
|
||
cancel()
|
||
j.Status = "completed"
|
||
j.Error = ""
|
||
if readErr != nil {
|
||
j.Error = readErr.Error()
|
||
j.Status = "queued"
|
||
j.NextAttemptAt = time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)
|
||
if j.Attempt >= 5 || (task != nil && s.annotationSupplierDeleting(context.WithoutCancel(ctx), task.SupplierID)) {
|
||
j.Status = "failed"
|
||
}
|
||
}
|
||
finishCtx, finishCancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
||
saved, finishErr := queue.FinishAnnotationIndex(finishCtx, *j)
|
||
finishCancel()
|
||
log.Printf("[annotation-validator] job=%s status=%s attempt=%d elapsed_ms=%d saved=%t error=%v finish_error=%v", j.ID, j.Status, j.Attempt, time.Since(started).Milliseconds(), saved, readErr, finishErr)
|
||
continue
|
||
}
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-time.After(2 * time.Second):
|
||
}
|
||
}
|
||
}()
|
||
}
|
||
wg.Wait()
|
||
return ctx.Err()
|
||
}
|
||
|
||
func (s *server) executeAnnotationJob(parent context.Context, claimed domain.HumanItemRecord) {
|
||
queue := s.store.(store.AnnotationProcessingStore)
|
||
j := *claimed.ProcessingJob
|
||
ctx, cancel := context.WithTimeout(parent, 24*time.Hour)
|
||
defer cancel()
|
||
ctx = context.WithValue(ctx, annotationJobLeaseKey{}, annotationJobLease{claimed.ID, j.ID, j.Owner})
|
||
done, stopped := make(chan struct{}), make(chan struct{})
|
||
go func() {
|
||
defer close(stopped)
|
||
tick := time.NewTicker(10 * time.Second)
|
||
defer tick.Stop()
|
||
for {
|
||
select {
|
||
case <-done:
|
||
return
|
||
case <-ctx.Done():
|
||
return
|
||
case <-tick.C:
|
||
if s.annotationSupplierDeleting(ctx, claimed.SupplierID) {
|
||
cancel()
|
||
return
|
||
}
|
||
stage, completed, total := "validating", 0, j.TotalSlices
|
||
if raw, ok := s.materializationProgress.Load(claimed.OrganizationID + ":" + j.TaskID); ok {
|
||
p := mapValue(raw)
|
||
stage = stringValue(p["stage"])
|
||
completed = intValue(p["completedSlices"])
|
||
total = intValue(p["totalSlices"])
|
||
}
|
||
ok, e := queue.TouchAnnotationProcessing(ctx, claimed.ID, j.ID, j.Owner, time.Now().Add(2*time.Minute).UTC().Format(time.RFC3339Nano), stage, completed, total)
|
||
if e != nil || !ok {
|
||
cancel()
|
||
return
|
||
}
|
||
}
|
||
}
|
||
}()
|
||
var e error
|
||
if s.annotationSupplierDeleting(ctx, claimed.SupplierID) {
|
||
e = fmt.Errorf("供应商正在删除")
|
||
} else {
|
||
e = s.processAnnotationJob(ctx, claimed)
|
||
}
|
||
close(done)
|
||
<-stopped
|
||
finishCtx, stop := context.WithTimeout(context.WithoutCancel(parent), 15*time.Second)
|
||
defer stop()
|
||
if s.annotationSupplierDeleting(finishCtx, claimed.SupplierID) {
|
||
j.Status = "failed"
|
||
j.Stage = "cancelled"
|
||
j.Error = "供应商正在级联删除,已停止处理"
|
||
j.UpdatedAt = nowISO()
|
||
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
|
||
return
|
||
}
|
||
if parent.Err() != nil || ctx.Err() != nil {
|
||
j.Status = "queued"
|
||
j.Stage = "queued"
|
||
j.Error = "processing interrupted; resuming"
|
||
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
|
||
return
|
||
}
|
||
if errors.Is(e, errSourceValidationPending) {
|
||
if deferForSourceValidation(&j, time.Now()) {
|
||
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
|
||
return
|
||
}
|
||
e = fmt.Errorf("源文件预校验等待超过 %s,编辑内容已保留;请确认校验进程在运行后重试", sourceValidationWaitLimit)
|
||
}
|
||
if j.Kind == "submit" && transientAnnotationStoreError(e) {
|
||
if deferForAnnotationStore(&j, time.Now()) {
|
||
if saved, saveErr := queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j); saveErr != nil || !saved {
|
||
log.Printf("[annotation-processor] submission retry persistence unavailable task=%s", j.TaskID)
|
||
}
|
||
return
|
||
}
|
||
}
|
||
j.Status = "completed"
|
||
j.Stage = "completed"
|
||
j.CompletedSlices = j.TotalSlices
|
||
j.UpdatedAt = nowISO()
|
||
if e != nil {
|
||
j.Status = "failed"
|
||
j.Stage = "failed"
|
||
j.Error = e.Error()
|
||
log.Printf("[annotation-processor] task=%s job=%s failed: %v", j.TaskID, j.ID, e)
|
||
} else {
|
||
j.Input = nil
|
||
}
|
||
_, _ = queue.SaveAnnotationProcessing(finishCtx, claimed.ID, j)
|
||
if e != nil {
|
||
s.failAnnotationJob(finishCtx, claimed.ID, j)
|
||
}
|
||
}
|
||
|
||
const (
|
||
sourceValidationWaitLimit = time.Hour
|
||
sourceValidationMaxBackoff = time.Minute
|
||
)
|
||
|
||
// deferForSourceValidation requeues a revision that waits on source
|
||
// validation with exponential backoff (2s doubling to 1m). Waiting does not
|
||
// consume a crash retry, but it is bounded: past the limit the job fails with
|
||
// its edits kept, so a missing validation process cannot spin it forever.
|
||
func deferForSourceValidation(j *domain.AnnotationProcessingJob, now time.Time) bool {
|
||
since, err := time.Parse(time.RFC3339Nano, j.ValidationSince)
|
||
if err != nil {
|
||
since = now
|
||
j.ValidationSince = now.UTC().Format(time.RFC3339Nano)
|
||
}
|
||
if now.Sub(since) >= sourceValidationWaitLimit {
|
||
return false
|
||
}
|
||
j.ValidationWaits++
|
||
delay := sourceValidationMaxBackoff
|
||
if j.ValidationWaits <= 5 {
|
||
delay = min(2*time.Second<<(j.ValidationWaits-1), sourceValidationMaxBackoff)
|
||
}
|
||
j.Status, j.Stage, j.Error = "queued", "validating", ""
|
||
j.NextAttemptAt = now.Add(delay).UTC().Format(time.RFC3339Nano)
|
||
j.Attempt = max(0, j.Attempt-1)
|
||
return true
|
||
}
|
||
|
||
func (s *server) processAnnotationJob(ctx context.Context, h domain.HumanItemRecord) error {
|
||
j := h.ProcessingJob
|
||
if j.Attempt > 3 {
|
||
return fmt.Errorf("后台处理连续中断,已保留编辑内容,请重试")
|
||
}
|
||
t, e := s.store.GetAnnotationTask(ctx, j.TaskID)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if t == nil || t.OrganizationID != h.OrganizationID || t.RunID != h.RunID {
|
||
return fmt.Errorf("annotation job task scope changed")
|
||
}
|
||
// Repair a crash after authoritative handoff but before legacy task publication.
|
||
if j.Kind == "submit" && h.PendingCombinedSubmissionID == "" && h.CurrentStageID == "quality_review" && h.CurrentRevision > 0 {
|
||
t.CurrentRevision = h.CurrentRevision
|
||
t.Status = domain.AnnotationTaskStatusPendingQuality
|
||
t.AssignedUserID = ""
|
||
return s.store.UpsertAnnotationTask(ctx, *t)
|
||
}
|
||
actor, e := s.store.GetUser(ctx, j.ActorID)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if actor == nil || actor.Status != domain.UserStatusActive || actor.OrganizationID != h.OrganizationID || h.AssignedUserID != actor.ID || h.CurrentStageID != "annotation_combined" {
|
||
return fmt.Errorf("annotation processing ownership or user status changed")
|
||
}
|
||
org, e := s.store.GetOrganization(ctx, h.OrganizationID)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if org == nil || org.Status != domain.OrganizationStatusActive {
|
||
return fmt.Errorf("annotation organization unavailable")
|
||
}
|
||
principal := &authenticatedPrincipal{User: *actor, Organization: *org}
|
||
r, _ := http.NewRequestWithContext(context.WithValue(ctx, authContextKey{}, principal), http.MethodPost, "http://annotation-processor/internal", nil)
|
||
w := &asynchronousAnnotationResponseWriter{}
|
||
if _, _, ok := s.ensureAnnotationTaskAccess(w, r, *t); !ok {
|
||
return fmt.Errorf("annotation job no longer authorized")
|
||
}
|
||
if !annotationManager(principal) && !s.canAccessCurrentHumanStage(ctx, r, h) {
|
||
return fmt.Errorf("annotation job role changed")
|
||
}
|
||
run, e := s.store.GetRun(ctx, t.RunID)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if run == nil {
|
||
return fmt.Errorf("annotation run unavailable")
|
||
}
|
||
key, e := s.annotationSourceFingerprint(ctx, *t, *run)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if key != j.SourceFingerprint {
|
||
return fmt.Errorf("源文件已改变,已保留编辑内容,请重新确认")
|
||
}
|
||
if s.annotationSourceIndex {
|
||
if e = s.requirePreparedAnnotationSource(ctx, *t, key, j.Stage == "queued"); e != nil {
|
||
return e
|
||
}
|
||
}
|
||
b, e := json.Marshal(normalizeStructuredValue(j.Input))
|
||
if e != nil {
|
||
return e
|
||
}
|
||
switch j.Kind {
|
||
case "slice":
|
||
var p sliceConfirmationInput
|
||
if e = json.Unmarshal(b, &p); e != nil {
|
||
return e
|
||
}
|
||
s.finishSliceConfirmation(r.Context(), t.ID, h.ID, j.ID, p, j.Revision, j.ActorID)
|
||
fresh, e := s.store.GetHumanItem(ctx, h.ID)
|
||
if e != nil {
|
||
return e
|
||
}
|
||
if fresh == nil {
|
||
return fmt.Errorf("annotation task removed")
|
||
}
|
||
flow := mapValue(fresh.Subject.Payload["sliceWorkflow"])
|
||
if stringValue(flow["status"]) != "completed" {
|
||
return fmt.Errorf("%s", firstNonEmptyText(stringValue(flow["error"]), "slice processing did not complete"))
|
||
}
|
||
case "submit":
|
||
if h.PendingCombinedSubmissionID != j.ID {
|
||
return fmt.Errorf("annotation submission superseded")
|
||
}
|
||
var p combinedAnnotationSubmissionInput
|
||
if e = json.Unmarshal(b, &p); e != nil {
|
||
return e
|
||
}
|
||
s.completeCombinedAnnotationSubmission(w, r, t, &h, p, principal, j.ID, j.Attempt > 1)
|
||
if w.status >= 400 {
|
||
if w.cause != nil {
|
||
return fmt.Errorf("annotation submission failed: %w", w.cause)
|
||
}
|
||
return fmt.Errorf("annotation submission failed: %s", w.body.String())
|
||
}
|
||
default:
|
||
return fmt.Errorf("unknown annotation job kind")
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func (s *server) failAnnotationJob(ctx context.Context, id string, j domain.AnnotationProcessingJob) {
|
||
h, e := s.store.GetHumanItem(ctx, id)
|
||
if e != nil || h == nil || h.ProcessingJob == nil || h.ProcessingJob.ID != j.ID || h.ProcessingJob.Owner != j.Owner {
|
||
return
|
||
}
|
||
if j.Kind == "submit" {
|
||
recovery := *h
|
||
recovery.ProcessingJob = &j
|
||
if err := s.archiveFailedCombinedSubmission(ctx, recovery); err != nil {
|
||
// The failed job input remains durable. Any replacement must
|
||
// archive it successfully before overwriting it.
|
||
log.Printf("[annotation-processor] failed input archive unavailable task=%s", j.TaskID)
|
||
}
|
||
}
|
||
// Persist terminal state in the fenced item CAS even if saving the job
|
||
// separately failed; retain the actual current durable input.
|
||
h.ProcessingJob.Status = "failed"
|
||
h.ProcessingJob.Stage = "failed"
|
||
h.ProcessingJob.Error = j.Error
|
||
h.ProcessingJob.UpdatedAt = nowISO()
|
||
version := h.Version
|
||
if j.Kind == "submit" && h.PendingCombinedSubmissionID == j.ID {
|
||
h.PendingCombinedSubmissionID = ""
|
||
h.PendingCombinedPayloadSHA = ""
|
||
}
|
||
if j.Kind == "slice" {
|
||
w := mapValue(h.Subject.Payload["sliceWorkflow"])
|
||
if stringValue(w["requestId"]) == j.ID {
|
||
w["status"] = "failed"
|
||
w["error"] = j.Error
|
||
h.Subject.Payload["sliceWorkflow"] = w
|
||
}
|
||
}
|
||
h.Version++
|
||
h.UpdatedAt = nowISO()
|
||
_, _ = s.store.UpdateHumanItemVersioned(ctx, *h, version)
|
||
}
|
||
|
||
func (s *server) annotationSupplierDeleting(ctx context.Context, id string) bool {
|
||
if id == "" {
|
||
return false
|
||
}
|
||
supplier, e := s.store.GetSupplier(ctx, id)
|
||
return e == nil && supplier != nil && (supplier.Status == "deleting" || supplier.Status == "deleted")
|
||
}
|
||
|
||
func (s *server) watchSupplierDeletion(ctx context.Context, id string, cancel context.CancelFunc) func() {
|
||
done := make(chan struct{})
|
||
stopped := make(chan struct{})
|
||
go func() {
|
||
defer close(stopped)
|
||
tick := time.NewTicker(time.Second)
|
||
defer tick.Stop()
|
||
for {
|
||
select {
|
||
case <-done:
|
||
return
|
||
case <-ctx.Done():
|
||
return
|
||
case <-tick.C:
|
||
if s.annotationSupplierDeleting(ctx, id) {
|
||
cancel()
|
||
return
|
||
}
|
||
}
|
||
}
|
||
}()
|
||
return func() { close(done); <-stopped }
|
||
}
|