forked from longtao.wu/droboticflow
70 lines
2.0 KiB
Go
70 lines
2.0 KiB
Go
package worker
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"sync"
|
||
"time"
|
||
|
||
"droboticflow/internal/dataops"
|
||
)
|
||
|
||
// Each lane is serial and cancellation-bound. Slow capture discovery cannot
|
||
// block dispatch or human release, and a new poll cannot start overlapping scans.
|
||
func (runtime *Runtime) startMaintenance(ctx context.Context) <-chan struct{} {
|
||
jobs := []struct {
|
||
name string
|
||
interval time.Duration
|
||
run func(context.Context) error
|
||
}{
|
||
{"capture_discovery", 5 * time.Second, func(ctx context.Context) error {
|
||
workflows, err := runtime.store.ListWorkerWorkflows(ctx, nil, workerWorkflowQueryLimit)
|
||
if err == nil {
|
||
runtime.enqueueCaptureAutomations(ctx, workflows)
|
||
}
|
||
return err
|
||
}},
|
||
{"capture_status", 5 * time.Second, runtime.reconcileCaptureStatuses},
|
||
{"human_release", 5 * time.Second, runtime.reconcileHumanReleasePartitions},
|
||
{"run_metrics", 30 * time.Second, func(ctx context.Context) error {
|
||
_, err := dataops.ReconcileRunMetrics(ctx, runtime.store, 20)
|
||
return err
|
||
}},
|
||
{"housekeeping", time.Minute, func(ctx context.Context) error {
|
||
runtime.reconcileOrphanedRuntimes(ctx)
|
||
runtime.archiveTerminalRuns(ctx)
|
||
return nil
|
||
}},
|
||
// maintenance-backlog#01:低频代际清理(每小时一批,限流不挤占业务流量)。
|
||
{"materialization_cleanup", time.Hour, runtime.cleanupStaleMaterializationGenerations},
|
||
}
|
||
var wg sync.WaitGroup
|
||
for _, job := range jobs {
|
||
wg.Add(1)
|
||
go func() {
|
||
defer wg.Done()
|
||
timer := time.NewTimer(0)
|
||
defer timer.Stop()
|
||
for {
|
||
select {
|
||
case <-ctx.Done():
|
||
return
|
||
case <-timer.C:
|
||
}
|
||
started := time.Now()
|
||
runtime.metrics.StartMaintenance(job.name, started)
|
||
err := job.run(ctx)
|
||
runtime.metrics.FinishMaintenance(job.name, time.Since(started), err)
|
||
if err != nil && !errors.Is(err, context.Canceled) {
|
||
fmt.Printf("[worker] maintenance %s failed: %v\n", job.name, err)
|
||
}
|
||
timer.Reset(job.interval)
|
||
}
|
||
}()
|
||
}
|
||
done := make(chan struct{})
|
||
go func() { wg.Wait(); close(done) }()
|
||
return done
|
||
}
|