droboticflow/internal/worker/maintenance.go

70 lines
2.0 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package worker
import (
"context"
"errors"
"fmt"
"sync"
"time"
"droboticflow/internal/dataops"
)
// Each lane is serial and cancellation-bound. Slow capture discovery cannot
// block dispatch or human release, and a new poll cannot start overlapping scans.
func (runtime *Runtime) startMaintenance(ctx context.Context) <-chan struct{} {
jobs := []struct {
name string
interval time.Duration
run func(context.Context) error
}{
{"capture_discovery", 5 * time.Second, func(ctx context.Context) error {
workflows, err := runtime.store.ListWorkerWorkflows(ctx, nil, workerWorkflowQueryLimit)
if err == nil {
runtime.enqueueCaptureAutomations(ctx, workflows)
}
return err
}},
{"capture_status", 5 * time.Second, runtime.reconcileCaptureStatuses},
{"human_release", 5 * time.Second, runtime.reconcileHumanReleasePartitions},
{"run_metrics", 30 * time.Second, func(ctx context.Context) error {
_, err := dataops.ReconcileRunMetrics(ctx, runtime.store, 20)
return err
}},
{"housekeeping", time.Minute, func(ctx context.Context) error {
runtime.reconcileOrphanedRuntimes(ctx)
runtime.archiveTerminalRuns(ctx)
return nil
}},
// maintenance-backlog#01:低频代际清理(每小时一批,限流不挤占业务流量)。
{"materialization_cleanup", time.Hour, runtime.cleanupStaleMaterializationGenerations},
}
var wg sync.WaitGroup
for _, job := range jobs {
wg.Add(1)
go func() {
defer wg.Done()
timer := time.NewTimer(0)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return
case <-timer.C:
}
started := time.Now()
runtime.metrics.StartMaintenance(job.name, started)
err := job.run(ctx)
runtime.metrics.FinishMaintenance(job.name, time.Since(started), err)
if err != nil && !errors.Is(err, context.Canceled) {
fmt.Printf("[worker] maintenance %s failed: %v\n", job.name, err)
}
timer.Reset(job.interval)
}
}()
}
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
return done
}