forked from longtao.wu/droboticflow
132 lines
3.8 KiB
Go
132 lines
3.8 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"droboticflow/internal/bootstrap"
|
|
"droboticflow/internal/config"
|
|
"droboticflow/internal/retry"
|
|
"droboticflow/internal/store"
|
|
"droboticflow/internal/worker"
|
|
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) {
|
|
return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) {
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := client.Ping(ctx, nil); err != nil {
|
|
_ = client.Disconnect(ctx)
|
|
return nil, err
|
|
}
|
|
return client, nil
|
|
})
|
|
}
|
|
|
|
func main() {
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
cfg, err := config.LoadValidated()
|
|
if err != nil {
|
|
fmt.Printf("[worker] invalid configuration: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
client, err := connectMongo(ctx, cfg.MongoURI, cfg.MongoConnectRetries, time.Duration(cfg.MongoConnectRetryDelayMS)*time.Millisecond)
|
|
if err != nil {
|
|
fmt.Printf("[worker] failed to connect mongo: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
defer func() {
|
|
_ = client.Disconnect(context.Background())
|
|
}()
|
|
|
|
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
|
|
dataStore.EnableQuerySourceTracking(os.Getenv("QUERY_SOURCE_TRACKING") == "1")
|
|
if err := bootstrap.Run(ctx, dataStore, os.Getenv("DROBOTICFLOW_DATABASE_BOOTSTRAP")); err != nil {
|
|
fmt.Printf("[worker] database bootstrap: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
if cfg.OperatorRunnerImage == "" {
|
|
fmt.Println("[worker] OPERATOR_RUNNER_IMAGE is required; set it explicitly or configure REGISTRY_HOST and REGISTRY_NAMESPACE for deploy/render scripts")
|
|
os.Exit(1)
|
|
}
|
|
runtime, err := worker.NewRuntime(
|
|
dataStore,
|
|
cfg.K8sNamespace,
|
|
cfg.K8sImagePullSecretName,
|
|
cfg.OperatorRunnerImage,
|
|
cfg.WorkspacePVCName,
|
|
cfg.WorkspaceMountPath,
|
|
cfg.OperatorGPUNodeSelector,
|
|
cfg.OperatorGPUTolerations,
|
|
cfg.OperatorGPURuntimeClass,
|
|
cfg.OperatorJobTTLSeconds,
|
|
cfg.GlobalBOSEndpoint,
|
|
cfg.QwenNodeCacheEnabled,
|
|
cfg.QwenNodeCachePVCName,
|
|
cfg.QwenNodeCacheMountPath,
|
|
cfg.QwenNodeCacheNodeSelector,
|
|
cfg.QwenNodeCacheStaleStagingSeconds,
|
|
)
|
|
if err != nil {
|
|
fmt.Printf("[worker] failed to initialize runtime: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
runtime.SetBuildInfo(worker.BuildInfo{
|
|
BuildCommit: cfg.BuildCommit,
|
|
BuildTime: cfg.BuildTime,
|
|
Image: cfg.Image,
|
|
})
|
|
runtime.SetRunnerBuildInfo(worker.BuildInfo{
|
|
BuildCommit: cfg.OperatorRunnerBuildCommit,
|
|
BuildTime: cfg.OperatorRunnerBuildTime,
|
|
Image: cfg.OperatorRunnerImage,
|
|
})
|
|
|
|
go runtime.RunScheduler(ctx, time.Duration(cfg.PollIntervalMS)*time.Millisecond)
|
|
|
|
server := &http.Server{
|
|
Addr: fmt.Sprintf("0.0.0.0:%d", cfg.WorkerPort),
|
|
Handler: worker.NewHTTPHandler(dataStore, runtime),
|
|
ReadHeaderTimeout: 10 * time.Second,
|
|
ReadTimeout: 30 * time.Second,
|
|
WriteTimeout: 2 * time.Minute,
|
|
IdleTimeout: 2 * time.Minute,
|
|
}
|
|
|
|
fmt.Printf("[worker] listening on http://0.0.0.0:%d\n", cfg.WorkerPort)
|
|
serverErrors := make(chan error, 1)
|
|
go func() {
|
|
serverErrors <- server.ListenAndServe()
|
|
}()
|
|
|
|
select {
|
|
case err := <-serverErrors:
|
|
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
fmt.Printf("[worker] failed to start: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
case <-ctx.Done():
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
fmt.Println("[worker] shutting down")
|
|
if err := server.Shutdown(shutdownCtx); err != nil {
|
|
fmt.Printf("[worker] graceful shutdown failed: %v\n", err)
|
|
_ = server.Close()
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
}
|