135 lines
3.7 KiB
Go

package main
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"droboticflow/internal/config"
"droboticflow/internal/retry"
"droboticflow/internal/store"
"droboticflow/internal/worker"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) {
return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) {
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI))
if err != nil {
return nil, err
}
if err := client.Ping(ctx, nil); err != nil {
_ = client.Disconnect(ctx)
return nil, err
}
return client, nil
})
}
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cfg, err := config.LoadValidated()
if err != nil {
fmt.Printf("[worker] invalid configuration: %v\n", err)
os.Exit(1)
}
client, err := connectMongo(ctx, cfg.MongoURI, cfg.MongoConnectRetries, time.Duration(cfg.MongoConnectRetryDelayMS)*time.Millisecond)
if err != nil {
fmt.Printf("[worker] failed to connect mongo: %v\n", err)
os.Exit(1)
}
defer func() {
_ = client.Disconnect(context.Background())
}()
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
if err := dataStore.RunMigrations(ctx); err != nil {
fmt.Printf("[worker] failed to run mongo migrations: %v\n", err)
os.Exit(1)
}
if err := dataStore.EnsureIndexes(ctx); err != nil {
fmt.Printf("[worker] failed to ensure mongo indexes: %v\n", err)
os.Exit(1)
}
if _, err := dataStore.EnsureDefaultOrganization(ctx); err != nil {
fmt.Printf("[worker] failed to ensure default organization: %v\n", err)
os.Exit(1)
}
if cfg.OperatorRunnerImage == "" {
fmt.Println("[worker] OPERATOR_RUNNER_IMAGE is required; set it explicitly or configure REGISTRY_HOST and REGISTRY_NAMESPACE for deploy/render scripts")
os.Exit(1)
}
runtime, err := worker.NewRuntime(
dataStore,
cfg.K8sNamespace,
cfg.K8sImagePullSecretName,
cfg.OperatorRunnerImage,
cfg.WorkspacePVCName,
cfg.WorkspaceMountPath,
cfg.OperatorGPUNodeSelector,
cfg.OperatorGPUTolerations,
cfg.OperatorGPURuntimeClass,
cfg.OperatorJobTTLSeconds,
cfg.GlobalBOSEndpoint,
)
if err != nil {
fmt.Printf("[worker] failed to initialize runtime: %v\n", err)
os.Exit(1)
}
go func() {
ticker := time.NewTicker(time.Duration(cfg.PollIntervalMS) * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := runtime.ProcessAvailable(ctx); err != nil {
fmt.Printf("[worker] processAvailable failed: %v\n", err)
}
}
}
}()
server := &http.Server{
Addr: fmt.Sprintf("0.0.0.0:%d", cfg.WorkerPort),
Handler: worker.NewHTTPHandler(dataStore, runtime),
ReadHeaderTimeout: 10 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 2 * time.Minute,
IdleTimeout: 2 * time.Minute,
}
fmt.Printf("[worker] listening on http://0.0.0.0:%d\n", cfg.WorkerPort)
serverErrors := make(chan error, 1)
go func() {
serverErrors <- server.ListenAndServe()
}()
select {
case err := <-serverErrors:
if err != nil && !errors.Is(err, http.ErrServerClosed) {
fmt.Printf("[worker] failed to start: %v\n", err)
os.Exit(1)
}
case <-ctx.Done():
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
fmt.Println("[worker] shutting down")
if err := server.Shutdown(shutdownCtx); err != nil {
fmt.Printf("[worker] graceful shutdown failed: %v\n", err)
_ = server.Close()
os.Exit(1)
}
}
}