forked from longtao.wu/droboticflow
135 lines
3.7 KiB
Go
135 lines
3.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"droboticflow/internal/config"
|
|
"droboticflow/internal/retry"
|
|
"droboticflow/internal/store"
|
|
"droboticflow/internal/worker"
|
|
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
func connectMongo(ctx context.Context, mongoURI string, attempts int, delay time.Duration) (*mongo.Client, error) {
|
|
return retry.Do(ctx, attempts, delay, "mongodb connection", func(ctx context.Context, _ int) (*mongo.Client, error) {
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoURI))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := client.Ping(ctx, nil); err != nil {
|
|
_ = client.Disconnect(ctx)
|
|
return nil, err
|
|
}
|
|
return client, nil
|
|
})
|
|
}
|
|
|
|
func main() {
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
cfg, err := config.LoadValidated()
|
|
if err != nil {
|
|
fmt.Printf("[worker] invalid configuration: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
client, err := connectMongo(ctx, cfg.MongoURI, cfg.MongoConnectRetries, time.Duration(cfg.MongoConnectRetryDelayMS)*time.Millisecond)
|
|
if err != nil {
|
|
fmt.Printf("[worker] failed to connect mongo: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
defer func() {
|
|
_ = client.Disconnect(context.Background())
|
|
}()
|
|
|
|
dataStore := store.NewMongoStore(client.Database(cfg.MongoDB))
|
|
if err := dataStore.RunMigrations(ctx); err != nil {
|
|
fmt.Printf("[worker] failed to run mongo migrations: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
if err := dataStore.EnsureIndexes(ctx); err != nil {
|
|
fmt.Printf("[worker] failed to ensure mongo indexes: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
if _, err := dataStore.EnsureDefaultOrganization(ctx); err != nil {
|
|
fmt.Printf("[worker] failed to ensure default organization: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
if cfg.OperatorRunnerImage == "" {
|
|
fmt.Println("[worker] OPERATOR_RUNNER_IMAGE is required; set it explicitly or configure REGISTRY_HOST and REGISTRY_NAMESPACE for deploy/render scripts")
|
|
os.Exit(1)
|
|
}
|
|
runtime, err := worker.NewRuntime(
|
|
dataStore,
|
|
cfg.K8sNamespace,
|
|
cfg.K8sImagePullSecretName,
|
|
cfg.OperatorRunnerImage,
|
|
cfg.WorkspacePVCName,
|
|
cfg.WorkspaceMountPath,
|
|
cfg.OperatorGPUNodeSelector,
|
|
cfg.OperatorGPUTolerations,
|
|
cfg.OperatorGPURuntimeClass,
|
|
cfg.OperatorJobTTLSeconds,
|
|
cfg.GlobalBOSEndpoint,
|
|
)
|
|
if err != nil {
|
|
fmt.Printf("[worker] failed to initialize runtime: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
|
|
go func() {
|
|
ticker := time.NewTicker(time.Duration(cfg.PollIntervalMS) * time.Millisecond)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if err := runtime.ProcessAvailable(ctx); err != nil {
|
|
fmt.Printf("[worker] processAvailable failed: %v\n", err)
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
server := &http.Server{
|
|
Addr: fmt.Sprintf("0.0.0.0:%d", cfg.WorkerPort),
|
|
Handler: worker.NewHTTPHandler(dataStore, runtime),
|
|
ReadHeaderTimeout: 10 * time.Second,
|
|
ReadTimeout: 30 * time.Second,
|
|
WriteTimeout: 2 * time.Minute,
|
|
IdleTimeout: 2 * time.Minute,
|
|
}
|
|
|
|
fmt.Printf("[worker] listening on http://0.0.0.0:%d\n", cfg.WorkerPort)
|
|
serverErrors := make(chan error, 1)
|
|
go func() {
|
|
serverErrors <- server.ListenAndServe()
|
|
}()
|
|
|
|
select {
|
|
case err := <-serverErrors:
|
|
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
|
fmt.Printf("[worker] failed to start: %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
case <-ctx.Done():
|
|
shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
fmt.Println("[worker] shutting down")
|
|
if err := server.Shutdown(shutdownCtx); err != nil {
|
|
fmt.Printf("[worker] graceful shutdown failed: %v\n", err)
|
|
_ = server.Close()
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
}
|