2026-11-11 03:51:18 -08:00

573 lines
16 KiB
Go

package store
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"droboticflow/internal/domain"
)
func nowISO() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}
func computeRunDuration(run *domain.RunRecord) {
if run.StartedAt == "" || run.EndedAt == "" {
return
}
startedAt, err := time.Parse(time.RFC3339Nano, run.StartedAt)
if err != nil {
return
}
endedAt, err := time.Parse(time.RFC3339Nano, run.EndedAt)
if err != nil {
return
}
run.DurationMS = endedAt.Sub(startedAt).Milliseconds()
}
func hydrateRunDerivedFields(run *domain.RunRecord) {
if run == nil {
return
}
if run.Mode == "" {
run.Mode = domain.RunModeProduction
}
run.Logs = make([]domain.LogEntryRecord, 0)
run.ResourceSeries = make([]domain.ResourceSample, 0)
run.Artifacts = make([]domain.ExecutionArtifact, 0)
for _, execution := range run.NodeExecutions {
if len(execution.Logs) > 0 {
run.Logs = append(run.Logs, execution.Logs...)
}
if len(execution.ResourceSeries) > 0 {
run.ResourceSeries = append(run.ResourceSeries, execution.ResourceSeries...)
}
if len(execution.Artifacts) > 0 {
run.Artifacts = append(run.Artifacts, execution.Artifacts...)
}
}
}
func hydrateRunArtifactsWhenNeeded(items []domain.RunRecord) {
for index := range items {
hydrateRunArtifactsForRecordWhenNeeded(&items[index])
}
}
func hydrateRunArtifactsForRecordWhenNeeded(run *domain.RunRecord) {
if run == nil || len(run.Artifacts) > 0 {
return
}
hasNodeArtifacts := false
for _, execution := range run.NodeExecutions {
if len(execution.Artifacts) > 0 {
hasNodeArtifacts = true
break
}
}
if hasNodeArtifacts {
hydrateRunDerivedFields(run)
}
}
func compactRunForStorage(run domain.RunRecord) domain.RunRecord {
stored := clone(run)
stored.CaptureDirectories = domain.CaptureDirectoriesFromRun(stored)
stored.Logs = nil
stored.ResourceSeries = nil
stored.Artifacts = nil
return stored
}
func runStorageFieldsToUnset(run domain.RunRecord) []string {
fields := make([]string, 0, 9)
if strings.TrimSpace(run.StartedAt) == "" {
fields = append(fields, "startedAt")
}
if strings.TrimSpace(run.CurrentNodeID) == "" {
fields = append(fields, "currentNodeId")
}
if strings.TrimSpace(run.Error) == "" {
fields = append(fields, "error")
}
if run.Failure == nil {
fields = append(fields, "failure")
}
if run.Progress == nil {
fields = append(fields, "progress")
}
if strings.TrimSpace(run.EndedAt) == "" {
fields = append(fields, "endedAt")
}
if run.DurationMS == 0 {
fields = append(fields, "durationMs")
}
if strings.TrimSpace(string(run.StopMode)) == "" {
fields = append(fields, "stopMode")
}
if strings.TrimSpace(run.StopRequestedAt) == "" {
fields = append(fields, "stopRequestedAt")
}
return fields
}
func finalizeNodeExecutionsForStop(run *domain.RunRecord, mode domain.RunStopMode, endedAt string) {
if run == nil {
return
}
message := "用户已停止工作流"
if mode == domain.RunStopModeForce {
message = "用户已强制停止工作流"
}
for index := range run.NodeExecutions {
execution := &run.NodeExecutions[index]
switch execution.Status {
case domain.NodeExecutionStatusQueued, domain.NodeExecutionStatusRunning, domain.NodeExecutionStatusWaitingForAudit, domain.NodeExecutionStatusWaitingForHuman:
execution.Status = domain.NodeExecutionStatusCancelled
if strings.TrimSpace(execution.EndedAt) == "" {
execution.EndedAt = endedAt
}
if strings.TrimSpace(execution.Error) == "" {
execution.Error = message
}
}
}
}
func parseISOTime(value string) (time.Time, bool) {
if value == "" {
return time.Time{}, false
}
parsed, err := time.Parse(time.RFC3339Nano, value)
if err != nil {
return time.Time{}, false
}
return parsed, true
}
func runAvailableAt(run *domain.RunRecord) time.Time {
if run == nil {
return time.Time{}
}
if parsed, ok := parseISOTime(run.AvailableAt); ok {
return parsed
}
if parsed, ok := parseISOTime(run.CreatedAt); ok {
return parsed
}
return time.Time{}
}
func runIsAvailable(run *domain.RunRecord, now time.Time) bool {
availableAt := runAvailableAt(run)
if availableAt.IsZero() {
return true
}
return !availableAt.After(now)
}
func runLeaseExpiresAt(run *domain.RunRecord) time.Time {
if run == nil {
return time.Time{}
}
if parsed, ok := parseISOTime(run.ExecutionLeaseExpiresAt); ok {
return parsed
}
return time.Time{}
}
func runHasActiveLease(run *domain.RunRecord, now time.Time) bool {
if run == nil || strings.TrimSpace(run.ExecutionLeaseOwner) == "" {
return false
}
expiresAt := runLeaseExpiresAt(run)
return !expiresAt.IsZero() && expiresAt.After(now)
}
func setRunLease(run *domain.RunRecord, owner string, ttl time.Duration, now time.Time) {
if run == nil {
return
}
if ttl <= 0 {
ttl = 5 * time.Minute
}
run.ExecutionLeaseOwner = strings.TrimSpace(owner)
run.ExecutionLeaseExpiresAt = now.Add(ttl).UTC().Format(time.RFC3339Nano)
}
func clearRunLease(run *domain.RunRecord) {
if run == nil {
return
}
run.ExecutionLeaseOwner = ""
run.ExecutionLeaseExpiresAt = ""
}
func clone[T any](value T) T {
data, _ := json.Marshal(value)
var out T
_ = json.Unmarshal(data, &out)
// Retry evidence contains private artifact locators and is excluded from JSON.
if item, ok := any(value).(domain.RunRecord); ok {
copied := any(out).(domain.RunRecord)
copied.RetryHistory = clone(item.RetryHistory)
return any(copied).(T)
}
// Private recovery evidence is never part of the event JSON response.
if event, ok := any(value).(domain.HumanEventRecord); ok {
copied := any(out).(domain.HumanEventRecord)
copied.RecoveryInput = clone(event.RecoveryInput)
return any(copied).(T)
}
// The durable processing outbox is intentionally absent from public JSON.
// Preserve it when cloning private in-memory records for store operations.
if item, ok := any(value).(domain.HumanItemRecord); ok {
copied := any(out).(domain.HumanItemRecord)
copied.ReworkRoutingPending = item.ReworkRoutingPending
copied.AnnotationEpoch = item.AnnotationEpoch
if item.AccountHandoff != nil {
handoff := *item.AccountHandoff
copied.AccountHandoff = &handoff
}
if item.ProcessingJob != nil {
job := *item.ProcessingJob
job.Input = clone(job.Input)
copied.ProcessingJob = &job
}
return any(copied).(T)
}
if item, ok := any(value).(domain.SliceRevisionRecord); ok {
copied := any(out).(domain.SliceRevisionRecord)
copied.AnnotationEpoch = item.AnnotationEpoch
copied.SupersededByAccountDeletion = item.SupersededByAccountDeletion
out = any(copied).(T)
}
if item, ok := any(value).(domain.AnnotationRevisionRecord); ok {
copied := any(out).(domain.AnnotationRevisionRecord)
copied.AnnotationEpoch = item.AnnotationEpoch
copied.SupersededByAccountDeletion = item.SupersededByAccountDeletion
out = any(copied).(T)
}
if item, ok := any(value).(domain.AnnotationTaskRecord); ok {
copied := any(out).(domain.AnnotationTaskRecord)
copied.AnnotationEpoch = item.AnnotationEpoch
copied.AnnotationRevisionFloor = item.AnnotationRevisionFloor
return any(copied).(T)
}
// LegacyEmail is deliberately hidden from JSON responses, but it must
// survive in-memory store clones so the pre-phone login compatibility path
// can exercise the same data shape as Mongo.
if item, ok := any(value).(domain.UserRecord); ok {
copied := any(out).(domain.UserRecord)
copied.LegacyEmail = item.LegacyEmail
copied.DeletionRequestedAt = item.DeletionRequestedAt
copied.DeletionActorID = item.DeletionActorID
return any(copied).(T)
}
return out
}
func normalizedSettingsOrganizationID(value string) string {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
return domain.DefaultOrganizationID
}
func settingsIDForOrganization(value string) string {
organizationID := normalizedSettingsOrganizationID(value)
if organizationID == domain.DefaultOrganizationID {
return "system"
}
return "system:" + organizationID
}
func defaultSettings() domain.SettingsRecord {
return domain.SettingsRecord{
ID: "system",
K8s: domain.K8sSettings{
Namespace: func() string {
if value := strings.TrimSpace(os.Getenv("K8S_NAMESPACE")); value != "" {
return value
}
return "droboticflow"
}(),
WorkspacePVCName: strings.TrimSpace(os.Getenv("WORKSPACE_PVC_NAME")),
WorkspaceMountPath: func() string {
if value := strings.TrimSpace(os.Getenv("WORKSPACE_MOUNT_PATH")); value != "" {
return value
}
return "/workspace"
}(),
},
Worker: domain.WorkerSettings{
PollIntervalMS: 2000,
MaxConcurrentRuns: 1,
},
Storage: domain.StorageSettings{
Endpoint: "",
Buckets: []string{},
},
AI: domain.AISettings{
Provider: "openai-compatible",
},
OperatorGeneration: domain.OperatorGenerationSettings{
Enabled: true,
RepositoryMode: "per_operator_repo",
RepositoryPrefix: "operator-external-",
DefaultBranch: "main",
ImageRegistry: strings.TrimRight(strings.TrimSpace(os.Getenv("CUSTOM_OPERATOR_IMAGE_REGISTRY")), "/"),
BuildStrategy: "kaniko",
KanikoNamespace: strings.TrimSpace(os.Getenv("K8S_NAMESPACE")),
KanikoServiceAccount: "droboticflow-worker",
RegistrySecretName: strings.TrimSpace(os.Getenv("K8S_IMAGE_PULL_SECRET_NAME")),
RequireHumanApproval: true,
},
BusinessMongo: domain.BusinessMongoSettings{
URI: strings.TrimSpace(os.Getenv("BUSINESS_MONGO_URI")),
Database: strings.TrimSpace(os.Getenv("BUSINESS_MONGO_DB")),
},
ExternalBusinessWrite: domain.ExternalBusinessWriteSettings{
Tokens: []domain.ExternalBusinessWriteTokenRecord{},
},
Timezone: "Asia/Shanghai",
Defaults: domain.DefaultResources{
CPURequest: "500m",
CPULimit: "1000m",
MemoryRequest: "1Gi",
MemoryLimit: "2Gi",
GPULimit: "",
},
UpdatedAt: nowISO(),
}
}
func ensureSettingsDefaults(item domain.SettingsRecord) domain.SettingsRecord {
defaults := defaultSettings()
if item.ID == "" {
item.ID = defaults.ID
}
if item.K8s.Namespace == "" {
item.K8s.Namespace = defaults.K8s.Namespace
}
if item.K8s.WorkspacePVCName == "" {
item.K8s.WorkspacePVCName = defaults.K8s.WorkspacePVCName
}
if item.K8s.WorkspaceMountPath == "" {
item.K8s.WorkspaceMountPath = defaults.K8s.WorkspaceMountPath
}
if item.Worker.PollIntervalMS <= 0 {
item.Worker.PollIntervalMS = defaults.Worker.PollIntervalMS
}
if item.Worker.MaxConcurrentRuns <= 0 {
item.Worker.MaxConcurrentRuns = defaults.Worker.MaxConcurrentRuns
}
if item.AI.Provider == "" {
item.AI.Provider = defaults.AI.Provider
}
if item.AI.BaseURL == "" {
item.AI.BaseURL = defaults.AI.BaseURL
}
if item.AI.Model == "" {
item.AI.Model = defaults.AI.Model
}
if isZeroOperatorGenerationSettings(item.OperatorGeneration) {
item.OperatorGeneration = defaults.OperatorGeneration
}
if item.OperatorGeneration.RepositoryMode == "" {
item.OperatorGeneration.RepositoryMode = defaults.OperatorGeneration.RepositoryMode
}
if item.OperatorGeneration.RepositoryPrefix == "" {
item.OperatorGeneration.RepositoryPrefix = defaults.OperatorGeneration.RepositoryPrefix
}
if item.OperatorGeneration.DefaultBranch == "" {
item.OperatorGeneration.DefaultBranch = defaults.OperatorGeneration.DefaultBranch
}
if item.OperatorGeneration.ImageRegistry == "" {
item.OperatorGeneration.ImageRegistry = defaults.OperatorGeneration.ImageRegistry
}
if item.OperatorGeneration.BuildStrategy == "" {
item.OperatorGeneration.BuildStrategy = defaults.OperatorGeneration.BuildStrategy
}
if item.OperatorGeneration.KanikoNamespace == "" {
item.OperatorGeneration.KanikoNamespace = defaults.OperatorGeneration.KanikoNamespace
}
if item.OperatorGeneration.KanikoServiceAccount == "" {
item.OperatorGeneration.KanikoServiceAccount = defaults.OperatorGeneration.KanikoServiceAccount
}
if item.OperatorGeneration.RegistrySecretName == "" {
item.OperatorGeneration.RegistrySecretName = defaults.OperatorGeneration.RegistrySecretName
}
if item.Timezone == "" {
item.Timezone = defaults.Timezone
}
if item.BusinessMongo.Database == "" {
item.BusinessMongo.Database = defaults.BusinessMongo.Database
}
if item.BusinessMongo.URI == "" {
item.BusinessMongo.URI = defaults.BusinessMongo.URI
}
if item.ExternalBusinessWrite.Tokens == nil {
item.ExternalBusinessWrite.Tokens = defaults.ExternalBusinessWrite.Tokens
}
if item.Defaults.CPURequest == "" {
item.Defaults.CPURequest = defaults.Defaults.CPURequest
}
if item.Defaults.CPULimit == "" {
item.Defaults.CPULimit = defaults.Defaults.CPULimit
}
if item.Defaults.MemoryRequest == "" {
item.Defaults.MemoryRequest = defaults.Defaults.MemoryRequest
}
if item.Defaults.MemoryLimit == "" {
item.Defaults.MemoryLimit = defaults.Defaults.MemoryLimit
}
if item.UpdatedAt == "" {
item.UpdatedAt = nowISO()
}
return item
}
func isZeroOperatorGenerationSettings(settings domain.OperatorGenerationSettings) bool {
return !settings.Enabled &&
strings.TrimSpace(settings.GiteaBaseURL) == "" &&
strings.TrimSpace(settings.GiteaOrganization) == "" &&
strings.TrimSpace(settings.GiteaToken) == "" &&
strings.TrimSpace(settings.RepositoryMode) == "" &&
strings.TrimSpace(settings.RepositoryPrefix) == "" &&
strings.TrimSpace(settings.MonorepoName) == "" &&
strings.TrimSpace(settings.DefaultBranch) == "" &&
strings.TrimSpace(settings.ImageRegistry) == "" &&
strings.TrimSpace(settings.BuildStrategy) == "" &&
strings.TrimSpace(settings.KanikoNamespace) == "" &&
strings.TrimSpace(settings.KanikoServiceAccount) == "" &&
strings.TrimSpace(settings.RegistrySecretName) == "" &&
!settings.RequireHumanApproval
}
func sortByCreatedAtDescending[T interface{ GetCreatedAt() string }](items []T) {
sort.Slice(items, func(left int, right int) bool {
return items[left].GetCreatedAt() > items[right].GetCreatedAt()
})
}
func validateWorkflowProject(project *domain.ProjectRecord, workflowID string) error {
if project == nil {
return fmt.Errorf("project for workflow %s not found", workflowID)
}
return nil
}
func storageRoot(sourcePath string) string {
trimmed := strings.TrimSpace(sourcePath)
if trimmed == "" {
return ""
}
if strings.HasPrefix(trimmed, "file://") {
return strings.TrimPrefix(trimmed, "file://")
}
if strings.Contains(trimmed, "://") {
return ""
}
return filepath.Clean(trimmed)
}
func normalizeStorageBucket(bucket string) (string, bool) {
trimmed := strings.Trim(strings.TrimSpace(bucket), "/\\")
if trimmed == "" {
return "", false
}
if strings.ContainsAny(trimmed, "/\\") || trimmed == "." || trimmed == ".." {
return "", false
}
return trimmed, true
}
func normalizeStoragePrefix(prefix string) (string, string, bool) {
trimmed := strings.Trim(strings.TrimSpace(prefix), "/\\")
if trimmed == "" {
return "", "", true
}
parts := strings.Split(strings.ReplaceAll(trimmed, "\\", "/"), "/")
cleanParts := make([]string, 0, len(parts))
for _, part := range parts {
switch part {
case "", ".":
continue
case "..":
return "", "", false
default:
cleanParts = append(cleanParts, part)
}
}
if len(cleanParts) == 0 {
return "", "", true
}
return strings.Join(cleanParts, "/"), filepath.Join(cleanParts...), true
}
func listBucketsFromSourcePath(sourcePath string) ([]string, error) {
root := storageRoot(sourcePath)
if root == "" {
return []string{}, nil
}
entries, err := os.ReadDir(root)
if err != nil {
if os.IsNotExist(err) {
return []string{}, nil
}
return nil, err
}
items := make([]string, 0, len(entries))
for _, entry := range entries {
if entry.IsDir() {
items = append(items, entry.Name())
}
}
sort.Strings(items)
return items, nil
}
func listObjectsFromSourcePath(sourcePath string, bucket string, prefix string) ([]string, error) {
root := storageRoot(sourcePath)
bucketName, ok := normalizeStorageBucket(bucket)
if root == "" || !ok {
return []string{}, nil
}
normalizedPrefix, localPrefix, ok := normalizeStoragePrefix(prefix)
if !ok {
return []string{}, nil
}
basePath := filepath.Join(root, bucketName)
if localPrefix != "" {
basePath = filepath.Join(basePath, localPrefix)
}
entries, err := os.ReadDir(basePath)
if err != nil {
if os.IsNotExist(err) {
return []string{}, nil
}
return nil, err
}
if normalizedPrefix != "" && !strings.HasSuffix(normalizedPrefix, "/") {
normalizedPrefix += "/"
}
items := make([]string, 0, len(entries))
for _, entry := range entries {
name := normalizedPrefix + entry.Name()
if entry.IsDir() {
name += "/"
}
items = append(items, name)
}
sort.Strings(items)
return items, nil
}