eust-w 43326d6a5e feat(platform): land workflow, data tooling, and docs updates
Include the accumulated workflow scheduling, run history, subtree rerun, operator UI, deployment, data asset, object storage, database browser, and documentation changes currently in the worktree.
2026-04-24 12:29:46 +08:00

3248 lines
114 KiB
Go

package api
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/mail"
"regexp"
"sort"
"strconv"
"strings"
"time"
"droboticflow/internal/ai"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/store"
"github.com/baidubce/bce-sdk-go/services/bos"
bosapi "github.com/baidubce/bce-sdk-go/services/bos/api"
"github.com/go-chi/chi/v5"
)
type validationIssue struct {
Field string `json:"field"`
Message string `json:"message"`
}
type server struct {
store store.Store
ai ai.Service
operatorRunnerImage string
}
type runJobTerminator interface {
TerminateRun(ctx context.Context, namespace string, runID string) error
TerminateJob(ctx context.Context, namespace string, jobName string) error
}
var storageConnectionBucketsResolver = listStorageBucketsFromConnection
var storageConnectionObjectsResolver = listStorageObjectsFromConnection
var runJobTerminatorFactory = func(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string) (runJobTerminator, error) {
return k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath)
}
var workerAPIBaseURLResolver = func() string {
return "http://droboticflow-worker:3002"
}
const maxStoragePathFavoritesPerScope = 12
var workflowScheduleTimeOfDayPattern = regexp.MustCompile(`^(?:[01]\d|2[0-3]):[0-5]\d$`)
func maskedSecretPreview(value string) string {
compact := strings.Join(strings.Fields(strings.TrimSpace(value)), "")
if compact == "" {
compact = strings.TrimSpace(value)
}
if compact == "" {
return ""
}
runes := []rune(compact)
switch length := len(runes); {
case length <= 2:
return strings.Repeat("*", length)
case length <= 6:
return string(runes[:1]) + "****" + string(runes[length-1:])
case length <= 10:
return string(runes[:2]) + "****" + string(runes[length-2:])
default:
return string(runes[:4]) + "****" + string(runes[length-4:])
}
}
func sanitizeScanConfig(item *domain.ScanConfig) *domain.ScanConfig {
if item == nil {
return nil
}
copy := *item
copy.HasAccessKeyID = strings.TrimSpace(copy.AccessKeyID) != ""
copy.HasSecretAccessKey = strings.TrimSpace(copy.SecretAccessKey) != ""
copy.AccessKeyPreview = maskedSecretPreview(copy.AccessKeyID)
copy.SecretAccessKeyPreview = maskedSecretPreview(copy.SecretAccessKey)
return &copy
}
func sanitizeStorageConnection(item *domain.StorageConnectionRecord) *domain.StorageConnectionRecord {
if item == nil {
return nil
}
copy := *item
copy.HasAccessKeyID = strings.TrimSpace(copy.AccessKeyID) != ""
copy.HasSecretAccessKey = strings.TrimSpace(copy.SecretAccessKey) != ""
copy.AccessKeyPreview = maskedSecretPreview(copy.AccessKeyID)
copy.SecretAccessKeyPreview = maskedSecretPreview(copy.SecretAccessKey)
copy.AccessKeyID = ""
copy.SecretAccessKey = ""
return &copy
}
func sanitizeStorageConnections(items []domain.StorageConnectionRecord) []domain.StorageConnectionRecord {
if len(items) == 0 {
return items
}
sanitized := make([]domain.StorageConnectionRecord, 0, len(items))
for _, item := range items {
copy := item
sanitized = append(sanitized, *sanitizeStorageConnection(&copy))
}
return sanitized
}
func sanitizeMongoConnection(item *domain.MongoConnectionRecord) *domain.MongoConnectionRecord {
if item == nil {
return nil
}
copy := *item
copy.HasURI = strings.TrimSpace(copy.URI) != ""
copy.URIPreview = maskedSecretPreview(copy.URI)
copy.URI = ""
return &copy
}
func sanitizeMongoConnections(items []domain.MongoConnectionRecord) []domain.MongoConnectionRecord {
if len(items) == 0 {
return items
}
sanitized := make([]domain.MongoConnectionRecord, 0, len(items))
for _, item := range items {
copy := item
sanitized = append(sanitized, *sanitizeMongoConnection(&copy))
}
return sanitized
}
func normalizeStoragePathFavoritePrefix(value string) string {
return strings.Trim(strings.TrimSpace(value), "/")
}
func normalizeStoragePathFavoriteRecord(item domain.StoragePathFavoriteRecord) (domain.StoragePathFavoriteRecord, bool) {
scopeKey := strings.TrimSpace(item.ScopeKey)
bucket := strings.TrimSpace(item.Bucket)
if scopeKey == "" || bucket == "" {
return domain.StoragePathFavoriteRecord{}, false
}
prefix := normalizeStoragePathFavoritePrefix(item.Prefix)
label := strings.TrimSpace(item.Label)
if label == "" {
label = bucket
if prefix != "" {
label += "/" + prefix
}
}
return domain.StoragePathFavoriteRecord{
ScopeKey: scopeKey,
Bucket: bucket,
Prefix: prefix,
Label: label,
}, true
}
func sanitizeStoragePathFavorites(items []domain.StoragePathFavoriteRecord) []domain.StoragePathFavoriteRecord {
if len(items) == 0 {
return []domain.StoragePathFavoriteRecord{}
}
sanitized := make([]domain.StoragePathFavoriteRecord, 0, len(items))
countsByScope := map[string]int{}
seen := map[string]struct{}{}
for _, item := range items {
normalized, ok := normalizeStoragePathFavoriteRecord(item)
if !ok {
continue
}
pathKey := normalized.Bucket
if normalized.Prefix != "" {
pathKey += "/" + normalized.Prefix
}
compositeKey := normalized.ScopeKey + "|" + pathKey
if _, exists := seen[compositeKey]; exists {
continue
}
if countsByScope[normalized.ScopeKey] >= maxStoragePathFavoritesPerScope {
continue
}
seen[compositeKey] = struct{}{}
countsByScope[normalized.ScopeKey] += 1
sanitized = append(sanitized, normalized)
}
return sanitized
}
func listStorageBucketsFromConnection(ctx context.Context, connection *domain.StorageConnectionRecord) ([]string, error) {
if connection == nil {
return []string{}, nil
}
client, err := bos.NewClient(
strings.TrimSpace(connection.AccessKeyID),
strings.TrimSpace(connection.SecretAccessKey),
storageEndpointForProvider(connection.Provider, connection.Endpoint),
)
if err != nil {
return nil, err
}
result, err := client.ListBucketsWithContext(ctx)
if err != nil {
return nil, err
}
items := make([]string, 0, len(result.Buckets))
for _, bucket := range result.Buckets {
name := strings.TrimSpace(bucket.Name)
if name != "" {
items = append(items, name)
}
}
sort.Strings(items)
return items, nil
}
func storageObjectRequestPrefix(prefix string) string {
normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/")
if normalizedPrefix == "" {
return ""
}
return normalizedPrefix + "/"
}
func appendStorageObjectItems(items map[string]struct{}, result *bosapi.ListObjectsResult, normalizedPrefix string, requestPrefix string) {
if result == nil {
return
}
for _, item := range result.CommonPrefixes {
prefixValue := strings.TrimSpace(item.Prefix)
if prefixValue == "" || prefixValue == requestPrefix {
continue
}
items[prefixValue] = struct{}{}
}
for _, item := range result.Contents {
key := strings.TrimSpace(item.Key)
if key == "" || key == normalizedPrefix || key == requestPrefix {
continue
}
items[key] = struct{}{}
}
}
func listStorageObjectsFromConnection(ctx context.Context, connection *domain.StorageConnectionRecord, bucket string, prefix string) ([]string, error) {
if connection == nil {
return []string{}, nil
}
normalizedBucket := strings.TrimSpace(bucket)
if normalizedBucket == "" {
return []string{}, nil
}
client, err := bos.NewClient(
strings.TrimSpace(connection.AccessKeyID),
strings.TrimSpace(connection.SecretAccessKey),
storageEndpointForProvider(connection.Provider, connection.Endpoint),
)
if err != nil {
return nil, err
}
normalizedPrefix := strings.Trim(strings.TrimSpace(prefix), "/")
requestPrefix := storageObjectRequestPrefix(prefix)
collected := make(map[string]struct{})
marker := ""
for {
result, err := client.ListObjectsWithContext(ctx, normalizedBucket, &bosapi.ListObjectsArgs{
Prefix: requestPrefix,
Delimiter: "/",
MaxKeys: 1000,
Marker: marker,
})
if err != nil {
return nil, err
}
appendStorageObjectItems(collected, result, normalizedPrefix, requestPrefix)
if result == nil || !result.IsTruncated {
break
}
nextMarker := strings.TrimSpace(result.NextMarker)
if nextMarker == "" || nextMarker == marker {
break
}
marker = nextMarker
}
items := make([]string, 0, len(collected))
for item := range collected {
items = append(items, item)
}
sort.Strings(items)
return items, nil
}
func cloneWorkflowNodes(items []domain.WorkflowNode) []domain.WorkflowNode {
if len(items) == 0 {
return items
}
cloned := make([]domain.WorkflowNode, len(items))
copy(cloned, items)
return cloned
}
func sanitizeWorkflow(item *domain.WorkflowRecord) *domain.WorkflowRecord {
if item == nil {
return nil
}
copy := *item
copy.Nodes = cloneWorkflowNodes(item.Nodes)
copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig)
return &copy
}
func sanitizeWorkflows(items []domain.WorkflowRecord) []domain.WorkflowRecord {
if len(items) == 0 {
return items
}
sanitized := make([]domain.WorkflowRecord, 0, len(items))
for _, item := range items {
copy := item
copy.Nodes = cloneWorkflowNodes(item.Nodes)
copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig)
sanitized = append(sanitized, copy)
}
return sanitized
}
type workflowNodeSourceStateResponse struct {
NodeID string `json:"nodeId"`
Supported bool `json:"supported"`
Stateful bool `json:"stateful"`
Reason string `json:"reason,omitempty"`
ScanMode string `json:"scanMode,omitempty"`
IncrementalUnit string `json:"incrementalUnit,omitempty"`
DirectoryDepth int `json:"directoryDepth,omitempty"`
DataSource string `json:"dataSource,omitempty"`
SourcePath string `json:"sourcePath,omitempty"`
Bucket string `json:"bucket,omitempty"`
Prefix string `json:"prefix,omitempty"`
Pattern string `json:"pattern,omitempty"`
CurrentFingerprint string `json:"currentFingerprint,omitempty"`
Current *domain.SourceStateRecord `json:"current,omitempty"`
Versions []domain.SourceStateRecord `json:"versions,omitempty"`
}
func workflowNodeByID(workflow *domain.WorkflowRecord, nodeID string) *domain.WorkflowNode {
if workflow == nil {
return nil
}
for index := range workflow.Nodes {
if workflow.Nodes[index].ID == nodeID {
return &workflow.Nodes[index]
}
}
return nil
}
func runSummaryFromRecord(run domain.RunRecord) domain.RunSummary {
return domain.RunSummary{
ID: run.ID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
Trigger: run.Trigger,
Attempt: run.Attempt,
ParentRunID: run.ParentRunID,
RestartFromNodeID: run.RestartFromNodeID,
AvailableAt: run.AvailableAt,
Status: run.Status,
StopMode: run.StopMode,
StopRequestedAt: run.StopRequestedAt,
StartedAt: run.StartedAt,
EndedAt: run.EndedAt,
DurationMS: run.DurationMS,
CurrentNodeID: run.CurrentNodeID,
Error: run.Error,
CreatedAt: run.CreatedAt,
UpdatedAt: run.UpdatedAt,
}
}
func rerunSubtreeNodeSet(edges []domain.WorkflowEdge, startNodeID string) map[string]struct{} {
visited := map[string]struct{}{}
queue := []string{startNodeID}
for len(queue) > 0 {
nodeID := queue[0]
queue = queue[1:]
if _, ok := visited[nodeID]; ok {
continue
}
visited[nodeID] = struct{}{}
for _, edge := range domain.GetOutgoingEdges(edges, nodeID) {
if strings.TrimSpace(edge.Target) != "" {
queue = append(queue, edge.Target)
}
}
}
return visited
}
func rerunBaselineNodeSet(edges []domain.WorkflowEdge, subtree map[string]struct{}) map[string]struct{} {
baseline := map[string]struct{}{}
seen := map[string]struct{}{}
queue := make([]string, 0, len(subtree))
for nodeID := range subtree {
queue = append(queue, nodeID)
}
for len(queue) > 0 {
nodeID := queue[0]
queue = queue[1:]
if _, ok := seen[nodeID]; ok {
continue
}
seen[nodeID] = struct{}{}
for _, edge := range domain.GetIncomingEdges(edges, nodeID) {
sourceID := strings.TrimSpace(edge.Source)
if sourceID == "" {
continue
}
if _, ok := subtree[sourceID]; ok {
continue
}
if _, ok := baseline[sourceID]; ok {
continue
}
baseline[sourceID] = struct{}{}
queue = append(queue, sourceID)
}
}
return baseline
}
func cloneLogEntries(items []domain.LogEntryRecord) []domain.LogEntryRecord {
if len(items) == 0 {
return []domain.LogEntryRecord{}
}
cloned := make([]domain.LogEntryRecord, len(items))
copy(cloned, items)
return cloned
}
func cloneExecutionArtifacts(items []domain.ExecutionArtifact) []domain.ExecutionArtifact {
if len(items) == 0 {
return []domain.ExecutionArtifact{}
}
cloned := make([]domain.ExecutionArtifact, len(items))
copy(cloned, items)
return cloned
}
func cloneDataRefs(items []domain.DataRefRecord) []domain.DataRefRecord {
if len(items) == 0 {
return []domain.DataRefRecord{}
}
cloned := make([]domain.DataRefRecord, len(items))
copy(cloned, items)
return cloned
}
func cloneResourceSamples(items []domain.ResourceSample) []domain.ResourceSample {
if len(items) == 0 {
return []domain.ResourceSample{}
}
cloned := make([]domain.ResourceSample, len(items))
copy(cloned, items)
return cloned
}
func cloneNodeExecutionRecord(item domain.NodeExecutionRecord) domain.NodeExecutionRecord {
cloned := item
cloned.Logs = cloneLogEntries(item.Logs)
cloned.Artifacts = cloneExecutionArtifacts(item.Artifacts)
cloned.DataRefs = cloneDataRefs(item.DataRefs)
cloned.ResourceSeries = cloneResourceSamples(item.ResourceSeries)
return cloned
}
func rebuildRunAggregates(run *domain.RunRecord) {
if run == nil {
return
}
run.Logs = []domain.LogEntryRecord{}
run.ResourceSeries = []domain.ResourceSample{}
run.Artifacts = []domain.ExecutionArtifact{}
for _, execution := range run.NodeExecutions {
run.Logs = append(run.Logs, cloneLogEntries(execution.Logs)...)
run.ResourceSeries = append(run.ResourceSeries, cloneResourceSamples(execution.ResourceSeries)...)
run.Artifacts = append(run.Artifacts, cloneExecutionArtifacts(execution.Artifacts)...)
}
}
func workflowScanSourceNode(node *domain.WorkflowNode) bool {
if node == nil {
return false
}
label := strings.TrimSpace(node.Data.Label)
return node.ID == "scan-source" || label == "BOS 增量扫描"
}
func sortSourceStatesDescending(items []domain.SourceStateRecord) {
sort.SliceStable(items, func(i int, j int) bool {
return items[i].UpdatedAt > items[j].UpdatedAt
})
}
func sourceStateFingerprintFromScanConfig(scanConfig *domain.ScanConfig) string {
if scanConfig == nil {
return ""
}
return domain.ComputeSourceFingerprint(domain.SourceFingerprintInput{
ScanMode: chooseWorkflowScanMode(scanConfig.ScanMode),
IncrementalUnit: chooseWorkflowIncrementalUnit(scanConfig.IncrementalUnit),
DirectoryDepth: chooseWorkflowDirectoryDepth(scanConfig.DirectoryDepth),
DataSource: chooseDataSource(scanConfig.DataSource),
SourcePath: scanConfig.SourcePath,
Bucket: scanConfig.Bucket,
Prefix: scanConfig.Prefix,
Pattern: scanConfig.Pattern,
Endpoint: scanConfig.Endpoint,
})
}
func (server *server) buildWorkflowNodeSourceStateResponse(ctx context.Context, workflow *domain.WorkflowRecord, nodeID string) (*workflowNodeSourceStateResponse, int, error) {
if workflow == nil {
return nil, http.StatusNotFound, nil
}
node := workflowNodeByID(workflow, nodeID)
if node == nil {
return nil, http.StatusNotFound, nil
}
response := &workflowNodeSourceStateResponse{
NodeID: nodeID,
Supported: workflowScanSourceNode(node),
}
if !response.Supported {
response.Reason = "当前节点没有可持久化的扫描状态"
return response, http.StatusOK, nil
}
resolvedScanConfig, err := server.resolveScanConfig(ctx, workflow.ScanConfig)
if err != nil {
return nil, http.StatusInternalServerError, err
}
if resolvedScanConfig == nil {
response.Reason = "当前工作流尚未配置扫描源"
return response, http.StatusOK, nil
}
response.ScanMode = chooseWorkflowScanMode(resolvedScanConfig.ScanMode)
response.IncrementalUnit = chooseWorkflowIncrementalUnit(resolvedScanConfig.IncrementalUnit)
response.DirectoryDepth = chooseWorkflowDirectoryDepth(resolvedScanConfig.DirectoryDepth)
response.DataSource = chooseDataSource(resolvedScanConfig.DataSource)
response.SourcePath = strings.TrimSpace(resolvedScanConfig.SourcePath)
response.Bucket = strings.TrimSpace(resolvedScanConfig.Bucket)
response.Prefix = strings.TrimSpace(resolvedScanConfig.Prefix)
response.Pattern = strings.TrimSpace(resolvedScanConfig.Pattern)
response.Stateful = response.ScanMode == "INCREMENTAL"
response.CurrentFingerprint = sourceStateFingerprintFromScanConfig(resolvedScanConfig)
versions, err := server.store.ListSourceStates(ctx, workflow.ID, nodeID)
if err != nil {
return nil, http.StatusInternalServerError, err
}
sortSourceStatesDescending(versions)
response.Versions = versions
for index := range versions {
if versions[index].SourceFingerprint == response.CurrentFingerprint {
state := versions[index]
response.Current = &state
break
}
}
if !response.Stateful {
response.Reason = "当前为 FULL 模式,不会读取或提交增量状态"
} else if response.Current == nil {
response.Reason = "当前配置下还没有已提交的增量状态"
}
return response, http.StatusOK, nil
}
func sanitizeTemplate(item *domain.TemplateRecord) *domain.TemplateRecord {
if item == nil {
return nil
}
copy := *item
copy.Nodes = cloneWorkflowNodes(item.Nodes)
copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig)
return &copy
}
func sanitizeTemplates(items []domain.TemplateRecord) []domain.TemplateRecord {
if len(items) == 0 {
return items
}
sanitized := make([]domain.TemplateRecord, 0, len(items))
for _, item := range items {
copy := item
copy.Nodes = cloneWorkflowNodes(item.Nodes)
copy.ScanConfig = sanitizeScanConfig(copy.ScanConfig)
sanitized = append(sanitized, copy)
}
return sanitized
}
func sanitizeOperator(item *domain.OperatorRecord) *domain.OperatorRecord {
if item == nil {
return nil
}
copy := *item
return &copy
}
func sanitizeOperators(items []domain.OperatorRecord) []domain.OperatorRecord {
if len(items) == 0 {
return items
}
sanitized := make([]domain.OperatorRecord, 0, len(items))
for _, item := range items {
copy := item
sanitized = append(sanitized, copy)
}
return sanitized
}
func sanitizeUser(item *domain.UserRecord) *domain.UserRecord {
if item == nil {
return nil
}
copy := *item
copy.StoragePathFavorites = nil
return &copy
}
func sanitizeUsers(items []domain.UserRecord) []domain.UserRecord {
if len(items) == 0 {
return items
}
sanitized := make([]domain.UserRecord, 0, len(items))
for _, item := range items {
copy := item
copy.StoragePathFavorites = nil
sanitized = append(sanitized, copy)
}
return sanitized
}
func sanitizeProject(item *domain.ProjectRecord) *domain.ProjectRecord {
if item == nil {
return nil
}
copy := *item
copy.StoragePathFavorites = nil
return &copy
}
func sanitizeProjects(items []domain.ProjectRecord) []domain.ProjectRecord {
if len(items) == 0 {
return items
}
sanitized := make([]domain.ProjectRecord, 0, len(items))
for _, item := range items {
copy := item
copy.StoragePathFavorites = nil
sanitized = append(sanitized, copy)
}
return sanitized
}
func NewHandler(dataStore store.Store, aiService ai.Service) http.Handler {
if aiService == nil {
aiService = ai.NewDisabledService("", "", "AI 服务未配置")
}
instance := &server{
store: dataStore,
ai: aiService,
operatorRunnerImage: config.Load().OperatorRunnerImage,
}
router := chi.NewRouter()
router.Get("/api/health", instance.handleHealth)
router.Get("/worker-api/health", instance.handleWorkerAPIHealth)
router.Get("/worker-api/stats", instance.handleWorkerAPIStats)
router.Post("/worker-api/process-once", instance.handleWorkerAPIProcessOnce)
router.Post("/worker-api/runs/{id}/retry", instance.handleWorkerAPIRetry)
router.Get("/api/ai/status", instance.handleAIStatus)
router.Post("/api/ai/operators/generate", instance.handleAIOperatorGenerate)
router.Post("/api/ai/workflows/generate", instance.handleAIWorkflowGenerate)
router.Get("/api/dashboard", instance.handleDashboard)
router.Get("/api/projects", instance.handleProjectsList)
router.Post("/api/projects", instance.handleProjectsCreate)
router.Get("/api/projects/{id}", instance.handleProjectsGet)
router.Put("/api/projects/{id}", instance.handleProjectsUpdate)
router.Delete("/api/projects/{id}", instance.handleProjectsDelete)
router.Get("/api/projects/{id}/storage-path-favorites", instance.handleProjectStoragePathFavoritesGet)
router.Put("/api/projects/{id}/storage-path-favorites", instance.handleProjectStoragePathFavoritesUpdate)
router.Get("/api/projects/{id}/workflows", instance.handleProjectWorkflowsList)
router.Post("/api/projects/{id}/workflows", instance.handleProjectWorkflowsCreate)
router.Get("/api/workflows/{id}", instance.handleWorkflowsGet)
router.Put("/api/workflows/{id}", instance.handleWorkflowsUpdate)
router.Delete("/api/workflows/{id}", instance.handleWorkflowsDelete)
router.Get("/api/workflows/{id}/runs", instance.handleWorkflowRunsList)
router.Get("/api/workflows/{id}/nodes/{nodeId}/source-state", instance.handleWorkflowNodeSourceStateGet)
router.Delete("/api/workflows/{id}/nodes/{nodeId}/source-state", instance.handleWorkflowNodeSourceStateDelete)
router.Post("/api/workflows/{id}/run", instance.handleWorkflowRun)
router.Get("/api/scan-configs", instance.handleScanConfigsList)
router.Post("/api/scan-configs", instance.handleScanConfigsCreate)
router.Get("/api/scan-configs/{id}", instance.handleScanConfigsGet)
router.Put("/api/scan-configs/{id}", instance.handleScanConfigsUpdate)
router.Delete("/api/scan-configs/{id}", instance.handleScanConfigsDelete)
router.Post("/api/scan-configs/{id}/run", instance.handleScanConfigsRun)
router.Get("/api/source-data/collections", instance.handleSourceCollectionsList)
router.Get("/api/source-data/collections/{runId}/{collectionId}", instance.handleSourceCollectionDetail)
router.Get("/api/source-data/collections/{runId}/{collectionId}/entries", instance.handleSourceCollectionEntries)
router.Get("/api/source-data/collections/{runId}/{collectionId}/content", instance.handleSourceCollectionContent)
router.Get("/api/packages", instance.handlePackagesList)
router.Get("/api/packages/{runId}/{packageName}", instance.handlePackageDetail)
router.Get("/api/packages/{runId}/{packageName}/content", instance.handlePackageContent)
router.Get("/api/sampling-tasks", instance.handleSamplingTasksList)
router.Get("/api/sampling-tasks/{id}", instance.handleSamplingTaskGet)
router.Post("/api/sampling-items/{id}/review", instance.handleSamplingItemReview)
router.Get("/api/templates", instance.handleTemplatesList)
router.Post("/api/templates", instance.handleTemplatesCreate)
router.Get("/api/templates/{id}", instance.handleTemplatesGet)
router.Put("/api/templates/{id}", instance.handleTemplatesUpdate)
router.Delete("/api/templates/{id}", instance.handleTemplatesDelete)
router.Get("/api/operators", instance.handleOperatorsList)
router.Post("/api/operators", instance.handleOperatorsCreate)
router.Get("/api/operators/{id}", instance.handleOperatorsGet)
router.Put("/api/operators/{id}", instance.handleOperatorsUpdate)
router.Delete("/api/operators/{id}", instance.handleOperatorsDelete)
router.Get("/api/database", instance.handleDatabaseOverview)
router.Post("/api/database/collections", instance.handleDatabaseCollectionCreate)
router.Patch("/api/database/collections/{name}", instance.handleDatabaseCollectionRename)
router.Delete("/api/database/collections/{name}", instance.handleDatabaseCollectionDelete)
router.Get("/api/database/collections/{name}/indexes", instance.handleDatabaseCollectionIndexes)
router.Post("/api/database/collections/{name}/indexes", instance.handleDatabaseCollectionIndexCreate)
router.Delete("/api/database/collections/{name}/indexes/{indexName}", instance.handleDatabaseCollectionIndexDelete)
router.Get("/api/database/collections/{name}/documents", instance.handleDatabaseCollectionDocuments)
router.Post("/api/database/collections/{name}/documents", instance.handleDatabaseDocumentCreate)
router.Put("/api/database/collections/{name}/documents/{documentId}", instance.handleDatabaseDocumentReplace)
router.Delete("/api/database/collections/{name}/documents/{documentId}", instance.handleDatabaseDocumentDelete)
router.Get("/api/runs", instance.handleRunsList)
router.Get("/api/runs/{id}", instance.handleRunsGet)
router.Post("/api/runs/{id}/rerun-subtree", instance.handleRunsRerunSubtree)
router.Post("/api/runs/{id}/stop", instance.handleRunsStop)
router.Post("/api/runs/{id}/force-stop", instance.handleRunsForceStop)
router.Get("/api/runs/{id}/logs", instance.handleRunsLogs)
router.Get("/api/runs/{id}/resources", instance.handleRunsResources)
router.Get("/api/runs/{id}/artifacts", instance.handleRunsArtifacts)
router.Get("/api/audit-tasks", instance.handleAuditTasksList)
router.Post("/api/audit-tasks/{id}/approve", instance.handleAuditTaskApprove)
router.Post("/api/audit-tasks/{id}/reject", instance.handleAuditTaskReject)
router.Get("/api/users", instance.handleUsersList)
router.Post("/api/users", instance.handleUsersCreate)
router.Put("/api/users/{id}", instance.handleUsersUpdate)
router.Delete("/api/users/{id}", instance.handleUsersDelete)
router.Get("/api/users/{id}/storage-path-favorites", instance.handleUserStoragePathFavoritesGet)
router.Put("/api/users/{id}/storage-path-favorites", instance.handleUserStoragePathFavoritesUpdate)
router.Get("/api/storage/connections", instance.handleStorageConnectionsList)
router.Post("/api/storage/connections", instance.handleStorageConnectionsCreate)
router.Get("/api/storage/connections/{id}", instance.handleStorageConnectionsGet)
router.Put("/api/storage/connections/{id}", instance.handleStorageConnectionsUpdate)
router.Delete("/api/storage/connections/{id}", instance.handleStorageConnectionsDelete)
router.Get("/api/mongo/connections", instance.handleMongoConnectionsList)
router.Post("/api/mongo/connections", instance.handleMongoConnectionsCreate)
router.Post("/api/mongo/connections/test", instance.handleMongoConnectionsTest)
router.Get("/api/mongo/connections/{id}", instance.handleMongoConnectionsGet)
router.Put("/api/mongo/connections/{id}", instance.handleMongoConnectionsUpdate)
router.Delete("/api/mongo/connections/{id}", instance.handleMongoConnectionsDelete)
router.Get("/api/external/business-db/tokens", instance.handleExternalBusinessWriteTokensList)
router.Post("/api/external/business-db/tokens", instance.handleExternalBusinessWriteTokensCreate)
router.Put("/api/external/business-db/tokens/{id}", instance.handleExternalBusinessWriteTokensUpdate)
router.Get("/api/external/business-db/tokens/{id}/reveal", instance.handleExternalBusinessWriteTokensReveal)
router.Post("/api/external/business-db/tokens/{id}/rotate", instance.handleExternalBusinessWriteTokensRotate)
router.Delete("/api/external/business-db/tokens/{id}", instance.handleExternalBusinessWriteTokensDelete)
router.Route("/api/external/business-db", func(router chi.Router) {
router.Use(instance.externalBusinessWriteAuth)
router.Post("/documents", instance.handleExternalBusinessWriteCreate)
router.Patch("/documents/{documentId}", instance.handleExternalBusinessWritePatch)
router.Post("/upserts", instance.handleExternalBusinessWriteUpsert)
})
router.Get("/api/settings", instance.handleSettingsGet)
router.Put("/api/settings", instance.handleSettingsUpdate)
router.Get("/api/storage/buckets", instance.handleStorageBuckets)
router.Get("/api/storage/objects", instance.handleStorageObjects)
router.Get("/api/object-storage", instance.handleObjectStorageOverview)
router.Get("/api/object-storage/objects", instance.handleObjectStorageObjects)
router.Post("/api/object-storage/buckets", instance.handleObjectStorageBucketCreate)
router.Delete("/api/object-storage/buckets/{name}", instance.handleObjectStorageBucketDelete)
router.Post("/api/object-storage/objects", instance.handleObjectStorageObjectCreate)
router.Post("/api/object-storage/objects/upload", instance.handleObjectStorageObjectUpload)
router.Patch("/api/object-storage/objects", instance.handleObjectStorageObjectRename)
router.Delete("/api/object-storage/objects", instance.handleObjectStorageObjectDelete)
router.Get("/api/object-storage/content", instance.handleObjectStorageContent)
return router
}
func (server *server) handleHealth(writer http.ResponseWriter, _ *http.Request) {
renderJSON(writer, http.StatusOK, map[string]string{"status": "ok", "service": "droboticflow-api"})
}
func workerAPIBaseURL() string {
return strings.TrimRight(strings.TrimSpace(workerAPIBaseURLResolver()), "/")
}
func proxyWorkerAPI(writer http.ResponseWriter, request *http.Request, path string) {
baseURL := workerAPIBaseURL()
if baseURL == "" {
renderJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "worker API is not configured"})
return
}
upstreamURL := baseURL + path
upstreamRequest, err := http.NewRequestWithContext(request.Context(), request.Method, upstreamURL, request.Body)
if err != nil {
renderError(writer, http.StatusBadGateway, err)
return
}
upstreamRequest.Header = request.Header.Clone()
response, err := http.DefaultClient.Do(upstreamRequest)
if err != nil {
renderJSON(writer, http.StatusBadGateway, map[string]string{
"error": "worker API request failed",
"details": err.Error(),
})
return
}
defer response.Body.Close()
for key, values := range response.Header {
for _, value := range values {
writer.Header().Add(key, value)
}
}
writer.WriteHeader(response.StatusCode)
if _, err := io.Copy(writer, response.Body); err != nil {
return
}
}
func (server *server) handleWorkerAPIHealth(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/health")
}
func (server *server) handleWorkerAPIStats(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/stats")
}
func (server *server) handleWorkerAPIProcessOnce(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/process-once")
}
func (server *server) handleWorkerAPIRetry(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/runs/"+chi.URLParam(request, "id")+"/retry")
}
func (server *server) handleAIStatus(writer http.ResponseWriter, _ *http.Request) {
renderJSON(writer, http.StatusOK, server.resolveAIService(context.Background()).Status())
}
type aiGenerateRequest struct {
Description string `json:"description"`
}
func (server *server) handleAIOperatorGenerate(writer http.ResponseWriter, request *http.Request) {
var payload aiGenerateRequest
if !decodeJSON(writer, request, &payload) {
return
}
if strings.TrimSpace(payload.Description) == "" {
renderValidationError(writer, validationIssue{Field: "description", Message: "description is required"})
return
}
item, err := server.resolveAIService(request.Context()).GenerateOperator(request.Context(), payload.Description)
if err != nil {
renderAIError(writer, err)
return
}
renderJSON(writer, http.StatusOK, item)
}
func (server *server) handleAIWorkflowGenerate(writer http.ResponseWriter, request *http.Request) {
var payload aiGenerateRequest
if !decodeJSON(writer, request, &payload) {
return
}
if strings.TrimSpace(payload.Description) == "" {
renderValidationError(writer, validationIssue{Field: "description", Message: "description is required"})
return
}
operators, err := server.listAvailableOperators(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if len(operators) == 0 {
renderValidationError(writer, validationIssue{Field: "operators", Message: "at least one operator is required to generate a workflow"})
return
}
item, err := server.resolveAIService(request.Context()).GenerateWorkflow(request.Context(), payload.Description, operators)
if err != nil {
renderAIError(writer, err)
return
}
renderJSON(writer, http.StatusOK, item)
}
func (server *server) handleDashboard(writer http.ResponseWriter, request *http.Request) {
summary, err := server.store.GetDashboardSummary(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, summary)
}
func (server *server) populateProjectWorkflowCounts(ctx context.Context, items []domain.ProjectRecord) ([]domain.ProjectRecord, error) {
if len(items) == 0 {
return items, nil
}
workflows, err := server.store.ListAllWorkflows(ctx)
if err != nil {
return nil, err
}
counts := make(map[string]int, len(items))
for _, workflow := range workflows {
if workflow.ProjectID == "" {
continue
}
counts[workflow.ProjectID]++
}
enriched := make([]domain.ProjectRecord, len(items))
for index, item := range items {
item.WorkflowCount = counts[item.ID]
enriched[index] = item
}
return enriched, nil
}
func (server *server) populateSingleProjectWorkflowCount(ctx context.Context, item *domain.ProjectRecord) (*domain.ProjectRecord, error) {
if item == nil {
return nil, nil
}
workflows, err := server.store.ListWorkflowsByProject(ctx, item.ID)
if err != nil {
return nil, err
}
enriched := *item
enriched.WorkflowCount = len(workflows)
return &enriched, nil
}
func (server *server) handleProjectsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListProjects(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
items, err = server.populateProjectWorkflowCounts(request.Context(), items)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeProjects(items)})
}
func (server *server) handleProjectsCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CreateProjectInput
if !decodeJSON(writer, request, &payload) {
return
}
if strings.TrimSpace(payload.Name) == "" {
renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"})
return
}
project, err := server.store.CreateProject(request.Context(), payload)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeProject(project))
}
func (server *server) handleProjectsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"})
return
}
item, err = server.populateSingleProjectWorkflowCount(request.Context(), item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeProject(item))
}
type projectPatch struct {
Name *string `json:"name"`
Description *string `json:"description"`
Status *string `json:"status"`
}
func (server *server) handleProjectsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"})
return
}
var payload projectPatch
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Name != nil {
item.Name = strings.TrimSpace(*payload.Name)
}
if payload.Description != nil {
item.Description = *payload.Description
}
if payload.Status != nil {
item.Status = *payload.Status
}
item.UpdatedAt = nowISO()
if err := server.store.UpsertProject(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
item, err = server.populateSingleProjectWorkflowCount(request.Context(), item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeProject(item))
}
func (server *server) handleProjectsDelete(writer http.ResponseWriter, request *http.Request) {
deleted, err := server.store.DeleteProject(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func (server *server) handleProjectStoragePathFavoritesGet(writer http.ResponseWriter, request *http.Request) {
project, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if project == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"})
return
}
renderJSON(writer, http.StatusOK, map[string]any{
"items": sanitizeStoragePathFavorites(project.StoragePathFavorites),
})
}
func (server *server) handleProjectStoragePathFavoritesUpdate(writer http.ResponseWriter, request *http.Request) {
project, err := server.store.GetProject(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if project == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Project not found"})
return
}
var payload storagePathFavoritesPayload
if !decodeJSON(writer, request, &payload) {
return
}
project.StoragePathFavorites = sanitizeStoragePathFavorites(payload.Items)
project.UpdatedAt = nowISO()
if err := server.store.UpsertProject(request.Context(), *project); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{
"items": project.StoragePathFavorites,
})
}
func (server *server) handleProjectWorkflowsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListWorkflowsByProject(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeWorkflows(items)})
}
func validWorkflowType(value domain.WorkflowType) bool {
return value == domain.WorkflowTypeBatch || value == domain.WorkflowTypeStreaming
}
func validateWorkflowExecutionConfig(maxConcurrentRuns int, nodeTimeoutSeconds int, runTimeoutSeconds int, maxRetryAttempts int, retryBackoffSeconds int) []validationIssue {
issues := make([]validationIssue, 0, 5)
if maxConcurrentRuns < 0 {
issues = append(issues, validationIssue{Field: "maxConcurrentRuns", Message: "maxConcurrentRuns must be zero or greater"})
}
if nodeTimeoutSeconds < 0 {
issues = append(issues, validationIssue{Field: "nodeTimeoutSeconds", Message: "nodeTimeoutSeconds must be zero or greater"})
}
if runTimeoutSeconds < 0 {
issues = append(issues, validationIssue{Field: "runTimeoutSeconds", Message: "runTimeoutSeconds must be zero or greater"})
}
if maxRetryAttempts < 0 {
issues = append(issues, validationIssue{Field: "maxRetryAttempts", Message: "maxRetryAttempts must be zero or greater"})
}
if retryBackoffSeconds < 0 {
issues = append(issues, validationIssue{Field: "retryBackoffSeconds", Message: "retryBackoffSeconds must be zero or greater"})
}
return issues
}
func cloneWorkflowSchedule(schedule *domain.WorkflowSchedule) *domain.WorkflowSchedule {
if schedule == nil {
return nil
}
cloned := *schedule
return &cloned
}
func normalizeWorkflowSchedule(existing *domain.WorkflowSchedule, incoming *domain.WorkflowSchedule, referenceTime string) *domain.WorkflowSchedule {
if incoming == nil {
return cloneWorkflowSchedule(existing)
}
next := *incoming
next.SourcePath = strings.TrimSpace(next.SourcePath)
next.TimeOfDay = strings.TrimSpace(next.TimeOfDay)
next.AnchorAt = strings.TrimSpace(next.AnchorAt)
next.Timezone = strings.TrimSpace(next.Timezone)
if normalizedMode := domain.NormalizeWorkflowScheduleMode(next.Mode); normalizedMode != "" {
next.Mode = normalizedMode
} else if strings.TrimSpace(string(next.Mode)) == "" {
next.Mode = domain.EffectiveWorkflowScheduleMode(&next)
}
effectiveMode := domain.EffectiveWorkflowScheduleMode(&next)
if effectiveMode == domain.WorkflowScheduleModeEveryDaysAtTime || next.Timezone != "" {
next.Timezone = domain.WorkflowScheduleTimezone(next.Timezone)
}
if (effectiveMode == domain.WorkflowScheduleModeEveryHours || effectiveMode == domain.WorkflowScheduleModeEveryDaysAtTime) &&
domain.WorkflowScheduleEnabled(&next) {
if next.AnchorAt == "" || domain.EffectiveWorkflowScheduleMode(existing) != effectiveMode || !domain.WorkflowScheduleEnabled(existing) {
next.AnchorAt = referenceTime
}
}
return &next
}
func validateWorkflowSchedule(schedule *domain.WorkflowSchedule) []validationIssue {
if schedule == nil {
return nil
}
issues := make([]validationIssue, 0, 6)
if rawMode := strings.TrimSpace(string(schedule.Mode)); rawMode != "" && domain.NormalizeWorkflowScheduleMode(schedule.Mode) == "" {
issues = append(issues, validationIssue{Field: "schedule.mode", Message: "schedule.mode must be interval_seconds, every_hours, or every_days_at_time"})
}
if schedule.Interval < 0 {
issues = append(issues, validationIssue{Field: "schedule.interval", Message: "schedule.interval must be zero or greater"})
}
if schedule.IntervalHours < 0 {
issues = append(issues, validationIssue{Field: "schedule.intervalHours", Message: "schedule.intervalHours must be zero or greater"})
}
if schedule.DayInterval < 0 {
issues = append(issues, validationIssue{Field: "schedule.dayInterval", Message: "schedule.dayInterval must be zero or greater"})
}
if timezone := strings.TrimSpace(schedule.Timezone); timezone != "" {
if _, err := time.LoadLocation(timezone); err != nil {
issues = append(issues, validationIssue{Field: "schedule.timezone", Message: "schedule.timezone must be a valid IANA timezone"})
}
}
switch domain.EffectiveWorkflowScheduleMode(schedule) {
case domain.WorkflowScheduleModeEveryHours:
if schedule.IntervalHours < 1 {
issues = append(issues, validationIssue{Field: "schedule.intervalHours", Message: "schedule.intervalHours must be greater than zero"})
}
case domain.WorkflowScheduleModeEveryDaysAtTime:
if schedule.DayInterval < 1 {
issues = append(issues, validationIssue{Field: "schedule.dayInterval", Message: "schedule.dayInterval must be greater than zero"})
}
if !workflowScheduleTimeOfDayPattern.MatchString(strings.TrimSpace(schedule.TimeOfDay)) {
issues = append(issues, validationIssue{Field: "schedule.timeOfDay", Message: "schedule.timeOfDay must use HH:mm format"})
}
}
return issues
}
func (server *server) handleProjectWorkflowsCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CreateWorkflowInput
if !decodeJSON(writer, request, &payload) {
return
}
payload.ProjectID = chi.URLParam(request, "id")
if payload.Type == "" {
payload.Type = domain.WorkflowTypeBatch
}
issues := make([]validationIssue, 0, 2)
if strings.TrimSpace(payload.Name) == "" {
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
}
if !validWorkflowType(payload.Type) {
issues = append(issues, validationIssue{Field: "type", Message: "type must be batch or streaming"})
}
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if issues := validateWorkflowExecutionConfig(payload.MaxConcurrentRuns, payload.NodeTimeoutSeconds, payload.RunTimeoutSeconds, payload.MaxRetryAttempts, payload.RetryBackoffSeconds); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
payload.Schedule = normalizeWorkflowSchedule(nil, payload.Schedule, nowISO())
if issues := validateWorkflowSchedule(payload.Schedule); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Kind == domain.WorkflowKindScan || payload.ScanConfig != nil || domain.FindScanSourceNode(payload.Nodes) != nil {
requestPayload := scanConfigRequestFromSourceNode(payload.Name, payload.Description, payload.Schedule, payload.ScanConfig, payload.Nodes)
if issues := validateScanConfigDraftPayload(requestPayload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if requestPayload.Interval > 0 && !scanConfigPayloadReady(requestPayload, nil) {
renderValidationError(writer, validateScanConfigPayload(requestPayload, nil)...)
return
}
payload.Kind = domain.WorkflowKindScan
payload.ScanConfig = normalizedScanConfigFromRequest(nil, requestPayload)
if issues := server.validateScanConnectionReference(request.Context(), payload.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
payload.Schedule = normalizedScanSchedule(payload.Schedule, payload.ScanConfig)
payload.Type = normalizedScanWorkflowType(payload.ScanConfig)
payload.MaxConcurrentRuns = normalizedScanMaxConcurrentRuns(payload.MaxConcurrentRuns)
if len(payload.Nodes) == 0 {
scaffold, err := server.buildScanWorkflowInput(request.Context(), payload.ProjectID, requestPayload, nil)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
payload.Nodes = scaffold.Nodes
payload.Edges = scaffold.Edges
}
}
if issues := server.validateNodeConnectionReferences(request.Context(), payload.Nodes); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if issues := validateInputPortConnectionTopology(payload.Nodes, payload.Edges); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item, err := server.store.CreateWorkflow(request.Context(), payload)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeWorkflow(item))
}
func (server *server) handleWorkflowsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"})
return
}
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleWorkflowNodeSourceStateGet(writer http.ResponseWriter, request *http.Request) {
workflow, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
response, status, err := server.buildWorkflowNodeSourceStateResponse(request.Context(), workflow, chi.URLParam(request, "nodeId"))
if err != nil {
renderError(writer, status, err)
return
}
if response == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow node not found"})
return
}
renderJSON(writer, status, response)
}
func (server *server) handleWorkflowNodeSourceStateDelete(writer http.ResponseWriter, request *http.Request) {
workflow, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
response, status, err := server.buildWorkflowNodeSourceStateResponse(request.Context(), workflow, chi.URLParam(request, "nodeId"))
if err != nil {
renderError(writer, status, err)
return
}
if response == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow node not found"})
return
}
if !response.Supported {
renderValidationError(writer, validationIssue{Field: "nodeId", Message: response.Reason})
return
}
if !response.Stateful || strings.TrimSpace(response.CurrentFingerprint) == "" {
writer.WriteHeader(http.StatusNoContent)
return
}
if err := server.store.DeleteSourceState(request.Context(), workflow.ID, response.NodeID, response.CurrentFingerprint); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
writer.WriteHeader(http.StatusNoContent)
}
type workflowPatch struct {
Name *string `json:"name"`
Description *string `json:"description"`
Type *domain.WorkflowType `json:"type"`
Kind *domain.WorkflowKind `json:"kind"`
ScanConfig *domain.ScanConfig `json:"scanConfig"`
Hyperparams *[]domain.HyperParam `json:"hyperparams"`
MaxConcurrentRuns *int `json:"maxConcurrentRuns"`
NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"`
RunTimeoutSeconds *int `json:"runTimeoutSeconds"`
MaxRetryAttempts *int `json:"maxRetryAttempts"`
RetryBackoffSeconds *int `json:"retryBackoffSeconds"`
Schedule *domain.WorkflowSchedule `json:"schedule"`
Nodes *[]domain.WorkflowNode `json:"nodes"`
Edges *[]domain.WorkflowEdge `json:"edges"`
Status *domain.WorkflowStatus `json:"status"`
}
func scanConfigRequestFromWorkflow(name string, description string, schedule *domain.WorkflowSchedule, scanConfig *domain.ScanConfig) scanConfigRequest {
requestPayload := scanConfigRequest{
Name: strings.TrimSpace(name),
Description: strings.TrimSpace(description),
}
if schedule != nil {
requestPayload.Interval = schedule.Interval
requestPayload.SourcePath = strings.TrimSpace(schedule.SourcePath)
requestPayload.Enabled = domain.WorkflowScheduleEnabled(schedule)
}
if scanConfig == nil {
return requestPayload
}
requestPayload.DataSource = scanConfig.DataSource
if sourcePath := strings.TrimSpace(scanConfig.SourcePath); sourcePath != "" {
requestPayload.SourcePath = sourcePath
}
requestPayload.ConnectionID = scanConfig.ConnectionID
requestPayload.Bucket = scanConfig.Bucket
requestPayload.Prefix = scanConfig.Prefix
requestPayload.Endpoint = scanConfig.Endpoint
requestPayload.AccessKeyID = scanConfig.AccessKeyID
requestPayload.SecretAccessKey = scanConfig.SecretAccessKey
requestPayload.ScanMode = scanConfig.ScanMode
requestPayload.IncrementalUnit = scanConfig.IncrementalUnit
requestPayload.DirectoryDepth = scanConfig.DirectoryDepth
requestPayload.Pattern = scanConfig.Pattern
requestPayload.Notifications = scanConfig.Notifications
requestPayload.Enabled = scanConfig.Enabled || requestPayload.Enabled
return requestPayload
}
func workflowHyperparamValue(node *domain.WorkflowNode, key string) string {
if node == nil {
return ""
}
target := strings.TrimSpace(key)
if target == "" {
return ""
}
for _, item := range node.Data.Hyperparams {
if strings.TrimSpace(item.Key) == target {
return strings.TrimSpace(item.Value)
}
}
return ""
}
func scanConfigRequestFromSourceNode(name string, description string, schedule *domain.WorkflowSchedule, scanConfig *domain.ScanConfig, nodes []domain.WorkflowNode) scanConfigRequest {
requestPayload := scanConfigRequestFromWorkflow(name, description, schedule, scanConfig)
node := domain.FindScanSourceNode(nodes)
if node == nil {
return requestPayload
}
if requestPayload.DataSource == "" {
requestPayload.DataSource = chooseDataSource(workflowHyperparamValue(node, "DATA_SOURCE"))
}
if requestPayload.ConnectionID == "" {
requestPayload.ConnectionID = strings.TrimSpace(node.Data.ConnectionID)
}
if requestPayload.Bucket == "" {
requestPayload.Bucket = strings.TrimSpace(node.Data.Bucket)
}
if requestPayload.Prefix == "" {
requestPayload.Prefix = cleanBOSPrefix(node.Data.Prefix)
}
if requestPayload.SourcePath == "" {
requestPayload.SourcePath = strings.TrimSpace(workflowHyperparamValue(node, "SOURCE_PATH"))
}
if requestPayload.Endpoint == "" {
requestPayload.Endpoint = strings.TrimSpace(workflowHyperparamValue(node, "BOS_ENDPOINT"))
}
if requestPayload.AccessKeyID == "" {
requestPayload.AccessKeyID = strings.TrimSpace(workflowHyperparamValue(node, "BOS_ACCESS_KEY_ID"))
}
if requestPayload.SecretAccessKey == "" {
requestPayload.SecretAccessKey = strings.TrimSpace(workflowHyperparamValue(node, "BOS_SECRET_ACCESS_KEY"))
}
if requestPayload.ScanMode == "" {
requestPayload.ScanMode = chooseWorkflowScanMode(workflowHyperparamValue(node, "SCAN_MODE"))
}
if requestPayload.IncrementalUnit == "" {
requestPayload.IncrementalUnit = chooseWorkflowIncrementalUnit(workflowHyperparamValue(node, "INCREMENTAL_UNIT"))
}
if requestPayload.DirectoryDepth <= 0 {
if raw := strings.TrimSpace(workflowHyperparamValue(node, "INCREMENTAL_DIRECTORY_DEPTH")); raw != "" {
if parsed, err := strconv.Atoi(raw); err == nil {
requestPayload.DirectoryDepth = parsed
}
}
}
if requestPayload.Pattern == "" {
requestPayload.Pattern = strings.TrimSpace(node.Data.Pattern)
}
if requestPayload.Enabled {
requestPayload.Enabled = true
}
return requestPayload
}
func normalizedScanConfigFromRequest(existing *domain.ScanConfig, requestPayload scanConfigRequest) *domain.ScanConfig {
dataSource := chooseDataSource(requestPayload.DataSource)
sourcePath := scanSourcePath(dataSource, requestPayload.SourcePath, requestPayload.Bucket, requestPayload.Prefix)
connectionID := ""
endpoint := ""
accessKeyID := ""
secretAccessKey := ""
if dataSource == "BOS" {
connectionID = mergeScanConnectionID(existing, requestPayload)
if connectionID == "" {
endpoint = chooseBOSEndpoint(requestPayload.Endpoint)
accessKeyID, secretAccessKey = mergeScanSecrets(existing, requestPayload)
}
}
return &domain.ScanConfig{
DataSource: dataSource,
SourcePath: sourcePath,
ConnectionID: connectionID,
Bucket: strings.TrimSpace(requestPayload.Bucket),
Prefix: cleanBOSPrefix(requestPayload.Prefix),
Endpoint: endpoint,
AccessKeyID: accessKeyID,
SecretAccessKey: secretAccessKey,
ScanMode: chooseWorkflowScanMode(requestPayload.ScanMode),
IncrementalUnit: chooseWorkflowIncrementalUnit(requestPayload.IncrementalUnit),
DirectoryDepth: chooseWorkflowDirectoryDepth(requestPayload.DirectoryDepth),
Pattern: strings.TrimSpace(requestPayload.Pattern),
Enabled: requestPayload.Enabled,
Notifications: requestPayload.Notifications,
}
}
func normalizedScanSchedule(existing *domain.WorkflowSchedule, scanConfig *domain.ScanConfig) *domain.WorkflowSchedule {
next := &domain.WorkflowSchedule{}
if existing != nil {
*next = *existing
}
if scanConfig == nil {
return next
}
next.SourcePath = scanConfig.SourcePath
if scanConfig.Enabled {
if !domain.WorkflowScheduleEnabled(next) {
next.Mode = domain.WorkflowScheduleModeIntervalSeconds
next.Interval = 60
}
return next
}
if domain.EffectiveWorkflowScheduleMode(next) == domain.WorkflowScheduleModeIntervalSeconds {
next.Interval = 0
}
return next
}
func normalizedScanWorkflowType(scanConfig *domain.ScanConfig) domain.WorkflowType {
if scanConfig != nil && scanConfig.Enabled {
return domain.WorkflowTypeStreaming
}
return domain.WorkflowTypeBatch
}
func normalizedScanMaxConcurrentRuns(value int) int {
if value > 0 {
return value
}
return 1
}
func (server *server) validateScanConnectionReference(ctx context.Context, scanConfig *domain.ScanConfig) []validationIssue {
if scanConfig == nil || strings.TrimSpace(scanConfig.ConnectionID) == "" {
return nil
}
if _, err := server.getStorageConnection(ctx, scanConfig.ConnectionID); err != nil {
return []validationIssue{{Field: "scanConfig.connectionId", Message: "connectionId must reference an existing storage connection"}}
}
return nil
}
func (server *server) validateNodeConnectionReferences(ctx context.Context, nodes []domain.WorkflowNode) []validationIssue {
issues := make([]validationIssue, 0)
for index, node := range nodes {
storageConnectionID := strings.TrimSpace(node.Data.ConnectionID)
if storageConnectionID != "" {
if _, err := server.getStorageConnection(ctx, storageConnectionID); err != nil {
issues = append(issues, validationIssue{
Field: "nodes[" + strconv.Itoa(index) + "].data.connectionId",
Message: "connectionId must reference an existing storage connection",
})
}
}
mongoConnectionID := ""
for _, item := range node.Data.Hyperparams {
if strings.EqualFold(strings.TrimSpace(item.Key), "MONGO_CONNECTION_ID") {
mongoConnectionID = strings.TrimSpace(item.Value)
break
}
}
if mongoConnectionID == "" {
continue
}
if _, err := server.getMongoConnection(ctx, mongoConnectionID); err != nil {
issues = append(issues, validationIssue{
Field: "nodes[" + strconv.Itoa(index) + "].data.hyperparams",
Message: "MONGO_CONNECTION_ID must reference an existing mongo connection",
})
}
}
return issues
}
func validateInputPortConnectionTopology(nodes []domain.WorkflowNode, edges []domain.WorkflowEdge) []validationIssue {
conflicts := domain.FindInputPortConflicts(nodes, edges)
if len(conflicts) == 0 {
return nil
}
issues := make([]validationIssue, 0, len(conflicts))
for _, conflict := range conflicts {
nodeLabel := strings.TrimSpace(conflict.TargetNodeLabel)
if nodeLabel == "" {
nodeLabel = conflict.TargetNodeID
}
portName := strings.TrimSpace(conflict.TargetPortName)
if portName == "" {
portName = "默认输入"
}
issues = append(issues, validationIssue{
Field: "edges",
Message: "节点“" + nodeLabel + "”的输入端口“" + portName + "”连接了多条上游连线。" +
"每个输入端口只能连接一条上游连线;如需上传多个结果,请为每个结果添加单独的上传算子。",
})
}
return issues
}
func (server *server) handleWorkflowsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"})
return
}
var payload workflowPatch
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Name != nil {
item.Name = *payload.Name
}
if payload.Description != nil {
item.Description = *payload.Description
}
if payload.Type != nil {
item.Type = *payload.Type
}
if payload.Kind != nil {
item.Kind = *payload.Kind
}
if payload.Hyperparams != nil {
item.Hyperparams = *payload.Hyperparams
}
if payload.MaxConcurrentRuns != nil {
if issues := validateWorkflowExecutionConfig(*payload.MaxConcurrentRuns, 0, 0, 0, 0); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.MaxConcurrentRuns = *payload.MaxConcurrentRuns
}
if payload.NodeTimeoutSeconds != nil {
if issues := validateWorkflowExecutionConfig(0, *payload.NodeTimeoutSeconds, 0, 0, 0); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.NodeTimeoutSeconds = *payload.NodeTimeoutSeconds
}
if payload.RunTimeoutSeconds != nil {
if issues := validateWorkflowExecutionConfig(0, 0, *payload.RunTimeoutSeconds, 0, 0); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.RunTimeoutSeconds = *payload.RunTimeoutSeconds
}
if payload.MaxRetryAttempts != nil {
if issues := validateWorkflowExecutionConfig(0, 0, 0, *payload.MaxRetryAttempts, 0); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.MaxRetryAttempts = *payload.MaxRetryAttempts
}
if payload.RetryBackoffSeconds != nil {
if issues := validateWorkflowExecutionConfig(0, 0, 0, 0, *payload.RetryBackoffSeconds); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.RetryBackoffSeconds = *payload.RetryBackoffSeconds
}
if payload.Schedule != nil {
item.Schedule = normalizeWorkflowSchedule(item.Schedule, payload.Schedule, nowISO())
if issues := validateWorkflowSchedule(item.Schedule); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
}
if payload.Nodes != nil {
item.Nodes = *payload.Nodes
}
if payload.Edges != nil {
item.Edges = *payload.Edges
}
if item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil || payload.ScanConfig != nil || domain.FindScanSourceNode(item.Nodes) != nil {
existingScanConfig := item.ScanConfig
effectiveScanConfig := item.ScanConfig
if payload.ScanConfig != nil {
effectiveScanConfig = payload.ScanConfig
}
requestPayload := scanConfigRequestFromSourceNode(item.Name, item.Description, item.Schedule, effectiveScanConfig, item.Nodes)
if issues := validateScanConfigDraftPayload(requestPayload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if requestPayload.Interval > 0 && !scanConfigPayloadReady(requestPayload, existingScanConfig) {
renderValidationError(writer, validateScanConfigPayload(requestPayload, existingScanConfig)...)
return
}
item.Kind = domain.WorkflowKindScan
item.ScanConfig = normalizedScanConfigFromRequest(existingScanConfig, requestPayload)
if issues := server.validateScanConnectionReference(request.Context(), item.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.Schedule = normalizedScanSchedule(item.Schedule, item.ScanConfig)
item.Type = normalizedScanWorkflowType(item.ScanConfig)
item.MaxConcurrentRuns = normalizedScanMaxConcurrentRuns(item.MaxConcurrentRuns)
}
if issues := server.validateNodeConnectionReferences(request.Context(), item.Nodes); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if issues := validateInputPortConnectionTopology(item.Nodes, item.Edges); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Status != nil {
item.Status = *payload.Status
}
item.UpdatedAt = nowISO()
if err := server.store.UpsertWorkflow(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeWorkflow(item))
}
func (server *server) handleWorkflowsDelete(writer http.ResponseWriter, request *http.Request) {
deleted, err := server.store.DeleteWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
type runRequest struct {
Trigger domain.RunTrigger `json:"trigger"`
}
func validRunTrigger(value domain.RunTrigger) bool {
return value == domain.RunTriggerManual || value == domain.RunTriggerScheduled || value == domain.RunTriggerRetry
}
type rerunSubtreeRequest struct {
NodeID string `json:"nodeId"`
}
func nextRunAttemptForChild(run *domain.RunRecord) int {
if run == nil || run.Attempt <= 0 {
return 2
}
return run.Attempt + 1
}
func (server *server) handleWorkflowRun(writer http.ResponseWriter, request *http.Request) {
payload := runRequest{Trigger: domain.RunTriggerManual}
if request.ContentLength != 0 {
if !decodeJSON(writer, request, &payload) {
return
}
}
if !validRunTrigger(payload.Trigger) {
renderValidationError(writer, validationIssue{Field: "trigger", Message: "trigger must be manual, scheduled, or retry"})
return
}
workflow, err := server.store.GetWorkflow(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if workflow == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"})
return
}
if workflow.Kind == domain.WorkflowKindScan || workflow.ScanConfig != nil {
requestPayload := scanConfigRequestFromWorkflow(workflow.Name, workflow.Description, workflow.Schedule, workflow.ScanConfig)
if !scanConfigPayloadReady(requestPayload, workflow.ScanConfig) {
renderJSON(writer, http.StatusBadRequest, map[string]string{"error": "扫描源配置未完成,请先在扫描源节点配置数据源、目标路径和访问密钥"})
return
}
if issues := server.validateScanConnectionReference(request.Context(), workflow.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
}
if issues := validateInputPortConnectionTopology(workflow.Nodes, workflow.Edges); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
run, err := server.store.CreateRun(request.Context(), chi.URLParam(request, "id"), payload.Trigger)
if err != nil {
if strings.Contains(err.Error(), "not found") {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()})
return
}
if errors.Is(err, store.ErrWorkflowAlreadyCompleted) {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流已成功执行,不支持再次手动运行"})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusAccepted, run)
}
func (server *server) handleWorkflowRunsList(writer http.ResponseWriter, request *http.Request) {
workflowID := chi.URLParam(request, "id")
workflow, err := server.store.GetWorkflow(request.Context(), workflowID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if workflow == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"})
return
}
items, err := server.store.ListRuns(request.Context(), "")
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
summaries := make([]domain.RunSummary, 0, len(items))
for _, item := range items {
if item.WorkflowID != workflowID {
continue
}
summaries = append(summaries, runSummaryFromRecord(item))
}
renderJSON(writer, http.StatusOK, map[string]any{"items": summaries})
}
func (server *server) handleRunsRerunSubtree(writer http.ResponseWriter, request *http.Request) {
sourceRun, err := server.loadRun(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if sourceRun == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
if sourceRun.Status != domain.RunStatusFailed {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "只有失败的运行才支持从指定节点重新执行下游"})
return
}
var payload rerunSubtreeRequest
if !decodeJSON(writer, request, &payload) {
return
}
payload.NodeID = strings.TrimSpace(payload.NodeID)
if payload.NodeID == "" {
renderValidationError(writer, validationIssue{Field: "nodeId", Message: "nodeId is required"})
return
}
workflow, err := server.store.GetWorkflow(request.Context(), sourceRun.WorkflowID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if workflow == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Workflow not found"})
return
}
if workflowNodeByID(workflow, payload.NodeID) == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Node not found"})
return
}
subtree := rerunSubtreeNodeSet(workflow.Edges, payload.NodeID)
baselineSet := rerunBaselineNodeSet(workflow.Edges, subtree)
sourceExecutionMap := make(map[string]domain.NodeExecutionRecord, len(sourceRun.NodeExecutions))
for _, execution := range sourceRun.NodeExecutions {
sourceExecutionMap[execution.NodeID] = execution
}
baselineExecutions := make([]domain.NodeExecutionRecord, 0, len(baselineSet))
for _, execution := range sourceRun.NodeExecutions {
if _, ok := baselineSet[execution.NodeID]; !ok {
continue
}
if execution.Status != domain.NodeExecutionStatusCompleted {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "选定节点缺少可复用的上游成功基线,无法从这里继续向后执行"})
return
}
baselineExecutions = append(baselineExecutions, cloneNodeExecutionRecord(execution))
}
for nodeID := range baselineSet {
execution, ok := sourceExecutionMap[nodeID]
if !ok || execution.Status != domain.NodeExecutionStatusCompleted {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "选定节点缺少可复用的上游成功基线,无法从这里继续向后执行"})
return
}
}
childRun, err := server.store.CreateRun(request.Context(), workflow.ID, domain.RunTriggerRerunSubtree)
if err != nil {
if strings.Contains(err.Error(), "not found") {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()})
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
childRun.ParentRunID = sourceRun.ID
childRun.RestartFromNodeID = payload.NodeID
childRun.Attempt = nextRunAttemptForChild(sourceRun)
childRun.NodeExecutions = baselineExecutions
childRun.K8sRefs = []domain.K8sRef{}
childRun.Error = ""
childRun.CurrentNodeID = ""
childRun.StartedAt = ""
childRun.EndedAt = ""
childRun.DurationMS = 0
childRun.Status = domain.RunStatusQueued
childRun.AvailableAt = nowISO()
childRun.UpdatedAt = nowISO()
rebuildRunAggregates(childRun)
if err := server.store.UpsertRun(request.Context(), *childRun); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusAccepted, childRun)
}
func (server *server) handleTemplatesList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListTemplates(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeTemplates(items)})
}
func (server *server) handleTemplatesCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CreateTemplateInput
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Type == "" {
payload.Type = domain.WorkflowTypeBatch
}
issues := make([]validationIssue, 0)
if strings.TrimSpace(payload.Name) == "" {
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
}
if !validWorkflowType(payload.Type) {
issues = append(issues, validationIssue{Field: "type", Message: "type must be batch or streaming"})
}
if configIssues := validateWorkflowExecutionConfig(payload.MaxConcurrentRuns, payload.NodeTimeoutSeconds, payload.RunTimeoutSeconds, payload.MaxRetryAttempts, payload.RetryBackoffSeconds); len(configIssues) > 0 {
issues = append(issues, configIssues...)
}
payload.Schedule = normalizeWorkflowSchedule(nil, payload.Schedule, nowISO())
issues = append(issues, validateWorkflowSchedule(payload.Schedule)...)
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if payload.Kind == domain.WorkflowKindScan || payload.ScanConfig != nil || domain.FindScanSourceNode(payload.Nodes) != nil {
requestPayload := scanConfigRequestFromSourceNode(payload.Name, payload.Description, payload.Schedule, payload.ScanConfig, payload.Nodes)
if issues := validateScanConfigDraftPayload(requestPayload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if requestPayload.Interval > 0 && !scanConfigPayloadReady(requestPayload, nil) {
renderValidationError(writer, validateScanConfigPayload(requestPayload, nil)...)
return
}
payload.Kind = domain.WorkflowKindScan
payload.ScanConfig = normalizedScanConfigFromRequest(nil, requestPayload)
if issues := server.validateScanConnectionReference(request.Context(), payload.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
payload.Schedule = normalizedScanSchedule(payload.Schedule, payload.ScanConfig)
payload.Type = normalizedScanWorkflowType(payload.ScanConfig)
payload.MaxConcurrentRuns = normalizedScanMaxConcurrentRuns(payload.MaxConcurrentRuns)
}
if issues := validateInputPortConnectionTopology(payload.Nodes, payload.Edges); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item, err := server.store.CreateTemplate(request.Context(), payload)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeTemplate(item))
}
func (server *server) handleTemplatesGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetTemplate(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Template not found"})
return
}
renderJSON(writer, http.StatusOK, sanitizeTemplate(item))
}
type templatePatch struct {
Name *string `json:"name"`
Description *string `json:"description"`
Type *domain.WorkflowType `json:"type"`
Kind *domain.WorkflowKind `json:"kind"`
ScanConfig *domain.ScanConfig `json:"scanConfig"`
Hyperparams *[]domain.HyperParam `json:"hyperparams"`
MaxConcurrentRuns *int `json:"maxConcurrentRuns"`
NodeTimeoutSeconds *int `json:"nodeTimeoutSeconds"`
RunTimeoutSeconds *int `json:"runTimeoutSeconds"`
MaxRetryAttempts *int `json:"maxRetryAttempts"`
RetryBackoffSeconds *int `json:"retryBackoffSeconds"`
Schedule *domain.WorkflowSchedule `json:"schedule"`
Nodes *[]domain.WorkflowNode `json:"nodes"`
Edges *[]domain.WorkflowEdge `json:"edges"`
}
func (server *server) handleTemplatesUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetTemplate(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Template not found"})
return
}
var payload templatePatch
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Name != nil {
item.Name = *payload.Name
}
if payload.Description != nil {
item.Description = *payload.Description
}
if payload.Type != nil {
if !validWorkflowType(*payload.Type) {
renderValidationError(writer, validationIssue{Field: "type", Message: "type must be batch or streaming"})
return
}
item.Type = *payload.Type
}
if payload.Kind != nil {
item.Kind = *payload.Kind
}
if payload.Hyperparams != nil {
item.Hyperparams = *payload.Hyperparams
}
if payload.MaxConcurrentRuns != nil {
if issues := validateWorkflowExecutionConfig(*payload.MaxConcurrentRuns, 0, 0, 0, 0); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.MaxConcurrentRuns = *payload.MaxConcurrentRuns
}
if payload.NodeTimeoutSeconds != nil {
if issues := validateWorkflowExecutionConfig(0, *payload.NodeTimeoutSeconds, 0, 0, 0); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.NodeTimeoutSeconds = *payload.NodeTimeoutSeconds
}
if payload.RunTimeoutSeconds != nil {
if issues := validateWorkflowExecutionConfig(0, 0, *payload.RunTimeoutSeconds, 0, 0); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.RunTimeoutSeconds = *payload.RunTimeoutSeconds
}
if payload.MaxRetryAttempts != nil {
if issues := validateWorkflowExecutionConfig(0, 0, 0, *payload.MaxRetryAttempts, 0); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.MaxRetryAttempts = *payload.MaxRetryAttempts
}
if payload.RetryBackoffSeconds != nil {
if issues := validateWorkflowExecutionConfig(0, 0, 0, 0, *payload.RetryBackoffSeconds); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.RetryBackoffSeconds = *payload.RetryBackoffSeconds
}
if payload.Schedule != nil {
item.Schedule = normalizeWorkflowSchedule(item.Schedule, payload.Schedule, nowISO())
if issues := validateWorkflowSchedule(item.Schedule); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
}
if payload.Nodes != nil {
item.Nodes = *payload.Nodes
}
if payload.Edges != nil {
item.Edges = *payload.Edges
}
if item.Kind == domain.WorkflowKindScan || item.ScanConfig != nil || payload.ScanConfig != nil || domain.FindScanSourceNode(item.Nodes) != nil {
existingScanConfig := item.ScanConfig
effectiveScanConfig := item.ScanConfig
if payload.ScanConfig != nil {
effectiveScanConfig = payload.ScanConfig
}
requestPayload := scanConfigRequestFromSourceNode(item.Name, item.Description, item.Schedule, effectiveScanConfig, item.Nodes)
if issues := validateScanConfigDraftPayload(requestPayload); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
if requestPayload.Interval > 0 && !scanConfigPayloadReady(requestPayload, existingScanConfig) {
renderValidationError(writer, validateScanConfigPayload(requestPayload, existingScanConfig)...)
return
}
item.Kind = domain.WorkflowKindScan
item.ScanConfig = normalizedScanConfigFromRequest(existingScanConfig, requestPayload)
if issues := server.validateScanConnectionReference(request.Context(), item.ScanConfig); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.Schedule = normalizedScanSchedule(item.Schedule, item.ScanConfig)
item.Type = normalizedScanWorkflowType(item.ScanConfig)
item.MaxConcurrentRuns = normalizedScanMaxConcurrentRuns(item.MaxConcurrentRuns)
}
if issues := validateInputPortConnectionTopology(item.Nodes, item.Edges); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.UpdatedAt = nowISO()
if err := server.store.UpsertTemplate(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeTemplate(item))
}
func (server *server) handleTemplatesDelete(writer http.ResponseWriter, request *http.Request) {
deleted, err := server.store.DeleteTemplate(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func (server *server) handleOperatorsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.listAvailableOperators(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeOperators(items)})
}
func (server *server) handleOperatorsCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CreateOperatorInput
if !decodeJSON(writer, request, &payload) {
return
}
if strings.TrimSpace(payload.Name) == "" || strings.TrimSpace(payload.Category) == "" || strings.TrimSpace(payload.Description) == "" {
renderValidationError(writer,
validationIssue{Field: "name", Message: "name is required"},
validationIssue{Field: "category", Message: "category is required"},
validationIssue{Field: "description", Message: "description is required"},
)
return
}
item, err := server.store.CreateOperator(request.Context(), payload)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeOperator(item))
}
func (server *server) handleOperatorsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetOperator(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Operator not found"})
return
}
renderJSON(writer, http.StatusOK, sanitizeOperator(item))
}
type operatorPatch struct {
Name *string `json:"name"`
Type *string `json:"type"`
Category *string `json:"category"`
Description *string `json:"description"`
Inputs *[]domain.PortDefinition `json:"inputs"`
Outputs *[]domain.PortDefinition `json:"outputs"`
ConfigSchema *map[string]any `json:"configSchema"`
Image *string `json:"image"`
SkillDefinition *string `json:"skillDefinition"`
Hyperparams *[]domain.HyperParam `json:"hyperparams"`
Resources *domain.OperatorResources `json:"resources"`
}
func (server *server) handleOperatorsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetOperator(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Operator not found"})
return
}
var payload operatorPatch
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Name != nil {
item.Name = *payload.Name
}
if payload.Type != nil {
item.Type = *payload.Type
}
if payload.Category != nil {
item.Category = *payload.Category
}
if payload.Description != nil {
item.Description = *payload.Description
}
if payload.Inputs != nil {
item.Inputs = *payload.Inputs
}
if payload.Outputs != nil {
item.Outputs = *payload.Outputs
}
if payload.ConfigSchema != nil {
item.ConfigSchema = *payload.ConfigSchema
}
if payload.Image != nil {
item.Image = *payload.Image
}
if payload.SkillDefinition != nil {
item.SkillDefinition = *payload.SkillDefinition
}
if payload.Hyperparams != nil {
item.Hyperparams = *payload.Hyperparams
}
if payload.Resources != nil {
item.Resources = payload.Resources
}
item.UpdatedAt = nowISO()
if err := server.store.UpsertOperator(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeOperator(item))
}
func (server *server) handleOperatorsDelete(writer http.ResponseWriter, request *http.Request) {
id := chi.URLParam(request, "id")
item, err := server.store.GetOperator(request.Context(), id)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
writer.WriteHeader(http.StatusNotFound)
return
}
if strings.EqualFold(strings.TrimSpace(item.Type), "builtin") {
if err := server.store.UpsertBuiltinOperatorDeletion(request.Context(), domain.BuiltinOperatorDeletionRecord{
Key: builtinOperatorDeletionKey(item.ID, item.Name),
OperatorID: item.ID,
OperatorName: item.Name,
DeletedAt: nowISO(),
}); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
deleted, err := server.store.DeleteOperator(request.Context(), id)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func (server *server) handleRunsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListRunSummaries(request.Context(), request.URL.Query().Get("status"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) loadRun(ctx context.Context, id string) (*domain.RunRecord, error) {
item, err := server.store.GetRun(ctx, id)
if err != nil || item == nil {
return item, err
}
return item, nil
}
func (server *server) handleRunsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.loadRun(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
renderJSON(writer, http.StatusOK, item)
}
func (server *server) markWorkflowStopped(ctx context.Context, run *domain.RunRecord) error {
if run == nil || strings.TrimSpace(run.WorkflowID) == "" {
return nil
}
workflow, err := server.store.GetWorkflow(ctx, run.WorkflowID)
if err != nil || workflow == nil {
return err
}
workflow.Status = domain.WorkflowStatusFailed
workflow.LastRunID = run.ID
workflow.LastRunAt = nowISO()
workflow.UpdatedAt = nowISO()
return server.store.UpsertWorkflow(ctx, *workflow)
}
func (server *server) terminateRunJobs(ctx context.Context, run *domain.RunRecord) error {
if run == nil || strings.TrimSpace(run.ID) == "" {
return nil
}
settings, err := server.store.GetSettings(ctx)
if err != nil || settings == nil {
return err
}
launcher, err := runJobTerminatorFactory(settings.K8s.Namespace, "", settings.K8s.WorkspacePVCName, settings.K8s.WorkspaceMountPath)
if err != nil {
return err
}
if err := launcher.TerminateRun(ctx, settings.K8s.Namespace, run.ID); err != nil {
return err
}
for _, ref := range run.K8sRefs {
if err := launcher.TerminateJob(ctx, ref.Namespace, ref.JobName); err != nil {
return err
}
}
return nil
}
func (server *server) handleRunStopWithMode(writer http.ResponseWriter, request *http.Request, mode domain.RunStopMode) {
run, err := server.store.RequestRunStop(request.Context(), chi.URLParam(request, "id"), mode)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if run == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
if mode == domain.RunStopModeForce && run.Status == domain.RunStatusCancelled {
if err := server.terminateRunJobs(request.Context(), run); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if err := server.markWorkflowStopped(request.Context(), run); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
if mode == domain.RunStopModeGraceful && run.Status == domain.RunStatusCancelled {
if err := server.markWorkflowStopped(request.Context(), run); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
}
renderJSON(writer, http.StatusOK, run)
}
func (server *server) handleRunsStop(writer http.ResponseWriter, request *http.Request) {
server.handleRunStopWithMode(writer, request, domain.RunStopModeGraceful)
}
func (server *server) handleRunsForceStop(writer http.ResponseWriter, request *http.Request) {
server.handleRunStopWithMode(writer, request, domain.RunStopModeForce)
}
func (server *server) handleRunsLogs(writer http.ResponseWriter, request *http.Request) {
item, err := server.loadRun(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": item.Logs})
}
func (server *server) handleRunsResources(writer http.ResponseWriter, request *http.Request) {
item, err := server.loadRun(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": item.ResourceSeries})
}
func (server *server) handleRunsArtifacts(writer http.ResponseWriter, request *http.Request) {
item, err := server.loadRun(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Run not found"})
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": item.Artifacts})
}
func (server *server) handleAuditTasksList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListAuditTasks(request.Context(), request.URL.Query().Get("status"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) resolveAuditTask(ctx context.Context, id string, decision domain.AuditStatus) (*domain.AuditTaskRecord, error) {
task, err := server.store.GetAuditTask(ctx, id)
if err != nil || task == nil {
return task, err
}
task.Status = decision
task.UpdatedAt = nowISO()
if err := server.store.UpsertAuditTask(ctx, *task); err != nil {
return nil, err
}
run, err := server.store.GetRun(ctx, task.RunID)
if err != nil {
return nil, err
}
if run != nil {
run.Status = domain.RunStatusQueued
run.UpdatedAt = nowISO()
if err := server.store.UpsertRun(ctx, *run); err != nil {
return nil, err
}
}
return task, nil
}
func (server *server) handleAuditTaskApprove(writer http.ResponseWriter, request *http.Request) {
task, err := server.resolveAuditTask(request.Context(), chi.URLParam(request, "id"), domain.AuditStatusApproved)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if task == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Audit task not found"})
return
}
renderJSON(writer, http.StatusOK, task)
}
func (server *server) handleAuditTaskReject(writer http.ResponseWriter, request *http.Request) {
task, err := server.resolveAuditTask(request.Context(), chi.URLParam(request, "id"), domain.AuditStatusRejected)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if task == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Audit task not found"})
return
}
renderJSON(writer, http.StatusOK, task)
}
func (server *server) handleUsersList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListUsers(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeUsers(items)})
}
func normalizeUserInput(name string, email string) (string, string) {
return strings.TrimSpace(name), strings.TrimSpace(strings.ToLower(email))
}
func (server *server) handleUsersCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CreateUserInput
if !decodeJSON(writer, request, &payload) {
return
}
payload.Name, payload.Email = normalizeUserInput(payload.Name, payload.Email)
if payload.Name == "" {
renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"})
return
}
if _, err := mail.ParseAddress(payload.Email); err != nil {
renderValidationError(writer, validationIssue{Field: "email", Message: "email must be valid"})
return
}
if !validUserRole(payload.Role) {
renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, auditor, or operator"})
return
}
item, err := server.store.CreateUser(request.Context(), payload)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeUser(item))
}
type userPatch struct {
Name *string `json:"name"`
Email *string `json:"email"`
Role *domain.UserRole `json:"role"`
}
func validUserRole(value domain.UserRole) bool {
return value == domain.UserRoleAdmin || value == domain.UserRoleAuditor || value == domain.UserRoleOperator
}
func (server *server) handleUsersUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetUser(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "User not found"})
return
}
var payload userPatch
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Name != nil {
item.Name = strings.TrimSpace(*payload.Name)
}
if payload.Email != nil {
normalizedEmail := strings.TrimSpace(strings.ToLower(*payload.Email))
if _, err := mail.ParseAddress(normalizedEmail); err != nil {
renderValidationError(writer, validationIssue{Field: "email", Message: "email must be valid"})
return
}
item.Email = normalizedEmail
}
if payload.Role != nil {
if !validUserRole(*payload.Role) {
renderValidationError(writer, validationIssue{Field: "role", Message: "role must be admin, auditor, or operator"})
return
}
item.Role = *payload.Role
}
item.UpdatedAt = nowISO()
if err := server.store.UpsertUser(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeUser(item))
}
func (server *server) handleUsersDelete(writer http.ResponseWriter, request *http.Request) {
deleted, err := server.store.DeleteUser(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
type storagePathFavoritesPayload struct {
Items []domain.StoragePathFavoriteRecord `json:"items"`
}
func (server *server) handleUserStoragePathFavoritesGet(writer http.ResponseWriter, request *http.Request) {
user, err := server.store.GetUser(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if user == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "User not found"})
return
}
renderJSON(writer, http.StatusOK, map[string]any{
"items": sanitizeStoragePathFavorites(user.StoragePathFavorites),
})
}
func (server *server) handleUserStoragePathFavoritesUpdate(writer http.ResponseWriter, request *http.Request) {
user, err := server.store.GetUser(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if user == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "User not found"})
return
}
var payload storagePathFavoritesPayload
if !decodeJSON(writer, request, &payload) {
return
}
user.StoragePathFavorites = sanitizeStoragePathFavorites(payload.Items)
user.UpdatedAt = nowISO()
if err := server.store.UpsertUser(request.Context(), *user); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{
"items": user.StoragePathFavorites,
})
}
type storageConnectionPatch struct {
Name *string `json:"name"`
Description *string `json:"description"`
Provider *domain.StorageProvider `json:"provider"`
Endpoint *string `json:"endpoint"`
Bucket *string `json:"bucket"`
AccessKeyID *string `json:"accessKeyId"`
SecretAccessKey *string `json:"secretAccessKey"`
}
type mongoConnectionPatch struct {
Name *string `json:"name"`
Description *string `json:"description"`
URI *string `json:"uri"`
Database *string `json:"database"`
}
func normalizeStorageProvider(value domain.StorageProvider) domain.StorageProvider {
switch domain.StorageProvider(strings.ToUpper(strings.TrimSpace(string(value)))) {
case domain.StorageProviderBOS:
return domain.StorageProviderBOS
default:
return ""
}
}
func validStorageProvider(value domain.StorageProvider) bool {
return normalizeStorageProvider(value) != ""
}
func storageEndpointForProvider(provider domain.StorageProvider, endpoint string) string {
switch normalizeStorageProvider(provider) {
case domain.StorageProviderBOS:
return chooseBOSEndpoint(endpoint)
default:
return strings.TrimSpace(endpoint)
}
}
func (server *server) handleStorageConnectionsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListStorageConnections(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeStorageConnections(items)})
}
func (server *server) handleStorageConnectionsCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CreateStorageConnectionInput
if !decodeJSON(writer, request, &payload) {
return
}
payload.Name = strings.TrimSpace(payload.Name)
payload.Description = strings.TrimSpace(payload.Description)
payload.Provider = normalizeStorageProvider(payload.Provider)
payload.Endpoint = strings.TrimSpace(payload.Endpoint)
payload.Bucket = strings.Trim(strings.TrimSpace(payload.Bucket), "/")
payload.AccessKeyID = strings.TrimSpace(payload.AccessKeyID)
payload.SecretAccessKey = strings.TrimSpace(payload.SecretAccessKey)
if payload.Name == "" {
renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"})
return
}
if !validStorageProvider(payload.Provider) {
renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS"})
return
}
if payload.AccessKeyID == "" {
renderValidationError(writer, validationIssue{Field: "accessKeyId", Message: "accessKeyId is required"})
return
}
if payload.SecretAccessKey == "" {
renderValidationError(writer, validationIssue{Field: "secretAccessKey", Message: "secretAccessKey is required"})
return
}
payload.Endpoint = storageEndpointForProvider(payload.Provider, payload.Endpoint)
item, err := server.store.CreateStorageConnection(request.Context(), payload)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeStorageConnection(item))
}
func (server *server) handleStorageConnectionsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetStorageConnection(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Storage connection not found"})
return
}
renderJSON(writer, http.StatusOK, sanitizeStorageConnection(item))
}
func (server *server) handleStorageConnectionsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetStorageConnection(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Storage connection not found"})
return
}
var payload storageConnectionPatch
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Name != nil {
item.Name = strings.TrimSpace(*payload.Name)
}
if payload.Description != nil {
item.Description = strings.TrimSpace(*payload.Description)
}
if payload.Provider != nil {
provider := normalizeStorageProvider(*payload.Provider)
if !validStorageProvider(provider) {
renderValidationError(writer, validationIssue{Field: "provider", Message: "provider must be BOS"})
return
}
item.Provider = provider
}
if payload.Endpoint != nil {
item.Endpoint = storageEndpointForProvider(item.Provider, *payload.Endpoint)
}
if payload.Bucket != nil {
item.Bucket = strings.Trim(strings.TrimSpace(*payload.Bucket), "/")
}
if payload.AccessKeyID != nil {
if trimmed := strings.TrimSpace(*payload.AccessKeyID); trimmed != "" {
item.AccessKeyID = trimmed
}
}
if payload.SecretAccessKey != nil {
if trimmed := strings.TrimSpace(*payload.SecretAccessKey); trimmed != "" {
item.SecretAccessKey = trimmed
}
}
if strings.TrimSpace(item.Name) == "" {
renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"})
return
}
item.Endpoint = storageEndpointForProvider(item.Provider, item.Endpoint)
item.UpdatedAt = nowISO()
if err := server.store.UpsertStorageConnection(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeStorageConnection(item))
}
func (server *server) handleStorageConnectionsDelete(writer http.ResponseWriter, request *http.Request) {
deleted, err := server.store.DeleteStorageConnection(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func (server *server) handleMongoConnectionsList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListMongoConnections(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": sanitizeMongoConnections(items)})
}
func (server *server) handleMongoConnectionsCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CreateMongoConnectionInput
if !decodeJSON(writer, request, &payload) {
return
}
payload.Name = strings.TrimSpace(payload.Name)
payload.Description = strings.TrimSpace(payload.Description)
payload.URI = strings.TrimSpace(payload.URI)
payload.Database = strings.TrimSpace(payload.Database)
if payload.Name == "" {
renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"})
return
}
if payload.URI == "" {
renderValidationError(writer, validationIssue{Field: "uri", Message: "uri is required"})
return
}
if payload.Database == "" {
renderValidationError(writer, validationIssue{Field: "database", Message: "database is required"})
return
}
item, err := server.store.CreateMongoConnection(request.Context(), payload)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, sanitizeMongoConnection(item))
}
func (server *server) handleMongoConnectionsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetMongoConnection(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Mongo connection not found"})
return
}
renderJSON(writer, http.StatusOK, sanitizeMongoConnection(item))
}
func (server *server) handleMongoConnectionsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetMongoConnection(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Mongo connection not found"})
return
}
var payload mongoConnectionPatch
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Name != nil {
item.Name = strings.TrimSpace(*payload.Name)
}
if payload.Description != nil {
item.Description = strings.TrimSpace(*payload.Description)
}
if payload.URI != nil {
if trimmed := strings.TrimSpace(*payload.URI); trimmed != "" {
item.URI = trimmed
}
}
if payload.Database != nil {
if trimmed := strings.TrimSpace(*payload.Database); trimmed != "" {
item.Database = trimmed
}
}
if strings.TrimSpace(item.Name) == "" {
renderValidationError(writer, validationIssue{Field: "name", Message: "name is required"})
return
}
if strings.TrimSpace(item.URI) == "" {
renderValidationError(writer, validationIssue{Field: "uri", Message: "uri is required"})
return
}
if strings.TrimSpace(item.Database) == "" {
renderValidationError(writer, validationIssue{Field: "database", Message: "database is required"})
return
}
item.UpdatedAt = nowISO()
if err := server.store.UpsertMongoConnection(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeMongoConnection(item))
}
func (server *server) handleMongoConnectionsDelete(writer http.ResponseWriter, request *http.Request) {
deleted, err := server.store.DeleteMongoConnection(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if !deleted {
writer.WriteHeader(http.StatusNotFound)
return
}
writer.WriteHeader(http.StatusNoContent)
}
func (server *server) handleSettingsGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetSettings(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeSettings(item))
}
type settingsPatch struct {
K8s *struct {
Endpoint *string `json:"endpoint"`
Namespace *string `json:"namespace"`
WorkspacePVCName *string `json:"workspacePVCName"`
WorkspaceMountPath *string `json:"workspaceMountPath"`
Kubeconfig *string `json:"kubeconfig"`
} `json:"k8s"`
Worker *struct {
PollIntervalMS *int `json:"pollIntervalMs"`
MaxConcurrentRuns *int `json:"maxConcurrentRuns"`
} `json:"worker"`
AI *struct {
Provider *string `json:"provider"`
BaseURL *string `json:"baseURL"`
Model *string `json:"model"`
APIKey *string `json:"apiKey"`
} `json:"ai"`
BusinessMongo *struct {
URI *string `json:"uri"`
Database *string `json:"database"`
} `json:"businessMongo"`
Defaults *struct {
CPURequest *string `json:"cpuRequest"`
CPULimit *string `json:"cpuLimit"`
MemoryRequest *string `json:"memoryRequest"`
MemoryLimit *string `json:"memoryLimit"`
} `json:"defaults"`
Timezone *string `json:"timezone"`
}
func (server *server) handleSettingsUpdate(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetSettings(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
var payload settingsPatch
if !decodeJSON(writer, request, &payload) {
return
}
if payload.Timezone != nil {
timezone := strings.TrimSpace(*payload.Timezone)
if timezone == "" {
renderValidationError(writer, validationIssue{Field: "timezone", Message: "timezone is required"})
return
}
if _, err := time.LoadLocation(timezone); err != nil {
renderValidationError(writer, validationIssue{Field: "timezone", Message: "timezone must be a valid IANA timezone"})
return
}
item.Timezone = timezone
}
if payload.K8s != nil {
if payload.K8s.Endpoint != nil {
item.K8s.Endpoint = *payload.K8s.Endpoint
}
if payload.K8s.Namespace != nil {
item.K8s.Namespace = *payload.K8s.Namespace
}
if payload.K8s.WorkspacePVCName != nil {
item.K8s.WorkspacePVCName = strings.TrimSpace(*payload.K8s.WorkspacePVCName)
}
if payload.K8s.WorkspaceMountPath != nil {
item.K8s.WorkspaceMountPath = strings.TrimSpace(*payload.K8s.WorkspaceMountPath)
}
if payload.K8s.Kubeconfig != nil {
item.K8s.Kubeconfig = strings.TrimSpace(*payload.K8s.Kubeconfig)
}
}
if strings.TrimSpace(item.K8s.WorkspacePVCName) == "" {
renderValidationError(writer, validationIssue{Field: "k8s.workspacePVCName", Message: "workspacePVCName is required"})
return
}
if strings.TrimSpace(item.K8s.WorkspaceMountPath) == "" {
item.K8s.WorkspaceMountPath = "/workspace"
}
if !strings.HasPrefix(item.K8s.WorkspaceMountPath, "/") {
renderValidationError(writer, validationIssue{Field: "k8s.workspaceMountPath", Message: "workspaceMountPath must be an absolute path"})
return
}
if payload.Worker != nil {
if payload.Worker.PollIntervalMS != nil && *payload.Worker.PollIntervalMS > 0 {
item.Worker.PollIntervalMS = *payload.Worker.PollIntervalMS
}
if payload.Worker.MaxConcurrentRuns != nil && *payload.Worker.MaxConcurrentRuns > 0 {
item.Worker.MaxConcurrentRuns = *payload.Worker.MaxConcurrentRuns
}
}
if payload.AI != nil {
if payload.AI.Provider != nil {
item.AI.Provider = strings.TrimSpace(*payload.AI.Provider)
}
if payload.AI.BaseURL != nil {
item.AI.BaseURL = strings.TrimSpace(*payload.AI.BaseURL)
}
if payload.AI.Model != nil {
item.AI.Model = strings.TrimSpace(*payload.AI.Model)
}
if payload.AI.APIKey != nil {
item.AI.APIKey = strings.TrimSpace(*payload.AI.APIKey)
}
}
if payload.BusinessMongo != nil {
if payload.BusinessMongo.URI != nil {
item.BusinessMongo.URI = strings.TrimSpace(*payload.BusinessMongo.URI)
}
if payload.BusinessMongo.Database != nil {
item.BusinessMongo.Database = strings.TrimSpace(*payload.BusinessMongo.Database)
}
}
if payload.Defaults != nil {
if payload.Defaults.CPURequest != nil {
item.Defaults.CPURequest = *payload.Defaults.CPURequest
}
if payload.Defaults.CPULimit != nil {
item.Defaults.CPULimit = *payload.Defaults.CPULimit
}
if payload.Defaults.MemoryRequest != nil {
item.Defaults.MemoryRequest = *payload.Defaults.MemoryRequest
}
if payload.Defaults.MemoryLimit != nil {
item.Defaults.MemoryLimit = *payload.Defaults.MemoryLimit
}
}
if strings.TrimSpace(item.BusinessMongo.URI) != "" && strings.TrimSpace(item.BusinessMongo.Database) == "" {
renderValidationError(writer, validationIssue{Field: "businessMongo.database", Message: "database is required when businessMongo.uri is configured"})
return
}
item.UpdatedAt = nowISO()
if err := server.store.UpsertSettings(request.Context(), *item); err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, sanitizeSettings(item))
}
func (server *server) resolveAIService(ctx context.Context) ai.Service {
settings, err := server.store.GetSettings(ctx)
if err != nil || settings == nil {
return server.ai
}
if strings.TrimSpace(settings.AI.APIKey) != "" {
return ai.NewService(settings.AI.BaseURL, settings.AI.APIKey, settings.AI.Model)
}
if server.ai != nil && server.ai.Status().Enabled {
return server.ai
}
return ai.NewDisabledService(settings.AI.BaseURL, settings.AI.Model, "服务端未配置 AI_API_KEY")
}
func sanitizeSettings(item *domain.SettingsRecord) *domain.SettingsView {
if item == nil {
return nil
}
view := &domain.SettingsView{
ID: item.ID,
K8s: item.K8s,
Worker: item.Worker,
AI: item.AI,
BusinessMongo: item.BusinessMongo,
Defaults: item.Defaults,
Timezone: item.Timezone,
UpdatedAt: item.UpdatedAt,
}
view.K8s.HasKubeconfig = strings.TrimSpace(view.K8s.Kubeconfig) != ""
view.K8s.KubeconfigPreview = maskedSecretPreview(view.K8s.Kubeconfig)
view.K8s.Kubeconfig = ""
view.AI.HasAPIKey = strings.TrimSpace(view.AI.APIKey) != ""
view.AI.APIKeyPreview = maskedSecretPreview(view.AI.APIKey)
view.AI.APIKey = ""
view.BusinessMongo.HasURI = strings.TrimSpace(view.BusinessMongo.URI) != ""
view.BusinessMongo.URIPreview = maskedSecretPreview(view.BusinessMongo.URI)
view.BusinessMongo.URI = ""
return view
}
func (server *server) handleStorageBuckets(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
if connectionID != "" {
connection, err := server.store.GetStorageConnection(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if connection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Storage connection not found"})
return
}
items, err := storageConnectionBucketsResolver(request.Context(), connection)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
return
}
sourcePath := strings.TrimSpace(request.URL.Query().Get("sourcePath"))
if sourcePath == "" {
renderValidationError(writer, validationIssue{Field: "sourcePath", Message: "sourcePath or connectionId is required"})
return
}
items, err := server.store.ListStorageBuckets(request.Context(), sourcePath)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func (server *server) handleStorageObjects(writer http.ResponseWriter, request *http.Request) {
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
if connectionID != "" {
connection, err := server.store.GetStorageConnection(request.Context(), connectionID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if connection == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Storage connection not found"})
return
}
items, err := storageConnectionObjectsResolver(
request.Context(),
connection,
request.URL.Query().Get("bucket"),
request.URL.Query().Get("prefix"),
)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
return
}
sourcePath := strings.TrimSpace(request.URL.Query().Get("sourcePath"))
if sourcePath == "" {
renderValidationError(writer, validationIssue{Field: "sourcePath", Message: "sourcePath or connectionId is required"})
return
}
items, err := server.store.ListStorageObjects(
request.Context(),
sourcePath,
request.URL.Query().Get("bucket"),
request.URL.Query().Get("prefix"),
)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func decodeJSON(writer http.ResponseWriter, request *http.Request, target any) bool {
defer request.Body.Close()
decoder := json.NewDecoder(request.Body)
if err := decoder.Decode(target); err != nil {
renderJSON(writer, http.StatusBadRequest, map[string]any{"error": "Validation failed", "issues": []validationIssue{{Field: "body", Message: "invalid JSON payload"}}})
return false
}
return true
}
func renderValidationError(writer http.ResponseWriter, issues ...validationIssue) {
renderJSON(writer, http.StatusBadRequest, map[string]any{"error": "Validation failed", "issues": issues})
}
func renderError(writer http.ResponseWriter, status int, err error) {
renderJSON(writer, status, map[string]string{"error": err.Error()})
}
func renderAIError(writer http.ResponseWriter, err error) {
switch {
case errors.Is(err, ai.ErrUnavailable):
renderJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "AI 服务未配置,请在服务端配置 AI_API_KEY 后再试"})
case strings.Contains(err.Error(), "AI 生成超时") || strings.Contains(err.Error(), "AI 服务响应超时"):
renderJSON(writer, http.StatusGatewayTimeout, map[string]string{"error": err.Error()})
case strings.Contains(strings.ToLower(err.Error()), "description is required"):
renderValidationError(writer, validationIssue{Field: "description", Message: "description is required"})
default:
renderJSON(writer, http.StatusBadGateway, map[string]string{"error": err.Error()})
}
}
func renderJSON(writer http.ResponseWriter, status int, payload any) {
writer.Header().Set("Content-Type", "application/json")
writer.WriteHeader(status)
_ = json.NewEncoder(writer).Encode(payload)
}
func nowISO() string {
return time.Now().UTC().Format(time.RFC3339Nano)
}