droboticflow/internal/api/data_operations.go

536 lines
19 KiB
Go

package api
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"droboticflow/internal/domain"
"github.com/go-chi/chi/v5"
)
func validCollectionTaskMode(value domain.CollectionTaskMode) bool {
return value == domain.CollectionTaskModeCollectionOnly || value == domain.CollectionTaskModeFullTraining
}
func validCollectionTaskStatus(value domain.CollectionTaskStatus) bool {
switch value {
case domain.CollectionTaskStatusDraft, domain.CollectionTaskStatusActive, domain.CollectionTaskStatusPaused,
domain.CollectionTaskStatusCompleted, domain.CollectionTaskStatusCancelled:
return true
default:
return false
}
}
func validDataMetricStage(value domain.DataMetricStage) bool {
for _, stage := range domain.DataMetricStages {
if value == stage {
return true
}
}
return false
}
func normalizedStringList(items []string) []string {
result := make([]string, 0, len(items))
seen := map[string]struct{}{}
for _, value := range items {
trimmed := strings.TrimSpace(value)
key := strings.ToLower(trimmed)
if trimmed == "" {
continue
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
result = append(result, trimmed)
}
return result
}
func validateNonNegativeVolume(field string, value domain.DataVolume) []validationIssue {
issues := make([]validationIssue, 0)
if value.TrajectoryCount != nil && *value.TrajectoryCount < 0 {
issues = append(issues, validationIssue{Field: field + ".trajectoryCount", Message: "must be greater than or equal to 0"})
}
if value.DurationSeconds != nil && (*value.DurationSeconds < 0 || math.IsNaN(*value.DurationSeconds) || math.IsInf(*value.DurationSeconds, 0)) {
issues = append(issues, validationIssue{Field: field + ".durationSeconds", Message: "must be a finite value greater than or equal to 0"})
}
if value.Bytes != nil && *value.Bytes < 0 {
issues = append(issues, validationIssue{Field: field + ".bytes", Message: "must be greater than or equal to 0"})
}
if value.FileCount != nil && *value.FileCount < 0 {
issues = append(issues, validationIssue{Field: field + ".fileCount", Message: "must be greater than or equal to 0"})
}
return issues
}
func collectionTaskHasTarget(value domain.DataVolume) bool {
return (value.TrajectoryCount != nil && *value.TrajectoryCount > 0) ||
(value.DurationSeconds != nil && *value.DurationSeconds > 0) ||
(value.Bytes != nil && *value.Bytes > 0)
}
func (server *server) normalizeCollectionTaskInput(ctxRequest *http.Request, payload *domain.CollectionTaskInput, existing *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, []validationIssue, error) {
payload.ProjectID = strings.TrimSpace(payload.ProjectID)
payload.Name = strings.TrimSpace(payload.Name)
payload.Description = strings.TrimSpace(payload.Description)
payload.RobotModel = strings.TrimSpace(payload.RobotModel)
payload.RobotVersion = strings.TrimSpace(payload.RobotVersion)
payload.Scene = strings.TrimSpace(payload.Scene)
payload.Format = strings.TrimSpace(payload.Format)
payload.WorkflowID = strings.TrimSpace(payload.WorkflowID)
payload.Modalities = normalizedStringList(payload.Modalities)
if payload.Mode == "" {
payload.Mode = domain.CollectionTaskModeFullTraining
}
if payload.Status == "" {
payload.Status = domain.CollectionTaskStatusDraft
}
issues := make([]validationIssue, 0)
if payload.ProjectID == "" {
issues = append(issues, validationIssue{Field: "projectId", Message: "projectId is required"})
}
if payload.Name == "" {
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
}
if payload.RobotModel == "" {
issues = append(issues, validationIssue{Field: "robotModel", Message: "robotModel is required"})
}
if payload.Scene == "" {
issues = append(issues, validationIssue{Field: "scene", Message: "scene is required"})
}
if !validCollectionTaskMode(payload.Mode) {
issues = append(issues, validationIssue{Field: "mode", Message: "mode must be collection_only or full_training"})
}
if !validCollectionTaskStatus(payload.Status) {
issues = append(issues, validationIssue{Field: "status", Message: "invalid task status"})
}
issues = append(issues, validateNonNegativeVolume("targets", payload.Targets)...)
if !collectionTaskHasTarget(payload.Targets) {
issues = append(issues, validationIssue{Field: "targets", Message: "at least one positive trajectory, duration, or bytes target is required"})
}
splitTotal := 0
for split, percent := range payload.SplitRules {
if split != "train" && split != "val" && split != "test" {
issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split must be train, val, or test"})
}
if percent < 0 || percent > 100 {
issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split percent must be between 0 and 100"})
}
splitTotal += percent
}
if len(payload.SplitRules) > 0 && splitTotal != 100 {
issues = append(issues, validationIssue{Field: "splitRules", Message: "split percentages must total 100"})
}
seenNodes := map[string]domain.DataMetricStage{}
for stage, nodeID := range payload.StageBindings {
nodeID = strings.TrimSpace(nodeID)
if !validDataMetricStage(stage) {
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "invalid business stage"})
continue
}
if nodeID == "" {
delete(payload.StageBindings, stage)
continue
}
if previous, exists := seenNodes[nodeID]; exists {
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: fmt.Sprintf("node is already the metric boundary for %s", previous)})
}
seenNodes[nodeID] = stage
payload.StageBindings[stage] = nodeID
}
if len(issues) > 0 {
return nil, issues, nil
}
project, err := server.store.GetProject(ctxRequest.Context(), payload.ProjectID)
if err != nil {
return nil, nil, err
}
if project == nil || !server.filterByTenant(ctxRequest, project.OrganizationID) {
return nil, []validationIssue{{Field: "projectId", Message: "project not found"}}, nil
}
workflowName := ""
if payload.WorkflowID != "" {
workflow, err := server.store.GetWorkflow(ctxRequest.Context(), payload.WorkflowID)
if err != nil {
return nil, nil, err
}
if workflow == nil || workflow.ProjectID != project.ID || !server.filterByTenant(ctxRequest, workflow.OrganizationID) {
return nil, []validationIssue{{Field: "workflowId", Message: "workflow not found in selected project"}}, nil
}
workflowName = workflow.Name
for stage, nodeID := range payload.StageBindings {
if domain.GetNodeByID(workflow.Nodes, nodeID) == nil {
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "workflow node not found"})
}
}
}
if len(issues) > 0 {
return nil, issues, nil
}
item := &domain.CollectionTaskRecord{
OrganizationID: project.OrganizationID,
ProjectID: project.ID,
ProjectName: project.Name,
Name: payload.Name,
Description: payload.Description,
RobotModel: payload.RobotModel,
RobotVersion: payload.RobotVersion,
Scene: payload.Scene,
Modalities: payload.Modalities,
Format: payload.Format,
Mode: payload.Mode,
Status: payload.Status,
WorkflowID: payload.WorkflowID,
WorkflowName: workflowName,
StageBindings: payload.StageBindings,
Targets: payload.Targets,
Requirements: payload.Requirements,
SplitRules: payload.SplitRules,
}
if existing != nil {
item.ID = existing.ID
item.CreatedAt = existing.CreatedAt
item.LastRunID = existing.LastRunID
}
return item, nil, nil
}
func (server *server) handleCollectionTasksList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListCollectionTasks(request.Context(), server.requestOrganizationID(request))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
projectID := strings.TrimSpace(request.URL.Query().Get("projectId"))
status := strings.TrimSpace(request.URL.Query().Get("status"))
filtered := make([]domain.CollectionTaskRecord, 0, len(items))
for _, item := range items {
if projectID != "" && item.ProjectID != projectID {
continue
}
if status != "" && string(item.Status) != status {
continue
}
filtered = append(filtered, item)
}
renderJSON(writer, http.StatusOK, map[string]any{"items": filtered})
}
func (server *server) handleCollectionTaskGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !server.ensureTenantAccess(writer, request, item.OrganizationID, "Data task") {
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
}
return
}
renderJSON(writer, http.StatusOK, item)
}
func (server *server) persistCollectionTask(request *http.Request, item *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, error) {
if err := server.store.UpsertCollectionTask(request.Context(), *item); err != nil {
return nil, err
}
return server.store.GetCollectionTask(request.Context(), item.ID)
}
func (server *server) handleCollectionTaskCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CollectionTaskInput
if !decodeJSON(writer, request, &payload) {
return
}
item, issues, err := server.normalizeCollectionTaskInput(request, &payload, nil)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.ID = domain.NewID("data-task")
saved, err := server.persistCollectionTask(request, item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, saved)
}
func (server *server) handleCollectionTaskUpdate(writer http.ResponseWriter, request *http.Request) {
existing, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if existing == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
return
}
if !server.ensureTenantAccess(writer, request, existing.OrganizationID, "Data task") {
return
}
var payload domain.CollectionTaskInput
if !decodeJSON(writer, request, &payload) {
return
}
item, issues, err := server.normalizeCollectionTaskInput(request, &payload, existing)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
saved, err := server.persistCollectionTask(request, item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, saved)
}
func (server *server) handleCollectionTaskStart(writer http.ResponseWriter, request *http.Request) {
task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if task == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
return
}
if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") {
return
}
if task.WorkflowID == "" {
renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bind a workflow before starting the task"})
return
}
requiredStages := []domain.DataMetricStage{domain.DataMetricStageCollect}
if task.Mode == domain.CollectionTaskModeFullTraining {
requiredStages = append(
requiredStages,
domain.DataMetricStageProcess,
domain.DataMetricStageClean,
domain.DataMetricStageValidate,
domain.DataMetricStageTrain,
)
if task.Requirements.RequireAnnotation {
requiredStages = append(requiredStages, domain.DataMetricStageAnnotate)
}
if task.Requirements.RequireAudit {
requiredStages = append(requiredStages, domain.DataMetricStageAudit)
}
}
missingStageIssues := make([]validationIssue, 0)
for _, stage := range requiredStages {
if strings.TrimSpace(task.StageBindings[stage]) == "" {
missingStageIssues = append(missingStageIssues, validationIssue{
Field: "stageBindings." + string(stage),
Message: "required stage is not bound to a workflow node",
})
}
}
if len(missingStageIssues) > 0 {
renderValidationError(writer, missingStageIssues...)
return
}
workflow, err := server.store.GetWorkflow(request.Context(), task.WorkflowID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if workflow == nil || workflow.ProjectID != task.ProjectID {
renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bound workflow is unavailable"})
return
}
active, err := server.store.HasActiveRun(request.Context(), workflow.ID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if active {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流已有进行中的运行", "code": "workflow_run_active"})
return
}
if issues := validateWorkflowTopology(workflow.Nodes, workflow.Edges); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
run, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerManual, task.ID)
if err != nil {
if renderRunCreationError(writer, err) {
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
task.Status = domain.CollectionTaskStatusActive
task.LastRunID = run.ID
saved, err := server.persistCollectionTask(request, task)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusAccepted, map[string]any{"task": saved, "run": run})
}
func (server *server) handleCollectionTaskComplete(writer http.ResponseWriter, request *http.Request) {
task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if task == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
return
}
if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") {
return
}
task.Status = domain.CollectionTaskStatusCompleted
saved, err := server.persistCollectionTask(request, task)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, saved)
}
func parseDataOperationsQuery(request *http.Request, settings *domain.SettingsRecord) (domain.DataOperationsQuery, []validationIssue) {
timezone := strings.TrimSpace(request.URL.Query().Get("timezone"))
if timezone == "" && settings != nil {
timezone = strings.TrimSpace(settings.Timezone)
}
if timezone == "" {
timezone = domain.DefaultWorkflowScheduleTimezone
}
location, err := time.LoadLocation(timezone)
if err != nil {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "timezone", Message: "invalid IANA timezone"}}
}
query := domain.DataOperationsQuery{
Timezone: timezone,
ProjectID: strings.TrimSpace(request.URL.Query().Get("projectId")),
TaskID: strings.TrimSpace(request.URL.Query().Get("taskId")),
RobotModel: strings.TrimSpace(request.URL.Query().Get("robotModel")),
Scene: strings.TrimSpace(request.URL.Query().Get("scene")),
Stage: domain.DataMetricStage(strings.TrimSpace(request.URL.Query().Get("stage"))),
Cursor: strings.TrimSpace(request.URL.Query().Get("cursor")),
}
if query.Stage != "" && !validDataMetricStage(query.Stage) {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "stage", Message: "invalid business stage"}}
}
if raw := strings.TrimSpace(request.URL.Query().Get("limit")); raw != "" {
value, parseErr := strconv.Atoi(raw)
if parseErr != nil || value < 1 || value > 100 {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "limit", Message: "limit must be between 1 and 100"}}
}
query.Limit = value
}
now := time.Now()
fromRaw := strings.TrimSpace(request.URL.Query().Get("from"))
toRaw := strings.TrimSpace(request.URL.Query().Get("to"))
if fromRaw == "" {
localNow := now.In(location)
fromRaw = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, location).UTC().Format(time.RFC3339Nano)
}
if toRaw == "" {
toRaw = now.UTC().Format(time.RFC3339Nano)
}
from, fromErr := time.Parse(time.RFC3339Nano, fromRaw)
to, toErr := time.Parse(time.RFC3339Nano, toRaw)
if fromErr != nil {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be RFC3339"}}
}
if toErr != nil {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "to", Message: "to must be RFC3339"}}
}
if !from.Before(to) {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be before to"}}
}
query.From = from.UTC().Format(time.RFC3339Nano)
query.To = to.UTC().Format(time.RFC3339Nano)
return query, nil
}
func (server *server) dataOperationsQuery(writer http.ResponseWriter, request *http.Request) (domain.DataOperationsQuery, bool) {
settings, err := server.store.GetSettings(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return domain.DataOperationsQuery{}, false
}
query, issues := parseDataOperationsQuery(request, settings)
if len(issues) > 0 {
renderValidationError(writer, issues...)
return domain.DataOperationsQuery{}, false
}
return query, true
}
func (server *server) handleDataOperationsSummary(writer http.ResponseWriter, request *http.Request) {
query, ok := server.dataOperationsQuery(writer, request)
if !ok {
return
}
summary, err := server.store.GetDataOperationsSummary(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, summary)
}
func (server *server) handleDataOperationsStageItems(writer http.ResponseWriter, request *http.Request) {
query, ok := server.dataOperationsQuery(writer, request)
if !ok {
return
}
page, err := server.store.ListDataMetricFactsPage(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, page)
}
func (server *server) handleDataOperationsTrainingUsage(writer http.ResponseWriter, request *http.Request) {
query, ok := server.dataOperationsQuery(writer, request)
if !ok {
return
}
page, err := server.store.ListTrainingUsagePage(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, page)
}
func (server *server) handleDataOperationsDatasets(writer http.ResponseWriter, request *http.Request) {
query, ok := server.dataOperationsQuery(writer, request)
if !ok {
return
}
items, err := server.store.ListDatasetVersions(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}