forked from longtao.wu/droboticflow
783 lines
27 KiB
Go
783 lines
27 KiB
Go
package api
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"droboticflow/internal/dataops"
|
|
"droboticflow/internal/domain"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
)
|
|
|
|
func validCollectionTaskMode(value domain.CollectionTaskMode) bool {
|
|
return value == domain.CollectionTaskModeCollectionOnly || value == domain.CollectionTaskModeFullTraining
|
|
}
|
|
|
|
func validCollectionTaskStatus(value domain.CollectionTaskStatus) bool {
|
|
switch value {
|
|
case domain.CollectionTaskStatusDraft, domain.CollectionTaskStatusActive, domain.CollectionTaskStatusPaused,
|
|
domain.CollectionTaskStatusCompleted, domain.CollectionTaskStatusCancelled:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func validDataMetricStage(value domain.DataMetricStage) bool {
|
|
for _, stage := range domain.DataMetricStages {
|
|
if value == stage {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func normalizedStringList(items []string) []string {
|
|
result := make([]string, 0, len(items))
|
|
seen := map[string]struct{}{}
|
|
for _, value := range items {
|
|
trimmed := strings.TrimSpace(value)
|
|
key := strings.ToLower(trimmed)
|
|
if trimmed == "" {
|
|
continue
|
|
}
|
|
if _, exists := seen[key]; exists {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
result = append(result, trimmed)
|
|
}
|
|
return result
|
|
}
|
|
|
|
func validateNonNegativeVolume(field string, value domain.DataVolume) []validationIssue {
|
|
issues := make([]validationIssue, 0)
|
|
if value.TrajectoryCount != nil && *value.TrajectoryCount < 0 {
|
|
issues = append(issues, validationIssue{Field: field + ".trajectoryCount", Message: "must be greater than or equal to 0"})
|
|
}
|
|
if value.DurationSeconds != nil && (*value.DurationSeconds < 0 || math.IsNaN(*value.DurationSeconds) || math.IsInf(*value.DurationSeconds, 0)) {
|
|
issues = append(issues, validationIssue{Field: field + ".durationSeconds", Message: "must be a finite value greater than or equal to 0"})
|
|
}
|
|
if value.Bytes != nil && *value.Bytes < 0 {
|
|
issues = append(issues, validationIssue{Field: field + ".bytes", Message: "must be greater than or equal to 0"})
|
|
}
|
|
if value.FileCount != nil && *value.FileCount < 0 {
|
|
issues = append(issues, validationIssue{Field: field + ".fileCount", Message: "must be greater than or equal to 0"})
|
|
}
|
|
return issues
|
|
}
|
|
|
|
func collectionTaskHasTarget(value domain.DataVolume) bool {
|
|
return (value.TrajectoryCount != nil && *value.TrajectoryCount > 0) ||
|
|
(value.DurationSeconds != nil && *value.DurationSeconds > 0) ||
|
|
(value.Bytes != nil && *value.Bytes > 0) ||
|
|
(value.FileCount != nil && *value.FileCount > 0)
|
|
}
|
|
|
|
func (server *server) normalizeCollectionTaskInput(ctxRequest *http.Request, payload *domain.CollectionTaskInput, existing *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, []validationIssue, error) {
|
|
payload.ProjectID = strings.TrimSpace(payload.ProjectID)
|
|
payload.Name = strings.TrimSpace(payload.Name)
|
|
payload.Description = strings.TrimSpace(payload.Description)
|
|
payload.RobotModel = strings.TrimSpace(payload.RobotModel)
|
|
payload.RobotVersion = strings.TrimSpace(payload.RobotVersion)
|
|
payload.Scene = strings.TrimSpace(payload.Scene)
|
|
payload.Format = strings.TrimSpace(payload.Format)
|
|
payload.WorkflowID = strings.TrimSpace(payload.WorkflowID)
|
|
payload.Modalities = normalizedStringList(payload.Modalities)
|
|
if payload.Mode == "" {
|
|
payload.Mode = domain.CollectionTaskModeFullTraining
|
|
}
|
|
if payload.Status == "" {
|
|
payload.Status = domain.CollectionTaskStatusDraft
|
|
}
|
|
issues := make([]validationIssue, 0)
|
|
if payload.ProjectID == "" {
|
|
issues = append(issues, validationIssue{Field: "projectId", Message: "projectId is required"})
|
|
}
|
|
if payload.Name == "" {
|
|
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
|
|
}
|
|
if payload.RobotModel == "" {
|
|
issues = append(issues, validationIssue{Field: "robotModel", Message: "robotModel is required"})
|
|
}
|
|
if payload.Scene == "" {
|
|
issues = append(issues, validationIssue{Field: "scene", Message: "scene is required"})
|
|
}
|
|
if !validCollectionTaskMode(payload.Mode) {
|
|
issues = append(issues, validationIssue{Field: "mode", Message: "mode must be collection_only or full_training"})
|
|
}
|
|
if !validCollectionTaskStatus(payload.Status) {
|
|
issues = append(issues, validationIssue{Field: "status", Message: "invalid task status"})
|
|
}
|
|
issues = append(issues, validateNonNegativeVolume("targets", payload.Targets)...)
|
|
if !collectionTaskHasTarget(payload.Targets) {
|
|
issues = append(issues, validationIssue{Field: "targets", Message: "at least one positive trajectory, duration, or bytes target is required"})
|
|
}
|
|
splitTotal := 0
|
|
for split, percent := range payload.SplitRules {
|
|
if split != "train" && split != "val" && split != "test" {
|
|
issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split must be train, val, or test"})
|
|
}
|
|
if percent < 0 || percent > 100 {
|
|
issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split percent must be between 0 and 100"})
|
|
}
|
|
splitTotal += percent
|
|
}
|
|
if len(payload.SplitRules) > 0 && splitTotal != 100 {
|
|
issues = append(issues, validationIssue{Field: "splitRules", Message: "split percentages must total 100"})
|
|
}
|
|
seenNodes := map[string]domain.DataMetricStage{}
|
|
for stage, nodeID := range payload.StageBindings {
|
|
nodeID = strings.TrimSpace(nodeID)
|
|
if !validDataMetricStage(stage) {
|
|
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "invalid business stage"})
|
|
continue
|
|
}
|
|
if nodeID == "" {
|
|
delete(payload.StageBindings, stage)
|
|
continue
|
|
}
|
|
if previous, exists := seenNodes[nodeID]; exists {
|
|
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: fmt.Sprintf("node is already the metric boundary for %s", previous)})
|
|
}
|
|
seenNodes[nodeID] = stage
|
|
payload.StageBindings[stage] = nodeID
|
|
}
|
|
if len(issues) > 0 {
|
|
return nil, issues, nil
|
|
}
|
|
project, err := server.store.GetProject(ctxRequest.Context(), payload.ProjectID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if project == nil || !server.filterByTenant(ctxRequest, project.OrganizationID) {
|
|
return nil, []validationIssue{{Field: "projectId", Message: "project not found"}}, nil
|
|
}
|
|
workflowName := ""
|
|
if payload.WorkflowID != "" {
|
|
workflow, err := server.store.GetWorkflow(ctxRequest.Context(), payload.WorkflowID)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if workflow == nil || workflow.ProjectID != project.ID || !server.filterByTenant(ctxRequest, workflow.OrganizationID) {
|
|
return nil, []validationIssue{{Field: "workflowId", Message: "workflow not found in selected project"}}, nil
|
|
}
|
|
workflowName = workflow.Name
|
|
for stage, nodeID := range payload.StageBindings {
|
|
if domain.GetNodeByID(workflow.Nodes, nodeID) == nil {
|
|
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "workflow node not found"})
|
|
}
|
|
}
|
|
}
|
|
if len(issues) > 0 {
|
|
return nil, issues, nil
|
|
}
|
|
item := &domain.CollectionTaskRecord{
|
|
OrganizationID: project.OrganizationID,
|
|
ProjectID: project.ID,
|
|
ProjectName: project.Name,
|
|
Name: payload.Name,
|
|
Description: payload.Description,
|
|
RobotModel: payload.RobotModel,
|
|
RobotVersion: payload.RobotVersion,
|
|
Scene: payload.Scene,
|
|
Modalities: payload.Modalities,
|
|
Format: payload.Format,
|
|
Mode: payload.Mode,
|
|
Status: payload.Status,
|
|
WorkflowID: payload.WorkflowID,
|
|
WorkflowName: workflowName,
|
|
StageBindings: payload.StageBindings,
|
|
Targets: payload.Targets,
|
|
Requirements: payload.Requirements,
|
|
SplitRules: payload.SplitRules,
|
|
}
|
|
if existing != nil {
|
|
item.ID = existing.ID
|
|
item.CreatedAt = existing.CreatedAt
|
|
item.LastRunID = existing.LastRunID
|
|
}
|
|
return item, nil, nil
|
|
}
|
|
|
|
func (server *server) handleCollectionTasksList(writer http.ResponseWriter, request *http.Request) {
|
|
items, err := server.store.ListCollectionTasks(request.Context(), server.requestOrganizationID(request))
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
projectID := strings.TrimSpace(request.URL.Query().Get("projectId"))
|
|
status := strings.TrimSpace(request.URL.Query().Get("status"))
|
|
filtered := make([]domain.CollectionTaskRecord, 0, len(items))
|
|
for _, item := range items {
|
|
if projectID != "" && item.ProjectID != projectID {
|
|
continue
|
|
}
|
|
if status != "" && string(item.Status) != status {
|
|
continue
|
|
}
|
|
filtered = append(filtered, item)
|
|
}
|
|
renderJSON(writer, http.StatusOK, map[string]any{"items": filtered})
|
|
}
|
|
|
|
func (server *server) handleCollectionTaskGet(writer http.ResponseWriter, request *http.Request) {
|
|
item, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if item == nil || !server.ensureTenantAccess(writer, request, item.OrganizationID, "Data task") {
|
|
if item == nil {
|
|
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
|
|
}
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, item)
|
|
}
|
|
|
|
func (server *server) persistCollectionTask(request *http.Request, item *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, error) {
|
|
if err := server.store.UpsertCollectionTask(request.Context(), *item); err != nil {
|
|
return nil, err
|
|
}
|
|
return server.store.GetCollectionTask(request.Context(), item.ID)
|
|
}
|
|
|
|
func (server *server) handleCollectionTaskCreate(writer http.ResponseWriter, request *http.Request) {
|
|
var payload domain.CollectionTaskInput
|
|
if !decodeJSON(writer, request, &payload) {
|
|
return
|
|
}
|
|
item, issues, err := server.normalizeCollectionTaskInput(request, &payload, nil)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if len(issues) > 0 {
|
|
renderValidationError(writer, issues...)
|
|
return
|
|
}
|
|
item.ID = domain.NewID("data-task")
|
|
saved, err := server.persistCollectionTask(request, item)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusCreated, saved)
|
|
}
|
|
|
|
func (server *server) handleCollectionTaskUpdate(writer http.ResponseWriter, request *http.Request) {
|
|
existing, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if existing == nil {
|
|
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
|
|
return
|
|
}
|
|
if !server.ensureTenantAccess(writer, request, existing.OrganizationID, "Data task") {
|
|
return
|
|
}
|
|
var payload domain.CollectionTaskInput
|
|
if !decodeJSON(writer, request, &payload) {
|
|
return
|
|
}
|
|
item, issues, err := server.normalizeCollectionTaskInput(request, &payload, existing)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if len(issues) > 0 {
|
|
renderValidationError(writer, issues...)
|
|
return
|
|
}
|
|
saved, err := server.persistCollectionTask(request, item)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, saved)
|
|
}
|
|
|
|
func (server *server) handleCollectionTaskStart(writer http.ResponseWriter, request *http.Request) {
|
|
task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if task == nil {
|
|
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
|
|
return
|
|
}
|
|
if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") {
|
|
return
|
|
}
|
|
if task.WorkflowID == "" {
|
|
renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bind a workflow before starting the task"})
|
|
return
|
|
}
|
|
requiredStages := []domain.DataMetricStage{domain.DataMetricStageCollect}
|
|
if task.Mode == domain.CollectionTaskModeFullTraining {
|
|
requiredStages = append(
|
|
requiredStages,
|
|
domain.DataMetricStageProcess,
|
|
domain.DataMetricStageClean,
|
|
domain.DataMetricStageValidate,
|
|
domain.DataMetricStageTrain,
|
|
)
|
|
if task.Requirements.RequireAnnotation {
|
|
requiredStages = append(requiredStages, domain.DataMetricStageAnnotate)
|
|
}
|
|
if task.Requirements.RequireAudit {
|
|
requiredStages = append(requiredStages, domain.DataMetricStageAudit)
|
|
}
|
|
}
|
|
missingStageIssues := make([]validationIssue, 0)
|
|
for _, stage := range requiredStages {
|
|
if strings.TrimSpace(task.StageBindings[stage]) == "" {
|
|
missingStageIssues = append(missingStageIssues, validationIssue{
|
|
Field: "stageBindings." + string(stage),
|
|
Message: "required stage is not bound to a workflow node",
|
|
})
|
|
}
|
|
}
|
|
if len(missingStageIssues) > 0 {
|
|
renderValidationError(writer, missingStageIssues...)
|
|
return
|
|
}
|
|
workflow, err := server.store.GetWorkflow(request.Context(), task.WorkflowID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if workflow == nil || workflow.ProjectID != task.ProjectID {
|
|
renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bound workflow is unavailable"})
|
|
return
|
|
}
|
|
active, err := server.store.HasActiveRun(request.Context(), workflow.ID)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if active {
|
|
renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流已有进行中的运行", "code": "workflow_run_active"})
|
|
return
|
|
}
|
|
if issues := validateWorkflowTopology(workflow.Nodes, workflow.Edges); len(issues) > 0 {
|
|
renderValidationError(writer, issues...)
|
|
return
|
|
}
|
|
run, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerManual, task.ID)
|
|
if err != nil {
|
|
if renderRunCreationError(writer, err) {
|
|
return
|
|
}
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
task.Status = domain.CollectionTaskStatusActive
|
|
task.LastRunID = run.ID
|
|
saved, err := server.persistCollectionTask(request, task)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusAccepted, map[string]any{"task": saved, "run": run})
|
|
}
|
|
|
|
func (server *server) handleCollectionTaskComplete(writer http.ResponseWriter, request *http.Request) {
|
|
task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
if task == nil {
|
|
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
|
|
return
|
|
}
|
|
if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") {
|
|
return
|
|
}
|
|
task.Status = domain.CollectionTaskStatusCompleted
|
|
saved, err := server.persistCollectionTask(request, task)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, saved)
|
|
}
|
|
|
|
func parseDataOperationsQuery(request *http.Request, settings *domain.SettingsRecord) (domain.DataOperationsQuery, []validationIssue) {
|
|
timezone := strings.TrimSpace(request.URL.Query().Get("timezone"))
|
|
if timezone == "" && settings != nil {
|
|
timezone = strings.TrimSpace(settings.Timezone)
|
|
}
|
|
if timezone == "" {
|
|
timezone = domain.DefaultWorkflowScheduleTimezone
|
|
}
|
|
location, err := time.LoadLocation(timezone)
|
|
if err != nil {
|
|
return domain.DataOperationsQuery{}, []validationIssue{{Field: "timezone", Message: "invalid IANA timezone"}}
|
|
}
|
|
query := domain.DataOperationsQuery{
|
|
Timezone: timezone,
|
|
ProjectID: strings.TrimSpace(request.URL.Query().Get("projectId")),
|
|
TaskID: strings.TrimSpace(request.URL.Query().Get("taskId")),
|
|
RobotModel: strings.TrimSpace(request.URL.Query().Get("robotModel")),
|
|
Scene: strings.TrimSpace(request.URL.Query().Get("scene")),
|
|
Stage: domain.DataMetricStage(strings.TrimSpace(request.URL.Query().Get("stage"))),
|
|
Cursor: strings.TrimSpace(request.URL.Query().Get("cursor")),
|
|
}
|
|
if query.Stage != "" && !validDataMetricStage(query.Stage) {
|
|
return domain.DataOperationsQuery{}, []validationIssue{{Field: "stage", Message: "invalid business stage"}}
|
|
}
|
|
if raw := strings.TrimSpace(request.URL.Query().Get("limit")); raw != "" {
|
|
value, parseErr := strconv.Atoi(raw)
|
|
if parseErr != nil || value < 1 || value > 100 {
|
|
return domain.DataOperationsQuery{}, []validationIssue{{Field: "limit", Message: "limit must be between 1 and 100"}}
|
|
}
|
|
query.Limit = value
|
|
}
|
|
now := time.Now()
|
|
fromRaw := strings.TrimSpace(request.URL.Query().Get("from"))
|
|
toRaw := strings.TrimSpace(request.URL.Query().Get("to"))
|
|
if fromRaw == "" {
|
|
localNow := now.In(location)
|
|
fromRaw = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, location).UTC().Format(time.RFC3339Nano)
|
|
}
|
|
if toRaw == "" {
|
|
toRaw = now.UTC().Format(time.RFC3339Nano)
|
|
}
|
|
from, fromErr := time.Parse(time.RFC3339Nano, fromRaw)
|
|
to, toErr := time.Parse(time.RFC3339Nano, toRaw)
|
|
if fromErr != nil {
|
|
return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be RFC3339"}}
|
|
}
|
|
if toErr != nil {
|
|
return domain.DataOperationsQuery{}, []validationIssue{{Field: "to", Message: "to must be RFC3339"}}
|
|
}
|
|
if !from.Before(to) {
|
|
return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be before to"}}
|
|
}
|
|
query.From = from.UTC().Format(time.RFC3339Nano)
|
|
query.To = to.UTC().Format(time.RFC3339Nano)
|
|
return query, nil
|
|
}
|
|
|
|
func (server *server) dataOperationsQuery(writer http.ResponseWriter, request *http.Request) (domain.DataOperationsQuery, bool) {
|
|
settings, err := server.store.GetSettings(request.Context())
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return domain.DataOperationsQuery{}, false
|
|
}
|
|
query, issues := parseDataOperationsQuery(request, settings)
|
|
if len(issues) > 0 {
|
|
renderValidationError(writer, issues...)
|
|
return domain.DataOperationsQuery{}, false
|
|
}
|
|
return query, true
|
|
}
|
|
|
|
func (server *server) handleDataOperationsSummary(writer http.ResponseWriter, request *http.Request) {
|
|
query, ok := server.dataOperationsQuery(writer, request)
|
|
if !ok {
|
|
return
|
|
}
|
|
summary, err := server.store.GetDataOperationsSummary(request.Context(), server.requestOrganizationID(request), query)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, summary)
|
|
}
|
|
|
|
func (server *server) handleDataOperationsStageItems(writer http.ResponseWriter, request *http.Request) {
|
|
query, ok := server.dataOperationsQuery(writer, request)
|
|
if !ok {
|
|
return
|
|
}
|
|
page, err := server.store.ListDataMetricFactsPage(request.Context(), server.requestOrganizationID(request), query)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, page)
|
|
}
|
|
|
|
func (server *server) handleDataOperationsTrainingUsage(writer http.ResponseWriter, request *http.Request) {
|
|
query, ok := server.dataOperationsQuery(writer, request)
|
|
if !ok {
|
|
return
|
|
}
|
|
page, err := server.store.ListTrainingUsagePage(request.Context(), server.requestOrganizationID(request), query)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, page)
|
|
}
|
|
|
|
func (server *server) handleDataOperationsDatasets(writer http.ResponseWriter, request *http.Request) {
|
|
query, ok := server.dataOperationsQuery(writer, request)
|
|
if !ok {
|
|
return
|
|
}
|
|
items, err := server.store.ListDatasetVersions(request.Context(), server.requestOrganizationID(request), query)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
|
|
}
|
|
|
|
func legacyStageFromExecution(execution domain.NodeExecutionRecord) domain.DataMetricStage {
|
|
label := strings.ToLower(strings.TrimSpace(execution.NodeLabel + " " + execution.OperatorID))
|
|
stageTokens := []struct {
|
|
stage domain.DataMetricStage
|
|
tokens []string
|
|
}{
|
|
{domain.DataMetricStageTrain, []string{"train", "训练"}},
|
|
{domain.DataMetricStageAudit, []string{"audit", "审核"}},
|
|
{domain.DataMetricStageAnnotate, []string{"annotat", "标注"}},
|
|
{domain.DataMetricStageValidate, []string{"validat", "quality", "校验", "质检"}},
|
|
{domain.DataMetricStageClean, []string{"clean", "filter", "清洗", "过滤"}},
|
|
{domain.DataMetricStageProcess, []string{"process", "convert", "package", "处理", "转换", "打包"}},
|
|
{domain.DataMetricStageCollect, []string{"collect", "scan", "source", "采集", "扫描", "数据源"}},
|
|
}
|
|
for _, item := range stageTokens {
|
|
for _, token := range item.tokens {
|
|
if strings.Contains(label, token) {
|
|
return item.stage
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func numericArtifactValue(payload map[string]any, keys ...string) (float64, bool) {
|
|
for _, key := range keys {
|
|
switch value := payload[key].(type) {
|
|
case int:
|
|
return float64(value), value >= 0
|
|
case int32:
|
|
return float64(value), value >= 0
|
|
case int64:
|
|
return float64(value), value >= 0
|
|
case float64:
|
|
return value, value >= 0 && !math.IsNaN(value) && !math.IsInf(value, 0)
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func legacyVolumeFromArtifacts(items []domain.ExecutionArtifact) domain.DataVolume {
|
|
var trajectory, duration, bytes, files float64
|
|
var hasTrajectory, hasDuration, hasBytes, hasFiles bool
|
|
for _, artifact := range items {
|
|
payload := artifact.Data
|
|
if value, ok := numericArtifactValue(payload, "trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"); ok && (!hasTrajectory || value > trajectory) {
|
|
trajectory, hasTrajectory = value, true
|
|
}
|
|
if value, ok := numericArtifactValue(payload, "durationSeconds", "totalDurationSeconds", "totalDuration", "duration"); ok && (!hasDuration || value > duration) {
|
|
duration, hasDuration = value, true
|
|
}
|
|
if value, ok := numericArtifactValue(payload, "totalBytes", "bytes", "sizeBytes"); ok && (!hasBytes || value > bytes) {
|
|
bytes, hasBytes = value, true
|
|
}
|
|
if value, ok := numericArtifactValue(payload, "fileCount", "filesCount"); ok && (!hasFiles || value > files) {
|
|
files, hasFiles = value, true
|
|
}
|
|
}
|
|
result := domain.DataVolume{}
|
|
if hasTrajectory {
|
|
value := int64(trajectory)
|
|
result.TrajectoryCount = &value
|
|
}
|
|
if hasDuration {
|
|
value := duration
|
|
result.DurationSeconds = &value
|
|
}
|
|
if hasBytes {
|
|
value := int64(bytes)
|
|
result.Bytes = &value
|
|
}
|
|
if hasFiles {
|
|
value := int64(files)
|
|
result.FileCount = &value
|
|
}
|
|
return result
|
|
}
|
|
|
|
func legacySourceCollectionMetrics(items []domain.ExecutionArtifact) (domain.DataVolume, domain.DataVolume, []domain.DatasetMetricRef, bool) {
|
|
var trajectories int64
|
|
var validTrajectories int64
|
|
var files int64
|
|
var bytes int64
|
|
refs := make([]domain.DatasetMetricRef, 0)
|
|
for artifactIndex := range items {
|
|
artifact := &items[artifactIndex]
|
|
payload := artifactStructuredData(artifact)
|
|
if payload == nil {
|
|
continue
|
|
}
|
|
for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") {
|
|
collection := mapValue(raw)
|
|
if collection == nil {
|
|
continue
|
|
}
|
|
trajectories++
|
|
fileCount := int64Value(collection["fileCount"])
|
|
totalBytes := int64Value(collection["totalBytes"])
|
|
files += fileCount
|
|
bytes += totalBytes
|
|
if boolValue(collection["valid"]) {
|
|
validTrajectories++
|
|
}
|
|
name := firstNonEmptyText(stringValue(collection["name"]), stringValue(collection["id"]), stringValue(collection["relativePath"]))
|
|
refs = append(refs, domain.DatasetMetricRef{
|
|
DatasetID: stringValue(collection["id"]),
|
|
Name: name,
|
|
Kind: "source-collection",
|
|
DatasetType: firstNonEmptyText(stringValue(collection["datasetType"]), "trajectory"),
|
|
PayloadPath: stringValue(collection["path"]),
|
|
ItemCount: 1,
|
|
})
|
|
}
|
|
}
|
|
if trajectories == 0 {
|
|
return domain.DataVolume{}, domain.DataVolume{}, nil, false
|
|
}
|
|
output := domain.DataVolume{
|
|
TrajectoryCount: &trajectories,
|
|
Bytes: &bytes,
|
|
FileCount: &files,
|
|
}
|
|
valid := domain.DataVolume{TrajectoryCount: &validTrajectories}
|
|
return output, valid, refs, true
|
|
}
|
|
|
|
func legacyRejectedVolume(input domain.DataVolume, output domain.DataVolume) domain.DataVolume {
|
|
result := domain.DataVolume{}
|
|
if input.TrajectoryCount != nil && output.TrajectoryCount != nil && *input.TrajectoryCount >= *output.TrajectoryCount {
|
|
value := *input.TrajectoryCount - *output.TrajectoryCount
|
|
result.TrajectoryCount = &value
|
|
}
|
|
if input.DurationSeconds != nil && output.DurationSeconds != nil && *input.DurationSeconds >= *output.DurationSeconds {
|
|
value := *input.DurationSeconds - *output.DurationSeconds
|
|
result.DurationSeconds = &value
|
|
}
|
|
if input.Bytes != nil && output.Bytes != nil && *input.Bytes >= *output.Bytes {
|
|
value := *input.Bytes - *output.Bytes
|
|
result.Bytes = &value
|
|
}
|
|
if input.FileCount != nil && output.FileCount != nil && *input.FileCount >= *output.FileCount {
|
|
value := *input.FileCount - *output.FileCount
|
|
result.FileCount = &value
|
|
}
|
|
return result
|
|
}
|
|
|
|
func legacyMetricsFromExecution(run domain.RunRecord, execution domain.NodeExecutionRecord) *domain.DataMetricsRecord {
|
|
stage := legacyStageFromExecution(execution)
|
|
if stage == "" {
|
|
return nil
|
|
}
|
|
input := legacyVolumeFromArtifacts(execution.InputArtifacts)
|
|
output := legacyVolumeFromArtifacts(execution.Artifacts)
|
|
technicalValid := domain.DataVolume{}
|
|
refs := []domain.DatasetMetricRef(nil)
|
|
if collectionOutput, valid, collectionRefs, ok := legacySourceCollectionMetrics(execution.Artifacts); ok {
|
|
output = collectionOutput
|
|
technicalValid = valid
|
|
refs = collectionRefs
|
|
}
|
|
metrics := &domain.DataMetricsRecord{
|
|
SchemaVersion: 1,
|
|
Stage: stage,
|
|
Input: input,
|
|
Output: output,
|
|
DatasetRefs: refs,
|
|
Quality: domain.DataMetricQualityEstimated,
|
|
OccurredAt: firstNonEmptyText(execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt),
|
|
}
|
|
switch stage {
|
|
case domain.DataMetricStageClean:
|
|
metrics.Rejected = legacyRejectedVolume(input, output)
|
|
case domain.DataMetricStageValidate:
|
|
if technicalValid.TrajectoryCount != nil {
|
|
metrics.TechnicalValid = technicalValid
|
|
} else {
|
|
metrics.TechnicalValid = output
|
|
}
|
|
case domain.DataMetricStageAudit:
|
|
metrics.TrainReady = output
|
|
}
|
|
return metrics
|
|
}
|
|
|
|
func (server *server) handleDataOperationsBackfill(writer http.ResponseWriter, request *http.Request) {
|
|
limit, issue := parseDatabaseBrowserPageQuery(request.URL.Query().Get("limit"), "limit", 100, 1, 500)
|
|
if issue != nil {
|
|
renderValidationError(writer, *issue)
|
|
return
|
|
}
|
|
cursor, err := decodeRunSummaryCursor(request.URL.Query().Get("cursor"))
|
|
if err != nil {
|
|
renderValidationError(writer, validationIssue{Field: "cursor", Message: err.Error()})
|
|
return
|
|
}
|
|
cutoff := time.Now().UTC().Add(-30 * 24 * time.Hour)
|
|
page, err := server.store.ListDataOperationsBackfillRuns(
|
|
request.Context(),
|
|
server.requestOrganizationID(request),
|
|
cutoff.Format(time.RFC3339Nano),
|
|
cursor,
|
|
limit,
|
|
)
|
|
if err != nil {
|
|
renderError(writer, http.StatusInternalServerError, err)
|
|
return
|
|
}
|
|
processed := 0
|
|
skipped := 0
|
|
failures := 0
|
|
for runIndex := range page.Items {
|
|
run := &page.Items[runIndex]
|
|
for executionIndex := range run.NodeExecutions {
|
|
execution := &run.NodeExecutions[executionIndex]
|
|
if execution.Status != domain.NodeExecutionStatusCompleted {
|
|
continue
|
|
}
|
|
legacy := execution.DataMetrics == nil
|
|
if legacy {
|
|
execution.DataMetrics = legacyMetricsFromExecution(*run, *execution)
|
|
if execution.DataMetrics == nil {
|
|
skipped++
|
|
continue
|
|
}
|
|
}
|
|
if err := dataops.MaterializeLegacyNodeMetrics(request.Context(), server.store, run, execution); err != nil {
|
|
failures++
|
|
continue
|
|
}
|
|
processed++
|
|
}
|
|
}
|
|
renderJSON(writer, http.StatusOK, map[string]any{
|
|
"windowDays": 30,
|
|
"scannedRuns": len(page.Items),
|
|
"processed": processed,
|
|
"skipped": skipped,
|
|
"failures": failures,
|
|
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
|
|
"hasMore": page.HasMore,
|
|
"completedAt": nowISO(),
|
|
})
|
|
}
|