📊 Add end-to-end data operations lifecycle

This commit is contained in:
eust-w 2026-07-23 21:11:15 +08:00
parent 89372faf6b
commit 9051f412c9
30 changed files with 5001 additions and 41 deletions

View File

@ -195,6 +195,151 @@ def load_input_artifacts() -> list[dict[str, Any]]:
return []
def _first_numeric(payload: dict[str, Any], keys: tuple[str, ...]) -> int | float | None:
for key in keys:
value = payload.get(key)
if isinstance(value, bool):
continue
if isinstance(value, (int, float)) and value >= 0:
return value
return None
def _metric_volume(payloads: list[dict[str, Any]]) -> dict[str, int | float]:
candidates: dict[str, list[int | float]] = {
"trajectoryCount": [],
"durationSeconds": [],
"bytes": [],
"fileCount": [],
}
for payload in payloads:
data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {}
trajectory_count = _first_numeric(
payload,
("trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"),
)
if trajectory_count is None and data_ref:
trajectory_count = _first_numeric(data_ref, ("itemCount",))
duration_seconds = _first_numeric(payload, ("durationSeconds", "totalDurationSeconds", "totalDuration", "duration"))
total_bytes = _first_numeric(payload, ("totalBytes", "bytes", "sizeBytes"))
file_count = _first_numeric(payload, ("fileCount", "filesCount"))
if trajectory_count is not None:
candidates["trajectoryCount"].append(int(trajectory_count))
if duration_seconds is not None:
candidates["durationSeconds"].append(float(duration_seconds))
if total_bytes is not None:
candidates["bytes"].append(int(total_bytes))
if file_count is not None:
candidates["fileCount"].append(int(file_count))
return {key: max(values) for key, values in candidates.items() if values}
def _artifact_payloads(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [item["data"] for item in artifacts if isinstance(item.get("data"), dict)]
def _dataset_refs(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]:
refs: list[dict[str, Any]] = []
seen: set[str] = set()
for artifact in artifacts:
payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {}
if not data_ref:
continue
item = {
"name": text_value(artifact.get("name")),
"kind": text_value(data_ref.get("kind")),
"datasetType": text_value(data_ref.get("datasetType")),
"manifestPath": text_value(data_ref.get("manifestPath")),
"payloadPath": text_value(data_ref.get("payloadPath")),
"partitionDir": text_value(data_ref.get("partitionDir")),
"partitionCount": int(data_ref.get("partitionCount") or 0),
"itemCount": int(data_ref.get("itemCount") or 0),
"datasetId": text_value(data_ref.get("datasetId")),
"datasetVersionId": text_value(data_ref.get("datasetVersionId")),
"split": text_value(data_ref.get("split")),
"fingerprint": text_value(data_ref.get("fingerprint")),
}
key = "|".join(
text_value(item.get(name))
for name in ("datasetVersionId", "datasetId", "manifestPath", "payloadPath", "partitionDir", "split")
)
if key in seen:
continue
seen.add(key)
refs.append(item)
return refs
def infer_data_metrics(
node_data: dict[str, Any],
artifacts: list[dict[str, Any]],
inputs: list[dict[str, Any]],
) -> dict[str, Any] | None:
stage = text_value(node_data.get("businessStage")).strip().lower()
if stage not in {"collect", "process", "clean", "validate", "annotate", "audit", "train"}:
return None
if node_data.get("metricBoundary") is False:
return None
output_payloads = _artifact_payloads(artifacts)
for payload in output_payloads:
explicit = payload.get("dataMetrics")
if isinstance(explicit, dict):
normalized = json_compatible(explicit)
normalized.setdefault("schemaVersion", 1)
normalized.setdefault("stage", stage)
normalized.setdefault("quality", "exact")
normalized.setdefault("occurredAt", now_iso())
return normalized
input_payloads = _artifact_payloads(inputs)
input_volume = _metric_volume(input_payloads)
output_volume = _metric_volume(output_payloads)
metrics: dict[str, Any] = {
"schemaVersion": 1,
"stage": stage,
"input": input_volume,
"output": output_volume,
"datasetRefs": _dataset_refs(artifacts),
"quality": "derived",
"occurredAt": now_iso(),
}
if stage == "collect" and not input_volume:
metrics.pop("input")
if stage == "validate":
valid = any(payload.get("valid") is True for payload in output_payloads)
metrics["technicalValid"] = output_volume if valid else {}
if not valid and input_volume:
metrics["rejected"] = input_volume
if stage in {"annotate", "audit"}:
metrics["trainReady"] = output_volume
if stage == "clean" and input_volume and output_volume:
rejected: dict[str, int | float] = {}
for key in ("trajectoryCount", "durationSeconds", "bytes", "fileCount"):
if key in input_volume and key in output_volume:
rejected[key] = max(0, input_volume[key] - output_volume[key])
if rejected:
metrics["rejected"] = rejected
if stage == "train":
training_inputs: list[dict[str, Any]] = []
for ref in _dataset_refs(inputs):
training_inputs.append(
{
"datasetId": ref.get("datasetId", ""),
"datasetVersionId": ref.get("datasetVersionId", ""),
"name": ref.get("name", ""),
"split": ref.get("split", ""),
"volume": input_volume,
"manifestPath": ref.get("manifestPath", ""),
"partitionDir": ref.get("partitionDir", ""),
"external": not bool(ref.get("datasetVersionId")),
"quality": "derived",
}
)
metrics["trainingInputs"] = training_inputs
return metrics
def stage_manifest_payload(
run_id: str,
workflow_id: str,
@ -202,6 +347,7 @@ def stage_manifest_payload(
node_label: str,
output_dir: pathlib.Path,
artifacts: list[dict[str, Any]],
data_metrics: dict[str, Any] | None = None,
) -> dict[str, Any]:
manifest_artifacts: list[dict[str, Any]] = []
data_refs: list[dict[str, Any]] = []
@ -232,7 +378,7 @@ def stage_manifest_payload(
}
)
manifest_artifacts.append(item)
return {
result = {
"runId": run_id,
"workflowId": workflow_id,
"nodeId": node_id,
@ -244,6 +390,9 @@ def stage_manifest_payload(
"primaryDataRefs": data_refs,
"generatedAt": now_iso(),
}
if data_metrics:
result["dataMetrics"] = json_compatible(data_metrics)
return result
def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path:
@ -5295,16 +5444,25 @@ def main() -> int:
resource_samples = sampler.stop(ended_at)
append_log("INFO", f"Produced {len(artifacts)} artifact(s)")
data_metrics = infer_data_metrics(node_data, artifacts, inputs)
result = {
"logs": logs,
"artifacts": artifacts,
"resourceSeries": resource_samples,
}
if data_metrics:
result["dataMetrics"] = data_metrics
stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip()
if stage_manifest_path:
write_json_file(stage_manifest_path, stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts))
write_json_file(
stage_manifest_path,
stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics),
)
else:
write_json_file(default_stage_manifest_path(output_dir), stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts))
write_json_file(
default_stage_manifest_path(output_dir),
stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics),
)
result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip()
if result_output_path:

View File

@ -646,6 +646,41 @@ class RuntimeFileContractTests(unittest.TestCase):
self.assertEqual(len(manifest["dataRefs"]), 1)
self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages")
def test_infer_data_metrics_uses_bound_stage_and_preserves_unknowns(self):
metrics = MODULE.infer_data_metrics(
{"businessStage": "clean", "metricBoundary": True},
[{"name": "filtered.json", "data": {"count": 8, "fileCount": 16}}],
[{"name": "source.json", "data": {"count": 10, "fileCount": 20}}],
)
self.assertEqual(metrics["stage"], "clean")
self.assertEqual(metrics["input"]["trajectoryCount"], 10)
self.assertEqual(metrics["output"]["trajectoryCount"], 8)
self.assertEqual(metrics["rejected"]["trajectoryCount"], 2)
self.assertEqual(metrics["quality"], "derived")
def test_stage_manifest_includes_standard_data_metrics(self):
with tempfile.TemporaryDirectory() as temp_dir:
output_dir = pathlib.Path(temp_dir)
metrics = {
"schemaVersion": 1,
"stage": "collect",
"output": {"trajectoryCount": 12},
"quality": "exact",
}
manifest = MODULE.stage_manifest_payload(
"run-1",
"wf-1",
"node-1",
"数据采集",
output_dir,
[],
metrics,
)
self.assertEqual(manifest["dataMetrics"]["stage"], "collect")
self.assertEqual(manifest["dataMetrics"]["output"]["trajectoryCount"], 12)
class XPlatformIndexingTests(unittest.TestCase):
def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self):

View File

@ -0,0 +1,781 @@
package api
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"droboticflow/internal/dataops"
"droboticflow/internal/domain"
"github.com/go-chi/chi/v5"
)
func validCollectionTaskMode(value domain.CollectionTaskMode) bool {
return value == domain.CollectionTaskModeCollectionOnly || value == domain.CollectionTaskModeFullTraining
}
func validCollectionTaskStatus(value domain.CollectionTaskStatus) bool {
switch value {
case domain.CollectionTaskStatusDraft, domain.CollectionTaskStatusActive, domain.CollectionTaskStatusPaused,
domain.CollectionTaskStatusCompleted, domain.CollectionTaskStatusCancelled:
return true
default:
return false
}
}
func validDataMetricStage(value domain.DataMetricStage) bool {
for _, stage := range domain.DataMetricStages {
if value == stage {
return true
}
}
return false
}
func normalizedStringList(items []string) []string {
result := make([]string, 0, len(items))
seen := map[string]struct{}{}
for _, value := range items {
trimmed := strings.TrimSpace(value)
key := strings.ToLower(trimmed)
if trimmed == "" {
continue
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
result = append(result, trimmed)
}
return result
}
func validateNonNegativeVolume(field string, value domain.DataVolume) []validationIssue {
issues := make([]validationIssue, 0)
if value.TrajectoryCount != nil && *value.TrajectoryCount < 0 {
issues = append(issues, validationIssue{Field: field + ".trajectoryCount", Message: "must be greater than or equal to 0"})
}
if value.DurationSeconds != nil && (*value.DurationSeconds < 0 || math.IsNaN(*value.DurationSeconds) || math.IsInf(*value.DurationSeconds, 0)) {
issues = append(issues, validationIssue{Field: field + ".durationSeconds", Message: "must be a finite value greater than or equal to 0"})
}
if value.Bytes != nil && *value.Bytes < 0 {
issues = append(issues, validationIssue{Field: field + ".bytes", Message: "must be greater than or equal to 0"})
}
if value.FileCount != nil && *value.FileCount < 0 {
issues = append(issues, validationIssue{Field: field + ".fileCount", Message: "must be greater than or equal to 0"})
}
return issues
}
func collectionTaskHasTarget(value domain.DataVolume) bool {
return (value.TrajectoryCount != nil && *value.TrajectoryCount > 0) ||
(value.DurationSeconds != nil && *value.DurationSeconds > 0) ||
(value.Bytes != nil && *value.Bytes > 0)
}
func (server *server) normalizeCollectionTaskInput(ctxRequest *http.Request, payload *domain.CollectionTaskInput, existing *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, []validationIssue, error) {
payload.ProjectID = strings.TrimSpace(payload.ProjectID)
payload.Name = strings.TrimSpace(payload.Name)
payload.Description = strings.TrimSpace(payload.Description)
payload.RobotModel = strings.TrimSpace(payload.RobotModel)
payload.RobotVersion = strings.TrimSpace(payload.RobotVersion)
payload.Scene = strings.TrimSpace(payload.Scene)
payload.Format = strings.TrimSpace(payload.Format)
payload.WorkflowID = strings.TrimSpace(payload.WorkflowID)
payload.Modalities = normalizedStringList(payload.Modalities)
if payload.Mode == "" {
payload.Mode = domain.CollectionTaskModeFullTraining
}
if payload.Status == "" {
payload.Status = domain.CollectionTaskStatusDraft
}
issues := make([]validationIssue, 0)
if payload.ProjectID == "" {
issues = append(issues, validationIssue{Field: "projectId", Message: "projectId is required"})
}
if payload.Name == "" {
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
}
if payload.RobotModel == "" {
issues = append(issues, validationIssue{Field: "robotModel", Message: "robotModel is required"})
}
if payload.Scene == "" {
issues = append(issues, validationIssue{Field: "scene", Message: "scene is required"})
}
if !validCollectionTaskMode(payload.Mode) {
issues = append(issues, validationIssue{Field: "mode", Message: "mode must be collection_only or full_training"})
}
if !validCollectionTaskStatus(payload.Status) {
issues = append(issues, validationIssue{Field: "status", Message: "invalid task status"})
}
issues = append(issues, validateNonNegativeVolume("targets", payload.Targets)...)
if !collectionTaskHasTarget(payload.Targets) {
issues = append(issues, validationIssue{Field: "targets", Message: "at least one positive trajectory, duration, or bytes target is required"})
}
splitTotal := 0
for split, percent := range payload.SplitRules {
if split != "train" && split != "val" && split != "test" {
issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split must be train, val, or test"})
}
if percent < 0 || percent > 100 {
issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split percent must be between 0 and 100"})
}
splitTotal += percent
}
if len(payload.SplitRules) > 0 && splitTotal != 100 {
issues = append(issues, validationIssue{Field: "splitRules", Message: "split percentages must total 100"})
}
seenNodes := map[string]domain.DataMetricStage{}
for stage, nodeID := range payload.StageBindings {
nodeID = strings.TrimSpace(nodeID)
if !validDataMetricStage(stage) {
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "invalid business stage"})
continue
}
if nodeID == "" {
delete(payload.StageBindings, stage)
continue
}
if previous, exists := seenNodes[nodeID]; exists {
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: fmt.Sprintf("node is already the metric boundary for %s", previous)})
}
seenNodes[nodeID] = stage
payload.StageBindings[stage] = nodeID
}
if len(issues) > 0 {
return nil, issues, nil
}
project, err := server.store.GetProject(ctxRequest.Context(), payload.ProjectID)
if err != nil {
return nil, nil, err
}
if project == nil || !server.filterByTenant(ctxRequest, project.OrganizationID) {
return nil, []validationIssue{{Field: "projectId", Message: "project not found"}}, nil
}
workflowName := ""
if payload.WorkflowID != "" {
workflow, err := server.store.GetWorkflow(ctxRequest.Context(), payload.WorkflowID)
if err != nil {
return nil, nil, err
}
if workflow == nil || workflow.ProjectID != project.ID || !server.filterByTenant(ctxRequest, workflow.OrganizationID) {
return nil, []validationIssue{{Field: "workflowId", Message: "workflow not found in selected project"}}, nil
}
workflowName = workflow.Name
for stage, nodeID := range payload.StageBindings {
if domain.GetNodeByID(workflow.Nodes, nodeID) == nil {
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "workflow node not found"})
}
}
}
if len(issues) > 0 {
return nil, issues, nil
}
item := &domain.CollectionTaskRecord{
OrganizationID: project.OrganizationID,
ProjectID: project.ID,
ProjectName: project.Name,
Name: payload.Name,
Description: payload.Description,
RobotModel: payload.RobotModel,
RobotVersion: payload.RobotVersion,
Scene: payload.Scene,
Modalities: payload.Modalities,
Format: payload.Format,
Mode: payload.Mode,
Status: payload.Status,
WorkflowID: payload.WorkflowID,
WorkflowName: workflowName,
StageBindings: payload.StageBindings,
Targets: payload.Targets,
Requirements: payload.Requirements,
SplitRules: payload.SplitRules,
}
if existing != nil {
item.ID = existing.ID
item.CreatedAt = existing.CreatedAt
item.LastRunID = existing.LastRunID
}
return item, nil, nil
}
func (server *server) handleCollectionTasksList(writer http.ResponseWriter, request *http.Request) {
items, err := server.store.ListCollectionTasks(request.Context(), server.requestOrganizationID(request))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
projectID := strings.TrimSpace(request.URL.Query().Get("projectId"))
status := strings.TrimSpace(request.URL.Query().Get("status"))
filtered := make([]domain.CollectionTaskRecord, 0, len(items))
for _, item := range items {
if projectID != "" && item.ProjectID != projectID {
continue
}
if status != "" && string(item.Status) != status {
continue
}
filtered = append(filtered, item)
}
renderJSON(writer, http.StatusOK, map[string]any{"items": filtered})
}
func (server *server) handleCollectionTaskGet(writer http.ResponseWriter, request *http.Request) {
item, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if item == nil || !server.ensureTenantAccess(writer, request, item.OrganizationID, "Data task") {
if item == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
}
return
}
renderJSON(writer, http.StatusOK, item)
}
func (server *server) persistCollectionTask(request *http.Request, item *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, error) {
if err := server.store.UpsertCollectionTask(request.Context(), *item); err != nil {
return nil, err
}
return server.store.GetCollectionTask(request.Context(), item.ID)
}
func (server *server) handleCollectionTaskCreate(writer http.ResponseWriter, request *http.Request) {
var payload domain.CollectionTaskInput
if !decodeJSON(writer, request, &payload) {
return
}
item, issues, err := server.normalizeCollectionTaskInput(request, &payload, nil)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
item.ID = domain.NewID("data-task")
saved, err := server.persistCollectionTask(request, item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusCreated, saved)
}
func (server *server) handleCollectionTaskUpdate(writer http.ResponseWriter, request *http.Request) {
existing, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if existing == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
return
}
if !server.ensureTenantAccess(writer, request, existing.OrganizationID, "Data task") {
return
}
var payload domain.CollectionTaskInput
if !decodeJSON(writer, request, &payload) {
return
}
item, issues, err := server.normalizeCollectionTaskInput(request, &payload, existing)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
saved, err := server.persistCollectionTask(request, item)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, saved)
}
func (server *server) handleCollectionTaskStart(writer http.ResponseWriter, request *http.Request) {
task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if task == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
return
}
if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") {
return
}
if task.WorkflowID == "" {
renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bind a workflow before starting the task"})
return
}
requiredStages := []domain.DataMetricStage{domain.DataMetricStageCollect}
if task.Mode == domain.CollectionTaskModeFullTraining {
requiredStages = append(
requiredStages,
domain.DataMetricStageProcess,
domain.DataMetricStageClean,
domain.DataMetricStageValidate,
domain.DataMetricStageTrain,
)
if task.Requirements.RequireAnnotation {
requiredStages = append(requiredStages, domain.DataMetricStageAnnotate)
}
if task.Requirements.RequireAudit {
requiredStages = append(requiredStages, domain.DataMetricStageAudit)
}
}
missingStageIssues := make([]validationIssue, 0)
for _, stage := range requiredStages {
if strings.TrimSpace(task.StageBindings[stage]) == "" {
missingStageIssues = append(missingStageIssues, validationIssue{
Field: "stageBindings." + string(stage),
Message: "required stage is not bound to a workflow node",
})
}
}
if len(missingStageIssues) > 0 {
renderValidationError(writer, missingStageIssues...)
return
}
workflow, err := server.store.GetWorkflow(request.Context(), task.WorkflowID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if workflow == nil || workflow.ProjectID != task.ProjectID {
renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bound workflow is unavailable"})
return
}
active, err := server.store.HasActiveRun(request.Context(), workflow.ID)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if active {
renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流已有进行中的运行", "code": "workflow_run_active"})
return
}
if issues := validateWorkflowTopology(workflow.Nodes, workflow.Edges); len(issues) > 0 {
renderValidationError(writer, issues...)
return
}
run, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerManual, task.ID)
if err != nil {
if renderRunCreationError(writer, err) {
return
}
renderError(writer, http.StatusInternalServerError, err)
return
}
task.Status = domain.CollectionTaskStatusActive
task.LastRunID = run.ID
saved, err := server.persistCollectionTask(request, task)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusAccepted, map[string]any{"task": saved, "run": run})
}
func (server *server) handleCollectionTaskComplete(writer http.ResponseWriter, request *http.Request) {
task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
if task == nil {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
return
}
if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") {
return
}
task.Status = domain.CollectionTaskStatusCompleted
saved, err := server.persistCollectionTask(request, task)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, saved)
}
func parseDataOperationsQuery(request *http.Request, settings *domain.SettingsRecord) (domain.DataOperationsQuery, []validationIssue) {
timezone := strings.TrimSpace(request.URL.Query().Get("timezone"))
if timezone == "" && settings != nil {
timezone = strings.TrimSpace(settings.Timezone)
}
if timezone == "" {
timezone = domain.DefaultWorkflowScheduleTimezone
}
location, err := time.LoadLocation(timezone)
if err != nil {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "timezone", Message: "invalid IANA timezone"}}
}
query := domain.DataOperationsQuery{
Timezone: timezone,
ProjectID: strings.TrimSpace(request.URL.Query().Get("projectId")),
TaskID: strings.TrimSpace(request.URL.Query().Get("taskId")),
RobotModel: strings.TrimSpace(request.URL.Query().Get("robotModel")),
Scene: strings.TrimSpace(request.URL.Query().Get("scene")),
Stage: domain.DataMetricStage(strings.TrimSpace(request.URL.Query().Get("stage"))),
Cursor: strings.TrimSpace(request.URL.Query().Get("cursor")),
}
if query.Stage != "" && !validDataMetricStage(query.Stage) {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "stage", Message: "invalid business stage"}}
}
if raw := strings.TrimSpace(request.URL.Query().Get("limit")); raw != "" {
value, parseErr := strconv.Atoi(raw)
if parseErr != nil || value < 1 || value > 100 {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "limit", Message: "limit must be between 1 and 100"}}
}
query.Limit = value
}
now := time.Now()
fromRaw := strings.TrimSpace(request.URL.Query().Get("from"))
toRaw := strings.TrimSpace(request.URL.Query().Get("to"))
if fromRaw == "" {
localNow := now.In(location)
fromRaw = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, location).UTC().Format(time.RFC3339Nano)
}
if toRaw == "" {
toRaw = now.UTC().Format(time.RFC3339Nano)
}
from, fromErr := time.Parse(time.RFC3339Nano, fromRaw)
to, toErr := time.Parse(time.RFC3339Nano, toRaw)
if fromErr != nil {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be RFC3339"}}
}
if toErr != nil {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "to", Message: "to must be RFC3339"}}
}
if !from.Before(to) {
return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be before to"}}
}
query.From = from.UTC().Format(time.RFC3339Nano)
query.To = to.UTC().Format(time.RFC3339Nano)
return query, nil
}
func (server *server) dataOperationsQuery(writer http.ResponseWriter, request *http.Request) (domain.DataOperationsQuery, bool) {
settings, err := server.store.GetSettings(request.Context())
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return domain.DataOperationsQuery{}, false
}
query, issues := parseDataOperationsQuery(request, settings)
if len(issues) > 0 {
renderValidationError(writer, issues...)
return domain.DataOperationsQuery{}, false
}
return query, true
}
func (server *server) handleDataOperationsSummary(writer http.ResponseWriter, request *http.Request) {
query, ok := server.dataOperationsQuery(writer, request)
if !ok {
return
}
summary, err := server.store.GetDataOperationsSummary(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, summary)
}
func (server *server) handleDataOperationsStageItems(writer http.ResponseWriter, request *http.Request) {
query, ok := server.dataOperationsQuery(writer, request)
if !ok {
return
}
page, err := server.store.ListDataMetricFactsPage(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, page)
}
func (server *server) handleDataOperationsTrainingUsage(writer http.ResponseWriter, request *http.Request) {
query, ok := server.dataOperationsQuery(writer, request)
if !ok {
return
}
page, err := server.store.ListTrainingUsagePage(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, page)
}
func (server *server) handleDataOperationsDatasets(writer http.ResponseWriter, request *http.Request) {
query, ok := server.dataOperationsQuery(writer, request)
if !ok {
return
}
items, err := server.store.ListDatasetVersions(request.Context(), server.requestOrganizationID(request), query)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
}
func legacyStageFromExecution(execution domain.NodeExecutionRecord) domain.DataMetricStage {
label := strings.ToLower(strings.TrimSpace(execution.NodeLabel + " " + execution.OperatorID))
stageTokens := []struct {
stage domain.DataMetricStage
tokens []string
}{
{domain.DataMetricStageTrain, []string{"train", "训练"}},
{domain.DataMetricStageAudit, []string{"audit", "审核"}},
{domain.DataMetricStageAnnotate, []string{"annotat", "标注"}},
{domain.DataMetricStageValidate, []string{"validat", "quality", "校验", "质检"}},
{domain.DataMetricStageClean, []string{"clean", "filter", "清洗", "过滤"}},
{domain.DataMetricStageProcess, []string{"process", "convert", "package", "处理", "转换", "打包"}},
{domain.DataMetricStageCollect, []string{"collect", "scan", "source", "采集", "扫描", "数据源"}},
}
for _, item := range stageTokens {
for _, token := range item.tokens {
if strings.Contains(label, token) {
return item.stage
}
}
}
return ""
}
func numericArtifactValue(payload map[string]any, keys ...string) (float64, bool) {
for _, key := range keys {
switch value := payload[key].(type) {
case int:
return float64(value), value >= 0
case int32:
return float64(value), value >= 0
case int64:
return float64(value), value >= 0
case float64:
return value, value >= 0 && !math.IsNaN(value) && !math.IsInf(value, 0)
}
}
return 0, false
}
func legacyVolumeFromArtifacts(items []domain.ExecutionArtifact) domain.DataVolume {
var trajectory, duration, bytes, files float64
var hasTrajectory, hasDuration, hasBytes, hasFiles bool
for _, artifact := range items {
payload := artifact.Data
if value, ok := numericArtifactValue(payload, "trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"); ok && (!hasTrajectory || value > trajectory) {
trajectory, hasTrajectory = value, true
}
if value, ok := numericArtifactValue(payload, "durationSeconds", "totalDurationSeconds", "totalDuration", "duration"); ok && (!hasDuration || value > duration) {
duration, hasDuration = value, true
}
if value, ok := numericArtifactValue(payload, "totalBytes", "bytes", "sizeBytes"); ok && (!hasBytes || value > bytes) {
bytes, hasBytes = value, true
}
if value, ok := numericArtifactValue(payload, "fileCount", "filesCount"); ok && (!hasFiles || value > files) {
files, hasFiles = value, true
}
}
result := domain.DataVolume{}
if hasTrajectory {
value := int64(trajectory)
result.TrajectoryCount = &value
}
if hasDuration {
value := duration
result.DurationSeconds = &value
}
if hasBytes {
value := int64(bytes)
result.Bytes = &value
}
if hasFiles {
value := int64(files)
result.FileCount = &value
}
return result
}
func legacySourceCollectionMetrics(items []domain.ExecutionArtifact) (domain.DataVolume, domain.DataVolume, []domain.DatasetMetricRef, bool) {
var trajectories int64
var validTrajectories int64
var files int64
var bytes int64
refs := make([]domain.DatasetMetricRef, 0)
for artifactIndex := range items {
artifact := &items[artifactIndex]
payload := artifactStructuredData(artifact)
if payload == nil {
continue
}
for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") {
collection := mapValue(raw)
if collection == nil {
continue
}
trajectories++
fileCount := int64Value(collection["fileCount"])
totalBytes := int64Value(collection["totalBytes"])
files += fileCount
bytes += totalBytes
if boolValue(collection["valid"]) {
validTrajectories++
}
name := firstNonEmptyText(stringValue(collection["name"]), stringValue(collection["id"]), stringValue(collection["relativePath"]))
refs = append(refs, domain.DatasetMetricRef{
DatasetID: stringValue(collection["id"]),
Name: name,
Kind: "source-collection",
DatasetType: firstNonEmptyText(stringValue(collection["datasetType"]), "trajectory"),
PayloadPath: stringValue(collection["path"]),
ItemCount: 1,
})
}
}
if trajectories == 0 {
return domain.DataVolume{}, domain.DataVolume{}, nil, false
}
output := domain.DataVolume{
TrajectoryCount: &trajectories,
Bytes: &bytes,
FileCount: &files,
}
valid := domain.DataVolume{TrajectoryCount: &validTrajectories}
return output, valid, refs, true
}
func legacyRejectedVolume(input domain.DataVolume, output domain.DataVolume) domain.DataVolume {
result := domain.DataVolume{}
if input.TrajectoryCount != nil && output.TrajectoryCount != nil && *input.TrajectoryCount >= *output.TrajectoryCount {
value := *input.TrajectoryCount - *output.TrajectoryCount
result.TrajectoryCount = &value
}
if input.DurationSeconds != nil && output.DurationSeconds != nil && *input.DurationSeconds >= *output.DurationSeconds {
value := *input.DurationSeconds - *output.DurationSeconds
result.DurationSeconds = &value
}
if input.Bytes != nil && output.Bytes != nil && *input.Bytes >= *output.Bytes {
value := *input.Bytes - *output.Bytes
result.Bytes = &value
}
if input.FileCount != nil && output.FileCount != nil && *input.FileCount >= *output.FileCount {
value := *input.FileCount - *output.FileCount
result.FileCount = &value
}
return result
}
func legacyMetricsFromExecution(run domain.RunRecord, execution domain.NodeExecutionRecord) *domain.DataMetricsRecord {
stage := legacyStageFromExecution(execution)
if stage == "" {
return nil
}
input := legacyVolumeFromArtifacts(execution.InputArtifacts)
output := legacyVolumeFromArtifacts(execution.Artifacts)
technicalValid := domain.DataVolume{}
refs := []domain.DatasetMetricRef(nil)
if collectionOutput, valid, collectionRefs, ok := legacySourceCollectionMetrics(execution.Artifacts); ok {
output = collectionOutput
technicalValid = valid
refs = collectionRefs
}
metrics := &domain.DataMetricsRecord{
SchemaVersion: 1,
Stage: stage,
Input: input,
Output: output,
DatasetRefs: refs,
Quality: domain.DataMetricQualityEstimated,
OccurredAt: firstNonEmptyText(execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt),
}
switch stage {
case domain.DataMetricStageClean:
metrics.Rejected = legacyRejectedVolume(input, output)
case domain.DataMetricStageValidate:
if technicalValid.TrajectoryCount != nil {
metrics.TechnicalValid = technicalValid
} else {
metrics.TechnicalValid = output
}
case domain.DataMetricStageAudit:
metrics.TrainReady = output
}
return metrics
}
func (server *server) handleDataOperationsBackfill(writer http.ResponseWriter, request *http.Request) {
limit, issue := parseDatabaseBrowserPageQuery(request.URL.Query().Get("limit"), "limit", 100, 1, 500)
if issue != nil {
renderValidationError(writer, *issue)
return
}
cursor, err := decodeRunSummaryCursor(request.URL.Query().Get("cursor"))
if err != nil {
renderValidationError(writer, validationIssue{Field: "cursor", Message: err.Error()})
return
}
cutoff := time.Now().UTC().Add(-30 * 24 * time.Hour)
page, err := server.store.ListDataOperationsBackfillRuns(
request.Context(),
server.requestOrganizationID(request),
cutoff.Format(time.RFC3339Nano),
cursor,
limit,
)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
processed := 0
skipped := 0
failures := 0
for runIndex := range page.Items {
run := &page.Items[runIndex]
for executionIndex := range run.NodeExecutions {
execution := &run.NodeExecutions[executionIndex]
if execution.Status != domain.NodeExecutionStatusCompleted {
continue
}
legacy := execution.DataMetrics == nil
if legacy {
execution.DataMetrics = legacyMetricsFromExecution(*run, *execution)
if execution.DataMetrics == nil {
skipped++
continue
}
}
if err := dataops.MaterializeLegacyNodeMetrics(request.Context(), server.store, run, execution); err != nil {
failures++
continue
}
processed++
}
}
renderJSON(writer, http.StatusOK, map[string]any{
"windowDays": 30,
"scannedRuns": len(page.Items),
"processed": processed,
"skipped": skipped,
"failures": failures,
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
"hasMore": page.HasMore,
"completedAt": nowISO(),
})
}

View File

@ -0,0 +1,150 @@
package api
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func apiDataOperationsInt64(value int64) *int64 {
return &value
}
func TestCollectionTaskCreateAndStartCarriesTaskIntoRun(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Y1 桌面整理"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "Y1 完整链路",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{{
ID: "collect-node",
Type: "default",
Data: domain.WorkflowNodeData{
Label: "数据采集",
Category: "DATA",
Description: "collect",
Inputs: []domain.PortDefinition{},
Outputs: []domain.PortDefinition{},
},
Position: domain.Position{},
}},
Edges: []domain.WorkflowEdge{},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
handler := NewHandlerWithConfig(dataStore, nil, config.AppConfig{WorkspaceMountPath: t.TempDir()})
payload := domain.CollectionTaskInput{
ProjectID: project.ID,
Name: "Y1 桌面整理数据闭环",
RobotModel: "Y1",
Scene: "桌面整理",
Modalities: []string{"RGB", "关节状态"},
Mode: domain.CollectionTaskModeCollectionOnly,
Status: domain.CollectionTaskStatusDraft,
WorkflowID: workflow.ID,
StageBindings: map[domain.DataMetricStage]string{
domain.DataMetricStageCollect: "collect-node",
},
Targets: domain.DataVolume{TrajectoryCount: apiDataOperationsInt64(100)},
Requirements: domain.CollectionTaskRequirements{
RequireTechnicalValidation: true,
RequireQuality: true,
RequireAnnotation: true,
RequireAudit: true,
},
SplitRules: map[string]int{"train": 80, "val": 10, "test": 10},
}
body, _ := json.Marshal(payload)
createRecorder := httptest.NewRecorder()
handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/data-collection-tasks", bytesReader(body)))
if createRecorder.Code != http.StatusCreated {
t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
}
var task domain.CollectionTaskRecord
if err := json.Unmarshal(createRecorder.Body.Bytes(), &task); err != nil {
t.Fatalf("decode task: %v", err)
}
if task.ID == "" || task.CreatedAt == "" || task.UpdatedAt == "" {
t.Fatalf("created task is missing persisted fields: %#v", task)
}
startRecorder := httptest.NewRecorder()
handler.ServeHTTP(startRecorder, httptest.NewRequest(http.MethodPost, "/api/data-collection-tasks/"+task.ID+"/start", nil))
if startRecorder.Code != http.StatusAccepted {
t.Fatalf("start status = %d body=%s", startRecorder.Code, startRecorder.Body.String())
}
var response struct {
Run domain.RunRecord `json:"run"`
}
if err := json.Unmarshal(startRecorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode start response: %v", err)
}
if response.Run.TaskID != task.ID {
t.Fatalf("run task id = %q, want %q", response.Run.TaskID, task.ID)
}
}
func TestWorkflowNodeBusinessStageValidation(t *testing.T) {
issues := validateWorkflowNodeResources([]domain.WorkflowNode{{
Data: domain.WorkflowNodeData{BusinessStage: domain.DataMetricStage("publish")},
}})
if len(issues) != 1 || issues[0].Field != "nodes[0].data.businessStage" {
t.Fatalf("issues = %#v", issues)
}
}
func TestLegacySourceCollectionsProduceEstimatedTrajectoryMetrics(t *testing.T) {
metrics := legacyMetricsFromExecution(
domain.RunRecord{CreatedAt: "2026-07-23T01:00:00Z"},
domain.NodeExecutionRecord{
NodeLabel: "BOS 增量扫描",
Artifacts: []domain.ExecutionArtifact{{
Name: "dataset-index.json",
Data: map[string]any{
"collections": []any{
map[string]any{"id": "trajectory-1", "path": "y1/1", "fileCount": float64(5), "totalBytes": float64(100), "valid": true},
map[string]any{"id": "trajectory-2", "path": "y1/2", "fileCount": float64(4), "totalBytes": float64(80), "valid": false},
},
},
}},
},
)
if metrics == nil || metrics.Output.TrajectoryCount == nil || *metrics.Output.TrajectoryCount != 2 {
t.Fatalf("metrics = %#v", metrics)
}
if metrics.Output.FileCount == nil || *metrics.Output.FileCount != 9 ||
metrics.Output.Bytes == nil || *metrics.Output.Bytes != 180 {
t.Fatalf("source collection volumes = %#v", metrics)
}
if len(metrics.DatasetRefs) != 2 || metrics.Quality != domain.DataMetricQualityEstimated {
t.Fatalf("source collection lineage = %#v", metrics)
}
}
func TestParseDataOperationsQueryUsesUTC8TodayByDefault(t *testing.T) {
request := httptest.NewRequest(http.MethodGet, "/api/data-operations/summary", nil)
query, issues := parseDataOperationsQuery(request, &domain.SettingsRecord{Timezone: "Asia/Shanghai"})
if len(issues) > 0 {
t.Fatalf("issues = %#v", issues)
}
if query.Timezone != "Asia/Shanghai" || query.From == "" || query.To == "" {
t.Fatalf("query = %#v", query)
}
}
func bytesReader(value []byte) *bytes.Reader {
return bytes.NewReader(value)
}

View File

@ -409,6 +409,7 @@ func runSummaryFromRecord(run domain.RunRecord) domain.RunSummary {
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
TaskID: run.TaskID,
Trigger: run.Trigger,
Attempt: run.Attempt,
ParentRunID: run.ParentRunID,
@ -1205,6 +1206,23 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
})
router.Get("/api/acceptance/poc-readiness", instance.handlePocAcceptanceReadiness)
router.Get("/api/dashboard", instance.handleDashboard)
router.Get("/api/data-collection-tasks", instance.handleCollectionTasksList)
router.Get("/api/data-collection-tasks/{id}", instance.handleCollectionTaskGet)
router.Get("/api/data-operations/summary", instance.handleDataOperationsSummary)
router.Get("/api/data-operations/stage-items", instance.handleDataOperationsStageItems)
router.Get("/api/data-operations/training-usage", instance.handleDataOperationsTrainingUsage)
router.Get("/api/data-operations/datasets", instance.handleDataOperationsDatasets)
router.Group(func(router chi.Router) {
router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleOperator))
router.Post("/api/data-collection-tasks", instance.handleCollectionTaskCreate)
router.Patch("/api/data-collection-tasks/{id}", instance.handleCollectionTaskUpdate)
router.Post("/api/data-collection-tasks/{id}/start", instance.handleCollectionTaskStart)
router.Post("/api/data-collection-tasks/{id}/complete", instance.handleCollectionTaskComplete)
})
router.Group(func(router chi.Router) {
router.Use(instance.requireBusinessRole(domain.UserRoleAdmin))
router.Post("/api/data-operations/backfill", instance.handleDataOperationsBackfill)
})
router.Get("/api/projects", instance.handleProjectsList)
router.Get("/api/projects/{id}", instance.handleProjectsGet)
@ -3481,6 +3499,12 @@ func validateWorkflowNodeResources(nodes []domain.WorkflowNode) []validationIssu
if err := k8s.ValidateResourceQuantity(field, node.Data.SharedMemorySize); err != nil {
issues = append(issues, validationIssue{Field: field, Message: err.Error()})
}
if node.Data.BusinessStage != "" && !validDataMetricStage(node.Data.BusinessStage) {
issues = append(issues, validationIssue{
Field: "nodes[" + strconv.Itoa(index) + "].data.businessStage",
Message: "invalid business stage",
})
}
}
return issues
}
@ -3739,7 +3763,7 @@ func validRunTrigger(value domain.RunTrigger) bool {
return value == domain.RunTriggerManual || value == domain.RunTriggerScheduled || value == domain.RunTriggerRetry
}
func (server *server) createRunWithLightweightManifests(ctx context.Context, workflow *domain.WorkflowRecord, trigger domain.RunTrigger) (*domain.RunRecord, error) {
func (server *server) createRunWithLightweightManifests(ctx context.Context, workflow *domain.WorkflowRecord, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) {
if workflow == nil {
return nil, fmt.Errorf("workflow not found")
}
@ -3747,7 +3771,7 @@ func (server *server) createRunWithLightweightManifests(ctx context.Context, wor
if err != nil {
return nil, err
}
run, err := server.store.CreateRun(ctx, workflow.ID, trigger)
run, err := server.store.CreateRun(ctx, workflow.ID, trigger, taskIDs...)
if err != nil {
return nil, err
}
@ -3980,7 +4004,7 @@ func (server *server) handleRunsRerunSubtree(writer http.ResponseWriter, request
}
}
childRun, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerRerunSubtree)
childRun, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerRerunSubtree, sourceRun.TaskID)
if err != nil {
if renderRunCreationError(writer, err) {
return
@ -4735,6 +4759,9 @@ func (server *server) handleOperatorsCreate(writer http.ResponseWriter, request
normalizeLightweightOperatorContract(&payload.RuntimeContract, containerCommandProvided, containerArgsProvided)
}
issues := validateOperatorMetadata(payload.Name, payload.Type, payload.Category, payload.Description, payload.Version, payload.Inputs, payload.Outputs, payload.Resources, payload.ContainerCommand, payload.ContainerArgs)
if payload.BusinessStage != "" && !validDataMetricStage(payload.BusinessStage) {
issues = append(issues, validationIssue{Field: "businessStage", Message: "invalid business stage"})
}
if payload.Type == "lightweight" {
issues = append(issues, validateLightweightOperatorMetadata(payload.Image, payload.RuntimeContract, payload.ContainerCommand, payload.ContainerArgs, server.workspaceMountPath)...)
}
@ -4801,6 +4828,7 @@ type operatorPatch struct {
SkillDefinition *string `json:"skillDefinition"`
Hyperparams *[]domain.HyperParam `json:"hyperparams"`
Resources *domain.OperatorResources `json:"resources"`
BusinessStage *domain.DataMetricStage `json:"businessStage"`
PublicationStatus *domain.OperatorPublicationStatus `json:"publicationStatus"`
GenerationSource *domain.OperatorGenerationSource `json:"generationSource"`
PublicationEvidence *domain.OperatorPublicationEvidence `json:"publicationEvidence"`
@ -4883,6 +4911,13 @@ func (server *server) handleOperatorsUpdate(writer http.ResponseWriter, request
if payload.Resources != nil {
next.Resources = normalizeOperatorResources(payload.Resources)
}
if payload.BusinessStage != nil {
if *payload.BusinessStage != "" && !validDataMetricStage(*payload.BusinessStage) {
renderValidationError(writer, validationIssue{Field: "businessStage", Message: "invalid business stage"})
return
}
next.BusinessStage = *payload.BusinessStage
}
if payload.GenerationSource != nil {
next.GenerationSource = normalizeOperatorGenerationSource(*payload.GenerationSource)
}

View File

@ -0,0 +1,292 @@
package dataops
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"strings"
"time"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func normalizedQuality(value domain.DataMetricQuality) domain.DataMetricQuality {
switch value {
case domain.DataMetricQualityExact, domain.DataMetricQualityDerived, domain.DataMetricQualityEstimated:
return value
default:
return domain.DataMetricQualityDerived
}
}
func stableID(prefix string, value string) string {
sum := sha256.Sum256([]byte(value))
return prefix + "-" + hex.EncodeToString(sum[:10])
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if trimmed := strings.TrimSpace(value); trimmed != "" {
return trimmed
}
}
return ""
}
func dataRefFingerprint(ref domain.DatasetMetricRef, runID string, nodeID string) string {
value := strings.Join([]string{
ref.DatasetVersionID,
ref.DatasetID,
ref.Fingerprint,
ref.ManifestPath,
ref.PayloadPath,
ref.PartitionDir,
ref.Split,
ref.Name,
}, "|")
if strings.Trim(value, "|") == "" {
value = runID + "|" + nodeID + "|" + ref.Name
}
sum := sha256.Sum256([]byte(value))
return hex.EncodeToString(sum[:])
}
func metricRefFromDataRef(item domain.DataRefRecord) domain.DatasetMetricRef {
return domain.DatasetMetricRef{
DatasetID: item.DatasetID,
DatasetVersionID: item.DatasetVersionID,
Name: item.ArtifactName,
Split: item.Split,
Fingerprint: item.Fingerprint,
Kind: item.Kind,
DatasetType: item.DatasetType,
ManifestPath: item.ManifestPath,
PayloadPath: item.PayloadPath,
PartitionDir: item.PartitionDir,
PartitionCount: item.PartitionCount,
ItemCount: item.ItemCount,
}
}
func int64Pointer(value int64) *int64 {
return &value
}
func volumeWithItemCount(volume domain.DataVolume, itemCount int) domain.DataVolume {
if volume.TrajectoryCount == nil && itemCount > 0 {
volume.TrajectoryCount = int64Pointer(int64(itemCount))
}
return volume
}
func anyString(value any) string {
if value == nil {
return ""
}
return strings.TrimSpace(fmt.Sprint(value))
}
func anyInt(value any) int {
switch typed := value.(type) {
case int:
return typed
case int32:
return int(typed)
case int64:
return int(typed)
case float64:
return int(typed)
default:
return 0
}
}
func metricRefFromArtifact(artifact domain.ExecutionArtifact) (domain.DatasetMetricRef, bool) {
if artifact.Data == nil {
return domain.DatasetMetricRef{}, false
}
raw, ok := artifact.Data["dataRef"].(map[string]any)
if !ok || len(raw) == 0 {
return domain.DatasetMetricRef{}, false
}
return domain.DatasetMetricRef{
Name: artifact.Name,
DatasetID: anyString(raw["datasetId"]),
DatasetVersionID: anyString(raw["datasetVersionId"]),
Split: anyString(raw["split"]),
Fingerprint: anyString(raw["fingerprint"]),
Kind: anyString(raw["kind"]),
DatasetType: anyString(raw["datasetType"]),
ManifestPath: anyString(raw["manifestPath"]),
PayloadPath: anyString(raw["payloadPath"]),
PartitionDir: anyString(raw["partitionDir"]),
PartitionCount: anyInt(raw["partitionCount"]),
ItemCount: anyInt(raw["itemCount"]),
}, true
}
func trainingInputsFromExecution(execution domain.NodeExecutionRecord, inputVolume domain.DataVolume) []domain.TrainingInputRecord {
result := make([]domain.TrainingInputRecord, 0)
seen := map[string]struct{}{}
for _, artifact := range execution.InputArtifacts {
ref, ok := metricRefFromArtifact(artifact)
if !ok {
continue
}
key := strings.Join([]string{ref.DatasetVersionID, ref.DatasetID, ref.ManifestPath, ref.PartitionDir, ref.Split}, "|")
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
result = append(result, domain.TrainingInputRecord{
DatasetID: ref.DatasetID,
DatasetVersionID: ref.DatasetVersionID,
Name: ref.Name,
Split: ref.Split,
Volume: volumeWithItemCount(inputVolume, ref.ItemCount),
ManifestPath: ref.ManifestPath,
PartitionDir: ref.PartitionDir,
External: ref.DatasetVersionID == "",
Quality: domain.DataMetricQualityDerived,
})
}
return result
}
func materializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord, markRun bool) error {
if run == nil || execution == nil || execution.DataMetrics == nil {
return nil
}
metrics := execution.DataMetrics
if metrics.Stage == "" {
return nil
}
task := (*domain.CollectionTaskRecord)(nil)
if strings.TrimSpace(run.TaskID) != "" {
item, err := dataStore.GetCollectionTask(ctx, run.TaskID)
if err != nil {
return err
}
task = item
}
refs := append([]domain.DatasetMetricRef(nil), metrics.DatasetRefs...)
if len(refs) == 0 {
for _, item := range execution.DataRefs {
refs = append(refs, metricRefFromDataRef(item))
}
}
quality := normalizedQuality(metrics.Quality)
for index := range refs {
ref := &refs[index]
fingerprint := firstNonEmpty(ref.Fingerprint, dataRefFingerprint(*ref, run.ID, execution.NodeID))
ref.Fingerprint = fingerprint
ref.DatasetID = firstNonEmpty(ref.DatasetID, stableID("dataset", firstNonEmpty(ref.Name, ref.DatasetType, fingerprint)))
ref.DatasetVersionID = firstNonEmpty(ref.DatasetVersionID, stableID("dataset-version", fingerprint))
volume := volumeWithItemCount(metrics.Output, ref.ItemCount)
version := domain.DatasetVersionRecord{
ID: ref.DatasetVersionID,
OrganizationID: run.OrganizationID,
DatasetID: ref.DatasetID,
TaskID: run.TaskID,
ProjectID: run.ProjectID,
Name: firstNonEmpty(ref.Name, ref.DatasetType, ref.DatasetID),
Split: ref.Split,
Fingerprint: fingerprint,
Volume: volume,
ManifestPath: ref.ManifestPath,
PartitionDir: ref.PartitionDir,
ProducerRunID: run.ID,
ProducerNodeID: execution.NodeID,
Quality: quality,
CreatedAt: firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.UpdatedAt),
}
if err := dataStore.UpsertDatasetVersion(ctx, version); err != nil {
return err
}
}
trainingInputs := append([]domain.TrainingInputRecord(nil), metrics.TrainingInputs...)
if metrics.Stage == domain.DataMetricStageTrain && len(trainingInputs) == 0 {
trainingInputs = trainingInputsFromExecution(*execution, metrics.Input)
}
occurredAt := firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt)
if parsed, err := time.Parse(time.RFC3339Nano, occurredAt); err != nil || parsed.IsZero() {
occurredAt = time.Now().UTC().Format(time.RFC3339Nano)
quality = domain.DataMetricQualityEstimated
} else {
occurredAt = parsed.UTC().Format(time.RFC3339Nano)
}
fact := domain.DataMetricFactRecord{
ID: stableID("data-fact", fmt.Sprintf("%s|%s|%d|%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage)),
SourceKey: fmt.Sprintf("%s/%s/%d/%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage),
OrganizationID: run.OrganizationID,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
TaskID: run.TaskID,
WorkflowID: run.WorkflowID,
WorkflowName: run.WorkflowName,
RunID: run.ID,
NodeID: execution.NodeID,
NodeLabel: execution.NodeLabel,
Attempt: execution.Attempt,
Stage: metrics.Stage,
Input: metrics.Input,
Output: metrics.Output,
Rejected: metrics.Rejected,
TechnicalValid: metrics.TechnicalValid,
TrainReady: metrics.TrainReady,
DatasetRefs: refs,
TrainingInputs: trainingInputs,
RejectionReasons: metrics.RejectionReasons,
Quality: quality,
OccurredAt: occurredAt,
RecordedAt: time.Now().UTC().Format(time.RFC3339Nano),
}
if task != nil {
fact.TaskName = task.Name
fact.RobotModel = task.RobotModel
fact.Scene = task.Scene
}
if err := dataStore.UpsertDataMetricFact(ctx, fact); err != nil {
return err
}
if !markRun {
return nil
}
return dataStore.MarkNodeMetricsMaterialized(ctx, run.ID, execution.NodeID, execution.Attempt, time.Now().UTC().Format(time.RFC3339Nano))
}
func MaterializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
return materializeNodeMetrics(ctx, dataStore, run, execution, true)
}
func MaterializeLegacyNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
return materializeNodeMetrics(ctx, dataStore, run, execution, false)
}
func ReconcileRunMetrics(ctx context.Context, dataStore store.Store, limit int) (int, error) {
runs, err := dataStore.ListRunsNeedingMetricMaterialization(ctx, limit)
if err != nil {
return 0, err
}
materialized := 0
var firstErr error
for runIndex := range runs {
run := &runs[runIndex]
for executionIndex := range run.NodeExecutions {
execution := &run.NodeExecutions[executionIndex]
if execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" {
continue
}
if err := MaterializeNodeMetrics(ctx, dataStore, run, execution); err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
materialized++
}
}
return materialized, firstErr
}

View File

@ -0,0 +1,89 @@
package dataops
import (
"context"
"testing"
"droboticflow/internal/domain"
"droboticflow/internal/store"
)
func materializerInt64(value int64) *int64 {
return &value
}
func TestMaterializeNodeMetricsCreatesFactDatasetAndMarksExecution(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Y1 项目"})
if err != nil {
t.Fatalf("create project: %v", err)
}
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
ProjectID: project.ID,
Name: "Y1 数据准备",
Type: domain.WorkflowTypeBatch,
Nodes: []domain.WorkflowNode{},
Edges: []domain.WorkflowEdge{},
})
if err != nil {
t.Fatalf("create workflow: %v", err)
}
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual, "task-y1")
if err != nil {
t.Fatalf("create run: %v", err)
}
execution := domain.NodeExecutionRecord{
NodeID: "collect",
NodeLabel: "采集索引",
Attempt: 1,
Status: domain.NodeExecutionStatusCompleted,
EndedAt: "2026-07-23T01:00:00Z",
DataMetrics: &domain.DataMetricsRecord{
SchemaVersion: 1,
Stage: domain.DataMetricStageCollect,
Output: domain.DataVolume{TrajectoryCount: materializerInt64(8)},
Quality: domain.DataMetricQualityExact,
DatasetRefs: []domain.DatasetMetricRef{{
Name: "Y1 batch",
DatasetType: "x-platform",
ManifestPath: "/workspace/y1/manifest.json",
ItemCount: 8,
}},
},
}
run.NodeExecutions = []domain.NodeExecutionRecord{execution}
if err := dataStore.UpsertRun(ctx, *run); err != nil {
t.Fatalf("upsert run: %v", err)
}
if err := MaterializeNodeMetrics(ctx, dataStore, run, &execution); err != nil {
t.Fatalf("materialize: %v", err)
}
page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{
From: "2026-07-23T00:00:00Z",
To: "2026-07-24T00:00:00Z",
})
if err != nil {
t.Fatalf("list facts: %v", err)
}
if len(page.Items) != 1 || len(page.Items[0].DatasetRefs) != 1 {
t.Fatalf("facts = %#v", page.Items)
}
if page.Items[0].DatasetRefs[0].DatasetVersionID == "" {
t.Fatal("dataset version id was not materialized")
}
versions, err := dataStore.ListDatasetVersions(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{Limit: 10})
if err != nil {
t.Fatalf("list versions: %v", err)
}
if len(versions) != 1 || versions[0].Volume.TrajectoryCount == nil || *versions[0].Volume.TrajectoryCount != 8 {
t.Fatalf("versions = %#v", versions)
}
persisted, err := dataStore.GetRun(ctx, run.ID)
if err != nil {
t.Fatalf("get run: %v", err)
}
if persisted.NodeExecutions[0].MetricsMaterializedAt == "" {
t.Fatal("node metrics were not marked materialized")
}
}

View File

@ -0,0 +1,295 @@
package domain
type DataMetricStage string
const (
DataMetricStageCollect DataMetricStage = "collect"
DataMetricStageProcess DataMetricStage = "process"
DataMetricStageClean DataMetricStage = "clean"
DataMetricStageValidate DataMetricStage = "validate"
DataMetricStageAnnotate DataMetricStage = "annotate"
DataMetricStageAudit DataMetricStage = "audit"
DataMetricStageTrain DataMetricStage = "train"
)
var DataMetricStages = []DataMetricStage{
DataMetricStageCollect,
DataMetricStageProcess,
DataMetricStageClean,
DataMetricStageValidate,
DataMetricStageAnnotate,
DataMetricStageAudit,
DataMetricStageTrain,
}
type DataMetricQuality string
const (
DataMetricQualityExact DataMetricQuality = "exact"
DataMetricQualityDerived DataMetricQuality = "derived"
DataMetricQualityEstimated DataMetricQuality = "estimated"
)
type DataVolume struct {
TrajectoryCount *int64 `json:"trajectoryCount,omitempty" bson:"trajectoryCount,omitempty"`
DurationSeconds *float64 `json:"durationSeconds,omitempty" bson:"durationSeconds,omitempty"`
Bytes *int64 `json:"bytes,omitempty" bson:"bytes,omitempty"`
FileCount *int64 `json:"fileCount,omitempty" bson:"fileCount,omitempty"`
}
type DatasetMetricRef struct {
DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"`
DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"`
Name string `json:"name,omitempty" bson:"name,omitempty"`
Split string `json:"split,omitempty" bson:"split,omitempty"`
Fingerprint string `json:"fingerprint,omitempty" bson:"fingerprint,omitempty"`
Kind string `json:"kind,omitempty" bson:"kind,omitempty"`
DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"`
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"`
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"`
ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"`
}
type TrainingInputRecord struct {
DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"`
DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"`
Name string `json:"name,omitempty" bson:"name,omitempty"`
Split string `json:"split,omitempty" bson:"split,omitempty"`
Volume DataVolume `json:"volume,omitempty" bson:"volume,omitempty"`
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
External bool `json:"external,omitempty" bson:"external,omitempty"`
Quality DataMetricQuality `json:"quality,omitempty" bson:"quality,omitempty"`
}
type DataMetricsRecord struct {
SchemaVersion int `json:"schemaVersion" bson:"schemaVersion"`
Stage DataMetricStage `json:"stage" bson:"stage"`
Input DataVolume `json:"input,omitempty" bson:"input,omitempty"`
Output DataVolume `json:"output,omitempty" bson:"output,omitempty"`
Rejected DataVolume `json:"rejected,omitempty" bson:"rejected,omitempty"`
TechnicalValid DataVolume `json:"technicalValid,omitempty" bson:"technicalValid,omitempty"`
TrainReady DataVolume `json:"trainReady,omitempty" bson:"trainReady,omitempty"`
DatasetRefs []DatasetMetricRef `json:"datasetRefs,omitempty" bson:"datasetRefs,omitempty"`
TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"`
RejectionReasons map[string]int64 `json:"rejectionReasons,omitempty" bson:"rejectionReasons,omitempty"`
Quality DataMetricQuality `json:"quality,omitempty" bson:"quality,omitempty"`
OccurredAt string `json:"occurredAt,omitempty" bson:"occurredAt,omitempty"`
}
type CollectionTaskStatus string
const (
CollectionTaskStatusDraft CollectionTaskStatus = "draft"
CollectionTaskStatusActive CollectionTaskStatus = "active"
CollectionTaskStatusPaused CollectionTaskStatus = "paused"
CollectionTaskStatusCompleted CollectionTaskStatus = "completed"
CollectionTaskStatusCancelled CollectionTaskStatus = "cancelled"
)
type CollectionTaskMode string
const (
CollectionTaskModeCollectionOnly CollectionTaskMode = "collection_only"
CollectionTaskModeFullTraining CollectionTaskMode = "full_training"
)
type CollectionTaskRequirements struct {
RequireTechnicalValidation bool `json:"requireTechnicalValidation" bson:"requireTechnicalValidation"`
RequireQuality bool `json:"requireQuality" bson:"requireQuality"`
RequireAnnotation bool `json:"requireAnnotation" bson:"requireAnnotation"`
RequireAudit bool `json:"requireAudit" bson:"requireAudit"`
}
type CollectionTaskRecord struct {
ID string `json:"id" bson:"id"`
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
ProjectID string `json:"projectId" bson:"projectId"`
ProjectName string `json:"projectName,omitempty" bson:"projectName,omitempty"`
Name string `json:"name" bson:"name"`
Description string `json:"description,omitempty" bson:"description,omitempty"`
RobotModel string `json:"robotModel" bson:"robotModel"`
RobotVersion string `json:"robotVersion,omitempty" bson:"robotVersion,omitempty"`
Scene string `json:"scene" bson:"scene"`
Modalities []string `json:"modalities" bson:"modalities"`
Format string `json:"format,omitempty" bson:"format,omitempty"`
Mode CollectionTaskMode `json:"mode" bson:"mode"`
Status CollectionTaskStatus `json:"status" bson:"status"`
WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"`
WorkflowName string `json:"workflowName,omitempty" bson:"workflowName,omitempty"`
StageBindings map[DataMetricStage]string `json:"stageBindings,omitempty" bson:"stageBindings,omitempty"`
Targets DataVolume `json:"targets" bson:"targets"`
Requirements CollectionTaskRequirements `json:"requirements" bson:"requirements"`
SplitRules map[string]int `json:"splitRules,omitempty" bson:"splitRules,omitempty"`
LastRunID string `json:"lastRunId,omitempty" bson:"lastRunId,omitempty"`
CreatedAt string `json:"createdAt" bson:"createdAt"`
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
}
type CollectionTaskInput struct {
ProjectID string `json:"projectId"`
Name string `json:"name"`
Description string `json:"description,omitempty"`
RobotModel string `json:"robotModel"`
RobotVersion string `json:"robotVersion,omitempty"`
Scene string `json:"scene"`
Modalities []string `json:"modalities"`
Format string `json:"format,omitempty"`
Mode CollectionTaskMode `json:"mode"`
Status CollectionTaskStatus `json:"status,omitempty"`
WorkflowID string `json:"workflowId,omitempty"`
StageBindings map[DataMetricStage]string `json:"stageBindings,omitempty"`
Targets DataVolume `json:"targets"`
Requirements CollectionTaskRequirements `json:"requirements"`
SplitRules map[string]int `json:"splitRules,omitempty"`
}
type DatasetVersionRecord struct {
ID string `json:"id" bson:"id"`
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
DatasetID string `json:"datasetId" bson:"datasetId"`
TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"`
ProjectID string `json:"projectId,omitempty" bson:"projectId,omitempty"`
Name string `json:"name" bson:"name"`
Split string `json:"split,omitempty" bson:"split,omitempty"`
Fingerprint string `json:"fingerprint" bson:"fingerprint"`
Volume DataVolume `json:"volume,omitempty" bson:"volume,omitempty"`
Storage AssetStorageLocation `json:"storage,omitempty" bson:"storage,omitempty"`
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
ProducerRunID string `json:"producerRunId,omitempty" bson:"producerRunId,omitempty"`
ProducerNodeID string `json:"producerNodeId,omitempty" bson:"producerNodeId,omitempty"`
ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"`
Quality DataMetricQuality `json:"quality" bson:"quality"`
CreatedAt string `json:"createdAt" bson:"createdAt"`
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
}
type DataMetricFactRecord struct {
ID string `json:"id" bson:"id"`
SourceKey string `json:"sourceKey" bson:"sourceKey"`
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
ProjectID string `json:"projectId,omitempty" bson:"projectId,omitempty"`
ProjectName string `json:"projectName,omitempty" bson:"projectName,omitempty"`
TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"`
TaskName string `json:"taskName,omitempty" bson:"taskName,omitempty"`
RobotModel string `json:"robotModel,omitempty" bson:"robotModel,omitempty"`
Scene string `json:"scene,omitempty" bson:"scene,omitempty"`
WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"`
WorkflowName string `json:"workflowName,omitempty" bson:"workflowName,omitempty"`
RunID string `json:"runId" bson:"runId"`
NodeID string `json:"nodeId" bson:"nodeId"`
NodeLabel string `json:"nodeLabel,omitempty" bson:"nodeLabel,omitempty"`
Attempt int `json:"attempt" bson:"attempt"`
Stage DataMetricStage `json:"stage" bson:"stage"`
Input DataVolume `json:"input,omitempty" bson:"input,omitempty"`
Output DataVolume `json:"output,omitempty" bson:"output,omitempty"`
Rejected DataVolume `json:"rejected,omitempty" bson:"rejected,omitempty"`
TechnicalValid DataVolume `json:"technicalValid,omitempty" bson:"technicalValid,omitempty"`
TrainReady DataVolume `json:"trainReady,omitempty" bson:"trainReady,omitempty"`
DatasetRefs []DatasetMetricRef `json:"datasetRefs,omitempty" bson:"datasetRefs,omitempty"`
TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"`
RejectionReasons map[string]int64 `json:"rejectionReasons,omitempty" bson:"rejectionReasons,omitempty"`
Quality DataMetricQuality `json:"quality" bson:"quality"`
OccurredAt string `json:"occurredAt" bson:"occurredAt"`
RecordedAt string `json:"recordedAt" bson:"recordedAt"`
}
type DataOperationsQuery struct {
From string
To string
Timezone string
ProjectID string
TaskID string
RobotModel string
Scene string
Stage DataMetricStage
Limit int
Cursor string
}
type DataStageSummary struct {
Stage DataMetricStage `json:"stage"`
Input DataVolume `json:"input,omitempty"`
Output DataVolume `json:"output,omitempty"`
Rejected DataVolume `json:"rejected,omitempty"`
TechnicalValid DataVolume `json:"technicalValid,omitempty"`
TrainReady DataVolume `json:"trainReady,omitempty"`
RunCount int64 `json:"runCount"`
}
type DataOperationsHourlyPoint struct {
Hour string `json:"hour"`
Stages map[DataMetricStage]DataVolume `json:"stages"`
}
type DataOperationsTrainingSummary struct {
Consumed DataVolume `json:"consumed"`
Unique DataVolume `json:"unique"`
RunCount int64 `json:"runCount"`
}
type CollectionTaskProgress struct {
TaskID string `json:"taskId"`
TaskName string `json:"taskName"`
Status CollectionTaskStatus `json:"status"`
Targets DataVolume `json:"targets"`
Collected DataVolume `json:"collected"`
Percent float64 `json:"percent"`
}
type DataOperationsSummary struct {
From string `json:"from"`
To string `json:"to"`
Timezone string `json:"timezone"`
UpdatedAt string `json:"updatedAt"`
Freshness string `json:"freshness"`
Stages []DataStageSummary `json:"stages"`
Training DataOperationsTrainingSummary `json:"training"`
Hourly []DataOperationsHourlyPoint `json:"hourly"`
RejectionReasons map[string]int64 `json:"rejectionReasons"`
Tasks []CollectionTaskProgress `json:"tasks"`
FactCount int64 `json:"factCount"`
ExactCount int64 `json:"exactCount"`
DerivedCount int64 `json:"derivedCount"`
EstimatedCount int64 `json:"estimatedCount"`
UnknownCount int64 `json:"unknownCount"`
}
type TrainingUsageRecord struct {
ID string `json:"id"`
TaskID string `json:"taskId,omitempty"`
TaskName string `json:"taskName,omitempty"`
ProjectID string `json:"projectId,omitempty"`
ProjectName string `json:"projectName,omitempty"`
WorkflowID string `json:"workflowId,omitempty"`
WorkflowName string `json:"workflowName,omitempty"`
RunID string `json:"runId"`
NodeID string `json:"nodeId"`
NodeLabel string `json:"nodeLabel,omitempty"`
Model string `json:"model,omitempty"`
Inputs []TrainingInputRecord `json:"inputs"`
Consumed DataVolume `json:"consumed"`
Quality DataMetricQuality `json:"quality"`
OccurredAt string `json:"occurredAt"`
}
type TrainingUsagePage struct {
Items []TrainingUsageRecord `json:"items"`
NextCursor string `json:"nextCursor,omitempty"`
HasMore bool `json:"hasMore"`
}
type DataMetricFactPage struct {
Items []DataMetricFactRecord `json:"items"`
NextCursor string `json:"nextCursor,omitempty"`
HasMore bool `json:"hasMore"`
}
type DataOperationsRunPage struct {
Items []RunRecord
NextCursor *RunSummaryCursor
HasMore bool
}

View File

@ -183,6 +183,8 @@ type WorkflowNodeData struct {
Memory string `json:"memory,omitempty" bson:"memory,omitempty"`
GPU string `json:"gpu,omitempty" bson:"gpu,omitempty"`
SharedMemorySize string `json:"sharedMemorySize,omitempty" bson:"sharedMemorySize,omitempty"`
BusinessStage DataMetricStage `json:"businessStage,omitempty" bson:"businessStage,omitempty"`
MetricBoundary bool `json:"metricBoundary,omitempty" bson:"metricBoundary,omitempty"`
}
type WorkflowNode struct {
@ -281,6 +283,7 @@ type OperatorRecord struct {
SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"`
Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"`
Resources *OperatorResources `json:"resources,omitempty" bson:"resources,omitempty"`
BusinessStage DataMetricStage `json:"businessStage,omitempty" bson:"businessStage,omitempty"`
PublicationStatus OperatorPublicationStatus `json:"publicationStatus,omitempty" bson:"publicationStatus,omitempty"`
GenerationSource OperatorGenerationSource `json:"generationSource,omitempty" bson:"generationSource,omitempty"`
PublicationEvidence *OperatorPublicationEvidence `json:"publicationEvidence,omitempty" bson:"publicationEvidence,omitempty"`
@ -463,37 +466,43 @@ func formatArtifactSize(bytes int64) string {
}
type DataRefRecord struct {
ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"`
ArtifactName string `json:"artifactName,omitempty" bson:"artifactName,omitempty"`
PortID string `json:"portId,omitempty" bson:"portId,omitempty"`
Kind string `json:"kind,omitempty" bson:"kind,omitempty"`
DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"`
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"`
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"`
ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"`
ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"`
ArtifactName string `json:"artifactName,omitempty" bson:"artifactName,omitempty"`
PortID string `json:"portId,omitempty" bson:"portId,omitempty"`
Kind string `json:"kind,omitempty" bson:"kind,omitempty"`
DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"`
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"`
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"`
ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"`
DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"`
DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"`
Split string `json:"split,omitempty" bson:"split,omitempty"`
Fingerprint string `json:"fingerprint,omitempty" bson:"fingerprint,omitempty"`
}
type NodeExecutionRecord struct {
NodeID string `json:"nodeId" bson:"nodeId"`
NodeLabel string `json:"nodeLabel" bson:"nodeLabel"`
OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"`
OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"`
OperatorImage string `json:"operatorImage,omitempty" bson:"operatorImage,omitempty"`
OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"`
Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"`
Status NodeExecutionStatus `json:"status" bson:"status"`
StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"`
EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"`
StageManifestPath string `json:"stageManifestPath,omitempty" bson:"stageManifestPath,omitempty"`
InputArtifacts []ExecutionArtifact `json:"inputArtifacts,omitempty" bson:"inputArtifacts,omitempty"`
InputLineage []InputLineageRecord `json:"inputLineage,omitempty" bson:"inputLineage,omitempty"`
Logs []LogEntryRecord `json:"logs" bson:"logs"`
Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"`
DataRefs []DataRefRecord `json:"dataRefs,omitempty" bson:"dataRefs,omitempty"`
ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"`
Error string `json:"error,omitempty" bson:"error,omitempty"`
NodeID string `json:"nodeId" bson:"nodeId"`
NodeLabel string `json:"nodeLabel" bson:"nodeLabel"`
OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"`
OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"`
OperatorImage string `json:"operatorImage,omitempty" bson:"operatorImage,omitempty"`
OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"`
Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"`
Status NodeExecutionStatus `json:"status" bson:"status"`
StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"`
EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"`
StageManifestPath string `json:"stageManifestPath,omitempty" bson:"stageManifestPath,omitempty"`
InputArtifacts []ExecutionArtifact `json:"inputArtifacts,omitempty" bson:"inputArtifacts,omitempty"`
InputLineage []InputLineageRecord `json:"inputLineage,omitempty" bson:"inputLineage,omitempty"`
Logs []LogEntryRecord `json:"logs" bson:"logs"`
Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"`
DataRefs []DataRefRecord `json:"dataRefs,omitempty" bson:"dataRefs,omitempty"`
DataMetrics *DataMetricsRecord `json:"dataMetrics,omitempty" bson:"dataMetrics,omitempty"`
MetricsMaterializedAt string `json:"metricsMaterializedAt,omitempty" bson:"metricsMaterializedAt,omitempty"`
ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"`
Error string `json:"error,omitempty" bson:"error,omitempty"`
}
type InputLineageRecord struct {
@ -597,6 +606,7 @@ type LightweightOperatorCodeManifest struct {
type RunMetadata struct {
LightweightOperatorCodeManifests map[string]LightweightOperatorCodeManifest `json:"lightweightOperatorCodeManifests,omitempty" bson:"lightweightOperatorCodeManifests,omitempty"`
TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"`
}
// RunFailure is the machine-readable counterpart to RunRecord.Error. Error is
@ -630,6 +640,7 @@ type RunRecord struct {
WorkflowName string `json:"workflowName" bson:"workflowName"`
ProjectID string `json:"projectId" bson:"projectId"`
ProjectName string `json:"projectName" bson:"projectName"`
TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"`
Trigger RunTrigger `json:"trigger" bson:"trigger"`
Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"`
ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"`
@ -666,6 +677,7 @@ type RunSummary struct {
WorkflowName string `json:"workflowName" bson:"workflowName"`
ProjectID string `json:"projectId" bson:"projectId"`
ProjectName string `json:"projectName" bson:"projectName"`
TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"`
Trigger RunTrigger `json:"trigger" bson:"trigger"`
Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"`
ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"`
@ -1076,6 +1088,7 @@ type CreateOperatorInput struct {
SkillDefinition string `json:"skillDefinition,omitempty"`
Hyperparams []HyperParam `json:"hyperparams,omitempty"`
Resources *OperatorResources `json:"resources,omitempty"`
BusinessStage DataMetricStage `json:"businessStage,omitempty"`
PublicationStatus OperatorPublicationStatus `json:"publicationStatus,omitempty"`
GenerationSource OperatorGenerationSource `json:"generationSource,omitempty"`
PublicationEvidence *OperatorPublicationEvidence `json:"publicationEvidence,omitempty"`

View File

@ -0,0 +1,408 @@
package store
import (
"encoding/base64"
"fmt"
"sort"
"strings"
"time"
"droboticflow/internal/domain"
)
type volumeAccumulator struct {
trajectoryCount int64
durationSeconds float64
bytes int64
fileCount int64
hasTrajectory bool
hasDuration bool
hasBytes bool
hasFileCount bool
}
func dataMetricQualityRank(value domain.DataMetricQuality) int {
switch value {
case domain.DataMetricQualityExact:
return 3
case domain.DataMetricQualityDerived:
return 2
case domain.DataMetricQualityEstimated:
return 1
default:
return 0
}
}
func (item *volumeAccumulator) add(value domain.DataVolume) {
if value.TrajectoryCount != nil {
item.trajectoryCount += *value.TrajectoryCount
item.hasTrajectory = true
}
if value.DurationSeconds != nil {
item.durationSeconds += *value.DurationSeconds
item.hasDuration = true
}
if value.Bytes != nil {
item.bytes += *value.Bytes
item.hasBytes = true
}
if value.FileCount != nil {
item.fileCount += *value.FileCount
item.hasFileCount = true
}
}
func (item volumeAccumulator) value() domain.DataVolume {
result := domain.DataVolume{}
if item.hasTrajectory {
value := item.trajectoryCount
result.TrajectoryCount = &value
}
if item.hasDuration {
value := item.durationSeconds
result.DurationSeconds = &value
}
if item.hasBytes {
value := item.bytes
result.Bytes = &value
}
if item.hasFileCount {
value := item.fileCount
result.FileCount = &value
}
return result
}
type stageAccumulator struct {
input volumeAccumulator
output volumeAccumulator
rejected volumeAccumulator
technicalValid volumeAccumulator
trainReady volumeAccumulator
runs map[string]struct{}
}
func normalizedDataOperationsTimezone(value string) *time.Location {
name := strings.TrimSpace(value)
if name == "" {
name = domain.DefaultWorkflowScheduleTimezone
}
location, err := time.LoadLocation(name)
if err != nil {
location, _ = time.LoadLocation(domain.DefaultWorkflowScheduleTimezone)
}
if location == nil {
location = time.FixedZone("UTC+8", 8*60*60)
}
return location
}
func dataOperationsFactMatches(item domain.DataMetricFactRecord, organizationID string, query domain.DataOperationsQuery) bool {
if !organizationMatches(item.OrganizationID, organizationID) {
return false
}
if query.ProjectID != "" && item.ProjectID != query.ProjectID {
return false
}
if query.TaskID != "" && item.TaskID != query.TaskID {
return false
}
if query.RobotModel != "" && !strings.EqualFold(item.RobotModel, query.RobotModel) {
return false
}
if query.Scene != "" && !strings.EqualFold(item.Scene, query.Scene) {
return false
}
if query.Stage != "" && item.Stage != query.Stage {
return false
}
if query.From != "" && item.OccurredAt < query.From {
return false
}
if query.To != "" && item.OccurredAt >= query.To {
return false
}
return true
}
func datasetVersionMatches(item domain.DatasetVersionRecord, organizationID string, query domain.DataOperationsQuery) bool {
if !organizationMatches(item.OrganizationID, organizationID) {
return false
}
if query.ProjectID != "" && item.ProjectID != query.ProjectID {
return false
}
if query.TaskID != "" && item.TaskID != query.TaskID {
return false
}
if query.From != "" && item.CreatedAt < query.From {
return false
}
if query.To != "" && item.CreatedAt >= query.To {
return false
}
return true
}
func encodeDataOperationsCursor(occurredAt string, id string) string {
return base64.RawURLEncoding.EncodeToString([]byte(occurredAt + "\n" + id))
}
func decodeDataOperationsCursor(value string) (string, string, bool) {
if strings.TrimSpace(value) == "" {
return "", "", true
}
payload, err := base64.RawURLEncoding.DecodeString(value)
if err != nil {
return "", "", false
}
parts := strings.SplitN(string(payload), "\n", 2)
if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" {
return "", "", false
}
return parts[0], parts[1], true
}
func factsPage(items []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.DataMetricFactPage {
sort.Slice(items, func(left, right int) bool {
if items[left].OccurredAt == items[right].OccurredAt {
return items[left].ID > items[right].ID
}
return items[left].OccurredAt > items[right].OccurredAt
})
cursorTime, cursorID, validCursor := decodeDataOperationsCursor(query.Cursor)
if !validCursor {
return domain.DataMetricFactPage{Items: []domain.DataMetricFactRecord{}}
}
if cursorTime != "" {
filtered := items[:0]
for _, item := range items {
if item.OccurredAt < cursorTime || (item.OccurredAt == cursorTime && item.ID < cursorID) {
filtered = append(filtered, item)
}
}
items = filtered
}
limit := query.Limit
if limit <= 0 {
limit = 50
}
if limit > 100 {
limit = 100
}
page := domain.DataMetricFactPage{Items: items, HasMore: len(items) > limit}
if page.HasMore {
page.Items = items[:limit]
last := page.Items[len(page.Items)-1]
page.NextCursor = encodeDataOperationsCursor(last.OccurredAt, last.ID)
}
if page.Items == nil {
page.Items = []domain.DataMetricFactRecord{}
}
return page
}
func trainingUsageFromFacts(facts []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.TrainingUsagePage {
filtered := make([]domain.DataMetricFactRecord, 0)
for _, item := range facts {
if item.Stage == domain.DataMetricStageTrain {
filtered = append(filtered, item)
}
}
page := factsPage(filtered, query)
items := make([]domain.TrainingUsageRecord, 0, len(page.Items))
for _, fact := range page.Items {
items = append(items, domain.TrainingUsageRecord{
ID: fact.ID,
TaskID: fact.TaskID,
TaskName: fact.TaskName,
ProjectID: fact.ProjectID,
ProjectName: fact.ProjectName,
WorkflowID: fact.WorkflowID,
WorkflowName: fact.WorkflowName,
RunID: fact.RunID,
NodeID: fact.NodeID,
NodeLabel: fact.NodeLabel,
Model: fact.NodeLabel,
Inputs: fact.TrainingInputs,
Consumed: fact.Input,
Quality: fact.Quality,
OccurredAt: fact.OccurredAt,
})
}
return domain.TrainingUsagePage{Items: items, NextCursor: page.NextCursor, HasMore: page.HasMore}
}
func taskProgressPercent(target domain.DataVolume, collected domain.DataVolume) float64 {
switch {
case target.TrajectoryCount != nil && *target.TrajectoryCount > 0 && collected.TrajectoryCount != nil:
return minFloat(100, float64(*collected.TrajectoryCount)*100/float64(*target.TrajectoryCount))
case target.DurationSeconds != nil && *target.DurationSeconds > 0 && collected.DurationSeconds != nil:
return minFloat(100, *collected.DurationSeconds*100 / *target.DurationSeconds)
case target.Bytes != nil && *target.Bytes > 0 && collected.Bytes != nil:
return minFloat(100, float64(*collected.Bytes)*100/float64(*target.Bytes))
default:
return 0
}
}
func minFloat(left float64, right float64) float64 {
if left < right {
return left
}
return right
}
func BuildDataOperationsSummary(
facts []domain.DataMetricFactRecord,
tasks []domain.CollectionTaskRecord,
query domain.DataOperationsQuery,
) domain.DataOperationsSummary {
location := normalizedDataOperationsTimezone(query.Timezone)
accumulators := make(map[domain.DataMetricStage]*stageAccumulator, len(domain.DataMetricStages))
for _, stage := range domain.DataMetricStages {
accumulators[stage] = &stageAccumulator{runs: map[string]struct{}{}}
}
hourly := map[string]map[domain.DataMetricStage]*volumeAccumulator{}
reasons := map[string]int64{}
uniqueTraining := map[string]domain.DataVolume{}
var trainingConsumed volumeAccumulator
trainingRuns := map[string]struct{}{}
taskCollected := map[string]*volumeAccumulator{}
summary := domain.DataOperationsSummary{
From: query.From,
To: query.To,
Timezone: location.String(),
UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano),
Freshness: "live",
RejectionReasons: reasons,
}
for _, fact := range facts {
accumulator, ok := accumulators[fact.Stage]
if !ok {
summary.UnknownCount++
continue
}
accumulator.input.add(fact.Input)
accumulator.output.add(fact.Output)
accumulator.rejected.add(fact.Rejected)
accumulator.technicalValid.add(fact.TechnicalValid)
accumulator.trainReady.add(fact.TrainReady)
accumulator.runs[fact.RunID] = struct{}{}
summary.FactCount++
switch fact.Quality {
case domain.DataMetricQualityExact:
summary.ExactCount++
case domain.DataMetricQualityEstimated:
summary.EstimatedCount++
case domain.DataMetricQualityDerived:
summary.DerivedCount++
default:
summary.UnknownCount++
}
for reason, count := range fact.RejectionReasons {
reasons[reason] += count
}
if occurredAt, err := time.Parse(time.RFC3339Nano, fact.OccurredAt); err == nil {
hour := occurredAt.In(location).Format("2006-01-02T15:00:00-07:00")
if hourly[hour] == nil {
hourly[hour] = map[domain.DataMetricStage]*volumeAccumulator{}
}
if hourly[hour][fact.Stage] == nil {
hourly[hour][fact.Stage] = &volumeAccumulator{}
}
hourly[hour][fact.Stage].add(fact.Output)
}
if fact.Stage == domain.DataMetricStageCollect && fact.TaskID != "" {
if taskCollected[fact.TaskID] == nil {
taskCollected[fact.TaskID] = &volumeAccumulator{}
}
taskCollected[fact.TaskID].add(fact.Output)
}
if fact.Stage == domain.DataMetricStageTrain {
trainingConsumed.add(fact.Input)
trainingRuns[fact.RunID] = struct{}{}
for _, input := range fact.TrainingInputs {
key := strings.TrimSpace(input.DatasetVersionID)
if key == "" {
key = strings.TrimSpace(input.DatasetID) + "|" + strings.TrimSpace(input.ManifestPath) + "|" + strings.TrimSpace(input.Split)
}
if key == "||" || key == "" {
key = fmt.Sprintf("%s|%s", fact.RunID, fact.NodeID)
}
if _, exists := uniqueTraining[key]; !exists {
uniqueTraining[key] = input.Volume
}
}
}
}
summary.Stages = make([]domain.DataStageSummary, 0, len(domain.DataMetricStages))
for _, stage := range domain.DataMetricStages {
item := accumulators[stage]
summary.Stages = append(summary.Stages, domain.DataStageSummary{
Stage: stage,
Input: item.input.value(),
Output: item.output.value(),
Rejected: item.rejected.value(),
TechnicalValid: item.technicalValid.value(),
TrainReady: item.trainReady.value(),
RunCount: int64(len(item.runs)),
})
}
var uniqueAccumulator volumeAccumulator
for _, volume := range uniqueTraining {
uniqueAccumulator.add(volume)
}
summary.Training = domain.DataOperationsTrainingSummary{
Consumed: trainingConsumed.value(),
Unique: uniqueAccumulator.value(),
RunCount: int64(len(trainingRuns)),
}
hours := make([]string, 0, len(hourly))
for hour := range hourly {
hours = append(hours, hour)
}
sort.Strings(hours)
summary.Hourly = make([]domain.DataOperationsHourlyPoint, 0, len(hours))
for _, hour := range hours {
point := domain.DataOperationsHourlyPoint{Hour: hour, Stages: map[domain.DataMetricStage]domain.DataVolume{}}
for stage, value := range hourly[hour] {
point.Stages[stage] = value.value()
}
summary.Hourly = append(summary.Hourly, point)
}
summary.Tasks = make([]domain.CollectionTaskProgress, 0)
for _, task := range tasks {
if query.ProjectID != "" && task.ProjectID != query.ProjectID {
continue
}
if query.TaskID != "" && task.ID != query.TaskID {
continue
}
if query.RobotModel != "" && !strings.EqualFold(task.RobotModel, query.RobotModel) {
continue
}
if query.Scene != "" && !strings.EqualFold(task.Scene, query.Scene) {
continue
}
collected := domain.DataVolume{}
if taskCollected[task.ID] != nil {
collected = taskCollected[task.ID].value()
}
summary.Tasks = append(summary.Tasks, domain.CollectionTaskProgress{
TaskID: task.ID,
TaskName: task.Name,
Status: task.Status,
Targets: task.Targets,
Collected: collected,
Percent: taskProgressPercent(task.Targets, collected),
})
}
sort.Slice(summary.Tasks, func(left, right int) bool { return summary.Tasks[left].TaskName < summary.Tasks[right].TaskName })
return summary
}

View File

@ -0,0 +1,240 @@
package store
import (
"context"
"fmt"
"sort"
"strings"
"droboticflow/internal/domain"
)
func (store *MemoryStore) ListCollectionTasks(_ context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
items := make([]domain.CollectionTaskRecord, 0)
for _, item := range store.collectionTasks {
if organizationMatches(item.OrganizationID, organizationID) {
items = append(items, clone(item))
}
}
sort.Slice(items, func(left, right int) bool { return items[left].UpdatedAt > items[right].UpdatedAt })
return items, nil
}
func (store *MemoryStore) GetCollectionTask(_ context.Context, id string) (*domain.CollectionTaskRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
item, ok := store.collectionTasks[id]
if !ok {
return nil, nil
}
cloned := clone(item)
return &cloned, nil
}
func (store *MemoryStore) UpsertCollectionTask(_ context.Context, item domain.CollectionTaskRecord) error {
store.mu.Lock()
defer store.mu.Unlock()
if strings.TrimSpace(item.ID) == "" {
item.ID = domain.NewID("data-task")
}
now := nowISO()
if item.CreatedAt == "" {
item.CreatedAt = now
}
item.UpdatedAt = now
store.collectionTasks[item.ID] = clone(item)
return nil
}
func (store *MemoryStore) UpsertDatasetVersion(_ context.Context, item domain.DatasetVersionRecord) error {
store.mu.Lock()
defer store.mu.Unlock()
if strings.TrimSpace(item.Fingerprint) == "" {
return fmt.Errorf("dataset fingerprint is required")
}
for id, existing := range store.datasetVersions {
if organizationMatches(existing.OrganizationID, item.OrganizationID) && existing.Fingerprint == item.Fingerprint {
if dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
return nil
}
item.ID = id
item.CreatedAt = existing.CreatedAt
break
}
}
if item.ID == "" {
item.ID = domain.NewID("dataset-version")
}
now := nowISO()
if item.CreatedAt == "" {
item.CreatedAt = now
}
item.UpdatedAt = now
store.datasetVersions[item.ID] = clone(item)
return nil
}
func (store *MemoryStore) ListDatasetVersions(_ context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
items := make([]domain.DatasetVersionRecord, 0)
for _, item := range store.datasetVersions {
if datasetVersionMatches(item, organizationID, query) {
items = append(items, clone(item))
}
}
sort.Slice(items, func(left, right int) bool { return items[left].CreatedAt > items[right].CreatedAt })
limit := query.Limit
if limit <= 0 || limit > 100 {
limit = 100
}
if len(items) > limit {
items = items[:limit]
}
return items, nil
}
func (store *MemoryStore) UpsertDataMetricFact(_ context.Context, item domain.DataMetricFactRecord) error {
store.mu.Lock()
defer store.mu.Unlock()
if strings.TrimSpace(item.SourceKey) == "" {
return fmt.Errorf("metric source key is required")
}
for id, existing := range store.dataMetricFacts {
if existing.SourceKey == item.SourceKey {
if dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
return nil
}
item.ID = id
break
}
}
if item.ID == "" {
item.ID = domain.NewID("data-fact")
}
if item.RecordedAt == "" {
item.RecordedAt = nowISO()
}
store.dataMetricFacts[item.ID] = clone(item)
return nil
}
func (store *MemoryStore) filteredDataMetricFactsLocked(organizationID string, query domain.DataOperationsQuery) []domain.DataMetricFactRecord {
items := make([]domain.DataMetricFactRecord, 0)
for _, item := range store.dataMetricFacts {
if dataOperationsFactMatches(item, organizationID, query) {
items = append(items, clone(item))
}
}
return items
}
func (store *MemoryStore) ListDataMetricFactsPage(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) {
store.mu.Lock()
defer store.mu.Unlock()
return factsPage(store.filteredDataMetricFactsLocked(organizationID, query), query), nil
}
func (store *MemoryStore) GetDataOperationsSummary(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) {
store.mu.Lock()
defer store.mu.Unlock()
facts := store.filteredDataMetricFactsLocked(organizationID, query)
tasks := make([]domain.CollectionTaskRecord, 0)
for _, item := range store.collectionTasks {
if organizationMatches(item.OrganizationID, organizationID) {
tasks = append(tasks, clone(item))
}
}
return BuildDataOperationsSummary(facts, tasks, query), nil
}
func (store *MemoryStore) ListTrainingUsagePage(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) {
store.mu.Lock()
defer store.mu.Unlock()
return trainingUsageFromFacts(store.filteredDataMetricFactsLocked(organizationID, query), query), nil
}
func (store *MemoryStore) ListDataOperationsBackfillRuns(
_ context.Context,
organizationID string,
from string,
cursor *domain.RunSummaryCursor,
limit int,
) (domain.DataOperationsRunPage, error) {
store.mu.Lock()
defer store.mu.Unlock()
if limit <= 0 {
limit = 100
}
if limit > 500 {
limit = 500
}
items := make([]domain.RunRecord, 0)
for _, run := range store.runs {
if !organizationMatches(run.OrganizationID, organizationID) || run.CreatedAt < from {
continue
}
if cursor != nil && !(run.CreatedAt < cursor.CreatedAt || (run.CreatedAt == cursor.CreatedAt && run.ID < cursor.ID)) {
continue
}
items = append(items, clone(run))
}
sort.Slice(items, func(left, right int) bool {
if items[left].CreatedAt == items[right].CreatedAt {
return items[left].ID > items[right].ID
}
return items[left].CreatedAt > items[right].CreatedAt
})
page := domain.DataOperationsRunPage{HasMore: len(items) > limit}
if page.HasMore {
items = items[:limit]
}
page.Items = items
if page.HasMore && len(items) > 0 {
last := items[len(items)-1]
page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
return page, nil
}
func (store *MemoryStore) ListRunsNeedingMetricMaterialization(_ context.Context, limit int) ([]domain.RunRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
items := make([]domain.RunRecord, 0)
for _, run := range store.runs {
for _, execution := range run.NodeExecutions {
if execution.DataMetrics != nil && execution.MetricsMaterializedAt == "" {
items = append(items, clone(run))
break
}
}
}
sortRuns(items)
if limit <= 0 {
limit = 20
}
if len(items) > limit {
items = items[:limit]
}
return items, nil
}
func (store *MemoryStore) MarkNodeMetricsMaterialized(_ context.Context, runID string, nodeID string, attempt int, at string) error {
store.mu.Lock()
defer store.mu.Unlock()
run, ok := store.runs[runID]
if !ok {
return fmt.Errorf("run %s not found", runID)
}
for index := range run.NodeExecutions {
execution := &run.NodeExecutions[index]
if execution.NodeID == nodeID && execution.Attempt == attempt {
execution.MetricsMaterializedAt = at
store.runs[runID] = run
return nil
}
}
return fmt.Errorf("node execution %s attempt %d not found", nodeID, attempt)
}

View File

@ -0,0 +1,300 @@
package store
import (
"context"
"fmt"
"strings"
"droboticflow/internal/domain"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
func dataOperationsMongoFilter(organizationID string, query domain.DataOperationsQuery) bson.M {
extra := bson.M{}
if query.ProjectID != "" {
extra["projectId"] = query.ProjectID
}
if query.TaskID != "" {
extra["taskId"] = query.TaskID
}
if query.RobotModel != "" {
extra["robotModel"] = query.RobotModel
}
if query.Scene != "" {
extra["scene"] = query.Scene
}
if query.Stage != "" {
extra["stage"] = query.Stage
}
timeRange := bson.M{}
if query.From != "" {
timeRange["$gte"] = query.From
}
if query.To != "" {
timeRange["$lt"] = query.To
}
if len(timeRange) > 0 {
extra["occurredAt"] = timeRange
}
return organizationScopedFilter(organizationID, extra)
}
func (store *MongoStore) ListCollectionTasks(ctx context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) {
cursor, err := store.collectionTasks.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}))
if err != nil {
return nil, err
}
return decodeMany[domain.CollectionTaskRecord](ctx, cursor)
}
func (store *MongoStore) GetCollectionTask(ctx context.Context, id string) (*domain.CollectionTaskRecord, error) {
var item domain.CollectionTaskRecord
err := store.collectionTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item)
if err == mongo.ErrNoDocuments {
return nil, nil
}
if err != nil {
return nil, err
}
return &item, nil
}
func (store *MongoStore) UpsertCollectionTask(ctx context.Context, item domain.CollectionTaskRecord) error {
if item.ID == "" {
item.ID = domain.NewID("data-task")
}
now := nowISO()
if item.CreatedAt == "" {
item.CreatedAt = now
}
item.UpdatedAt = now
_, err := store.collectionTasks.ReplaceOne(ctx, bson.M{"id": item.ID}, item, options.Replace().SetUpsert(true))
return err
}
func (store *MongoStore) UpsertDatasetVersion(ctx context.Context, item domain.DatasetVersionRecord) error {
if strings.TrimSpace(item.Fingerprint) == "" {
return fmt.Errorf("dataset fingerprint is required")
}
var existing domain.DatasetVersionRecord
existingErr := store.datasetVersions.FindOne(
ctx,
organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint}),
options.FindOne().SetProjection(bson.M{"quality": 1}),
).Decode(&existing)
if existingErr != nil && existingErr != mongo.ErrNoDocuments {
return existingErr
}
if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
return nil
}
now := nowISO()
if item.ID == "" {
item.ID = domain.NewID("dataset-version")
}
if item.CreatedAt == "" {
item.CreatedAt = now
}
item.UpdatedAt = now
filter := organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint})
update := bson.M{
"$set": bson.M{
"datasetId": item.DatasetID, "taskId": item.TaskID, "projectId": item.ProjectID,
"name": item.Name, "split": item.Split, "fingerprint": item.Fingerprint, "volume": item.Volume,
"storage": item.Storage, "manifestPath": item.ManifestPath, "partitionDir": item.PartitionDir,
"producerRunId": item.ProducerRunID, "producerNodeId": item.ProducerNodeID, "artifactId": item.ArtifactID,
"quality": item.Quality, "updatedAt": now, "organizationId": normalizedOrganizationID(item.OrganizationID),
},
"$setOnInsert": bson.M{"id": item.ID, "createdAt": item.CreatedAt},
}
_, err := store.datasetVersions.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true))
return err
}
func (store *MongoStore) ListDatasetVersions(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) {
extra := bson.M{}
if query.ProjectID != "" {
extra["projectId"] = query.ProjectID
}
if query.TaskID != "" {
extra["taskId"] = query.TaskID
}
timeRange := bson.M{}
if query.From != "" {
timeRange["$gte"] = query.From
}
if query.To != "" {
timeRange["$lt"] = query.To
}
if len(timeRange) > 0 {
extra["createdAt"] = timeRange
}
limit := int64(query.Limit)
if limit <= 0 || limit > 100 {
limit = 100
}
cursor, err := store.datasetVersions.Find(
ctx,
organizationScopedFilter(organizationID, extra),
options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(limit),
)
if err != nil {
return nil, err
}
return decodeMany[domain.DatasetVersionRecord](ctx, cursor)
}
func (store *MongoStore) UpsertDataMetricFact(ctx context.Context, item domain.DataMetricFactRecord) error {
if strings.TrimSpace(item.SourceKey) == "" {
return fmt.Errorf("metric source key is required")
}
if item.ID == "" {
item.ID = domain.NewID("data-fact")
}
if item.RecordedAt == "" {
item.RecordedAt = nowISO()
}
var existing domain.DataMetricFactRecord
existingErr := store.dataMetricFacts.FindOne(
ctx,
bson.M{"sourceKey": item.SourceKey},
options.FindOne().SetProjection(bson.M{"quality": 1}),
).Decode(&existing)
if existingErr != nil && existingErr != mongo.ErrNoDocuments {
return existingErr
}
if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
return nil
}
_, err := store.dataMetricFacts.ReplaceOne(ctx, bson.M{"sourceKey": item.SourceKey}, item, options.Replace().SetUpsert(true))
return err
}
func (store *MongoStore) findDataMetricFacts(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DataMetricFactRecord, error) {
cursor, err := store.dataMetricFacts.Find(
ctx,
dataOperationsMongoFilter(organizationID, query),
options.Find().SetSort(bson.D{{Key: "occurredAt", Value: -1}, {Key: "id", Value: -1}}),
)
if err != nil {
return nil, err
}
return decodeMany[domain.DataMetricFactRecord](ctx, cursor)
}
func (store *MongoStore) ListDataMetricFactsPage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) {
items, err := store.findDataMetricFacts(ctx, organizationID, query)
if err != nil {
return domain.DataMetricFactPage{}, err
}
return factsPage(items, query), nil
}
func (store *MongoStore) GetDataOperationsSummary(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) {
facts, err := store.findDataMetricFacts(ctx, organizationID, query)
if err != nil {
return domain.DataOperationsSummary{}, err
}
tasks, err := store.ListCollectionTasks(ctx, organizationID)
if err != nil {
return domain.DataOperationsSummary{}, err
}
return BuildDataOperationsSummary(facts, tasks, query), nil
}
func (store *MongoStore) ListTrainingUsagePage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) {
query.Stage = domain.DataMetricStageTrain
items, err := store.findDataMetricFacts(ctx, organizationID, query)
if err != nil {
return domain.TrainingUsagePage{}, err
}
return trainingUsageFromFacts(items, query), nil
}
func (store *MongoStore) ListDataOperationsBackfillRuns(
ctx context.Context,
organizationID string,
from string,
pageCursor *domain.RunSummaryCursor,
limit int,
) (domain.DataOperationsRunPage, error) {
if limit <= 0 {
limit = 100
}
if limit > 500 {
limit = 500
}
filter := organizationScopedFilter(organizationID, bson.M{"createdAt": bson.M{"$gte": from}})
filter = combineRunSummaryFilters(filter, runSummaryCursorFilter(pageCursor))
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(bson.M{
"id": 1, "organizationId": 1, "workflowId": 1, "workflowName": 1,
"projectId": 1, "projectName": 1, "taskId": 1, "status": 1,
"createdAt": 1, "updatedAt": 1, "endedAt": 1, "nodeExecutions": 1,
}).
SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}).
SetLimit(int64(limit+1)),
)
if err != nil {
return domain.DataOperationsRunPage{}, err
}
items, err := decodeMany[domain.RunRecord](ctx, cursor)
if err != nil {
return domain.DataOperationsRunPage{}, err
}
page := domain.DataOperationsRunPage{HasMore: len(items) > limit}
if page.HasMore {
items = items[:limit]
}
page.Items = items
if page.HasMore && len(items) > 0 {
last := items[len(items)-1]
page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
return page, nil
}
func (store *MongoStore) ListRunsNeedingMetricMaterialization(ctx context.Context, limit int) ([]domain.RunRecord, error) {
if limit <= 0 {
limit = 20
}
cursor, err := store.runs.Find(
ctx,
bson.M{"nodeExecutions": bson.M{"$elemMatch": bson.M{
"dataMetrics": bson.M{"$exists": true},
"$or": bson.A{
bson.M{"metricsMaterializedAt": bson.M{"$exists": false}},
bson.M{"metricsMaterializedAt": ""},
},
}}},
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)),
)
if err != nil {
return nil, err
}
return decodeMany[domain.RunRecord](ctx, cursor)
}
func (store *MongoStore) MarkNodeMetricsMaterialized(ctx context.Context, runID string, nodeID string, attempt int, at string) error {
result, err := store.runs.UpdateOne(
ctx,
bson.M{"id": runID},
bson.M{"$set": bson.M{"nodeExecutions.$[execution].metricsMaterializedAt": at}},
options.Update().SetArrayFilters(options.ArrayFilters{Filters: []any{
bson.M{"execution.nodeId": nodeID, "execution.attempt": attempt},
}}),
)
if err != nil {
return err
}
if result.MatchedCount == 0 {
return fmt.Errorf("run %s not found", runID)
}
return nil
}

View File

@ -0,0 +1,192 @@
package store
import (
"context"
"testing"
"droboticflow/internal/domain"
)
func dataOperationsInt64(value int64) *int64 {
return &value
}
func dataOperationsFloat64(value float64) *float64 {
return &value
}
func TestMemoryDataOperationsSummarySeparatesConsumptionAndUniqueData(t *testing.T) {
ctx := context.Background()
dataStore := NewMemoryStore()
task := domain.CollectionTaskRecord{
ID: "task-y1",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "project-y1",
Name: "Y1 桌面整理",
RobotModel: "Y1",
Scene: "桌面整理",
Status: domain.CollectionTaskStatusActive,
Targets: domain.DataVolume{TrajectoryCount: dataOperationsInt64(100)},
}
if err := dataStore.UpsertCollectionTask(ctx, task); err != nil {
t.Fatalf("upsert task: %v", err)
}
for _, item := range []domain.DataMetricFactRecord{
{
ID: "fact-collect",
SourceKey: "run-collect/node/1/collect",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "project-y1",
TaskID: task.ID,
TaskName: task.Name,
RobotModel: "Y1",
Scene: "桌面整理",
RunID: "run-collect",
NodeID: "collect",
Attempt: 1,
Stage: domain.DataMetricStageCollect,
Output: domain.DataVolume{
TrajectoryCount: dataOperationsInt64(40),
DurationSeconds: dataOperationsFloat64(1200),
},
Quality: domain.DataMetricQualityExact,
OccurredAt: "2026-07-23T01:00:00Z",
},
{
ID: "fact-train-1",
SourceKey: "run-train-1/node/1/train",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "project-y1",
TaskID: task.ID,
RunID: "run-train-1",
NodeID: "train",
Attempt: 1,
Stage: domain.DataMetricStageTrain,
Input: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)},
TrainingInputs: []domain.TrainingInputRecord{{
DatasetVersionID: "dataset-version-y1",
Split: "train",
Volume: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)},
}},
Quality: domain.DataMetricQualityExact,
OccurredAt: "2026-07-23T02:00:00Z",
},
{
ID: "fact-train-2",
SourceKey: "run-train-2/node/1/train",
OrganizationID: domain.DefaultOrganizationID,
ProjectID: "project-y1",
TaskID: task.ID,
RunID: "run-train-2",
NodeID: "train",
Attempt: 1,
Stage: domain.DataMetricStageTrain,
Input: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)},
TrainingInputs: []domain.TrainingInputRecord{{
DatasetVersionID: "dataset-version-y1",
Split: "train",
Volume: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)},
}},
Quality: domain.DataMetricQualityDerived,
OccurredAt: "2026-07-23T03:00:00Z",
},
} {
if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil {
t.Fatalf("upsert fact %s: %v", item.ID, err)
}
}
summary, err := dataStore.GetDataOperationsSummary(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{
From: "2026-07-22T16:00:00Z",
To: "2026-07-23T16:00:00Z",
Timezone: "Asia/Shanghai",
})
if err != nil {
t.Fatalf("get summary: %v", err)
}
if summary.Training.Consumed.TrajectoryCount == nil || *summary.Training.Consumed.TrajectoryCount != 80 {
t.Fatalf("consumed trajectories = %#v, want 80", summary.Training.Consumed.TrajectoryCount)
}
if summary.Training.Unique.TrajectoryCount == nil || *summary.Training.Unique.TrajectoryCount != 40 {
t.Fatalf("unique trajectories = %#v, want 40", summary.Training.Unique.TrajectoryCount)
}
if summary.Training.RunCount != 2 {
t.Fatalf("training run count = %d, want 2", summary.Training.RunCount)
}
if len(summary.Tasks) != 1 || summary.Tasks[0].Percent != 40 {
t.Fatalf("task progress = %#v, want 40 percent", summary.Tasks)
}
if summary.ExactCount != 2 || summary.DerivedCount != 1 {
t.Fatalf("quality counts = exact %d derived %d", summary.ExactCount, summary.DerivedCount)
}
}
func TestMemoryDataMetricUpsertIsIdempotent(t *testing.T) {
ctx := context.Background()
dataStore := NewMemoryStore()
item := domain.DataMetricFactRecord{
ID: "fact-one",
SourceKey: "run/node/1/collect",
OrganizationID: domain.DefaultOrganizationID,
RunID: "run",
NodeID: "node",
Attempt: 1,
Stage: domain.DataMetricStageCollect,
Output: domain.DataVolume{TrajectoryCount: dataOperationsInt64(10)},
Quality: domain.DataMetricQualityDerived,
OccurredAt: "2026-07-23T01:00:00Z",
}
if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil {
t.Fatalf("first upsert: %v", err)
}
item.ID = "fact-two"
item.Output = domain.DataVolume{TrajectoryCount: dataOperationsInt64(12)}
if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil {
t.Fatalf("second upsert: %v", err)
}
page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{
From: "2026-07-23T00:00:00Z",
To: "2026-07-24T00:00:00Z",
})
if err != nil {
t.Fatalf("list facts: %v", err)
}
if len(page.Items) != 1 || page.Items[0].Output.TrajectoryCount == nil || *page.Items[0].Output.TrajectoryCount != 12 {
t.Fatalf("idempotent items = %#v", page.Items)
}
}
func TestMemoryExactDataMetricIsNotDowngradedByBackfill(t *testing.T) {
ctx := context.Background()
dataStore := NewMemoryStore()
exact := domain.DataMetricFactRecord{
ID: "fact-exact",
SourceKey: "run/node/1/collect",
OrganizationID: domain.DefaultOrganizationID,
RunID: "run",
NodeID: "node",
Attempt: 1,
Stage: domain.DataMetricStageCollect,
Output: domain.DataVolume{TrajectoryCount: dataOperationsInt64(10)},
Quality: domain.DataMetricQualityExact,
OccurredAt: "2026-07-23T01:00:00Z",
}
if err := dataStore.UpsertDataMetricFact(ctx, exact); err != nil {
t.Fatalf("upsert exact fact: %v", err)
}
estimated := exact
estimated.ID = "fact-estimated"
estimated.Output = domain.DataVolume{TrajectoryCount: dataOperationsInt64(3)}
estimated.Quality = domain.DataMetricQualityEstimated
if err := dataStore.UpsertDataMetricFact(ctx, estimated); err != nil {
t.Fatalf("upsert estimated fact: %v", err)
}
page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{})
if err != nil {
t.Fatalf("list facts: %v", err)
}
if len(page.Items) != 1 || page.Items[0].Quality != domain.DataMetricQualityExact ||
page.Items[0].Output.TrajectoryCount == nil || *page.Items[0].Output.TrajectoryCount != 10 {
t.Fatalf("facts = %#v", page.Items)
}
}

View File

@ -21,6 +21,9 @@ type MemoryStore struct {
templates map[string]domain.TemplateRecord
runs map[string]domain.RunRecord
sourceStates map[string]domain.SourceStateRecord
collectionTasks map[string]domain.CollectionTaskRecord
datasetVersions map[string]domain.DatasetVersionRecord
dataMetricFacts map[string]domain.DataMetricFactRecord
auditTasks map[string]domain.AuditTaskRecord
samplingTasks map[string]domain.SamplingTaskRecord
users map[string]domain.UserRecord
@ -42,6 +45,9 @@ func NewMemoryStore() *MemoryStore {
templates: map[string]domain.TemplateRecord{},
runs: map[string]domain.RunRecord{},
sourceStates: map[string]domain.SourceStateRecord{},
collectionTasks: map[string]domain.CollectionTaskRecord{},
datasetVersions: map[string]domain.DatasetVersionRecord{},
dataMetricFacts: map[string]domain.DataMetricFactRecord{},
auditTasks: map[string]domain.AuditTaskRecord{},
samplingTasks: map[string]domain.SamplingTaskRecord{},
users: map[string]domain.UserRecord{},
@ -276,6 +282,21 @@ func (store *MemoryStore) DeleteOrganizationCascade(_ context.Context, id string
delete(store.sourceStates, key)
}
}
for key, item := range store.collectionTasks {
if organizationMatches(item.OrganizationID, organizationID) {
delete(store.collectionTasks, key)
}
}
for key, item := range store.datasetVersions {
if organizationMatches(item.OrganizationID, organizationID) {
delete(store.datasetVersions, key)
}
}
for key, item := range store.dataMetricFacts {
if organizationMatches(item.OrganizationID, organizationID) {
delete(store.dataMetricFacts, key)
}
}
for key, item := range store.auditTasks {
if organizationMatches(item.OrganizationID, organizationID) {
delete(store.auditTasks, key)
@ -486,6 +507,7 @@ func (store *MemoryStore) CreateOperator(_ context.Context, input domain.CreateO
SkillDefinition: input.SkillDefinition,
Hyperparams: clone(input.Hyperparams),
Resources: clone(input.Resources),
BusinessStage: input.BusinessStage,
PublicationStatus: input.PublicationStatus,
GenerationSource: input.GenerationSource,
PublicationEvidence: clone(input.PublicationEvidence),
@ -842,6 +864,7 @@ func summarizeRun(run domain.RunRecord) domain.RunSummary {
WorkflowName: run.WorkflowName,
ProjectID: run.ProjectID,
ProjectName: run.ProjectName,
TaskID: run.TaskID,
Trigger: run.Trigger,
Attempt: run.Attempt,
ParentRunID: run.ParentRunID,
@ -1140,7 +1163,7 @@ func (store *MemoryStore) GetRun(_ context.Context, id string) (*domain.RunRecor
return &cloned, nil
}
func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigger domain.RunTrigger) (*domain.RunRecord, error) {
func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) {
store.mu.Lock()
defer store.mu.Unlock()
workflow, ok := store.workflows[workflowID]
@ -1187,6 +1210,9 @@ func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigge
CreatedAt: now,
UpdatedAt: now,
}
if len(taskIDs) > 0 {
run.TaskID = strings.TrimSpace(taskIDs[0])
}
store.runs[run.ID] = clone(run)
workflow.LastRunID = run.ID

View File

@ -28,6 +28,9 @@ type MongoStore struct {
templates *mongo.Collection
runs *mongo.Collection
sourceStates *mongo.Collection
collectionTasks *mongo.Collection
datasetVersions *mongo.Collection
dataMetricFacts *mongo.Collection
auditTasks *mongo.Collection
samplingTasks *mongo.Collection
users *mongo.Collection
@ -49,6 +52,9 @@ func NewMongoStore(db *mongo.Database) *MongoStore {
templates: db.Collection("templates"),
runs: db.Collection("runs"),
sourceStates: db.Collection("sourceStates"),
collectionTasks: db.Collection("collectionTasks"),
datasetVersions: db.Collection("datasetVersions"),
dataMetricFacts: db.Collection("dataMetricFacts"),
auditTasks: db.Collection("auditTasks"),
samplingTasks: db.Collection("samplingTasks"),
users: db.Collection("users"),
@ -350,6 +356,7 @@ func (store *MongoStore) CreateOperator(ctx context.Context, input domain.Create
SkillDefinition: input.SkillDefinition,
Hyperparams: input.Hyperparams,
Resources: input.Resources,
BusinessStage: input.BusinessStage,
PublicationStatus: input.PublicationStatus,
GenerationSource: input.GenerationSource,
PublicationEvidence: input.PublicationEvidence,
@ -725,6 +732,7 @@ func runSummaryProjection() bson.M {
"workflowName": 1,
"projectId": 1,
"projectName": 1,
"taskId": 1,
"trigger": 1,
"attempt": 1,
"parentRunId": 1,
@ -1104,7 +1112,7 @@ func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunReco
return &item, nil
}
func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger) (*domain.RunRecord, error) {
func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) {
workflow, err := store.GetWorkflow(ctx, workflowID)
if err != nil {
return nil, err
@ -1152,6 +1160,9 @@ func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigg
CreatedAt: now,
UpdatedAt: now,
}
if len(taskIDs) > 0 {
run.TaskID = strings.TrimSpace(taskIDs[0])
}
if err := store.UpsertRun(ctx, run); err != nil {
return nil, err
}

View File

@ -88,6 +88,23 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
index("organizationId_workflow_node", bson.D{{Key: "organizationId", Value: 1}, {Key: "workflowId", Value: 1}, {Key: "nodeId", Value: 1}}),
index("workflowId_nodeId", bson.D{{Key: "workflowId", Value: 1}, {Key: "nodeId", Value: 1}}),
},
store.collectionTasks: {
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
index("organizationId_status_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}),
index("organizationId_projectId_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "updatedAt", Value: -1}}),
},
store.datasetVersions: {
uniqueIndex("organizationId_fingerprint_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "fingerprint", Value: 1}}),
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
index("organizationId_projectId_taskId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "createdAt", Value: -1}}),
},
store.dataMetricFacts: {
uniqueIndex("sourceKey_unique", bson.D{{Key: "sourceKey", Value: 1}}),
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
index("organizationId_projectId_taskId_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "occurredAt", Value: -1}}),
index("organizationId_stage_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "stage", Value: 1}, {Key: "occurredAt", Value: -1}}),
index("organizationId_robotModel_scene_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "robotModel", Value: 1}, {Key: "scene", Value: 1}, {Key: "occurredAt", Value: -1}}),
},
store.auditTasks: {
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}),

View File

@ -160,6 +160,9 @@ func (store *MongoStore) normalizeLegacyDefaultOrganizationReferences(ctx contex
store.templates,
store.runs,
store.sourceStates,
store.collectionTasks,
store.datasetVersions,
store.dataMetricFacts,
store.auditTasks,
store.samplingTasks,
store.users,
@ -228,7 +231,9 @@ func (store *MongoStore) dropLegacyBuiltinOperatorDeletionIDIndex(ctx context.Co
return nil
}
var commandError mongo.CommandError
if errors.As(err, &commandError) && (commandError.Name == "IndexNotFound" || commandError.HasErrorCode(27)) {
if errors.As(err, &commandError) &&
(commandError.Name == "IndexNotFound" || commandError.Name == "NamespaceNotFound" ||
commandError.HasErrorCode(27) || commandError.HasErrorCode(26)) {
return nil
}
return err

View File

@ -80,7 +80,7 @@ type Store interface {
CountWorkerRuns(context.Context) (WorkerRunCounts, error)
HasActiveRun(context.Context, string) (bool, error)
GetRun(context.Context, string) (*domain.RunRecord, error)
CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error)
CreateRun(context.Context, string, domain.RunTrigger, ...string) (*domain.RunRecord, error)
ClaimNextQueuedRun(context.Context, string, time.Duration) (*domain.RunRecord, error)
ClaimQueuedRun(context.Context, string, string, time.Duration) (*domain.RunRecord, error)
AcquireRunLease(context.Context, string, string, time.Duration) (*domain.RunRecord, bool, error)
@ -101,6 +101,19 @@ type Store interface {
DeleteSourceState(context.Context, string, string, string) error
DeleteSourceStatesByWorkflow(context.Context, string) error
ListCollectionTasks(context.Context, string) ([]domain.CollectionTaskRecord, error)
GetCollectionTask(context.Context, string) (*domain.CollectionTaskRecord, error)
UpsertCollectionTask(context.Context, domain.CollectionTaskRecord) error
UpsertDatasetVersion(context.Context, domain.DatasetVersionRecord) error
ListDatasetVersions(context.Context, string, domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error)
UpsertDataMetricFact(context.Context, domain.DataMetricFactRecord) error
ListDataMetricFactsPage(context.Context, string, domain.DataOperationsQuery) (domain.DataMetricFactPage, error)
GetDataOperationsSummary(context.Context, string, domain.DataOperationsQuery) (domain.DataOperationsSummary, error)
ListTrainingUsagePage(context.Context, string, domain.DataOperationsQuery) (domain.TrainingUsagePage, error)
ListDataOperationsBackfillRuns(context.Context, string, string, *domain.RunSummaryCursor, int) (domain.DataOperationsRunPage, error)
ListRunsNeedingMetricMaterialization(context.Context, int) ([]domain.RunRecord, error)
MarkNodeMetricsMaterialized(context.Context, string, string, int, string) error
ListAuditTasks(context.Context, string) ([]domain.AuditTaskRecord, error)
ListAuditTasksByOrganization(context.Context, string, string) ([]domain.AuditTaskRecord, error)
GetAuditTask(context.Context, string) (*domain.AuditTaskRecord, error)

View File

@ -33,12 +33,14 @@ type executionResult struct {
Logs []domain.LogEntryRecord `json:"logs"`
Artifacts []domain.ExecutionArtifact `json:"artifacts"`
DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"`
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
StageManifestPath string `json:"stageManifestPath,omitempty"`
ResourceSeries []domain.ResourceSample `json:"resourceSeries"`
}
type stageManifestRecord struct {
DataRefs []domain.DataRefRecord `json:"dataRefs"`
DataRefs []domain.DataRefRecord `json:"dataRefs"`
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
}
var protectedSystemEnv = map[string]struct{}{
@ -593,7 +595,49 @@ func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) {
return manifest.DataRefs, nil
}
func loadStageManifestDataMetrics(path string) (*domain.DataMetricsRecord, error) {
trimmed := strings.TrimSpace(path)
if trimmed == "" {
return nil, nil
}
payload, err := os.ReadFile(trimmed)
if err != nil {
return nil, err
}
var manifest stageManifestRecord
if err := json.Unmarshal(payload, &manifest); err != nil {
return nil, err
}
return manifest.DataMetrics, nil
}
func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) domain.WorkflowNode {
if run != nil && strings.TrimSpace(run.TaskID) != "" {
if task, err := runtime.store.GetCollectionTask(ctx, run.TaskID); err == nil && task != nil {
for stage, nodeID := range task.StageBindings {
if strings.TrimSpace(nodeID) == node.ID {
node.Data.BusinessStage = stage
node.Data.MetricBoundary = true
return node
}
}
}
}
if node.Data.BusinessStage != "" {
node.Data.MetricBoundary = true
return node
}
if strings.TrimSpace(node.Data.OperatorID) != "" {
if operator, err := runtime.store.GetOperator(ctx, node.Data.OperatorID); err == nil && operator != nil && operator.BusinessStage != "" {
node.Data.BusinessStage = operator.BusinessStage
node.Data.MetricBoundary = true
}
}
return node
}
func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) {
node = runtime.resolveNodeBusinessStage(ctx, run, node)
workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID)
if err != nil {
return executionResult{}, nil, err
@ -675,6 +719,9 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord,
if err == nil {
parsed.StageManifestPath = workspace.StageManifestPath
parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath)
if err == nil && parsed.DataMetrics == nil {
parsed.DataMetrics, err = loadStageManifestDataMetrics(workspace.StageManifestPath)
}
}
// Custom-container operators bypass operator-runner's ResourceSampler, so they
// emit no resourceSeries in result.json. The launcher samples their GPU usage

View File

@ -15,6 +15,7 @@ import (
"sync/atomic"
"time"
"droboticflow/internal/dataops"
"droboticflow/internal/domain"
"droboticflow/internal/k8s"
"droboticflow/internal/lightweight"
@ -564,6 +565,12 @@ func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord)
} else {
run.NodeExecutions = append(run.NodeExecutions, execution)
}
if execution.DataMetrics != nil && execution.DataMetrics.Stage == domain.DataMetricStageTrain && len(execution.DataMetrics.TrainingInputs) > 0 {
if run.Metadata == nil {
run.Metadata = &domain.RunMetadata{}
}
run.Metadata.TrainingInputs = append([]domain.TrainingInputRecord(nil), execution.DataMetrics.TrainingInputs...)
}
run.Logs = []domain.LogEntryRecord{}
run.ResourceSeries = []domain.ResourceSample{}
run.Artifacts = []domain.ExecutionArtifact{}
@ -901,6 +908,7 @@ func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord {
WorkflowName: summary.WorkflowName,
ProjectID: summary.ProjectID,
ProjectName: summary.ProjectName,
TaskID: summary.TaskID,
Trigger: summary.Trigger,
Attempt: summary.Attempt,
ParentRunID: summary.ParentRunID,
@ -923,6 +931,18 @@ func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord {
return runs
}
func (runtime *Runtime) materializeNodeMetricsBestEffort(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) {
if run == nil || execution == nil || execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" {
return
}
if err := dataops.MaterializeNodeMetrics(ctx, runtime.store, run, execution); err != nil {
fmt.Printf("[worker] materialize data metrics for run %s node %s failed: %v\n", run.ID, execution.NodeID, err)
return
}
execution.MetricsMaterializedAt = nowISO()
applyExecution(run, *execution)
}
func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) {
if claimed == nil {
return
@ -1090,6 +1110,9 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
return processErr
}
runtime.updateEffectivePollInterval(settings)
if _, err := dataops.ReconcileRunMetrics(ctx, runtime.store, 20); err != nil {
fmt.Printf("[worker] reconcile data metrics failed: %v\n", err)
}
reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
if err != nil {
processErr = err
@ -1391,10 +1414,16 @@ func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.Run
execution.Logs = append(execution.Logs, parsed.Logs...)
execution.Artifacts = parsed.Artifacts
execution.DataRefs = parsed.DataRefs
execution.DataMetrics = parsed.DataMetrics
execution.ResourceSeries = parsed.ResourceSeries
execution.Error = ""
executionMap[node.ID] = execution
applyExecution(run, execution)
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
return err
}
runtime.materializeNodeMetricsBestEffort(ctx, run, &execution)
executionMap[node.ID] = execution
if run.Status == domain.RunStatusStopping {
_, err := runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流")
return err
@ -2124,7 +2153,7 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo
if err != nil {
return nil, err
}
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry)
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry, failedRun.TaskID)
if err != nil {
return nil, err
}
@ -2375,6 +2404,27 @@ func enqueueReadyNodes(workflow domain.WorkflowRecord, executionMap map[string]d
}
}
func upstreamBusinessVolume(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) domain.DataVolume {
for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
execution, ok := executionMap[edge.Source]
if !ok || execution.DataMetrics == nil {
continue
}
if execution.DataMetrics.TrainReady.TrajectoryCount != nil ||
execution.DataMetrics.TrainReady.DurationSeconds != nil ||
execution.DataMetrics.TrainReady.Bytes != nil {
return execution.DataMetrics.TrainReady
}
if execution.DataMetrics.TechnicalValid.TrajectoryCount != nil ||
execution.DataMetrics.TechnicalValid.DurationSeconds != nil ||
execution.DataMetrics.TechnicalValid.Bytes != nil {
return execution.DataMetrics.TechnicalValid
}
return execution.DataMetrics.Output
}
return domain.DataVolume{}
}
func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) {
if _, ok, err := runtime.store.AcquireRunLease(ctx, claimed.ID, runtime.leaseOwner(), runtime.leaseTTL()); err != nil {
return nil, err
@ -2469,6 +2519,25 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
execution.Status = domain.NodeExecutionStatusCompleted
execution.EndedAt = nowISO()
execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label))
resolvedNode := runtime.resolveNodeBusinessStage(ctx, currentRun, *node)
if resolvedNode.Data.BusinessStage == domain.DataMetricStageAudit {
volume := upstreamBusinessVolume(*workflow, executionMap, node.ID)
execution.DataMetrics = &domain.DataMetricsRecord{
SchemaVersion: 1,
Stage: domain.DataMetricStageAudit,
Input: volume,
Output: volume,
Quality: domain.DataMetricQualityExact,
OccurredAt: execution.EndedAt,
}
if task.Status == domain.AuditStatusApproved {
execution.DataMetrics.TrainReady = volume
} else {
execution.DataMetrics.Output = domain.DataVolume{}
execution.DataMetrics.Rejected = volume
execution.DataMetrics.RejectionReasons = map[string]int64{"人工审核拒绝": 1}
}
}
executionMap[node.ID] = execution
applyExecution(currentRun, execution)
for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) {
@ -2480,6 +2549,8 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution)
executionMap[node.ID] = execution
}
break
}
@ -2868,6 +2939,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
execution.Logs = append(execution.Logs, outcome.result.Logs...)
execution.Artifacts = outcome.result.Artifacts
execution.DataRefs = outcome.result.DataRefs
execution.DataMetrics = outcome.result.DataMetrics
execution.StageManifestPath = outcome.result.StageManifestPath
execution.ResourceSeries = outcome.result.ResourceSeries
executionMap[outcome.nodeID] = execution
@ -2890,6 +2962,8 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
return nil, err
}
runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution)
executionMap[outcome.nodeID] = execution
if forceStopRequested && len(runningNodes) == 0 {
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
}

View File

@ -86,6 +86,7 @@ const Templates = lazyRoute(() => import("./pages/Templates"));
const Users = lazyRoute(() => import("./pages/Users"));
const Audit = lazyRoute(() => import("./pages/Audit"));
const DataAssets = lazyRoute(() => import("./pages/DataAssets"));
const DataOperations = lazyRoute(() => import("./pages/DataOperations"));
const DatabaseBrowser = lazyRoute(() => import("./pages/DatabaseBrowser"));
const ObjectStorageBrowser = lazyRoute(() => import("./pages/ObjectStorageBrowser"));
const LegacyDataAssetsRedirect = lazyRoute(() => import("./pages/LegacyDataAssetsRedirect"));
@ -110,6 +111,7 @@ export default function App() {
<Route path="/operators" element={<Operators />} />
<Route path="/workflow/:id" element={<WorkflowEditor />} />
<Route path="/data-assets" element={<DataAssets />} />
<Route path="/data-operations" element={<DataOperations />} />
<Route path="/database" element={<DatabaseBrowser />} />
<Route path="/object-storage" element={<ObjectStorageBrowser />} />
<Route path="/source-data" element={<LegacyDataAssetsRedirect />} />

View File

@ -21,6 +21,7 @@ import {
Database,
Cloudy,
DatabaseZap,
BarChart3,
LogOut,
Loader2,
} from "lucide-react";
@ -41,6 +42,12 @@ const navItems = [
path: "/projects",
roles: ["admin", "pipeline_engineer", "data_operator", "operator"],
},
{
icon: BarChart3,
label: "数据运营",
path: "/data-operations",
roles: ["admin", "pipeline_engineer", "data_operator", "qa_reviewer", "compliance_auditor", "auditor", "operator"],
},
{
icon: FileCode,
label: "工作流模板",

View File

@ -1,6 +1,7 @@
const operatorTypes = new Set(["builtin", "custom", "lightweight"]);
const portTypes = new Set(["json", "image", "video", "tensor", "file"]);
const portTransmissions = new Set(["", "bus", "oss", "shm", "stream"]);
const businessStages = new Set(["", "collect", "process", "clean", "validate", "annotate", "audit", "train"]);
const positiveInteger = /^[1-9]\d*$/;
const positiveResourceQuantity = /^(?:[1-9]\d*(?:\.\d+)?|0\.\d+)(?:(?:[eE][+-]?\d+)|(?:[numkKMGTP](?:i)?))?$/;
@ -73,6 +74,7 @@ export function buildOperatorPayload(source = {}, overrides = {}) {
publicationEvidence: deepClone(overrides.publicationEvidence ?? source.publicationEvidence),
hyperparams: deepClone(overrides.hyperparams ?? source.hyperparams ?? []),
resources: normalizeResources(deepClone(overrides.resources ?? source.resources)),
businessStage: trimString(overrides.businessStage ?? source.businessStage).toLowerCase(),
};
}
@ -133,6 +135,9 @@ export function getOperatorPayloadIssue(source = {}, overrides = {}) {
if (payload.version && !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(payload.version)) {
return "算子版本只能包含字母、数字、下划线、点和短横线,长度不超过 128";
}
if (!businessStages.has(payload.businessStage)) {
return "业务阶段必须是采集、处理、清洗、校验、标注、审核或训练";
}
return (
validatePorts("inputs", payload.inputs) ||
validatePorts("outputs", payload.outputs) ||

1337
src/pages/DataOperations.tsx Normal file

File diff suppressed because it is too large Load Diff

View File

@ -39,7 +39,7 @@ import {
import { buildOperatorPayload, getOperatorPayloadIssue } from "../lib/operators.js";
import { cn } from "../lib/utils";
import { canManagePipelineDesign } from "../lib/user-roles";
import { Operator, PortDefinition, PortTransmission, PortType } from "../types";
import { DataMetricStage, Operator, PortDefinition, PortTransmission, PortType } from "../types";
import { generateOperatorFromDescription } from "../services/aiService";
import type { GeneratedOperator } from "../services/api/ai";
import {
@ -76,6 +76,19 @@ const gpuTypeOptions = [
"NVIDIA T4 16GB",
"NVIDIA GPU",
];
const businessStageOptions: Array<{ value: DataMetricStage; label: string }> = [
{ value: "collect", label: "采集" },
{ value: "process", label: "处理 / 转换" },
{ value: "clean", label: "清洗" },
{ value: "validate", label: "技术校验" },
{ value: "annotate", label: "标注" },
{ value: "audit", label: "审核 / 训练可用" },
{ value: "train", label: "训练" },
];
function businessStageLabel(stage?: DataMetricStage) {
return businessStageOptions.find((option) => option.value === stage)?.label || "";
}
type GeneratedImplementation = NonNullable<GeneratedOperator["implementation"]>;
@ -1399,6 +1412,11 @@ export default function Operators() {
SHM {trimResource(op.resources?.sharedMemorySize)}
</span>
)}
{op.businessStage && (
<span className="text-[10px] px-2 py-0.5 rounded-full font-mono uppercase tracking-wider bg-emerald-500/10 text-emerald-300 border border-emerald-500/30">
阶段 {businessStageLabel(op.businessStage)}
</span>
)}
</div>
</div>
@ -2010,6 +2028,34 @@ export default function Operators() {
</label>
</div>
<div className="grid gap-2 md:grid-cols-[minmax(0,280px)_minmax(0,1fr)] md:items-end">
<div className="space-y-1.5">
<label className="text-[10px] font-medium uppercase tracking-wider text-gray-500">
默认业务阶段
</label>
<select
className="w-full rounded-md border border-[#2a2b2f] bg-[#151619] px-3 py-2 text-xs focus:border-blue-500 focus:outline-none"
value={newOp.businessStage || ""}
onChange={(event) =>
setNewOp((current) => ({
...current,
businessStage: (event.target.value || undefined) as DataMetricStage | undefined,
}))
}
>
<option value="">不指定(由任务或节点决定)</option>
{businessStageOptions.map((option) => (
<option key={option.value} value={option.value}>
{option.label}
</option>
))}
</select>
</div>
<p className="text-[10px] leading-relaxed text-gray-600">
用于算子独立运行时的指标归类;完整任务绑定和工作流节点覆盖值优先。
</p>
</div>
<div className="grid grid-cols-1 gap-3 md:grid-cols-5">
<div className="space-y-1.5">
<label className="text-[10px] font-medium uppercase tracking-wider text-gray-500">

View File

@ -145,6 +145,7 @@ import { workflowsApi } from "../services/api/workflows";
import { templatesApi } from "../services/api/templates";
import { projectsApi } from "../services/api/projects";
import type {
DataMetricStage,
ExecutionData,
ExecutionRun,
ExecutionRunSummary,
@ -219,9 +220,21 @@ interface OperatorNodeData {
memory?: string;
gpu?: string;
sharedMemorySize?: string;
businessStage?: DataMetricStage;
metricBoundary?: boolean;
executionState?: OperatorExecutionState;
}
const nodeBusinessStageOptions: Array<{ value: DataMetricStage; label: string }> = [
{ value: "collect", label: "采集" },
{ value: "process", label: "处理 / 转换" },
{ value: "clean", label: "清洗" },
{ value: "validate", label: "技术校验" },
{ value: "annotate", label: "标注" },
{ value: "audit", label: "审核 / 训练可用" },
{ value: "train", label: "训练" },
];
function isOperatorCandidate(value: unknown): value is Operator {
if (!value || typeof value !== "object") {
return false;
@ -794,6 +807,10 @@ export default function WorkflowEditor() {
}, [operators, selectedNode?.data]);
const inheritedSharedMemorySize = String(selectedNodeOperator?.resources?.sharedMemorySize || "").trim();
const sharedMemoryPlaceholder = inheritedSharedMemorySize ? `继承算子 ${inheritedSharedMemorySize}` : "继承平台默认";
const inheritedBusinessStage = selectedNodeOperator?.businessStage;
const businessStagePlaceholder = inheritedBusinessStage
? `继承算子:${nodeBusinessStageOptions.find((option) => option.value === inheritedBusinessStage)?.label || inheritedBusinessStage}`
: "不指定";
const selectedNodeRequestsGPU = hasPositiveGPURequest((selectedNode?.data as OperatorNodeData | undefined)?.gpu);
const selectedNodeGPUBlocked = selectedNodeRequestsGPU && workerCapacity?.gpu && !workerCapacity.gpu.available;
const workflowGPUCapacityBlocker = useMemo(
@ -2479,6 +2496,7 @@ export default function WorkflowEditor() {
memory: op.resources?.memoryLimit || "",
gpu: op.resources?.gpuLimit || "",
sharedMemorySize: "",
businessStage: undefined,
},
};
@ -6543,6 +6561,32 @@ export default function WorkflowEditor() {
</div>
)}
<div className="grid gap-2 md:grid-cols-[minmax(0,260px)_minmax(0,1fr)] md:items-end">
<div className="space-y-1">
<label className="text-[10px] text-gray-500 uppercase font-mono">业务阶段</label>
<select
className="w-full rounded border border-[#2a2b2f] bg-[#0a0a0a] px-2 py-1 text-xs focus:border-blue-500 focus:outline-none disabled:opacity-50"
value={selectedNode.data.businessStage || ""}
onChange={(event) =>
updateNodeData({
businessStage: (event.target.value || undefined) as DataMetricStage | undefined,
})
}
disabled={isReadOnly}
>
<option value="">{businessStagePlaceholder}</option>
{nodeBusinessStageOptions.map((option) => (
<option key={option.value} value={option.value}>
{option.label}
</option>
))}
</select>
</div>
<p className="text-[10px] leading-snug text-gray-600">
完整任务绑定优先,其次使用当前节点;留空时继承算子默认阶段。
</p>
</div>
<div className={cn("grid gap-2", isConfigPanelExpanded ? "grid-cols-4" : "grid-cols-2")}>
<div className="space-y-1">
<label className="text-[10px] text-gray-500 uppercase font-mono">资源限制 (CPU)</label>

View File

@ -0,0 +1,78 @@
import type {
CollectionTask,
CollectionTaskInput,
DataMetricFact,
DataMetricStage,
DataOperationsSummary,
DatasetVersion,
ExecutionRun,
TrainingUsage,
} from "../../types";
import { apiRequest } from "./client";
export type DataOperationsFilters = {
from?: string;
to?: string;
timezone?: string;
projectId?: string;
taskId?: string;
robotModel?: string;
scene?: string;
stage?: DataMetricStage;
limit?: number;
cursor?: string;
};
const queryString = (filters: DataOperationsFilters = {}) => {
const params = new URLSearchParams();
for (const [key, value] of Object.entries(filters)) {
if (value !== undefined && value !== "") {
params.set(key, String(value));
}
}
const value = params.toString();
return value ? `?${value}` : "";
};
export const dataOperationsApi = {
listTasks: async (filters: Pick<DataOperationsFilters, "projectId"> & { status?: string } = {}) =>
(
await apiRequest<{ items: CollectionTask[] }>(
`/api/data-collection-tasks${queryString(filters as DataOperationsFilters)}`,
)
).items,
getTask: (id: string) => apiRequest<CollectionTask>(`/api/data-collection-tasks/${id}`),
createTask: (payload: CollectionTaskInput) =>
apiRequest<CollectionTask>("/api/data-collection-tasks", { method: "POST" }, payload),
updateTask: (id: string, payload: CollectionTaskInput) =>
apiRequest<CollectionTask>(`/api/data-collection-tasks/${id}`, { method: "PATCH" }, payload),
startTask: (id: string) =>
apiRequest<{ task: CollectionTask; run: ExecutionRun }>(`/api/data-collection-tasks/${id}/start`, {
method: "POST",
}),
completeTask: (id: string) =>
apiRequest<CollectionTask>(`/api/data-collection-tasks/${id}/complete`, { method: "POST" }),
getSummary: (filters: DataOperationsFilters = {}, signal?: AbortSignal) =>
apiRequest<DataOperationsSummary>(`/api/data-operations/summary${queryString(filters)}`, { signal }),
listStageItems: (filters: DataOperationsFilters = {}, signal?: AbortSignal) =>
apiRequest<{ items: DataMetricFact[]; nextCursor?: string; hasMore: boolean }>(
`/api/data-operations/stage-items${queryString(filters)}`,
{ signal },
),
listTrainingUsage: (filters: DataOperationsFilters = {}, signal?: AbortSignal) =>
apiRequest<{ items: TrainingUsage[]; nextCursor?: string; hasMore: boolean }>(
`/api/data-operations/training-usage${queryString(filters)}`,
{ signal },
),
listDatasets: async (filters: DataOperationsFilters = {}, signal?: AbortSignal) =>
(
await apiRequest<{ items: DatasetVersion[] }>(`/api/data-operations/datasets${queryString(filters)}`, {
signal,
})
).items,
backfill: () =>
apiRequest<{ windowDays: number; processed: number; skipped: number; failures: number; completedAt: string }>(
"/api/data-operations/backfill",
{ method: "POST" },
),
};

View File

@ -261,6 +261,7 @@ export interface Operator {
gpuLimit?: string;
sharedMemorySize?: string;
};
businessStage?: DataMetricStage;
}
export interface HyperParam {
@ -609,6 +610,7 @@ export interface LightweightOperatorCodeManifest {
export interface ExecutionRunMetadata {
lightweightOperatorCodeManifests?: Record<string, LightweightOperatorCodeManifest>;
trainingInputs?: TrainingInput[];
}
export interface ScanConfig {
@ -817,6 +819,8 @@ export interface NodeExecution {
logs: LogEntry[];
artifacts: ExecutionData[];
dataRefs?: DataRef[];
dataMetrics?: DataMetrics;
metricsMaterializedAt?: string;
resourceSeries: ResourceUsage[];
error?: string;
}
@ -839,6 +843,207 @@ export interface DataRef {
partitionDir?: string;
partitionCount?: number;
itemCount?: number;
datasetId?: string;
datasetVersionId?: string;
split?: string;
fingerprint?: string;
}
export type DataMetricStage = "collect" | "process" | "clean" | "validate" | "annotate" | "audit" | "train";
export type DataMetricQuality = "exact" | "derived" | "estimated";
export type CollectionTaskStatus = "draft" | "active" | "paused" | "completed" | "cancelled";
export type CollectionTaskMode = "collection_only" | "full_training";
export interface DataVolume {
trajectoryCount?: number;
durationSeconds?: number;
bytes?: number;
fileCount?: number;
}
export interface DatasetMetricRef {
datasetId?: string;
datasetVersionId?: string;
name?: string;
split?: string;
fingerprint?: string;
kind?: string;
datasetType?: string;
manifestPath?: string;
payloadPath?: string;
partitionDir?: string;
partitionCount?: number;
itemCount?: number;
}
export interface TrainingInput {
datasetId?: string;
datasetVersionId?: string;
name?: string;
split?: string;
volume?: DataVolume;
manifestPath?: string;
partitionDir?: string;
external?: boolean;
quality?: DataMetricQuality;
}
export interface DataMetrics {
schemaVersion: number;
stage: DataMetricStage;
input?: DataVolume;
output?: DataVolume;
rejected?: DataVolume;
technicalValid?: DataVolume;
trainReady?: DataVolume;
datasetRefs?: DatasetMetricRef[];
trainingInputs?: TrainingInput[];
rejectionReasons?: Record<string, number>;
quality?: DataMetricQuality;
occurredAt?: string;
}
export interface CollectionTaskRequirements {
requireTechnicalValidation: boolean;
requireQuality: boolean;
requireAnnotation: boolean;
requireAudit: boolean;
}
export interface CollectionTask {
id: string;
organizationId?: string;
projectId: string;
projectName?: string;
name: string;
description?: string;
robotModel: string;
robotVersion?: string;
scene: string;
modalities: string[];
format?: string;
mode: CollectionTaskMode;
status: CollectionTaskStatus;
workflowId?: string;
workflowName?: string;
stageBindings?: Partial<Record<DataMetricStage, string>>;
targets: DataVolume;
requirements: CollectionTaskRequirements;
splitRules?: Partial<Record<"train" | "val" | "test", number>>;
lastRunId?: string;
createdAt?: string;
updatedAt?: string;
}
export type CollectionTaskInput = Omit<
CollectionTask,
"id" | "organizationId" | "projectName" | "workflowName" | "lastRunId" | "createdAt" | "updatedAt"
>;
export interface DatasetVersion {
id: string;
datasetId: string;
taskId?: string;
projectId?: string;
name: string;
split?: string;
fingerprint: string;
volume?: DataVolume;
storage?: AssetStorageLocation;
manifestPath?: string;
partitionDir?: string;
producerRunId?: string;
producerNodeId?: string;
artifactId?: string;
quality: DataMetricQuality;
createdAt: string;
updatedAt?: string;
}
export interface DataMetricFact {
id: string;
sourceKey: string;
projectId?: string;
projectName?: string;
taskId?: string;
taskName?: string;
robotModel?: string;
scene?: string;
workflowId?: string;
workflowName?: string;
runId: string;
nodeId: string;
nodeLabel?: string;
attempt: number;
stage: DataMetricStage;
input?: DataVolume;
output?: DataVolume;
rejected?: DataVolume;
technicalValid?: DataVolume;
trainReady?: DataVolume;
datasetRefs?: DatasetMetricRef[];
trainingInputs?: TrainingInput[];
rejectionReasons?: Record<string, number>;
quality: DataMetricQuality;
occurredAt: string;
recordedAt: string;
}
export interface DataStageSummary {
stage: DataMetricStage;
input?: DataVolume;
output?: DataVolume;
rejected?: DataVolume;
technicalValid?: DataVolume;
trainReady?: DataVolume;
runCount: number;
}
export interface DataOperationsSummary {
from: string;
to: string;
timezone: string;
updatedAt: string;
freshness: string;
stages: DataStageSummary[];
training: {
consumed: DataVolume;
unique: DataVolume;
runCount: number;
};
hourly: Array<{ hour: string; stages: Partial<Record<DataMetricStage, DataVolume>> }>;
rejectionReasons: Record<string, number>;
tasks: Array<{
taskId: string;
taskName: string;
status: CollectionTaskStatus;
targets: DataVolume;
collected: DataVolume;
percent: number;
}>;
factCount: number;
exactCount: number;
derivedCount: number;
estimatedCount: number;
unknownCount: number;
}
export interface TrainingUsage {
id: string;
taskId?: string;
taskName?: string;
projectId?: string;
projectName?: string;
workflowId?: string;
workflowName?: string;
runId: string;
nodeId: string;
nodeLabel?: string;
model?: string;
inputs: TrainingInput[];
consumed: DataVolume;
quality: DataMetricQuality;
occurredAt: string;
}
export interface InputLineage {
@ -915,6 +1120,7 @@ export interface ExecutionRun {
workflowName: string;
projectId: string;
projectName: string;
taskId?: string;
trigger: "manual" | "scheduled" | "retry" | "rerun_subtree";
attempt?: number;
parentRunId?: string;
@ -968,6 +1174,7 @@ export interface ExecutionRunSummary {
workflowName: string;
projectId: string;
projectName: string;
taskId?: string;
trigger: "manual" | "scheduled" | "retry" | "rerun_subtree";
attempt?: number;
parentRunId?: string;

View File

@ -0,0 +1,37 @@
import test from "node:test";
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
const pageURL = new URL("../../src/pages/DataOperations.tsx", import.meta.url);
const serviceURL = new URL("../../src/services/api/data-operations.ts", import.meta.url);
test("data operations page exposes complete task and business metric surfaces", async () => {
const source = await readFile(pageURL, "utf8");
assert.match(source, /数据总览/);
assert.match(source, /完整任务/);
assert.match(source, /数据版本/);
assert.match(source, /训练使用/);
assert.match(source, /技术有效/);
assert.match(source, /训练可用/);
assert.match(source, /训练消费/);
});
test("data operations polling pauses while the page is hidden", async () => {
const source = await readFile(pageURL, "utf8");
assert.match(source, /document\.hidden/);
assert.match(source, /visibilitychange/);
assert.match(source, /30_000/);
assert.match(source, /controller\?\.abort\(\)/);
});
test("data operations client uses additive compatibility endpoints", async () => {
const source = await readFile(serviceURL, "utf8");
assert.match(source, /\/api\/data-collection-tasks/);
assert.match(source, /\/api\/data-operations\/summary/);
assert.match(source, /\/api\/data-operations\/stage-items/);
assert.match(source, /\/api\/data-operations\/training-usage/);
assert.match(source, /\/api\/data-operations\/datasets/);
});

View File

@ -55,6 +55,22 @@ test("buildOperatorPayload deep clones mutable operator fields", () => {
assert.equal(source.hyperparams[0].value, "fast");
});
test("buildOperatorPayload normalizes and validates business stage", () => {
const base = {
name: "cleaner",
type: "custom",
category: "DATA",
description: "clean data",
inputs: [],
outputs: [],
businessStage: " CLEAN ",
};
const payload = buildOperatorPayload(base);
assert.equal(payload.businessStage, "clean");
assert.equal(getOperatorPayloadIssue(base), "");
assert.match(getOperatorPayloadIssue({ ...base, businessStage: "publish" }), /业务阶段/);
});
test("operator payload validation rejects missing metadata and invalid ports", () => {
assert.equal(getOperatorPayloadIssue({ name: " ", description: "ok" }), "算子名称不能为空");
assert.equal(