forked from longtao.wu/droboticflow
📊 Add end-to-end data operations lifecycle
This commit is contained in:
parent
89372faf6b
commit
9051f412c9
@ -195,6 +195,151 @@ def load_input_artifacts() -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
|
||||
def _first_numeric(payload: dict[str, Any], keys: tuple[str, ...]) -> int | float | None:
|
||||
for key in keys:
|
||||
value = payload.get(key)
|
||||
if isinstance(value, bool):
|
||||
continue
|
||||
if isinstance(value, (int, float)) and value >= 0:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _metric_volume(payloads: list[dict[str, Any]]) -> dict[str, int | float]:
|
||||
candidates: dict[str, list[int | float]] = {
|
||||
"trajectoryCount": [],
|
||||
"durationSeconds": [],
|
||||
"bytes": [],
|
||||
"fileCount": [],
|
||||
}
|
||||
for payload in payloads:
|
||||
data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {}
|
||||
trajectory_count = _first_numeric(
|
||||
payload,
|
||||
("trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"),
|
||||
)
|
||||
if trajectory_count is None and data_ref:
|
||||
trajectory_count = _first_numeric(data_ref, ("itemCount",))
|
||||
duration_seconds = _first_numeric(payload, ("durationSeconds", "totalDurationSeconds", "totalDuration", "duration"))
|
||||
total_bytes = _first_numeric(payload, ("totalBytes", "bytes", "sizeBytes"))
|
||||
file_count = _first_numeric(payload, ("fileCount", "filesCount"))
|
||||
if trajectory_count is not None:
|
||||
candidates["trajectoryCount"].append(int(trajectory_count))
|
||||
if duration_seconds is not None:
|
||||
candidates["durationSeconds"].append(float(duration_seconds))
|
||||
if total_bytes is not None:
|
||||
candidates["bytes"].append(int(total_bytes))
|
||||
if file_count is not None:
|
||||
candidates["fileCount"].append(int(file_count))
|
||||
return {key: max(values) for key, values in candidates.items() if values}
|
||||
|
||||
|
||||
def _artifact_payloads(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [item["data"] for item in artifacts if isinstance(item.get("data"), dict)]
|
||||
|
||||
|
||||
def _dataset_refs(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
refs: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for artifact in artifacts:
|
||||
payload = artifact.get("data") if isinstance(artifact.get("data"), dict) else {}
|
||||
data_ref = payload.get("dataRef") if isinstance(payload.get("dataRef"), dict) else {}
|
||||
if not data_ref:
|
||||
continue
|
||||
item = {
|
||||
"name": text_value(artifact.get("name")),
|
||||
"kind": text_value(data_ref.get("kind")),
|
||||
"datasetType": text_value(data_ref.get("datasetType")),
|
||||
"manifestPath": text_value(data_ref.get("manifestPath")),
|
||||
"payloadPath": text_value(data_ref.get("payloadPath")),
|
||||
"partitionDir": text_value(data_ref.get("partitionDir")),
|
||||
"partitionCount": int(data_ref.get("partitionCount") or 0),
|
||||
"itemCount": int(data_ref.get("itemCount") or 0),
|
||||
"datasetId": text_value(data_ref.get("datasetId")),
|
||||
"datasetVersionId": text_value(data_ref.get("datasetVersionId")),
|
||||
"split": text_value(data_ref.get("split")),
|
||||
"fingerprint": text_value(data_ref.get("fingerprint")),
|
||||
}
|
||||
key = "|".join(
|
||||
text_value(item.get(name))
|
||||
for name in ("datasetVersionId", "datasetId", "manifestPath", "payloadPath", "partitionDir", "split")
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
refs.append(item)
|
||||
return refs
|
||||
|
||||
|
||||
def infer_data_metrics(
|
||||
node_data: dict[str, Any],
|
||||
artifacts: list[dict[str, Any]],
|
||||
inputs: list[dict[str, Any]],
|
||||
) -> dict[str, Any] | None:
|
||||
stage = text_value(node_data.get("businessStage")).strip().lower()
|
||||
if stage not in {"collect", "process", "clean", "validate", "annotate", "audit", "train"}:
|
||||
return None
|
||||
if node_data.get("metricBoundary") is False:
|
||||
return None
|
||||
output_payloads = _artifact_payloads(artifacts)
|
||||
for payload in output_payloads:
|
||||
explicit = payload.get("dataMetrics")
|
||||
if isinstance(explicit, dict):
|
||||
normalized = json_compatible(explicit)
|
||||
normalized.setdefault("schemaVersion", 1)
|
||||
normalized.setdefault("stage", stage)
|
||||
normalized.setdefault("quality", "exact")
|
||||
normalized.setdefault("occurredAt", now_iso())
|
||||
return normalized
|
||||
|
||||
input_payloads = _artifact_payloads(inputs)
|
||||
input_volume = _metric_volume(input_payloads)
|
||||
output_volume = _metric_volume(output_payloads)
|
||||
metrics: dict[str, Any] = {
|
||||
"schemaVersion": 1,
|
||||
"stage": stage,
|
||||
"input": input_volume,
|
||||
"output": output_volume,
|
||||
"datasetRefs": _dataset_refs(artifacts),
|
||||
"quality": "derived",
|
||||
"occurredAt": now_iso(),
|
||||
}
|
||||
if stage == "collect" and not input_volume:
|
||||
metrics.pop("input")
|
||||
if stage == "validate":
|
||||
valid = any(payload.get("valid") is True for payload in output_payloads)
|
||||
metrics["technicalValid"] = output_volume if valid else {}
|
||||
if not valid and input_volume:
|
||||
metrics["rejected"] = input_volume
|
||||
if stage in {"annotate", "audit"}:
|
||||
metrics["trainReady"] = output_volume
|
||||
if stage == "clean" and input_volume and output_volume:
|
||||
rejected: dict[str, int | float] = {}
|
||||
for key in ("trajectoryCount", "durationSeconds", "bytes", "fileCount"):
|
||||
if key in input_volume and key in output_volume:
|
||||
rejected[key] = max(0, input_volume[key] - output_volume[key])
|
||||
if rejected:
|
||||
metrics["rejected"] = rejected
|
||||
if stage == "train":
|
||||
training_inputs: list[dict[str, Any]] = []
|
||||
for ref in _dataset_refs(inputs):
|
||||
training_inputs.append(
|
||||
{
|
||||
"datasetId": ref.get("datasetId", ""),
|
||||
"datasetVersionId": ref.get("datasetVersionId", ""),
|
||||
"name": ref.get("name", ""),
|
||||
"split": ref.get("split", ""),
|
||||
"volume": input_volume,
|
||||
"manifestPath": ref.get("manifestPath", ""),
|
||||
"partitionDir": ref.get("partitionDir", ""),
|
||||
"external": not bool(ref.get("datasetVersionId")),
|
||||
"quality": "derived",
|
||||
}
|
||||
)
|
||||
metrics["trainingInputs"] = training_inputs
|
||||
return metrics
|
||||
|
||||
|
||||
def stage_manifest_payload(
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
@ -202,6 +347,7 @@ def stage_manifest_payload(
|
||||
node_label: str,
|
||||
output_dir: pathlib.Path,
|
||||
artifacts: list[dict[str, Any]],
|
||||
data_metrics: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
manifest_artifacts: list[dict[str, Any]] = []
|
||||
data_refs: list[dict[str, Any]] = []
|
||||
@ -232,7 +378,7 @@ def stage_manifest_payload(
|
||||
}
|
||||
)
|
||||
manifest_artifacts.append(item)
|
||||
return {
|
||||
result = {
|
||||
"runId": run_id,
|
||||
"workflowId": workflow_id,
|
||||
"nodeId": node_id,
|
||||
@ -244,6 +390,9 @@ def stage_manifest_payload(
|
||||
"primaryDataRefs": data_refs,
|
||||
"generatedAt": now_iso(),
|
||||
}
|
||||
if data_metrics:
|
||||
result["dataMetrics"] = json_compatible(data_metrics)
|
||||
return result
|
||||
|
||||
|
||||
def default_result_output_path(output_dir: pathlib.Path) -> pathlib.Path:
|
||||
@ -5295,16 +5444,25 @@ def main() -> int:
|
||||
resource_samples = sampler.stop(ended_at)
|
||||
|
||||
append_log("INFO", f"Produced {len(artifacts)} artifact(s)")
|
||||
data_metrics = infer_data_metrics(node_data, artifacts, inputs)
|
||||
result = {
|
||||
"logs": logs,
|
||||
"artifacts": artifacts,
|
||||
"resourceSeries": resource_samples,
|
||||
}
|
||||
if data_metrics:
|
||||
result["dataMetrics"] = data_metrics
|
||||
stage_manifest_path = os.getenv("STAGE_MANIFEST_PATH", "").strip()
|
||||
if stage_manifest_path:
|
||||
write_json_file(stage_manifest_path, stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts))
|
||||
write_json_file(
|
||||
stage_manifest_path,
|
||||
stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics),
|
||||
)
|
||||
else:
|
||||
write_json_file(default_stage_manifest_path(output_dir), stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts))
|
||||
write_json_file(
|
||||
default_stage_manifest_path(output_dir),
|
||||
stage_manifest_payload(run_id, workflow_id, node_id, node_label, output_dir, artifacts, data_metrics),
|
||||
)
|
||||
|
||||
result_output_path = os.getenv("RESULT_OUTPUT_PATH", "").strip()
|
||||
if result_output_path:
|
||||
|
||||
@ -646,6 +646,41 @@ class RuntimeFileContractTests(unittest.TestCase):
|
||||
self.assertEqual(len(manifest["dataRefs"]), 1)
|
||||
self.assertEqual(manifest["dataRefs"][0]["datasetType"], "x-platform-packages")
|
||||
|
||||
def test_infer_data_metrics_uses_bound_stage_and_preserves_unknowns(self):
|
||||
metrics = MODULE.infer_data_metrics(
|
||||
{"businessStage": "clean", "metricBoundary": True},
|
||||
[{"name": "filtered.json", "data": {"count": 8, "fileCount": 16}}],
|
||||
[{"name": "source.json", "data": {"count": 10, "fileCount": 20}}],
|
||||
)
|
||||
|
||||
self.assertEqual(metrics["stage"], "clean")
|
||||
self.assertEqual(metrics["input"]["trajectoryCount"], 10)
|
||||
self.assertEqual(metrics["output"]["trajectoryCount"], 8)
|
||||
self.assertEqual(metrics["rejected"]["trajectoryCount"], 2)
|
||||
self.assertEqual(metrics["quality"], "derived")
|
||||
|
||||
def test_stage_manifest_includes_standard_data_metrics(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
output_dir = pathlib.Path(temp_dir)
|
||||
metrics = {
|
||||
"schemaVersion": 1,
|
||||
"stage": "collect",
|
||||
"output": {"trajectoryCount": 12},
|
||||
"quality": "exact",
|
||||
}
|
||||
manifest = MODULE.stage_manifest_payload(
|
||||
"run-1",
|
||||
"wf-1",
|
||||
"node-1",
|
||||
"数据采集",
|
||||
output_dir,
|
||||
[],
|
||||
metrics,
|
||||
)
|
||||
|
||||
self.assertEqual(manifest["dataMetrics"]["stage"], "collect")
|
||||
self.assertEqual(manifest["dataMetrics"]["output"]["trajectoryCount"], 12)
|
||||
|
||||
|
||||
class XPlatformIndexingTests(unittest.TestCase):
|
||||
def test_load_index_payload_prefers_manifest_entries_over_local_manifest_path(self):
|
||||
|
||||
781
internal/api/data_operations.go
Normal file
781
internal/api/data_operations.go
Normal file
@ -0,0 +1,781 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/dataops"
|
||||
"droboticflow/internal/domain"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
func validCollectionTaskMode(value domain.CollectionTaskMode) bool {
|
||||
return value == domain.CollectionTaskModeCollectionOnly || value == domain.CollectionTaskModeFullTraining
|
||||
}
|
||||
|
||||
func validCollectionTaskStatus(value domain.CollectionTaskStatus) bool {
|
||||
switch value {
|
||||
case domain.CollectionTaskStatusDraft, domain.CollectionTaskStatusActive, domain.CollectionTaskStatusPaused,
|
||||
domain.CollectionTaskStatusCompleted, domain.CollectionTaskStatusCancelled:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func validDataMetricStage(value domain.DataMetricStage) bool {
|
||||
for _, stage := range domain.DataMetricStages {
|
||||
if value == stage {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizedStringList(items []string) []string {
|
||||
result := make([]string, 0, len(items))
|
||||
seen := map[string]struct{}{}
|
||||
for _, value := range items {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
key := strings.ToLower(trimmed)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
result = append(result, trimmed)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func validateNonNegativeVolume(field string, value domain.DataVolume) []validationIssue {
|
||||
issues := make([]validationIssue, 0)
|
||||
if value.TrajectoryCount != nil && *value.TrajectoryCount < 0 {
|
||||
issues = append(issues, validationIssue{Field: field + ".trajectoryCount", Message: "must be greater than or equal to 0"})
|
||||
}
|
||||
if value.DurationSeconds != nil && (*value.DurationSeconds < 0 || math.IsNaN(*value.DurationSeconds) || math.IsInf(*value.DurationSeconds, 0)) {
|
||||
issues = append(issues, validationIssue{Field: field + ".durationSeconds", Message: "must be a finite value greater than or equal to 0"})
|
||||
}
|
||||
if value.Bytes != nil && *value.Bytes < 0 {
|
||||
issues = append(issues, validationIssue{Field: field + ".bytes", Message: "must be greater than or equal to 0"})
|
||||
}
|
||||
if value.FileCount != nil && *value.FileCount < 0 {
|
||||
issues = append(issues, validationIssue{Field: field + ".fileCount", Message: "must be greater than or equal to 0"})
|
||||
}
|
||||
return issues
|
||||
}
|
||||
|
||||
func collectionTaskHasTarget(value domain.DataVolume) bool {
|
||||
return (value.TrajectoryCount != nil && *value.TrajectoryCount > 0) ||
|
||||
(value.DurationSeconds != nil && *value.DurationSeconds > 0) ||
|
||||
(value.Bytes != nil && *value.Bytes > 0)
|
||||
}
|
||||
|
||||
func (server *server) normalizeCollectionTaskInput(ctxRequest *http.Request, payload *domain.CollectionTaskInput, existing *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, []validationIssue, error) {
|
||||
payload.ProjectID = strings.TrimSpace(payload.ProjectID)
|
||||
payload.Name = strings.TrimSpace(payload.Name)
|
||||
payload.Description = strings.TrimSpace(payload.Description)
|
||||
payload.RobotModel = strings.TrimSpace(payload.RobotModel)
|
||||
payload.RobotVersion = strings.TrimSpace(payload.RobotVersion)
|
||||
payload.Scene = strings.TrimSpace(payload.Scene)
|
||||
payload.Format = strings.TrimSpace(payload.Format)
|
||||
payload.WorkflowID = strings.TrimSpace(payload.WorkflowID)
|
||||
payload.Modalities = normalizedStringList(payload.Modalities)
|
||||
if payload.Mode == "" {
|
||||
payload.Mode = domain.CollectionTaskModeFullTraining
|
||||
}
|
||||
if payload.Status == "" {
|
||||
payload.Status = domain.CollectionTaskStatusDraft
|
||||
}
|
||||
issues := make([]validationIssue, 0)
|
||||
if payload.ProjectID == "" {
|
||||
issues = append(issues, validationIssue{Field: "projectId", Message: "projectId is required"})
|
||||
}
|
||||
if payload.Name == "" {
|
||||
issues = append(issues, validationIssue{Field: "name", Message: "name is required"})
|
||||
}
|
||||
if payload.RobotModel == "" {
|
||||
issues = append(issues, validationIssue{Field: "robotModel", Message: "robotModel is required"})
|
||||
}
|
||||
if payload.Scene == "" {
|
||||
issues = append(issues, validationIssue{Field: "scene", Message: "scene is required"})
|
||||
}
|
||||
if !validCollectionTaskMode(payload.Mode) {
|
||||
issues = append(issues, validationIssue{Field: "mode", Message: "mode must be collection_only or full_training"})
|
||||
}
|
||||
if !validCollectionTaskStatus(payload.Status) {
|
||||
issues = append(issues, validationIssue{Field: "status", Message: "invalid task status"})
|
||||
}
|
||||
issues = append(issues, validateNonNegativeVolume("targets", payload.Targets)...)
|
||||
if !collectionTaskHasTarget(payload.Targets) {
|
||||
issues = append(issues, validationIssue{Field: "targets", Message: "at least one positive trajectory, duration, or bytes target is required"})
|
||||
}
|
||||
splitTotal := 0
|
||||
for split, percent := range payload.SplitRules {
|
||||
if split != "train" && split != "val" && split != "test" {
|
||||
issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split must be train, val, or test"})
|
||||
}
|
||||
if percent < 0 || percent > 100 {
|
||||
issues = append(issues, validationIssue{Field: "splitRules." + split, Message: "split percent must be between 0 and 100"})
|
||||
}
|
||||
splitTotal += percent
|
||||
}
|
||||
if len(payload.SplitRules) > 0 && splitTotal != 100 {
|
||||
issues = append(issues, validationIssue{Field: "splitRules", Message: "split percentages must total 100"})
|
||||
}
|
||||
seenNodes := map[string]domain.DataMetricStage{}
|
||||
for stage, nodeID := range payload.StageBindings {
|
||||
nodeID = strings.TrimSpace(nodeID)
|
||||
if !validDataMetricStage(stage) {
|
||||
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "invalid business stage"})
|
||||
continue
|
||||
}
|
||||
if nodeID == "" {
|
||||
delete(payload.StageBindings, stage)
|
||||
continue
|
||||
}
|
||||
if previous, exists := seenNodes[nodeID]; exists {
|
||||
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: fmt.Sprintf("node is already the metric boundary for %s", previous)})
|
||||
}
|
||||
seenNodes[nodeID] = stage
|
||||
payload.StageBindings[stage] = nodeID
|
||||
}
|
||||
if len(issues) > 0 {
|
||||
return nil, issues, nil
|
||||
}
|
||||
project, err := server.store.GetProject(ctxRequest.Context(), payload.ProjectID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if project == nil || !server.filterByTenant(ctxRequest, project.OrganizationID) {
|
||||
return nil, []validationIssue{{Field: "projectId", Message: "project not found"}}, nil
|
||||
}
|
||||
workflowName := ""
|
||||
if payload.WorkflowID != "" {
|
||||
workflow, err := server.store.GetWorkflow(ctxRequest.Context(), payload.WorkflowID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if workflow == nil || workflow.ProjectID != project.ID || !server.filterByTenant(ctxRequest, workflow.OrganizationID) {
|
||||
return nil, []validationIssue{{Field: "workflowId", Message: "workflow not found in selected project"}}, nil
|
||||
}
|
||||
workflowName = workflow.Name
|
||||
for stage, nodeID := range payload.StageBindings {
|
||||
if domain.GetNodeByID(workflow.Nodes, nodeID) == nil {
|
||||
issues = append(issues, validationIssue{Field: "stageBindings." + string(stage), Message: "workflow node not found"})
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(issues) > 0 {
|
||||
return nil, issues, nil
|
||||
}
|
||||
item := &domain.CollectionTaskRecord{
|
||||
OrganizationID: project.OrganizationID,
|
||||
ProjectID: project.ID,
|
||||
ProjectName: project.Name,
|
||||
Name: payload.Name,
|
||||
Description: payload.Description,
|
||||
RobotModel: payload.RobotModel,
|
||||
RobotVersion: payload.RobotVersion,
|
||||
Scene: payload.Scene,
|
||||
Modalities: payload.Modalities,
|
||||
Format: payload.Format,
|
||||
Mode: payload.Mode,
|
||||
Status: payload.Status,
|
||||
WorkflowID: payload.WorkflowID,
|
||||
WorkflowName: workflowName,
|
||||
StageBindings: payload.StageBindings,
|
||||
Targets: payload.Targets,
|
||||
Requirements: payload.Requirements,
|
||||
SplitRules: payload.SplitRules,
|
||||
}
|
||||
if existing != nil {
|
||||
item.ID = existing.ID
|
||||
item.CreatedAt = existing.CreatedAt
|
||||
item.LastRunID = existing.LastRunID
|
||||
}
|
||||
return item, nil, nil
|
||||
}
|
||||
|
||||
func (server *server) handleCollectionTasksList(writer http.ResponseWriter, request *http.Request) {
|
||||
items, err := server.store.ListCollectionTasks(request.Context(), server.requestOrganizationID(request))
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
projectID := strings.TrimSpace(request.URL.Query().Get("projectId"))
|
||||
status := strings.TrimSpace(request.URL.Query().Get("status"))
|
||||
filtered := make([]domain.CollectionTaskRecord, 0, len(items))
|
||||
for _, item := range items {
|
||||
if projectID != "" && item.ProjectID != projectID {
|
||||
continue
|
||||
}
|
||||
if status != "" && string(item.Status) != status {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": filtered})
|
||||
}
|
||||
|
||||
func (server *server) handleCollectionTaskGet(writer http.ResponseWriter, request *http.Request) {
|
||||
item, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if item == nil || !server.ensureTenantAccess(writer, request, item.OrganizationID, "Data task") {
|
||||
if item == nil {
|
||||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
|
||||
}
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, item)
|
||||
}
|
||||
|
||||
func (server *server) persistCollectionTask(request *http.Request, item *domain.CollectionTaskRecord) (*domain.CollectionTaskRecord, error) {
|
||||
if err := server.store.UpsertCollectionTask(request.Context(), *item); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return server.store.GetCollectionTask(request.Context(), item.ID)
|
||||
}
|
||||
|
||||
func (server *server) handleCollectionTaskCreate(writer http.ResponseWriter, request *http.Request) {
|
||||
var payload domain.CollectionTaskInput
|
||||
if !decodeJSON(writer, request, &payload) {
|
||||
return
|
||||
}
|
||||
item, issues, err := server.normalizeCollectionTaskInput(request, &payload, nil)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if len(issues) > 0 {
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
item.ID = domain.NewID("data-task")
|
||||
saved, err := server.persistCollectionTask(request, item)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusCreated, saved)
|
||||
}
|
||||
|
||||
func (server *server) handleCollectionTaskUpdate(writer http.ResponseWriter, request *http.Request) {
|
||||
existing, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if existing == nil {
|
||||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
|
||||
return
|
||||
}
|
||||
if !server.ensureTenantAccess(writer, request, existing.OrganizationID, "Data task") {
|
||||
return
|
||||
}
|
||||
var payload domain.CollectionTaskInput
|
||||
if !decodeJSON(writer, request, &payload) {
|
||||
return
|
||||
}
|
||||
item, issues, err := server.normalizeCollectionTaskInput(request, &payload, existing)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if len(issues) > 0 {
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
saved, err := server.persistCollectionTask(request, item)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, saved)
|
||||
}
|
||||
|
||||
func (server *server) handleCollectionTaskStart(writer http.ResponseWriter, request *http.Request) {
|
||||
task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
|
||||
return
|
||||
}
|
||||
if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") {
|
||||
return
|
||||
}
|
||||
if task.WorkflowID == "" {
|
||||
renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bind a workflow before starting the task"})
|
||||
return
|
||||
}
|
||||
requiredStages := []domain.DataMetricStage{domain.DataMetricStageCollect}
|
||||
if task.Mode == domain.CollectionTaskModeFullTraining {
|
||||
requiredStages = append(
|
||||
requiredStages,
|
||||
domain.DataMetricStageProcess,
|
||||
domain.DataMetricStageClean,
|
||||
domain.DataMetricStageValidate,
|
||||
domain.DataMetricStageTrain,
|
||||
)
|
||||
if task.Requirements.RequireAnnotation {
|
||||
requiredStages = append(requiredStages, domain.DataMetricStageAnnotate)
|
||||
}
|
||||
if task.Requirements.RequireAudit {
|
||||
requiredStages = append(requiredStages, domain.DataMetricStageAudit)
|
||||
}
|
||||
}
|
||||
missingStageIssues := make([]validationIssue, 0)
|
||||
for _, stage := range requiredStages {
|
||||
if strings.TrimSpace(task.StageBindings[stage]) == "" {
|
||||
missingStageIssues = append(missingStageIssues, validationIssue{
|
||||
Field: "stageBindings." + string(stage),
|
||||
Message: "required stage is not bound to a workflow node",
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(missingStageIssues) > 0 {
|
||||
renderValidationError(writer, missingStageIssues...)
|
||||
return
|
||||
}
|
||||
workflow, err := server.store.GetWorkflow(request.Context(), task.WorkflowID)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if workflow == nil || workflow.ProjectID != task.ProjectID {
|
||||
renderValidationError(writer, validationIssue{Field: "workflowId", Message: "bound workflow is unavailable"})
|
||||
return
|
||||
}
|
||||
active, err := server.store.HasActiveRun(request.Context(), workflow.ID)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if active {
|
||||
renderJSON(writer, http.StatusConflict, map[string]string{"error": "工作流已有进行中的运行", "code": "workflow_run_active"})
|
||||
return
|
||||
}
|
||||
if issues := validateWorkflowTopology(workflow.Nodes, workflow.Edges); len(issues) > 0 {
|
||||
renderValidationError(writer, issues...)
|
||||
return
|
||||
}
|
||||
run, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerManual, task.ID)
|
||||
if err != nil {
|
||||
if renderRunCreationError(writer, err) {
|
||||
return
|
||||
}
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
task.Status = domain.CollectionTaskStatusActive
|
||||
task.LastRunID = run.ID
|
||||
saved, err := server.persistCollectionTask(request, task)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusAccepted, map[string]any{"task": saved, "run": run})
|
||||
}
|
||||
|
||||
func (server *server) handleCollectionTaskComplete(writer http.ResponseWriter, request *http.Request) {
|
||||
task, err := server.store.GetCollectionTask(request.Context(), chi.URLParam(request, "id"))
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": "Data task not found"})
|
||||
return
|
||||
}
|
||||
if !server.ensureTenantAccess(writer, request, task.OrganizationID, "Data task") {
|
||||
return
|
||||
}
|
||||
task.Status = domain.CollectionTaskStatusCompleted
|
||||
saved, err := server.persistCollectionTask(request, task)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, saved)
|
||||
}
|
||||
|
||||
func parseDataOperationsQuery(request *http.Request, settings *domain.SettingsRecord) (domain.DataOperationsQuery, []validationIssue) {
|
||||
timezone := strings.TrimSpace(request.URL.Query().Get("timezone"))
|
||||
if timezone == "" && settings != nil {
|
||||
timezone = strings.TrimSpace(settings.Timezone)
|
||||
}
|
||||
if timezone == "" {
|
||||
timezone = domain.DefaultWorkflowScheduleTimezone
|
||||
}
|
||||
location, err := time.LoadLocation(timezone)
|
||||
if err != nil {
|
||||
return domain.DataOperationsQuery{}, []validationIssue{{Field: "timezone", Message: "invalid IANA timezone"}}
|
||||
}
|
||||
query := domain.DataOperationsQuery{
|
||||
Timezone: timezone,
|
||||
ProjectID: strings.TrimSpace(request.URL.Query().Get("projectId")),
|
||||
TaskID: strings.TrimSpace(request.URL.Query().Get("taskId")),
|
||||
RobotModel: strings.TrimSpace(request.URL.Query().Get("robotModel")),
|
||||
Scene: strings.TrimSpace(request.URL.Query().Get("scene")),
|
||||
Stage: domain.DataMetricStage(strings.TrimSpace(request.URL.Query().Get("stage"))),
|
||||
Cursor: strings.TrimSpace(request.URL.Query().Get("cursor")),
|
||||
}
|
||||
if query.Stage != "" && !validDataMetricStage(query.Stage) {
|
||||
return domain.DataOperationsQuery{}, []validationIssue{{Field: "stage", Message: "invalid business stage"}}
|
||||
}
|
||||
if raw := strings.TrimSpace(request.URL.Query().Get("limit")); raw != "" {
|
||||
value, parseErr := strconv.Atoi(raw)
|
||||
if parseErr != nil || value < 1 || value > 100 {
|
||||
return domain.DataOperationsQuery{}, []validationIssue{{Field: "limit", Message: "limit must be between 1 and 100"}}
|
||||
}
|
||||
query.Limit = value
|
||||
}
|
||||
now := time.Now()
|
||||
fromRaw := strings.TrimSpace(request.URL.Query().Get("from"))
|
||||
toRaw := strings.TrimSpace(request.URL.Query().Get("to"))
|
||||
if fromRaw == "" {
|
||||
localNow := now.In(location)
|
||||
fromRaw = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), 0, 0, 0, 0, location).UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
if toRaw == "" {
|
||||
toRaw = now.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
from, fromErr := time.Parse(time.RFC3339Nano, fromRaw)
|
||||
to, toErr := time.Parse(time.RFC3339Nano, toRaw)
|
||||
if fromErr != nil {
|
||||
return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be RFC3339"}}
|
||||
}
|
||||
if toErr != nil {
|
||||
return domain.DataOperationsQuery{}, []validationIssue{{Field: "to", Message: "to must be RFC3339"}}
|
||||
}
|
||||
if !from.Before(to) {
|
||||
return domain.DataOperationsQuery{}, []validationIssue{{Field: "from", Message: "from must be before to"}}
|
||||
}
|
||||
query.From = from.UTC().Format(time.RFC3339Nano)
|
||||
query.To = to.UTC().Format(time.RFC3339Nano)
|
||||
return query, nil
|
||||
}
|
||||
|
||||
func (server *server) dataOperationsQuery(writer http.ResponseWriter, request *http.Request) (domain.DataOperationsQuery, bool) {
|
||||
settings, err := server.store.GetSettings(request.Context())
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return domain.DataOperationsQuery{}, false
|
||||
}
|
||||
query, issues := parseDataOperationsQuery(request, settings)
|
||||
if len(issues) > 0 {
|
||||
renderValidationError(writer, issues...)
|
||||
return domain.DataOperationsQuery{}, false
|
||||
}
|
||||
return query, true
|
||||
}
|
||||
|
||||
func (server *server) handleDataOperationsSummary(writer http.ResponseWriter, request *http.Request) {
|
||||
query, ok := server.dataOperationsQuery(writer, request)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
summary, err := server.store.GetDataOperationsSummary(request.Context(), server.requestOrganizationID(request), query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, summary)
|
||||
}
|
||||
|
||||
func (server *server) handleDataOperationsStageItems(writer http.ResponseWriter, request *http.Request) {
|
||||
query, ok := server.dataOperationsQuery(writer, request)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
page, err := server.store.ListDataMetricFactsPage(request.Context(), server.requestOrganizationID(request), query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, page)
|
||||
}
|
||||
|
||||
func (server *server) handleDataOperationsTrainingUsage(writer http.ResponseWriter, request *http.Request) {
|
||||
query, ok := server.dataOperationsQuery(writer, request)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
page, err := server.store.ListTrainingUsagePage(request.Context(), server.requestOrganizationID(request), query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, page)
|
||||
}
|
||||
|
||||
func (server *server) handleDataOperationsDatasets(writer http.ResponseWriter, request *http.Request) {
|
||||
query, ok := server.dataOperationsQuery(writer, request)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
items, err := server.store.ListDatasetVersions(request.Context(), server.requestOrganizationID(request), query)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": items})
|
||||
}
|
||||
|
||||
func legacyStageFromExecution(execution domain.NodeExecutionRecord) domain.DataMetricStage {
|
||||
label := strings.ToLower(strings.TrimSpace(execution.NodeLabel + " " + execution.OperatorID))
|
||||
stageTokens := []struct {
|
||||
stage domain.DataMetricStage
|
||||
tokens []string
|
||||
}{
|
||||
{domain.DataMetricStageTrain, []string{"train", "训练"}},
|
||||
{domain.DataMetricStageAudit, []string{"audit", "审核"}},
|
||||
{domain.DataMetricStageAnnotate, []string{"annotat", "标注"}},
|
||||
{domain.DataMetricStageValidate, []string{"validat", "quality", "校验", "质检"}},
|
||||
{domain.DataMetricStageClean, []string{"clean", "filter", "清洗", "过滤"}},
|
||||
{domain.DataMetricStageProcess, []string{"process", "convert", "package", "处理", "转换", "打包"}},
|
||||
{domain.DataMetricStageCollect, []string{"collect", "scan", "source", "采集", "扫描", "数据源"}},
|
||||
}
|
||||
for _, item := range stageTokens {
|
||||
for _, token := range item.tokens {
|
||||
if strings.Contains(label, token) {
|
||||
return item.stage
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func numericArtifactValue(payload map[string]any, keys ...string) (float64, bool) {
|
||||
for _, key := range keys {
|
||||
switch value := payload[key].(type) {
|
||||
case int:
|
||||
return float64(value), value >= 0
|
||||
case int32:
|
||||
return float64(value), value >= 0
|
||||
case int64:
|
||||
return float64(value), value >= 0
|
||||
case float64:
|
||||
return value, value >= 0 && !math.IsNaN(value) && !math.IsInf(value, 0)
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func legacyVolumeFromArtifacts(items []domain.ExecutionArtifact) domain.DataVolume {
|
||||
var trajectory, duration, bytes, files float64
|
||||
var hasTrajectory, hasDuration, hasBytes, hasFiles bool
|
||||
for _, artifact := range items {
|
||||
payload := artifact.Data
|
||||
if value, ok := numericArtifactValue(payload, "trajectoryCount", "episodeCount", "collectionCount", "sampleCount", "itemCount", "count", "videoCount"); ok && (!hasTrajectory || value > trajectory) {
|
||||
trajectory, hasTrajectory = value, true
|
||||
}
|
||||
if value, ok := numericArtifactValue(payload, "durationSeconds", "totalDurationSeconds", "totalDuration", "duration"); ok && (!hasDuration || value > duration) {
|
||||
duration, hasDuration = value, true
|
||||
}
|
||||
if value, ok := numericArtifactValue(payload, "totalBytes", "bytes", "sizeBytes"); ok && (!hasBytes || value > bytes) {
|
||||
bytes, hasBytes = value, true
|
||||
}
|
||||
if value, ok := numericArtifactValue(payload, "fileCount", "filesCount"); ok && (!hasFiles || value > files) {
|
||||
files, hasFiles = value, true
|
||||
}
|
||||
}
|
||||
result := domain.DataVolume{}
|
||||
if hasTrajectory {
|
||||
value := int64(trajectory)
|
||||
result.TrajectoryCount = &value
|
||||
}
|
||||
if hasDuration {
|
||||
value := duration
|
||||
result.DurationSeconds = &value
|
||||
}
|
||||
if hasBytes {
|
||||
value := int64(bytes)
|
||||
result.Bytes = &value
|
||||
}
|
||||
if hasFiles {
|
||||
value := int64(files)
|
||||
result.FileCount = &value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func legacySourceCollectionMetrics(items []domain.ExecutionArtifact) (domain.DataVolume, domain.DataVolume, []domain.DatasetMetricRef, bool) {
|
||||
var trajectories int64
|
||||
var validTrajectories int64
|
||||
var files int64
|
||||
var bytes int64
|
||||
refs := make([]domain.DatasetMetricRef, 0)
|
||||
for artifactIndex := range items {
|
||||
artifact := &items[artifactIndex]
|
||||
payload := artifactStructuredData(artifact)
|
||||
if payload == nil {
|
||||
continue
|
||||
}
|
||||
for _, raw := range sourceDatasetList(payload, "catalogCollections", "collections") {
|
||||
collection := mapValue(raw)
|
||||
if collection == nil {
|
||||
continue
|
||||
}
|
||||
trajectories++
|
||||
fileCount := int64Value(collection["fileCount"])
|
||||
totalBytes := int64Value(collection["totalBytes"])
|
||||
files += fileCount
|
||||
bytes += totalBytes
|
||||
if boolValue(collection["valid"]) {
|
||||
validTrajectories++
|
||||
}
|
||||
name := firstNonEmptyText(stringValue(collection["name"]), stringValue(collection["id"]), stringValue(collection["relativePath"]))
|
||||
refs = append(refs, domain.DatasetMetricRef{
|
||||
DatasetID: stringValue(collection["id"]),
|
||||
Name: name,
|
||||
Kind: "source-collection",
|
||||
DatasetType: firstNonEmptyText(stringValue(collection["datasetType"]), "trajectory"),
|
||||
PayloadPath: stringValue(collection["path"]),
|
||||
ItemCount: 1,
|
||||
})
|
||||
}
|
||||
}
|
||||
if trajectories == 0 {
|
||||
return domain.DataVolume{}, domain.DataVolume{}, nil, false
|
||||
}
|
||||
output := domain.DataVolume{
|
||||
TrajectoryCount: &trajectories,
|
||||
Bytes: &bytes,
|
||||
FileCount: &files,
|
||||
}
|
||||
valid := domain.DataVolume{TrajectoryCount: &validTrajectories}
|
||||
return output, valid, refs, true
|
||||
}
|
||||
|
||||
func legacyRejectedVolume(input domain.DataVolume, output domain.DataVolume) domain.DataVolume {
|
||||
result := domain.DataVolume{}
|
||||
if input.TrajectoryCount != nil && output.TrajectoryCount != nil && *input.TrajectoryCount >= *output.TrajectoryCount {
|
||||
value := *input.TrajectoryCount - *output.TrajectoryCount
|
||||
result.TrajectoryCount = &value
|
||||
}
|
||||
if input.DurationSeconds != nil && output.DurationSeconds != nil && *input.DurationSeconds >= *output.DurationSeconds {
|
||||
value := *input.DurationSeconds - *output.DurationSeconds
|
||||
result.DurationSeconds = &value
|
||||
}
|
||||
if input.Bytes != nil && output.Bytes != nil && *input.Bytes >= *output.Bytes {
|
||||
value := *input.Bytes - *output.Bytes
|
||||
result.Bytes = &value
|
||||
}
|
||||
if input.FileCount != nil && output.FileCount != nil && *input.FileCount >= *output.FileCount {
|
||||
value := *input.FileCount - *output.FileCount
|
||||
result.FileCount = &value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func legacyMetricsFromExecution(run domain.RunRecord, execution domain.NodeExecutionRecord) *domain.DataMetricsRecord {
|
||||
stage := legacyStageFromExecution(execution)
|
||||
if stage == "" {
|
||||
return nil
|
||||
}
|
||||
input := legacyVolumeFromArtifacts(execution.InputArtifacts)
|
||||
output := legacyVolumeFromArtifacts(execution.Artifacts)
|
||||
technicalValid := domain.DataVolume{}
|
||||
refs := []domain.DatasetMetricRef(nil)
|
||||
if collectionOutput, valid, collectionRefs, ok := legacySourceCollectionMetrics(execution.Artifacts); ok {
|
||||
output = collectionOutput
|
||||
technicalValid = valid
|
||||
refs = collectionRefs
|
||||
}
|
||||
metrics := &domain.DataMetricsRecord{
|
||||
SchemaVersion: 1,
|
||||
Stage: stage,
|
||||
Input: input,
|
||||
Output: output,
|
||||
DatasetRefs: refs,
|
||||
Quality: domain.DataMetricQualityEstimated,
|
||||
OccurredAt: firstNonEmptyText(execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt),
|
||||
}
|
||||
switch stage {
|
||||
case domain.DataMetricStageClean:
|
||||
metrics.Rejected = legacyRejectedVolume(input, output)
|
||||
case domain.DataMetricStageValidate:
|
||||
if technicalValid.TrajectoryCount != nil {
|
||||
metrics.TechnicalValid = technicalValid
|
||||
} else {
|
||||
metrics.TechnicalValid = output
|
||||
}
|
||||
case domain.DataMetricStageAudit:
|
||||
metrics.TrainReady = output
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
func (server *server) handleDataOperationsBackfill(writer http.ResponseWriter, request *http.Request) {
|
||||
limit, issue := parseDatabaseBrowserPageQuery(request.URL.Query().Get("limit"), "limit", 100, 1, 500)
|
||||
if issue != nil {
|
||||
renderValidationError(writer, *issue)
|
||||
return
|
||||
}
|
||||
cursor, err := decodeRunSummaryCursor(request.URL.Query().Get("cursor"))
|
||||
if err != nil {
|
||||
renderValidationError(writer, validationIssue{Field: "cursor", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
cutoff := time.Now().UTC().Add(-30 * 24 * time.Hour)
|
||||
page, err := server.store.ListDataOperationsBackfillRuns(
|
||||
request.Context(),
|
||||
server.requestOrganizationID(request),
|
||||
cutoff.Format(time.RFC3339Nano),
|
||||
cursor,
|
||||
limit,
|
||||
)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
processed := 0
|
||||
skipped := 0
|
||||
failures := 0
|
||||
for runIndex := range page.Items {
|
||||
run := &page.Items[runIndex]
|
||||
for executionIndex := range run.NodeExecutions {
|
||||
execution := &run.NodeExecutions[executionIndex]
|
||||
if execution.Status != domain.NodeExecutionStatusCompleted {
|
||||
continue
|
||||
}
|
||||
legacy := execution.DataMetrics == nil
|
||||
if legacy {
|
||||
execution.DataMetrics = legacyMetricsFromExecution(*run, *execution)
|
||||
if execution.DataMetrics == nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := dataops.MaterializeLegacyNodeMetrics(request.Context(), server.store, run, execution); err != nil {
|
||||
failures++
|
||||
continue
|
||||
}
|
||||
processed++
|
||||
}
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{
|
||||
"windowDays": 30,
|
||||
"scannedRuns": len(page.Items),
|
||||
"processed": processed,
|
||||
"skipped": skipped,
|
||||
"failures": failures,
|
||||
"nextCursor": encodeRunSummaryCursor(page.NextCursor),
|
||||
"hasMore": page.HasMore,
|
||||
"completedAt": nowISO(),
|
||||
})
|
||||
}
|
||||
150
internal/api/data_operations_test.go
Normal file
150
internal/api/data_operations_test.go
Normal file
@ -0,0 +1,150 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func apiDataOperationsInt64(value int64) *int64 {
|
||||
return &value
|
||||
}
|
||||
|
||||
func TestCollectionTaskCreateAndStartCarriesTaskIntoRun(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Y1 桌面整理"})
|
||||
if err != nil {
|
||||
t.Fatalf("create project: %v", err)
|
||||
}
|
||||
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
||||
ProjectID: project.ID,
|
||||
Name: "Y1 完整链路",
|
||||
Type: domain.WorkflowTypeBatch,
|
||||
Nodes: []domain.WorkflowNode{{
|
||||
ID: "collect-node",
|
||||
Type: "default",
|
||||
Data: domain.WorkflowNodeData{
|
||||
Label: "数据采集",
|
||||
Category: "DATA",
|
||||
Description: "collect",
|
||||
Inputs: []domain.PortDefinition{},
|
||||
Outputs: []domain.PortDefinition{},
|
||||
},
|
||||
Position: domain.Position{},
|
||||
}},
|
||||
Edges: []domain.WorkflowEdge{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create workflow: %v", err)
|
||||
}
|
||||
handler := NewHandlerWithConfig(dataStore, nil, config.AppConfig{WorkspaceMountPath: t.TempDir()})
|
||||
payload := domain.CollectionTaskInput{
|
||||
ProjectID: project.ID,
|
||||
Name: "Y1 桌面整理数据闭环",
|
||||
RobotModel: "Y1",
|
||||
Scene: "桌面整理",
|
||||
Modalities: []string{"RGB", "关节状态"},
|
||||
Mode: domain.CollectionTaskModeCollectionOnly,
|
||||
Status: domain.CollectionTaskStatusDraft,
|
||||
WorkflowID: workflow.ID,
|
||||
StageBindings: map[domain.DataMetricStage]string{
|
||||
domain.DataMetricStageCollect: "collect-node",
|
||||
},
|
||||
Targets: domain.DataVolume{TrajectoryCount: apiDataOperationsInt64(100)},
|
||||
Requirements: domain.CollectionTaskRequirements{
|
||||
RequireTechnicalValidation: true,
|
||||
RequireQuality: true,
|
||||
RequireAnnotation: true,
|
||||
RequireAudit: true,
|
||||
},
|
||||
SplitRules: map[string]int{"train": 80, "val": 10, "test": 10},
|
||||
}
|
||||
body, _ := json.Marshal(payload)
|
||||
createRecorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(createRecorder, httptest.NewRequest(http.MethodPost, "/api/data-collection-tasks", bytesReader(body)))
|
||||
if createRecorder.Code != http.StatusCreated {
|
||||
t.Fatalf("create status = %d body=%s", createRecorder.Code, createRecorder.Body.String())
|
||||
}
|
||||
var task domain.CollectionTaskRecord
|
||||
if err := json.Unmarshal(createRecorder.Body.Bytes(), &task); err != nil {
|
||||
t.Fatalf("decode task: %v", err)
|
||||
}
|
||||
if task.ID == "" || task.CreatedAt == "" || task.UpdatedAt == "" {
|
||||
t.Fatalf("created task is missing persisted fields: %#v", task)
|
||||
}
|
||||
|
||||
startRecorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(startRecorder, httptest.NewRequest(http.MethodPost, "/api/data-collection-tasks/"+task.ID+"/start", nil))
|
||||
if startRecorder.Code != http.StatusAccepted {
|
||||
t.Fatalf("start status = %d body=%s", startRecorder.Code, startRecorder.Body.String())
|
||||
}
|
||||
var response struct {
|
||||
Run domain.RunRecord `json:"run"`
|
||||
}
|
||||
if err := json.Unmarshal(startRecorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode start response: %v", err)
|
||||
}
|
||||
if response.Run.TaskID != task.ID {
|
||||
t.Fatalf("run task id = %q, want %q", response.Run.TaskID, task.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkflowNodeBusinessStageValidation(t *testing.T) {
|
||||
issues := validateWorkflowNodeResources([]domain.WorkflowNode{{
|
||||
Data: domain.WorkflowNodeData{BusinessStage: domain.DataMetricStage("publish")},
|
||||
}})
|
||||
if len(issues) != 1 || issues[0].Field != "nodes[0].data.businessStage" {
|
||||
t.Fatalf("issues = %#v", issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacySourceCollectionsProduceEstimatedTrajectoryMetrics(t *testing.T) {
|
||||
metrics := legacyMetricsFromExecution(
|
||||
domain.RunRecord{CreatedAt: "2026-07-23T01:00:00Z"},
|
||||
domain.NodeExecutionRecord{
|
||||
NodeLabel: "BOS 增量扫描",
|
||||
Artifacts: []domain.ExecutionArtifact{{
|
||||
Name: "dataset-index.json",
|
||||
Data: map[string]any{
|
||||
"collections": []any{
|
||||
map[string]any{"id": "trajectory-1", "path": "y1/1", "fileCount": float64(5), "totalBytes": float64(100), "valid": true},
|
||||
map[string]any{"id": "trajectory-2", "path": "y1/2", "fileCount": float64(4), "totalBytes": float64(80), "valid": false},
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
)
|
||||
if metrics == nil || metrics.Output.TrajectoryCount == nil || *metrics.Output.TrajectoryCount != 2 {
|
||||
t.Fatalf("metrics = %#v", metrics)
|
||||
}
|
||||
if metrics.Output.FileCount == nil || *metrics.Output.FileCount != 9 ||
|
||||
metrics.Output.Bytes == nil || *metrics.Output.Bytes != 180 {
|
||||
t.Fatalf("source collection volumes = %#v", metrics)
|
||||
}
|
||||
if len(metrics.DatasetRefs) != 2 || metrics.Quality != domain.DataMetricQualityEstimated {
|
||||
t.Fatalf("source collection lineage = %#v", metrics)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDataOperationsQueryUsesUTC8TodayByDefault(t *testing.T) {
|
||||
request := httptest.NewRequest(http.MethodGet, "/api/data-operations/summary", nil)
|
||||
query, issues := parseDataOperationsQuery(request, &domain.SettingsRecord{Timezone: "Asia/Shanghai"})
|
||||
if len(issues) > 0 {
|
||||
t.Fatalf("issues = %#v", issues)
|
||||
}
|
||||
if query.Timezone != "Asia/Shanghai" || query.From == "" || query.To == "" {
|
||||
t.Fatalf("query = %#v", query)
|
||||
}
|
||||
}
|
||||
|
||||
func bytesReader(value []byte) *bytes.Reader {
|
||||
return bytes.NewReader(value)
|
||||
}
|
||||
@ -409,6 +409,7 @@ func runSummaryFromRecord(run domain.RunRecord) domain.RunSummary {
|
||||
WorkflowName: run.WorkflowName,
|
||||
ProjectID: run.ProjectID,
|
||||
ProjectName: run.ProjectName,
|
||||
TaskID: run.TaskID,
|
||||
Trigger: run.Trigger,
|
||||
Attempt: run.Attempt,
|
||||
ParentRunID: run.ParentRunID,
|
||||
@ -1205,6 +1206,23 @@ func NewHandlerWithConfig(dataStore store.Store, aiService ai.Service, appConfig
|
||||
})
|
||||
router.Get("/api/acceptance/poc-readiness", instance.handlePocAcceptanceReadiness)
|
||||
router.Get("/api/dashboard", instance.handleDashboard)
|
||||
router.Get("/api/data-collection-tasks", instance.handleCollectionTasksList)
|
||||
router.Get("/api/data-collection-tasks/{id}", instance.handleCollectionTaskGet)
|
||||
router.Get("/api/data-operations/summary", instance.handleDataOperationsSummary)
|
||||
router.Get("/api/data-operations/stage-items", instance.handleDataOperationsStageItems)
|
||||
router.Get("/api/data-operations/training-usage", instance.handleDataOperationsTrainingUsage)
|
||||
router.Get("/api/data-operations/datasets", instance.handleDataOperationsDatasets)
|
||||
router.Group(func(router chi.Router) {
|
||||
router.Use(instance.requireBusinessRole(domain.UserRoleAdmin, domain.UserRolePipelineEngineer, domain.UserRoleDataOperator, domain.UserRoleOperator))
|
||||
router.Post("/api/data-collection-tasks", instance.handleCollectionTaskCreate)
|
||||
router.Patch("/api/data-collection-tasks/{id}", instance.handleCollectionTaskUpdate)
|
||||
router.Post("/api/data-collection-tasks/{id}/start", instance.handleCollectionTaskStart)
|
||||
router.Post("/api/data-collection-tasks/{id}/complete", instance.handleCollectionTaskComplete)
|
||||
})
|
||||
router.Group(func(router chi.Router) {
|
||||
router.Use(instance.requireBusinessRole(domain.UserRoleAdmin))
|
||||
router.Post("/api/data-operations/backfill", instance.handleDataOperationsBackfill)
|
||||
})
|
||||
|
||||
router.Get("/api/projects", instance.handleProjectsList)
|
||||
router.Get("/api/projects/{id}", instance.handleProjectsGet)
|
||||
@ -3481,6 +3499,12 @@ func validateWorkflowNodeResources(nodes []domain.WorkflowNode) []validationIssu
|
||||
if err := k8s.ValidateResourceQuantity(field, node.Data.SharedMemorySize); err != nil {
|
||||
issues = append(issues, validationIssue{Field: field, Message: err.Error()})
|
||||
}
|
||||
if node.Data.BusinessStage != "" && !validDataMetricStage(node.Data.BusinessStage) {
|
||||
issues = append(issues, validationIssue{
|
||||
Field: "nodes[" + strconv.Itoa(index) + "].data.businessStage",
|
||||
Message: "invalid business stage",
|
||||
})
|
||||
}
|
||||
}
|
||||
return issues
|
||||
}
|
||||
@ -3739,7 +3763,7 @@ func validRunTrigger(value domain.RunTrigger) bool {
|
||||
return value == domain.RunTriggerManual || value == domain.RunTriggerScheduled || value == domain.RunTriggerRetry
|
||||
}
|
||||
|
||||
func (server *server) createRunWithLightweightManifests(ctx context.Context, workflow *domain.WorkflowRecord, trigger domain.RunTrigger) (*domain.RunRecord, error) {
|
||||
func (server *server) createRunWithLightweightManifests(ctx context.Context, workflow *domain.WorkflowRecord, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) {
|
||||
if workflow == nil {
|
||||
return nil, fmt.Errorf("workflow not found")
|
||||
}
|
||||
@ -3747,7 +3771,7 @@ func (server *server) createRunWithLightweightManifests(ctx context.Context, wor
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
run, err := server.store.CreateRun(ctx, workflow.ID, trigger)
|
||||
run, err := server.store.CreateRun(ctx, workflow.ID, trigger, taskIDs...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -3980,7 +4004,7 @@ func (server *server) handleRunsRerunSubtree(writer http.ResponseWriter, request
|
||||
}
|
||||
}
|
||||
|
||||
childRun, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerRerunSubtree)
|
||||
childRun, err := server.createRunWithLightweightManifests(request.Context(), workflow, domain.RunTriggerRerunSubtree, sourceRun.TaskID)
|
||||
if err != nil {
|
||||
if renderRunCreationError(writer, err) {
|
||||
return
|
||||
@ -4735,6 +4759,9 @@ func (server *server) handleOperatorsCreate(writer http.ResponseWriter, request
|
||||
normalizeLightweightOperatorContract(&payload.RuntimeContract, containerCommandProvided, containerArgsProvided)
|
||||
}
|
||||
issues := validateOperatorMetadata(payload.Name, payload.Type, payload.Category, payload.Description, payload.Version, payload.Inputs, payload.Outputs, payload.Resources, payload.ContainerCommand, payload.ContainerArgs)
|
||||
if payload.BusinessStage != "" && !validDataMetricStage(payload.BusinessStage) {
|
||||
issues = append(issues, validationIssue{Field: "businessStage", Message: "invalid business stage"})
|
||||
}
|
||||
if payload.Type == "lightweight" {
|
||||
issues = append(issues, validateLightweightOperatorMetadata(payload.Image, payload.RuntimeContract, payload.ContainerCommand, payload.ContainerArgs, server.workspaceMountPath)...)
|
||||
}
|
||||
@ -4801,6 +4828,7 @@ type operatorPatch struct {
|
||||
SkillDefinition *string `json:"skillDefinition"`
|
||||
Hyperparams *[]domain.HyperParam `json:"hyperparams"`
|
||||
Resources *domain.OperatorResources `json:"resources"`
|
||||
BusinessStage *domain.DataMetricStage `json:"businessStage"`
|
||||
PublicationStatus *domain.OperatorPublicationStatus `json:"publicationStatus"`
|
||||
GenerationSource *domain.OperatorGenerationSource `json:"generationSource"`
|
||||
PublicationEvidence *domain.OperatorPublicationEvidence `json:"publicationEvidence"`
|
||||
@ -4883,6 +4911,13 @@ func (server *server) handleOperatorsUpdate(writer http.ResponseWriter, request
|
||||
if payload.Resources != nil {
|
||||
next.Resources = normalizeOperatorResources(payload.Resources)
|
||||
}
|
||||
if payload.BusinessStage != nil {
|
||||
if *payload.BusinessStage != "" && !validDataMetricStage(*payload.BusinessStage) {
|
||||
renderValidationError(writer, validationIssue{Field: "businessStage", Message: "invalid business stage"})
|
||||
return
|
||||
}
|
||||
next.BusinessStage = *payload.BusinessStage
|
||||
}
|
||||
if payload.GenerationSource != nil {
|
||||
next.GenerationSource = normalizeOperatorGenerationSource(*payload.GenerationSource)
|
||||
}
|
||||
|
||||
292
internal/dataops/materializer.go
Normal file
292
internal/dataops/materializer.go
Normal file
@ -0,0 +1,292 @@
|
||||
package dataops
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func normalizedQuality(value domain.DataMetricQuality) domain.DataMetricQuality {
|
||||
switch value {
|
||||
case domain.DataMetricQualityExact, domain.DataMetricQualityDerived, domain.DataMetricQualityEstimated:
|
||||
return value
|
||||
default:
|
||||
return domain.DataMetricQualityDerived
|
||||
}
|
||||
}
|
||||
|
||||
func stableID(prefix string, value string) string {
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return prefix + "-" + hex.EncodeToString(sum[:10])
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func dataRefFingerprint(ref domain.DatasetMetricRef, runID string, nodeID string) string {
|
||||
value := strings.Join([]string{
|
||||
ref.DatasetVersionID,
|
||||
ref.DatasetID,
|
||||
ref.Fingerprint,
|
||||
ref.ManifestPath,
|
||||
ref.PayloadPath,
|
||||
ref.PartitionDir,
|
||||
ref.Split,
|
||||
ref.Name,
|
||||
}, "|")
|
||||
if strings.Trim(value, "|") == "" {
|
||||
value = runID + "|" + nodeID + "|" + ref.Name
|
||||
}
|
||||
sum := sha256.Sum256([]byte(value))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func metricRefFromDataRef(item domain.DataRefRecord) domain.DatasetMetricRef {
|
||||
return domain.DatasetMetricRef{
|
||||
DatasetID: item.DatasetID,
|
||||
DatasetVersionID: item.DatasetVersionID,
|
||||
Name: item.ArtifactName,
|
||||
Split: item.Split,
|
||||
Fingerprint: item.Fingerprint,
|
||||
Kind: item.Kind,
|
||||
DatasetType: item.DatasetType,
|
||||
ManifestPath: item.ManifestPath,
|
||||
PayloadPath: item.PayloadPath,
|
||||
PartitionDir: item.PartitionDir,
|
||||
PartitionCount: item.PartitionCount,
|
||||
ItemCount: item.ItemCount,
|
||||
}
|
||||
}
|
||||
|
||||
func int64Pointer(value int64) *int64 {
|
||||
return &value
|
||||
}
|
||||
|
||||
func volumeWithItemCount(volume domain.DataVolume, itemCount int) domain.DataVolume {
|
||||
if volume.TrajectoryCount == nil && itemCount > 0 {
|
||||
volume.TrajectoryCount = int64Pointer(int64(itemCount))
|
||||
}
|
||||
return volume
|
||||
}
|
||||
|
||||
func anyString(value any) string {
|
||||
if value == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(fmt.Sprint(value))
|
||||
}
|
||||
|
||||
func anyInt(value any) int {
|
||||
switch typed := value.(type) {
|
||||
case int:
|
||||
return typed
|
||||
case int32:
|
||||
return int(typed)
|
||||
case int64:
|
||||
return int(typed)
|
||||
case float64:
|
||||
return int(typed)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func metricRefFromArtifact(artifact domain.ExecutionArtifact) (domain.DatasetMetricRef, bool) {
|
||||
if artifact.Data == nil {
|
||||
return domain.DatasetMetricRef{}, false
|
||||
}
|
||||
raw, ok := artifact.Data["dataRef"].(map[string]any)
|
||||
if !ok || len(raw) == 0 {
|
||||
return domain.DatasetMetricRef{}, false
|
||||
}
|
||||
return domain.DatasetMetricRef{
|
||||
Name: artifact.Name,
|
||||
DatasetID: anyString(raw["datasetId"]),
|
||||
DatasetVersionID: anyString(raw["datasetVersionId"]),
|
||||
Split: anyString(raw["split"]),
|
||||
Fingerprint: anyString(raw["fingerprint"]),
|
||||
Kind: anyString(raw["kind"]),
|
||||
DatasetType: anyString(raw["datasetType"]),
|
||||
ManifestPath: anyString(raw["manifestPath"]),
|
||||
PayloadPath: anyString(raw["payloadPath"]),
|
||||
PartitionDir: anyString(raw["partitionDir"]),
|
||||
PartitionCount: anyInt(raw["partitionCount"]),
|
||||
ItemCount: anyInt(raw["itemCount"]),
|
||||
}, true
|
||||
}
|
||||
|
||||
func trainingInputsFromExecution(execution domain.NodeExecutionRecord, inputVolume domain.DataVolume) []domain.TrainingInputRecord {
|
||||
result := make([]domain.TrainingInputRecord, 0)
|
||||
seen := map[string]struct{}{}
|
||||
for _, artifact := range execution.InputArtifacts {
|
||||
ref, ok := metricRefFromArtifact(artifact)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key := strings.Join([]string{ref.DatasetVersionID, ref.DatasetID, ref.ManifestPath, ref.PartitionDir, ref.Split}, "|")
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
result = append(result, domain.TrainingInputRecord{
|
||||
DatasetID: ref.DatasetID,
|
||||
DatasetVersionID: ref.DatasetVersionID,
|
||||
Name: ref.Name,
|
||||
Split: ref.Split,
|
||||
Volume: volumeWithItemCount(inputVolume, ref.ItemCount),
|
||||
ManifestPath: ref.ManifestPath,
|
||||
PartitionDir: ref.PartitionDir,
|
||||
External: ref.DatasetVersionID == "",
|
||||
Quality: domain.DataMetricQualityDerived,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func materializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord, markRun bool) error {
|
||||
if run == nil || execution == nil || execution.DataMetrics == nil {
|
||||
return nil
|
||||
}
|
||||
metrics := execution.DataMetrics
|
||||
if metrics.Stage == "" {
|
||||
return nil
|
||||
}
|
||||
task := (*domain.CollectionTaskRecord)(nil)
|
||||
if strings.TrimSpace(run.TaskID) != "" {
|
||||
item, err := dataStore.GetCollectionTask(ctx, run.TaskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
task = item
|
||||
}
|
||||
refs := append([]domain.DatasetMetricRef(nil), metrics.DatasetRefs...)
|
||||
if len(refs) == 0 {
|
||||
for _, item := range execution.DataRefs {
|
||||
refs = append(refs, metricRefFromDataRef(item))
|
||||
}
|
||||
}
|
||||
quality := normalizedQuality(metrics.Quality)
|
||||
for index := range refs {
|
||||
ref := &refs[index]
|
||||
fingerprint := firstNonEmpty(ref.Fingerprint, dataRefFingerprint(*ref, run.ID, execution.NodeID))
|
||||
ref.Fingerprint = fingerprint
|
||||
ref.DatasetID = firstNonEmpty(ref.DatasetID, stableID("dataset", firstNonEmpty(ref.Name, ref.DatasetType, fingerprint)))
|
||||
ref.DatasetVersionID = firstNonEmpty(ref.DatasetVersionID, stableID("dataset-version", fingerprint))
|
||||
volume := volumeWithItemCount(metrics.Output, ref.ItemCount)
|
||||
version := domain.DatasetVersionRecord{
|
||||
ID: ref.DatasetVersionID,
|
||||
OrganizationID: run.OrganizationID,
|
||||
DatasetID: ref.DatasetID,
|
||||
TaskID: run.TaskID,
|
||||
ProjectID: run.ProjectID,
|
||||
Name: firstNonEmpty(ref.Name, ref.DatasetType, ref.DatasetID),
|
||||
Split: ref.Split,
|
||||
Fingerprint: fingerprint,
|
||||
Volume: volume,
|
||||
ManifestPath: ref.ManifestPath,
|
||||
PartitionDir: ref.PartitionDir,
|
||||
ProducerRunID: run.ID,
|
||||
ProducerNodeID: execution.NodeID,
|
||||
Quality: quality,
|
||||
CreatedAt: firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.UpdatedAt),
|
||||
}
|
||||
if err := dataStore.UpsertDatasetVersion(ctx, version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
trainingInputs := append([]domain.TrainingInputRecord(nil), metrics.TrainingInputs...)
|
||||
if metrics.Stage == domain.DataMetricStageTrain && len(trainingInputs) == 0 {
|
||||
trainingInputs = trainingInputsFromExecution(*execution, metrics.Input)
|
||||
}
|
||||
occurredAt := firstNonEmpty(metrics.OccurredAt, execution.EndedAt, run.EndedAt, run.UpdatedAt, run.CreatedAt)
|
||||
if parsed, err := time.Parse(time.RFC3339Nano, occurredAt); err != nil || parsed.IsZero() {
|
||||
occurredAt = time.Now().UTC().Format(time.RFC3339Nano)
|
||||
quality = domain.DataMetricQualityEstimated
|
||||
} else {
|
||||
occurredAt = parsed.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
fact := domain.DataMetricFactRecord{
|
||||
ID: stableID("data-fact", fmt.Sprintf("%s|%s|%d|%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage)),
|
||||
SourceKey: fmt.Sprintf("%s/%s/%d/%s", run.ID, execution.NodeID, execution.Attempt, metrics.Stage),
|
||||
OrganizationID: run.OrganizationID,
|
||||
ProjectID: run.ProjectID,
|
||||
ProjectName: run.ProjectName,
|
||||
TaskID: run.TaskID,
|
||||
WorkflowID: run.WorkflowID,
|
||||
WorkflowName: run.WorkflowName,
|
||||
RunID: run.ID,
|
||||
NodeID: execution.NodeID,
|
||||
NodeLabel: execution.NodeLabel,
|
||||
Attempt: execution.Attempt,
|
||||
Stage: metrics.Stage,
|
||||
Input: metrics.Input,
|
||||
Output: metrics.Output,
|
||||
Rejected: metrics.Rejected,
|
||||
TechnicalValid: metrics.TechnicalValid,
|
||||
TrainReady: metrics.TrainReady,
|
||||
DatasetRefs: refs,
|
||||
TrainingInputs: trainingInputs,
|
||||
RejectionReasons: metrics.RejectionReasons,
|
||||
Quality: quality,
|
||||
OccurredAt: occurredAt,
|
||||
RecordedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
}
|
||||
if task != nil {
|
||||
fact.TaskName = task.Name
|
||||
fact.RobotModel = task.RobotModel
|
||||
fact.Scene = task.Scene
|
||||
}
|
||||
if err := dataStore.UpsertDataMetricFact(ctx, fact); err != nil {
|
||||
return err
|
||||
}
|
||||
if !markRun {
|
||||
return nil
|
||||
}
|
||||
return dataStore.MarkNodeMetricsMaterialized(ctx, run.ID, execution.NodeID, execution.Attempt, time.Now().UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
|
||||
func MaterializeNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
|
||||
return materializeNodeMetrics(ctx, dataStore, run, execution, true)
|
||||
}
|
||||
|
||||
func MaterializeLegacyNodeMetrics(ctx context.Context, dataStore store.Store, run *domain.RunRecord, execution *domain.NodeExecutionRecord) error {
|
||||
return materializeNodeMetrics(ctx, dataStore, run, execution, false)
|
||||
}
|
||||
|
||||
func ReconcileRunMetrics(ctx context.Context, dataStore store.Store, limit int) (int, error) {
|
||||
runs, err := dataStore.ListRunsNeedingMetricMaterialization(ctx, limit)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
materialized := 0
|
||||
var firstErr error
|
||||
for runIndex := range runs {
|
||||
run := &runs[runIndex]
|
||||
for executionIndex := range run.NodeExecutions {
|
||||
execution := &run.NodeExecutions[executionIndex]
|
||||
if execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" {
|
||||
continue
|
||||
}
|
||||
if err := MaterializeNodeMetrics(ctx, dataStore, run, execution); err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
materialized++
|
||||
}
|
||||
}
|
||||
return materialized, firstErr
|
||||
}
|
||||
89
internal/dataops/materializer_test.go
Normal file
89
internal/dataops/materializer_test.go
Normal file
@ -0,0 +1,89 @@
|
||||
package dataops
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
)
|
||||
|
||||
func materializerInt64(value int64) *int64 {
|
||||
return &value
|
||||
}
|
||||
|
||||
func TestMaterializeNodeMetricsCreatesFactDatasetAndMarksExecution(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
project, err := dataStore.CreateProject(ctx, domain.CreateProjectInput{Name: "Y1 项目"})
|
||||
if err != nil {
|
||||
t.Fatalf("create project: %v", err)
|
||||
}
|
||||
workflow, err := dataStore.CreateWorkflow(ctx, domain.CreateWorkflowInput{
|
||||
ProjectID: project.ID,
|
||||
Name: "Y1 数据准备",
|
||||
Type: domain.WorkflowTypeBatch,
|
||||
Nodes: []domain.WorkflowNode{},
|
||||
Edges: []domain.WorkflowEdge{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create workflow: %v", err)
|
||||
}
|
||||
run, err := dataStore.CreateRun(ctx, workflow.ID, domain.RunTriggerManual, "task-y1")
|
||||
if err != nil {
|
||||
t.Fatalf("create run: %v", err)
|
||||
}
|
||||
execution := domain.NodeExecutionRecord{
|
||||
NodeID: "collect",
|
||||
NodeLabel: "采集索引",
|
||||
Attempt: 1,
|
||||
Status: domain.NodeExecutionStatusCompleted,
|
||||
EndedAt: "2026-07-23T01:00:00Z",
|
||||
DataMetrics: &domain.DataMetricsRecord{
|
||||
SchemaVersion: 1,
|
||||
Stage: domain.DataMetricStageCollect,
|
||||
Output: domain.DataVolume{TrajectoryCount: materializerInt64(8)},
|
||||
Quality: domain.DataMetricQualityExact,
|
||||
DatasetRefs: []domain.DatasetMetricRef{{
|
||||
Name: "Y1 batch",
|
||||
DatasetType: "x-platform",
|
||||
ManifestPath: "/workspace/y1/manifest.json",
|
||||
ItemCount: 8,
|
||||
}},
|
||||
},
|
||||
}
|
||||
run.NodeExecutions = []domain.NodeExecutionRecord{execution}
|
||||
if err := dataStore.UpsertRun(ctx, *run); err != nil {
|
||||
t.Fatalf("upsert run: %v", err)
|
||||
}
|
||||
if err := MaterializeNodeMetrics(ctx, dataStore, run, &execution); err != nil {
|
||||
t.Fatalf("materialize: %v", err)
|
||||
}
|
||||
page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{
|
||||
From: "2026-07-23T00:00:00Z",
|
||||
To: "2026-07-24T00:00:00Z",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list facts: %v", err)
|
||||
}
|
||||
if len(page.Items) != 1 || len(page.Items[0].DatasetRefs) != 1 {
|
||||
t.Fatalf("facts = %#v", page.Items)
|
||||
}
|
||||
if page.Items[0].DatasetRefs[0].DatasetVersionID == "" {
|
||||
t.Fatal("dataset version id was not materialized")
|
||||
}
|
||||
versions, err := dataStore.ListDatasetVersions(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{Limit: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("list versions: %v", err)
|
||||
}
|
||||
if len(versions) != 1 || versions[0].Volume.TrajectoryCount == nil || *versions[0].Volume.TrajectoryCount != 8 {
|
||||
t.Fatalf("versions = %#v", versions)
|
||||
}
|
||||
persisted, err := dataStore.GetRun(ctx, run.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("get run: %v", err)
|
||||
}
|
||||
if persisted.NodeExecutions[0].MetricsMaterializedAt == "" {
|
||||
t.Fatal("node metrics were not marked materialized")
|
||||
}
|
||||
}
|
||||
295
internal/domain/data_operations.go
Normal file
295
internal/domain/data_operations.go
Normal file
@ -0,0 +1,295 @@
|
||||
package domain
|
||||
|
||||
type DataMetricStage string
|
||||
|
||||
const (
|
||||
DataMetricStageCollect DataMetricStage = "collect"
|
||||
DataMetricStageProcess DataMetricStage = "process"
|
||||
DataMetricStageClean DataMetricStage = "clean"
|
||||
DataMetricStageValidate DataMetricStage = "validate"
|
||||
DataMetricStageAnnotate DataMetricStage = "annotate"
|
||||
DataMetricStageAudit DataMetricStage = "audit"
|
||||
DataMetricStageTrain DataMetricStage = "train"
|
||||
)
|
||||
|
||||
var DataMetricStages = []DataMetricStage{
|
||||
DataMetricStageCollect,
|
||||
DataMetricStageProcess,
|
||||
DataMetricStageClean,
|
||||
DataMetricStageValidate,
|
||||
DataMetricStageAnnotate,
|
||||
DataMetricStageAudit,
|
||||
DataMetricStageTrain,
|
||||
}
|
||||
|
||||
type DataMetricQuality string
|
||||
|
||||
const (
|
||||
DataMetricQualityExact DataMetricQuality = "exact"
|
||||
DataMetricQualityDerived DataMetricQuality = "derived"
|
||||
DataMetricQualityEstimated DataMetricQuality = "estimated"
|
||||
)
|
||||
|
||||
type DataVolume struct {
|
||||
TrajectoryCount *int64 `json:"trajectoryCount,omitempty" bson:"trajectoryCount,omitempty"`
|
||||
DurationSeconds *float64 `json:"durationSeconds,omitempty" bson:"durationSeconds,omitempty"`
|
||||
Bytes *int64 `json:"bytes,omitempty" bson:"bytes,omitempty"`
|
||||
FileCount *int64 `json:"fileCount,omitempty" bson:"fileCount,omitempty"`
|
||||
}
|
||||
|
||||
type DatasetMetricRef struct {
|
||||
DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"`
|
||||
DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"`
|
||||
Name string `json:"name,omitempty" bson:"name,omitempty"`
|
||||
Split string `json:"split,omitempty" bson:"split,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty" bson:"fingerprint,omitempty"`
|
||||
Kind string `json:"kind,omitempty" bson:"kind,omitempty"`
|
||||
DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"`
|
||||
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
|
||||
PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"`
|
||||
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
|
||||
PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"`
|
||||
ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"`
|
||||
}
|
||||
|
||||
type TrainingInputRecord struct {
|
||||
DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"`
|
||||
DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"`
|
||||
Name string `json:"name,omitempty" bson:"name,omitempty"`
|
||||
Split string `json:"split,omitempty" bson:"split,omitempty"`
|
||||
Volume DataVolume `json:"volume,omitempty" bson:"volume,omitempty"`
|
||||
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
|
||||
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
|
||||
External bool `json:"external,omitempty" bson:"external,omitempty"`
|
||||
Quality DataMetricQuality `json:"quality,omitempty" bson:"quality,omitempty"`
|
||||
}
|
||||
|
||||
type DataMetricsRecord struct {
|
||||
SchemaVersion int `json:"schemaVersion" bson:"schemaVersion"`
|
||||
Stage DataMetricStage `json:"stage" bson:"stage"`
|
||||
Input DataVolume `json:"input,omitempty" bson:"input,omitempty"`
|
||||
Output DataVolume `json:"output,omitempty" bson:"output,omitempty"`
|
||||
Rejected DataVolume `json:"rejected,omitempty" bson:"rejected,omitempty"`
|
||||
TechnicalValid DataVolume `json:"technicalValid,omitempty" bson:"technicalValid,omitempty"`
|
||||
TrainReady DataVolume `json:"trainReady,omitempty" bson:"trainReady,omitempty"`
|
||||
DatasetRefs []DatasetMetricRef `json:"datasetRefs,omitempty" bson:"datasetRefs,omitempty"`
|
||||
TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"`
|
||||
RejectionReasons map[string]int64 `json:"rejectionReasons,omitempty" bson:"rejectionReasons,omitempty"`
|
||||
Quality DataMetricQuality `json:"quality,omitempty" bson:"quality,omitempty"`
|
||||
OccurredAt string `json:"occurredAt,omitempty" bson:"occurredAt,omitempty"`
|
||||
}
|
||||
|
||||
type CollectionTaskStatus string
|
||||
|
||||
const (
|
||||
CollectionTaskStatusDraft CollectionTaskStatus = "draft"
|
||||
CollectionTaskStatusActive CollectionTaskStatus = "active"
|
||||
CollectionTaskStatusPaused CollectionTaskStatus = "paused"
|
||||
CollectionTaskStatusCompleted CollectionTaskStatus = "completed"
|
||||
CollectionTaskStatusCancelled CollectionTaskStatus = "cancelled"
|
||||
)
|
||||
|
||||
type CollectionTaskMode string
|
||||
|
||||
const (
|
||||
CollectionTaskModeCollectionOnly CollectionTaskMode = "collection_only"
|
||||
CollectionTaskModeFullTraining CollectionTaskMode = "full_training"
|
||||
)
|
||||
|
||||
type CollectionTaskRequirements struct {
|
||||
RequireTechnicalValidation bool `json:"requireTechnicalValidation" bson:"requireTechnicalValidation"`
|
||||
RequireQuality bool `json:"requireQuality" bson:"requireQuality"`
|
||||
RequireAnnotation bool `json:"requireAnnotation" bson:"requireAnnotation"`
|
||||
RequireAudit bool `json:"requireAudit" bson:"requireAudit"`
|
||||
}
|
||||
|
||||
type CollectionTaskRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
|
||||
ProjectID string `json:"projectId" bson:"projectId"`
|
||||
ProjectName string `json:"projectName,omitempty" bson:"projectName,omitempty"`
|
||||
Name string `json:"name" bson:"name"`
|
||||
Description string `json:"description,omitempty" bson:"description,omitempty"`
|
||||
RobotModel string `json:"robotModel" bson:"robotModel"`
|
||||
RobotVersion string `json:"robotVersion,omitempty" bson:"robotVersion,omitempty"`
|
||||
Scene string `json:"scene" bson:"scene"`
|
||||
Modalities []string `json:"modalities" bson:"modalities"`
|
||||
Format string `json:"format,omitempty" bson:"format,omitempty"`
|
||||
Mode CollectionTaskMode `json:"mode" bson:"mode"`
|
||||
Status CollectionTaskStatus `json:"status" bson:"status"`
|
||||
WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"`
|
||||
WorkflowName string `json:"workflowName,omitempty" bson:"workflowName,omitempty"`
|
||||
StageBindings map[DataMetricStage]string `json:"stageBindings,omitempty" bson:"stageBindings,omitempty"`
|
||||
Targets DataVolume `json:"targets" bson:"targets"`
|
||||
Requirements CollectionTaskRequirements `json:"requirements" bson:"requirements"`
|
||||
SplitRules map[string]int `json:"splitRules,omitempty" bson:"splitRules,omitempty"`
|
||||
LastRunID string `json:"lastRunId,omitempty" bson:"lastRunId,omitempty"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
}
|
||||
|
||||
type CollectionTaskInput struct {
|
||||
ProjectID string `json:"projectId"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
RobotModel string `json:"robotModel"`
|
||||
RobotVersion string `json:"robotVersion,omitempty"`
|
||||
Scene string `json:"scene"`
|
||||
Modalities []string `json:"modalities"`
|
||||
Format string `json:"format,omitempty"`
|
||||
Mode CollectionTaskMode `json:"mode"`
|
||||
Status CollectionTaskStatus `json:"status,omitempty"`
|
||||
WorkflowID string `json:"workflowId,omitempty"`
|
||||
StageBindings map[DataMetricStage]string `json:"stageBindings,omitempty"`
|
||||
Targets DataVolume `json:"targets"`
|
||||
Requirements CollectionTaskRequirements `json:"requirements"`
|
||||
SplitRules map[string]int `json:"splitRules,omitempty"`
|
||||
}
|
||||
|
||||
type DatasetVersionRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
|
||||
DatasetID string `json:"datasetId" bson:"datasetId"`
|
||||
TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"`
|
||||
ProjectID string `json:"projectId,omitempty" bson:"projectId,omitempty"`
|
||||
Name string `json:"name" bson:"name"`
|
||||
Split string `json:"split,omitempty" bson:"split,omitempty"`
|
||||
Fingerprint string `json:"fingerprint" bson:"fingerprint"`
|
||||
Volume DataVolume `json:"volume,omitempty" bson:"volume,omitempty"`
|
||||
Storage AssetStorageLocation `json:"storage,omitempty" bson:"storage,omitempty"`
|
||||
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
|
||||
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
|
||||
ProducerRunID string `json:"producerRunId,omitempty" bson:"producerRunId,omitempty"`
|
||||
ProducerNodeID string `json:"producerNodeId,omitempty" bson:"producerNodeId,omitempty"`
|
||||
ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"`
|
||||
Quality DataMetricQuality `json:"quality" bson:"quality"`
|
||||
CreatedAt string `json:"createdAt" bson:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt" bson:"updatedAt"`
|
||||
}
|
||||
|
||||
type DataMetricFactRecord struct {
|
||||
ID string `json:"id" bson:"id"`
|
||||
SourceKey string `json:"sourceKey" bson:"sourceKey"`
|
||||
OrganizationID string `json:"organizationId,omitempty" bson:"organizationId,omitempty"`
|
||||
ProjectID string `json:"projectId,omitempty" bson:"projectId,omitempty"`
|
||||
ProjectName string `json:"projectName,omitempty" bson:"projectName,omitempty"`
|
||||
TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"`
|
||||
TaskName string `json:"taskName,omitempty" bson:"taskName,omitempty"`
|
||||
RobotModel string `json:"robotModel,omitempty" bson:"robotModel,omitempty"`
|
||||
Scene string `json:"scene,omitempty" bson:"scene,omitempty"`
|
||||
WorkflowID string `json:"workflowId,omitempty" bson:"workflowId,omitempty"`
|
||||
WorkflowName string `json:"workflowName,omitempty" bson:"workflowName,omitempty"`
|
||||
RunID string `json:"runId" bson:"runId"`
|
||||
NodeID string `json:"nodeId" bson:"nodeId"`
|
||||
NodeLabel string `json:"nodeLabel,omitempty" bson:"nodeLabel,omitempty"`
|
||||
Attempt int `json:"attempt" bson:"attempt"`
|
||||
Stage DataMetricStage `json:"stage" bson:"stage"`
|
||||
Input DataVolume `json:"input,omitempty" bson:"input,omitempty"`
|
||||
Output DataVolume `json:"output,omitempty" bson:"output,omitempty"`
|
||||
Rejected DataVolume `json:"rejected,omitempty" bson:"rejected,omitempty"`
|
||||
TechnicalValid DataVolume `json:"technicalValid,omitempty" bson:"technicalValid,omitempty"`
|
||||
TrainReady DataVolume `json:"trainReady,omitempty" bson:"trainReady,omitempty"`
|
||||
DatasetRefs []DatasetMetricRef `json:"datasetRefs,omitempty" bson:"datasetRefs,omitempty"`
|
||||
TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"`
|
||||
RejectionReasons map[string]int64 `json:"rejectionReasons,omitempty" bson:"rejectionReasons,omitempty"`
|
||||
Quality DataMetricQuality `json:"quality" bson:"quality"`
|
||||
OccurredAt string `json:"occurredAt" bson:"occurredAt"`
|
||||
RecordedAt string `json:"recordedAt" bson:"recordedAt"`
|
||||
}
|
||||
|
||||
type DataOperationsQuery struct {
|
||||
From string
|
||||
To string
|
||||
Timezone string
|
||||
ProjectID string
|
||||
TaskID string
|
||||
RobotModel string
|
||||
Scene string
|
||||
Stage DataMetricStage
|
||||
Limit int
|
||||
Cursor string
|
||||
}
|
||||
|
||||
type DataStageSummary struct {
|
||||
Stage DataMetricStage `json:"stage"`
|
||||
Input DataVolume `json:"input,omitempty"`
|
||||
Output DataVolume `json:"output,omitempty"`
|
||||
Rejected DataVolume `json:"rejected,omitempty"`
|
||||
TechnicalValid DataVolume `json:"technicalValid,omitempty"`
|
||||
TrainReady DataVolume `json:"trainReady,omitempty"`
|
||||
RunCount int64 `json:"runCount"`
|
||||
}
|
||||
|
||||
type DataOperationsHourlyPoint struct {
|
||||
Hour string `json:"hour"`
|
||||
Stages map[DataMetricStage]DataVolume `json:"stages"`
|
||||
}
|
||||
|
||||
type DataOperationsTrainingSummary struct {
|
||||
Consumed DataVolume `json:"consumed"`
|
||||
Unique DataVolume `json:"unique"`
|
||||
RunCount int64 `json:"runCount"`
|
||||
}
|
||||
|
||||
type CollectionTaskProgress struct {
|
||||
TaskID string `json:"taskId"`
|
||||
TaskName string `json:"taskName"`
|
||||
Status CollectionTaskStatus `json:"status"`
|
||||
Targets DataVolume `json:"targets"`
|
||||
Collected DataVolume `json:"collected"`
|
||||
Percent float64 `json:"percent"`
|
||||
}
|
||||
|
||||
type DataOperationsSummary struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
Timezone string `json:"timezone"`
|
||||
UpdatedAt string `json:"updatedAt"`
|
||||
Freshness string `json:"freshness"`
|
||||
Stages []DataStageSummary `json:"stages"`
|
||||
Training DataOperationsTrainingSummary `json:"training"`
|
||||
Hourly []DataOperationsHourlyPoint `json:"hourly"`
|
||||
RejectionReasons map[string]int64 `json:"rejectionReasons"`
|
||||
Tasks []CollectionTaskProgress `json:"tasks"`
|
||||
FactCount int64 `json:"factCount"`
|
||||
ExactCount int64 `json:"exactCount"`
|
||||
DerivedCount int64 `json:"derivedCount"`
|
||||
EstimatedCount int64 `json:"estimatedCount"`
|
||||
UnknownCount int64 `json:"unknownCount"`
|
||||
}
|
||||
|
||||
type TrainingUsageRecord struct {
|
||||
ID string `json:"id"`
|
||||
TaskID string `json:"taskId,omitempty"`
|
||||
TaskName string `json:"taskName,omitempty"`
|
||||
ProjectID string `json:"projectId,omitempty"`
|
||||
ProjectName string `json:"projectName,omitempty"`
|
||||
WorkflowID string `json:"workflowId,omitempty"`
|
||||
WorkflowName string `json:"workflowName,omitempty"`
|
||||
RunID string `json:"runId"`
|
||||
NodeID string `json:"nodeId"`
|
||||
NodeLabel string `json:"nodeLabel,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Inputs []TrainingInputRecord `json:"inputs"`
|
||||
Consumed DataVolume `json:"consumed"`
|
||||
Quality DataMetricQuality `json:"quality"`
|
||||
OccurredAt string `json:"occurredAt"`
|
||||
}
|
||||
|
||||
type TrainingUsagePage struct {
|
||||
Items []TrainingUsageRecord `json:"items"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
|
||||
type DataMetricFactPage struct {
|
||||
Items []DataMetricFactRecord `json:"items"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
|
||||
type DataOperationsRunPage struct {
|
||||
Items []RunRecord
|
||||
NextCursor *RunSummaryCursor
|
||||
HasMore bool
|
||||
}
|
||||
@ -183,6 +183,8 @@ type WorkflowNodeData struct {
|
||||
Memory string `json:"memory,omitempty" bson:"memory,omitempty"`
|
||||
GPU string `json:"gpu,omitempty" bson:"gpu,omitempty"`
|
||||
SharedMemorySize string `json:"sharedMemorySize,omitempty" bson:"sharedMemorySize,omitempty"`
|
||||
BusinessStage DataMetricStage `json:"businessStage,omitempty" bson:"businessStage,omitempty"`
|
||||
MetricBoundary bool `json:"metricBoundary,omitempty" bson:"metricBoundary,omitempty"`
|
||||
}
|
||||
|
||||
type WorkflowNode struct {
|
||||
@ -281,6 +283,7 @@ type OperatorRecord struct {
|
||||
SkillDefinition string `json:"skillDefinition,omitempty" bson:"skillDefinition,omitempty"`
|
||||
Hyperparams []HyperParam `json:"hyperparams,omitempty" bson:"hyperparams,omitempty"`
|
||||
Resources *OperatorResources `json:"resources,omitempty" bson:"resources,omitempty"`
|
||||
BusinessStage DataMetricStage `json:"businessStage,omitempty" bson:"businessStage,omitempty"`
|
||||
PublicationStatus OperatorPublicationStatus `json:"publicationStatus,omitempty" bson:"publicationStatus,omitempty"`
|
||||
GenerationSource OperatorGenerationSource `json:"generationSource,omitempty" bson:"generationSource,omitempty"`
|
||||
PublicationEvidence *OperatorPublicationEvidence `json:"publicationEvidence,omitempty" bson:"publicationEvidence,omitempty"`
|
||||
@ -463,37 +466,43 @@ func formatArtifactSize(bytes int64) string {
|
||||
}
|
||||
|
||||
type DataRefRecord struct {
|
||||
ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"`
|
||||
ArtifactName string `json:"artifactName,omitempty" bson:"artifactName,omitempty"`
|
||||
PortID string `json:"portId,omitempty" bson:"portId,omitempty"`
|
||||
Kind string `json:"kind,omitempty" bson:"kind,omitempty"`
|
||||
DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"`
|
||||
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
|
||||
PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"`
|
||||
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
|
||||
PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"`
|
||||
ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"`
|
||||
ArtifactID string `json:"artifactId,omitempty" bson:"artifactId,omitempty"`
|
||||
ArtifactName string `json:"artifactName,omitempty" bson:"artifactName,omitempty"`
|
||||
PortID string `json:"portId,omitempty" bson:"portId,omitempty"`
|
||||
Kind string `json:"kind,omitempty" bson:"kind,omitempty"`
|
||||
DatasetType string `json:"datasetType,omitempty" bson:"datasetType,omitempty"`
|
||||
ManifestPath string `json:"manifestPath,omitempty" bson:"manifestPath,omitempty"`
|
||||
PayloadPath string `json:"payloadPath,omitempty" bson:"payloadPath,omitempty"`
|
||||
PartitionDir string `json:"partitionDir,omitempty" bson:"partitionDir,omitempty"`
|
||||
PartitionCount int `json:"partitionCount,omitempty" bson:"partitionCount,omitempty"`
|
||||
ItemCount int `json:"itemCount,omitempty" bson:"itemCount,omitempty"`
|
||||
DatasetID string `json:"datasetId,omitempty" bson:"datasetId,omitempty"`
|
||||
DatasetVersionID string `json:"datasetVersionId,omitempty" bson:"datasetVersionId,omitempty"`
|
||||
Split string `json:"split,omitempty" bson:"split,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty" bson:"fingerprint,omitempty"`
|
||||
}
|
||||
|
||||
type NodeExecutionRecord struct {
|
||||
NodeID string `json:"nodeId" bson:"nodeId"`
|
||||
NodeLabel string `json:"nodeLabel" bson:"nodeLabel"`
|
||||
OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"`
|
||||
OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"`
|
||||
OperatorImage string `json:"operatorImage,omitempty" bson:"operatorImage,omitempty"`
|
||||
OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"`
|
||||
Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"`
|
||||
Status NodeExecutionStatus `json:"status" bson:"status"`
|
||||
StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"`
|
||||
EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"`
|
||||
StageManifestPath string `json:"stageManifestPath,omitempty" bson:"stageManifestPath,omitempty"`
|
||||
InputArtifacts []ExecutionArtifact `json:"inputArtifacts,omitempty" bson:"inputArtifacts,omitempty"`
|
||||
InputLineage []InputLineageRecord `json:"inputLineage,omitempty" bson:"inputLineage,omitempty"`
|
||||
Logs []LogEntryRecord `json:"logs" bson:"logs"`
|
||||
Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"`
|
||||
DataRefs []DataRefRecord `json:"dataRefs,omitempty" bson:"dataRefs,omitempty"`
|
||||
ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"`
|
||||
Error string `json:"error,omitempty" bson:"error,omitempty"`
|
||||
NodeID string `json:"nodeId" bson:"nodeId"`
|
||||
NodeLabel string `json:"nodeLabel" bson:"nodeLabel"`
|
||||
OperatorID string `json:"operatorId,omitempty" bson:"operatorId,omitempty"`
|
||||
OperatorVersion string `json:"operatorVersion,omitempty" bson:"operatorVersion,omitempty"`
|
||||
OperatorImage string `json:"operatorImage,omitempty" bson:"operatorImage,omitempty"`
|
||||
OperatorImageID string `json:"operatorImageId,omitempty" bson:"operatorImageId,omitempty"`
|
||||
Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"`
|
||||
Status NodeExecutionStatus `json:"status" bson:"status"`
|
||||
StartedAt string `json:"startedAt,omitempty" bson:"startedAt,omitempty"`
|
||||
EndedAt string `json:"endedAt,omitempty" bson:"endedAt,omitempty"`
|
||||
StageManifestPath string `json:"stageManifestPath,omitempty" bson:"stageManifestPath,omitempty"`
|
||||
InputArtifacts []ExecutionArtifact `json:"inputArtifacts,omitempty" bson:"inputArtifacts,omitempty"`
|
||||
InputLineage []InputLineageRecord `json:"inputLineage,omitempty" bson:"inputLineage,omitempty"`
|
||||
Logs []LogEntryRecord `json:"logs" bson:"logs"`
|
||||
Artifacts []ExecutionArtifact `json:"artifacts" bson:"artifacts"`
|
||||
DataRefs []DataRefRecord `json:"dataRefs,omitempty" bson:"dataRefs,omitempty"`
|
||||
DataMetrics *DataMetricsRecord `json:"dataMetrics,omitempty" bson:"dataMetrics,omitempty"`
|
||||
MetricsMaterializedAt string `json:"metricsMaterializedAt,omitempty" bson:"metricsMaterializedAt,omitempty"`
|
||||
ResourceSeries []ResourceSample `json:"resourceSeries" bson:"resourceSeries"`
|
||||
Error string `json:"error,omitempty" bson:"error,omitempty"`
|
||||
}
|
||||
|
||||
type InputLineageRecord struct {
|
||||
@ -597,6 +606,7 @@ type LightweightOperatorCodeManifest struct {
|
||||
|
||||
type RunMetadata struct {
|
||||
LightweightOperatorCodeManifests map[string]LightweightOperatorCodeManifest `json:"lightweightOperatorCodeManifests,omitempty" bson:"lightweightOperatorCodeManifests,omitempty"`
|
||||
TrainingInputs []TrainingInputRecord `json:"trainingInputs,omitempty" bson:"trainingInputs,omitempty"`
|
||||
}
|
||||
|
||||
// RunFailure is the machine-readable counterpart to RunRecord.Error. Error is
|
||||
@ -630,6 +640,7 @@ type RunRecord struct {
|
||||
WorkflowName string `json:"workflowName" bson:"workflowName"`
|
||||
ProjectID string `json:"projectId" bson:"projectId"`
|
||||
ProjectName string `json:"projectName" bson:"projectName"`
|
||||
TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"`
|
||||
Trigger RunTrigger `json:"trigger" bson:"trigger"`
|
||||
Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"`
|
||||
ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"`
|
||||
@ -666,6 +677,7 @@ type RunSummary struct {
|
||||
WorkflowName string `json:"workflowName" bson:"workflowName"`
|
||||
ProjectID string `json:"projectId" bson:"projectId"`
|
||||
ProjectName string `json:"projectName" bson:"projectName"`
|
||||
TaskID string `json:"taskId,omitempty" bson:"taskId,omitempty"`
|
||||
Trigger RunTrigger `json:"trigger" bson:"trigger"`
|
||||
Attempt int `json:"attempt,omitempty" bson:"attempt,omitempty"`
|
||||
ParentRunID string `json:"parentRunId,omitempty" bson:"parentRunId,omitempty"`
|
||||
@ -1076,6 +1088,7 @@ type CreateOperatorInput struct {
|
||||
SkillDefinition string `json:"skillDefinition,omitempty"`
|
||||
Hyperparams []HyperParam `json:"hyperparams,omitempty"`
|
||||
Resources *OperatorResources `json:"resources,omitempty"`
|
||||
BusinessStage DataMetricStage `json:"businessStage,omitempty"`
|
||||
PublicationStatus OperatorPublicationStatus `json:"publicationStatus,omitempty"`
|
||||
GenerationSource OperatorGenerationSource `json:"generationSource,omitempty"`
|
||||
PublicationEvidence *OperatorPublicationEvidence `json:"publicationEvidence,omitempty"`
|
||||
|
||||
408
internal/store/data_operations_common.go
Normal file
408
internal/store/data_operations_common.go
Normal file
@ -0,0 +1,408 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
type volumeAccumulator struct {
|
||||
trajectoryCount int64
|
||||
durationSeconds float64
|
||||
bytes int64
|
||||
fileCount int64
|
||||
hasTrajectory bool
|
||||
hasDuration bool
|
||||
hasBytes bool
|
||||
hasFileCount bool
|
||||
}
|
||||
|
||||
func dataMetricQualityRank(value domain.DataMetricQuality) int {
|
||||
switch value {
|
||||
case domain.DataMetricQualityExact:
|
||||
return 3
|
||||
case domain.DataMetricQualityDerived:
|
||||
return 2
|
||||
case domain.DataMetricQualityEstimated:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func (item *volumeAccumulator) add(value domain.DataVolume) {
|
||||
if value.TrajectoryCount != nil {
|
||||
item.trajectoryCount += *value.TrajectoryCount
|
||||
item.hasTrajectory = true
|
||||
}
|
||||
if value.DurationSeconds != nil {
|
||||
item.durationSeconds += *value.DurationSeconds
|
||||
item.hasDuration = true
|
||||
}
|
||||
if value.Bytes != nil {
|
||||
item.bytes += *value.Bytes
|
||||
item.hasBytes = true
|
||||
}
|
||||
if value.FileCount != nil {
|
||||
item.fileCount += *value.FileCount
|
||||
item.hasFileCount = true
|
||||
}
|
||||
}
|
||||
|
||||
func (item volumeAccumulator) value() domain.DataVolume {
|
||||
result := domain.DataVolume{}
|
||||
if item.hasTrajectory {
|
||||
value := item.trajectoryCount
|
||||
result.TrajectoryCount = &value
|
||||
}
|
||||
if item.hasDuration {
|
||||
value := item.durationSeconds
|
||||
result.DurationSeconds = &value
|
||||
}
|
||||
if item.hasBytes {
|
||||
value := item.bytes
|
||||
result.Bytes = &value
|
||||
}
|
||||
if item.hasFileCount {
|
||||
value := item.fileCount
|
||||
result.FileCount = &value
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type stageAccumulator struct {
|
||||
input volumeAccumulator
|
||||
output volumeAccumulator
|
||||
rejected volumeAccumulator
|
||||
technicalValid volumeAccumulator
|
||||
trainReady volumeAccumulator
|
||||
runs map[string]struct{}
|
||||
}
|
||||
|
||||
func normalizedDataOperationsTimezone(value string) *time.Location {
|
||||
name := strings.TrimSpace(value)
|
||||
if name == "" {
|
||||
name = domain.DefaultWorkflowScheduleTimezone
|
||||
}
|
||||
location, err := time.LoadLocation(name)
|
||||
if err != nil {
|
||||
location, _ = time.LoadLocation(domain.DefaultWorkflowScheduleTimezone)
|
||||
}
|
||||
if location == nil {
|
||||
location = time.FixedZone("UTC+8", 8*60*60)
|
||||
}
|
||||
return location
|
||||
}
|
||||
|
||||
func dataOperationsFactMatches(item domain.DataMetricFactRecord, organizationID string, query domain.DataOperationsQuery) bool {
|
||||
if !organizationMatches(item.OrganizationID, organizationID) {
|
||||
return false
|
||||
}
|
||||
if query.ProjectID != "" && item.ProjectID != query.ProjectID {
|
||||
return false
|
||||
}
|
||||
if query.TaskID != "" && item.TaskID != query.TaskID {
|
||||
return false
|
||||
}
|
||||
if query.RobotModel != "" && !strings.EqualFold(item.RobotModel, query.RobotModel) {
|
||||
return false
|
||||
}
|
||||
if query.Scene != "" && !strings.EqualFold(item.Scene, query.Scene) {
|
||||
return false
|
||||
}
|
||||
if query.Stage != "" && item.Stage != query.Stage {
|
||||
return false
|
||||
}
|
||||
if query.From != "" && item.OccurredAt < query.From {
|
||||
return false
|
||||
}
|
||||
if query.To != "" && item.OccurredAt >= query.To {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func datasetVersionMatches(item domain.DatasetVersionRecord, organizationID string, query domain.DataOperationsQuery) bool {
|
||||
if !organizationMatches(item.OrganizationID, organizationID) {
|
||||
return false
|
||||
}
|
||||
if query.ProjectID != "" && item.ProjectID != query.ProjectID {
|
||||
return false
|
||||
}
|
||||
if query.TaskID != "" && item.TaskID != query.TaskID {
|
||||
return false
|
||||
}
|
||||
if query.From != "" && item.CreatedAt < query.From {
|
||||
return false
|
||||
}
|
||||
if query.To != "" && item.CreatedAt >= query.To {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func encodeDataOperationsCursor(occurredAt string, id string) string {
|
||||
return base64.RawURLEncoding.EncodeToString([]byte(occurredAt + "\n" + id))
|
||||
}
|
||||
|
||||
func decodeDataOperationsCursor(value string) (string, string, bool) {
|
||||
if strings.TrimSpace(value) == "" {
|
||||
return "", "", true
|
||||
}
|
||||
payload, err := base64.RawURLEncoding.DecodeString(value)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
parts := strings.SplitN(string(payload), "\n", 2)
|
||||
if len(parts) != 2 || strings.TrimSpace(parts[0]) == "" || strings.TrimSpace(parts[1]) == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
|
||||
func factsPage(items []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.DataMetricFactPage {
|
||||
sort.Slice(items, func(left, right int) bool {
|
||||
if items[left].OccurredAt == items[right].OccurredAt {
|
||||
return items[left].ID > items[right].ID
|
||||
}
|
||||
return items[left].OccurredAt > items[right].OccurredAt
|
||||
})
|
||||
cursorTime, cursorID, validCursor := decodeDataOperationsCursor(query.Cursor)
|
||||
if !validCursor {
|
||||
return domain.DataMetricFactPage{Items: []domain.DataMetricFactRecord{}}
|
||||
}
|
||||
if cursorTime != "" {
|
||||
filtered := items[:0]
|
||||
for _, item := range items {
|
||||
if item.OccurredAt < cursorTime || (item.OccurredAt == cursorTime && item.ID < cursorID) {
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
}
|
||||
items = filtered
|
||||
}
|
||||
limit := query.Limit
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
if limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
page := domain.DataMetricFactPage{Items: items, HasMore: len(items) > limit}
|
||||
if page.HasMore {
|
||||
page.Items = items[:limit]
|
||||
last := page.Items[len(page.Items)-1]
|
||||
page.NextCursor = encodeDataOperationsCursor(last.OccurredAt, last.ID)
|
||||
}
|
||||
if page.Items == nil {
|
||||
page.Items = []domain.DataMetricFactRecord{}
|
||||
}
|
||||
return page
|
||||
}
|
||||
|
||||
func trainingUsageFromFacts(facts []domain.DataMetricFactRecord, query domain.DataOperationsQuery) domain.TrainingUsagePage {
|
||||
filtered := make([]domain.DataMetricFactRecord, 0)
|
||||
for _, item := range facts {
|
||||
if item.Stage == domain.DataMetricStageTrain {
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
}
|
||||
page := factsPage(filtered, query)
|
||||
items := make([]domain.TrainingUsageRecord, 0, len(page.Items))
|
||||
for _, fact := range page.Items {
|
||||
items = append(items, domain.TrainingUsageRecord{
|
||||
ID: fact.ID,
|
||||
TaskID: fact.TaskID,
|
||||
TaskName: fact.TaskName,
|
||||
ProjectID: fact.ProjectID,
|
||||
ProjectName: fact.ProjectName,
|
||||
WorkflowID: fact.WorkflowID,
|
||||
WorkflowName: fact.WorkflowName,
|
||||
RunID: fact.RunID,
|
||||
NodeID: fact.NodeID,
|
||||
NodeLabel: fact.NodeLabel,
|
||||
Model: fact.NodeLabel,
|
||||
Inputs: fact.TrainingInputs,
|
||||
Consumed: fact.Input,
|
||||
Quality: fact.Quality,
|
||||
OccurredAt: fact.OccurredAt,
|
||||
})
|
||||
}
|
||||
return domain.TrainingUsagePage{Items: items, NextCursor: page.NextCursor, HasMore: page.HasMore}
|
||||
}
|
||||
|
||||
func taskProgressPercent(target domain.DataVolume, collected domain.DataVolume) float64 {
|
||||
switch {
|
||||
case target.TrajectoryCount != nil && *target.TrajectoryCount > 0 && collected.TrajectoryCount != nil:
|
||||
return minFloat(100, float64(*collected.TrajectoryCount)*100/float64(*target.TrajectoryCount))
|
||||
case target.DurationSeconds != nil && *target.DurationSeconds > 0 && collected.DurationSeconds != nil:
|
||||
return minFloat(100, *collected.DurationSeconds*100 / *target.DurationSeconds)
|
||||
case target.Bytes != nil && *target.Bytes > 0 && collected.Bytes != nil:
|
||||
return minFloat(100, float64(*collected.Bytes)*100/float64(*target.Bytes))
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func minFloat(left float64, right float64) float64 {
|
||||
if left < right {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
|
||||
func BuildDataOperationsSummary(
|
||||
facts []domain.DataMetricFactRecord,
|
||||
tasks []domain.CollectionTaskRecord,
|
||||
query domain.DataOperationsQuery,
|
||||
) domain.DataOperationsSummary {
|
||||
location := normalizedDataOperationsTimezone(query.Timezone)
|
||||
accumulators := make(map[domain.DataMetricStage]*stageAccumulator, len(domain.DataMetricStages))
|
||||
for _, stage := range domain.DataMetricStages {
|
||||
accumulators[stage] = &stageAccumulator{runs: map[string]struct{}{}}
|
||||
}
|
||||
hourly := map[string]map[domain.DataMetricStage]*volumeAccumulator{}
|
||||
reasons := map[string]int64{}
|
||||
uniqueTraining := map[string]domain.DataVolume{}
|
||||
var trainingConsumed volumeAccumulator
|
||||
trainingRuns := map[string]struct{}{}
|
||||
taskCollected := map[string]*volumeAccumulator{}
|
||||
summary := domain.DataOperationsSummary{
|
||||
From: query.From,
|
||||
To: query.To,
|
||||
Timezone: location.String(),
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Freshness: "live",
|
||||
RejectionReasons: reasons,
|
||||
}
|
||||
|
||||
for _, fact := range facts {
|
||||
accumulator, ok := accumulators[fact.Stage]
|
||||
if !ok {
|
||||
summary.UnknownCount++
|
||||
continue
|
||||
}
|
||||
accumulator.input.add(fact.Input)
|
||||
accumulator.output.add(fact.Output)
|
||||
accumulator.rejected.add(fact.Rejected)
|
||||
accumulator.technicalValid.add(fact.TechnicalValid)
|
||||
accumulator.trainReady.add(fact.TrainReady)
|
||||
accumulator.runs[fact.RunID] = struct{}{}
|
||||
summary.FactCount++
|
||||
switch fact.Quality {
|
||||
case domain.DataMetricQualityExact:
|
||||
summary.ExactCount++
|
||||
case domain.DataMetricQualityEstimated:
|
||||
summary.EstimatedCount++
|
||||
case domain.DataMetricQualityDerived:
|
||||
summary.DerivedCount++
|
||||
default:
|
||||
summary.UnknownCount++
|
||||
}
|
||||
for reason, count := range fact.RejectionReasons {
|
||||
reasons[reason] += count
|
||||
}
|
||||
if occurredAt, err := time.Parse(time.RFC3339Nano, fact.OccurredAt); err == nil {
|
||||
hour := occurredAt.In(location).Format("2006-01-02T15:00:00-07:00")
|
||||
if hourly[hour] == nil {
|
||||
hourly[hour] = map[domain.DataMetricStage]*volumeAccumulator{}
|
||||
}
|
||||
if hourly[hour][fact.Stage] == nil {
|
||||
hourly[hour][fact.Stage] = &volumeAccumulator{}
|
||||
}
|
||||
hourly[hour][fact.Stage].add(fact.Output)
|
||||
}
|
||||
if fact.Stage == domain.DataMetricStageCollect && fact.TaskID != "" {
|
||||
if taskCollected[fact.TaskID] == nil {
|
||||
taskCollected[fact.TaskID] = &volumeAccumulator{}
|
||||
}
|
||||
taskCollected[fact.TaskID].add(fact.Output)
|
||||
}
|
||||
if fact.Stage == domain.DataMetricStageTrain {
|
||||
trainingConsumed.add(fact.Input)
|
||||
trainingRuns[fact.RunID] = struct{}{}
|
||||
for _, input := range fact.TrainingInputs {
|
||||
key := strings.TrimSpace(input.DatasetVersionID)
|
||||
if key == "" {
|
||||
key = strings.TrimSpace(input.DatasetID) + "|" + strings.TrimSpace(input.ManifestPath) + "|" + strings.TrimSpace(input.Split)
|
||||
}
|
||||
if key == "||" || key == "" {
|
||||
key = fmt.Sprintf("%s|%s", fact.RunID, fact.NodeID)
|
||||
}
|
||||
if _, exists := uniqueTraining[key]; !exists {
|
||||
uniqueTraining[key] = input.Volume
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
summary.Stages = make([]domain.DataStageSummary, 0, len(domain.DataMetricStages))
|
||||
for _, stage := range domain.DataMetricStages {
|
||||
item := accumulators[stage]
|
||||
summary.Stages = append(summary.Stages, domain.DataStageSummary{
|
||||
Stage: stage,
|
||||
Input: item.input.value(),
|
||||
Output: item.output.value(),
|
||||
Rejected: item.rejected.value(),
|
||||
TechnicalValid: item.technicalValid.value(),
|
||||
TrainReady: item.trainReady.value(),
|
||||
RunCount: int64(len(item.runs)),
|
||||
})
|
||||
}
|
||||
var uniqueAccumulator volumeAccumulator
|
||||
for _, volume := range uniqueTraining {
|
||||
uniqueAccumulator.add(volume)
|
||||
}
|
||||
summary.Training = domain.DataOperationsTrainingSummary{
|
||||
Consumed: trainingConsumed.value(),
|
||||
Unique: uniqueAccumulator.value(),
|
||||
RunCount: int64(len(trainingRuns)),
|
||||
}
|
||||
|
||||
hours := make([]string, 0, len(hourly))
|
||||
for hour := range hourly {
|
||||
hours = append(hours, hour)
|
||||
}
|
||||
sort.Strings(hours)
|
||||
summary.Hourly = make([]domain.DataOperationsHourlyPoint, 0, len(hours))
|
||||
for _, hour := range hours {
|
||||
point := domain.DataOperationsHourlyPoint{Hour: hour, Stages: map[domain.DataMetricStage]domain.DataVolume{}}
|
||||
for stage, value := range hourly[hour] {
|
||||
point.Stages[stage] = value.value()
|
||||
}
|
||||
summary.Hourly = append(summary.Hourly, point)
|
||||
}
|
||||
|
||||
summary.Tasks = make([]domain.CollectionTaskProgress, 0)
|
||||
for _, task := range tasks {
|
||||
if query.ProjectID != "" && task.ProjectID != query.ProjectID {
|
||||
continue
|
||||
}
|
||||
if query.TaskID != "" && task.ID != query.TaskID {
|
||||
continue
|
||||
}
|
||||
if query.RobotModel != "" && !strings.EqualFold(task.RobotModel, query.RobotModel) {
|
||||
continue
|
||||
}
|
||||
if query.Scene != "" && !strings.EqualFold(task.Scene, query.Scene) {
|
||||
continue
|
||||
}
|
||||
collected := domain.DataVolume{}
|
||||
if taskCollected[task.ID] != nil {
|
||||
collected = taskCollected[task.ID].value()
|
||||
}
|
||||
summary.Tasks = append(summary.Tasks, domain.CollectionTaskProgress{
|
||||
TaskID: task.ID,
|
||||
TaskName: task.Name,
|
||||
Status: task.Status,
|
||||
Targets: task.Targets,
|
||||
Collected: collected,
|
||||
Percent: taskProgressPercent(task.Targets, collected),
|
||||
})
|
||||
}
|
||||
sort.Slice(summary.Tasks, func(left, right int) bool { return summary.Tasks[left].TaskName < summary.Tasks[right].TaskName })
|
||||
return summary
|
||||
}
|
||||
240
internal/store/data_operations_memory.go
Normal file
240
internal/store/data_operations_memory.go
Normal file
@ -0,0 +1,240 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
func (store *MemoryStore) ListCollectionTasks(_ context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
items := make([]domain.CollectionTaskRecord, 0)
|
||||
for _, item := range store.collectionTasks {
|
||||
if organizationMatches(item.OrganizationID, organizationID) {
|
||||
items = append(items, clone(item))
|
||||
}
|
||||
}
|
||||
sort.Slice(items, func(left, right int) bool { return items[left].UpdatedAt > items[right].UpdatedAt })
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) GetCollectionTask(_ context.Context, id string) (*domain.CollectionTaskRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
item, ok := store.collectionTasks[id]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
cloned := clone(item)
|
||||
return &cloned, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) UpsertCollectionTask(_ context.Context, item domain.CollectionTaskRecord) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
if strings.TrimSpace(item.ID) == "" {
|
||||
item.ID = domain.NewID("data-task")
|
||||
}
|
||||
now := nowISO()
|
||||
if item.CreatedAt == "" {
|
||||
item.CreatedAt = now
|
||||
}
|
||||
item.UpdatedAt = now
|
||||
store.collectionTasks[item.ID] = clone(item)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) UpsertDatasetVersion(_ context.Context, item domain.DatasetVersionRecord) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
if strings.TrimSpace(item.Fingerprint) == "" {
|
||||
return fmt.Errorf("dataset fingerprint is required")
|
||||
}
|
||||
for id, existing := range store.datasetVersions {
|
||||
if organizationMatches(existing.OrganizationID, item.OrganizationID) && existing.Fingerprint == item.Fingerprint {
|
||||
if dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
|
||||
return nil
|
||||
}
|
||||
item.ID = id
|
||||
item.CreatedAt = existing.CreatedAt
|
||||
break
|
||||
}
|
||||
}
|
||||
if item.ID == "" {
|
||||
item.ID = domain.NewID("dataset-version")
|
||||
}
|
||||
now := nowISO()
|
||||
if item.CreatedAt == "" {
|
||||
item.CreatedAt = now
|
||||
}
|
||||
item.UpdatedAt = now
|
||||
store.datasetVersions[item.ID] = clone(item)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListDatasetVersions(_ context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
items := make([]domain.DatasetVersionRecord, 0)
|
||||
for _, item := range store.datasetVersions {
|
||||
if datasetVersionMatches(item, organizationID, query) {
|
||||
items = append(items, clone(item))
|
||||
}
|
||||
}
|
||||
sort.Slice(items, func(left, right int) bool { return items[left].CreatedAt > items[right].CreatedAt })
|
||||
limit := query.Limit
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) UpsertDataMetricFact(_ context.Context, item domain.DataMetricFactRecord) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
if strings.TrimSpace(item.SourceKey) == "" {
|
||||
return fmt.Errorf("metric source key is required")
|
||||
}
|
||||
for id, existing := range store.dataMetricFacts {
|
||||
if existing.SourceKey == item.SourceKey {
|
||||
if dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
|
||||
return nil
|
||||
}
|
||||
item.ID = id
|
||||
break
|
||||
}
|
||||
}
|
||||
if item.ID == "" {
|
||||
item.ID = domain.NewID("data-fact")
|
||||
}
|
||||
if item.RecordedAt == "" {
|
||||
item.RecordedAt = nowISO()
|
||||
}
|
||||
store.dataMetricFacts[item.ID] = clone(item)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) filteredDataMetricFactsLocked(organizationID string, query domain.DataOperationsQuery) []domain.DataMetricFactRecord {
|
||||
items := make([]domain.DataMetricFactRecord, 0)
|
||||
for _, item := range store.dataMetricFacts {
|
||||
if dataOperationsFactMatches(item, organizationID, query) {
|
||||
items = append(items, clone(item))
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListDataMetricFactsPage(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
return factsPage(store.filteredDataMetricFactsLocked(organizationID, query), query), nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) GetDataOperationsSummary(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
facts := store.filteredDataMetricFactsLocked(organizationID, query)
|
||||
tasks := make([]domain.CollectionTaskRecord, 0)
|
||||
for _, item := range store.collectionTasks {
|
||||
if organizationMatches(item.OrganizationID, organizationID) {
|
||||
tasks = append(tasks, clone(item))
|
||||
}
|
||||
}
|
||||
return BuildDataOperationsSummary(facts, tasks, query), nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListTrainingUsagePage(_ context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
return trainingUsageFromFacts(store.filteredDataMetricFactsLocked(organizationID, query), query), nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListDataOperationsBackfillRuns(
|
||||
_ context.Context,
|
||||
organizationID string,
|
||||
from string,
|
||||
cursor *domain.RunSummaryCursor,
|
||||
limit int,
|
||||
) (domain.DataOperationsRunPage, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
if limit > 500 {
|
||||
limit = 500
|
||||
}
|
||||
items := make([]domain.RunRecord, 0)
|
||||
for _, run := range store.runs {
|
||||
if !organizationMatches(run.OrganizationID, organizationID) || run.CreatedAt < from {
|
||||
continue
|
||||
}
|
||||
if cursor != nil && !(run.CreatedAt < cursor.CreatedAt || (run.CreatedAt == cursor.CreatedAt && run.ID < cursor.ID)) {
|
||||
continue
|
||||
}
|
||||
items = append(items, clone(run))
|
||||
}
|
||||
sort.Slice(items, func(left, right int) bool {
|
||||
if items[left].CreatedAt == items[right].CreatedAt {
|
||||
return items[left].ID > items[right].ID
|
||||
}
|
||||
return items[left].CreatedAt > items[right].CreatedAt
|
||||
})
|
||||
page := domain.DataOperationsRunPage{HasMore: len(items) > limit}
|
||||
if page.HasMore {
|
||||
items = items[:limit]
|
||||
}
|
||||
page.Items = items
|
||||
if page.HasMore && len(items) > 0 {
|
||||
last := items[len(items)-1]
|
||||
page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListRunsNeedingMetricMaterialization(_ context.Context, limit int) ([]domain.RunRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
items := make([]domain.RunRecord, 0)
|
||||
for _, run := range store.runs {
|
||||
for _, execution := range run.NodeExecutions {
|
||||
if execution.DataMetrics != nil && execution.MetricsMaterializedAt == "" {
|
||||
items = append(items, clone(run))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sortRuns(items)
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if len(items) > limit {
|
||||
items = items[:limit]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) MarkNodeMetricsMaterialized(_ context.Context, runID string, nodeID string, attempt int, at string) error {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
run, ok := store.runs[runID]
|
||||
if !ok {
|
||||
return fmt.Errorf("run %s not found", runID)
|
||||
}
|
||||
for index := range run.NodeExecutions {
|
||||
execution := &run.NodeExecutions[index]
|
||||
if execution.NodeID == nodeID && execution.Attempt == attempt {
|
||||
execution.MetricsMaterializedAt = at
|
||||
store.runs[runID] = run
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("node execution %s attempt %d not found", nodeID, attempt)
|
||||
}
|
||||
300
internal/store/data_operations_mongo.go
Normal file
300
internal/store/data_operations_mongo.go
Normal file
@ -0,0 +1,300 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
func dataOperationsMongoFilter(organizationID string, query domain.DataOperationsQuery) bson.M {
|
||||
extra := bson.M{}
|
||||
if query.ProjectID != "" {
|
||||
extra["projectId"] = query.ProjectID
|
||||
}
|
||||
if query.TaskID != "" {
|
||||
extra["taskId"] = query.TaskID
|
||||
}
|
||||
if query.RobotModel != "" {
|
||||
extra["robotModel"] = query.RobotModel
|
||||
}
|
||||
if query.Scene != "" {
|
||||
extra["scene"] = query.Scene
|
||||
}
|
||||
if query.Stage != "" {
|
||||
extra["stage"] = query.Stage
|
||||
}
|
||||
timeRange := bson.M{}
|
||||
if query.From != "" {
|
||||
timeRange["$gte"] = query.From
|
||||
}
|
||||
if query.To != "" {
|
||||
timeRange["$lt"] = query.To
|
||||
}
|
||||
if len(timeRange) > 0 {
|
||||
extra["occurredAt"] = timeRange
|
||||
}
|
||||
return organizationScopedFilter(organizationID, extra)
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListCollectionTasks(ctx context.Context, organizationID string) ([]domain.CollectionTaskRecord, error) {
|
||||
cursor, err := store.collectionTasks.Find(ctx, orgCompatibilityFilter(organizationID), options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeMany[domain.CollectionTaskRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) GetCollectionTask(ctx context.Context, id string) (*domain.CollectionTaskRecord, error) {
|
||||
var item domain.CollectionTaskRecord
|
||||
err := store.collectionTasks.FindOne(ctx, bson.M{"id": id}).Decode(&item)
|
||||
if err == mongo.ErrNoDocuments {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) UpsertCollectionTask(ctx context.Context, item domain.CollectionTaskRecord) error {
|
||||
if item.ID == "" {
|
||||
item.ID = domain.NewID("data-task")
|
||||
}
|
||||
now := nowISO()
|
||||
if item.CreatedAt == "" {
|
||||
item.CreatedAt = now
|
||||
}
|
||||
item.UpdatedAt = now
|
||||
_, err := store.collectionTasks.ReplaceOne(ctx, bson.M{"id": item.ID}, item, options.Replace().SetUpsert(true))
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *MongoStore) UpsertDatasetVersion(ctx context.Context, item domain.DatasetVersionRecord) error {
|
||||
if strings.TrimSpace(item.Fingerprint) == "" {
|
||||
return fmt.Errorf("dataset fingerprint is required")
|
||||
}
|
||||
var existing domain.DatasetVersionRecord
|
||||
existingErr := store.datasetVersions.FindOne(
|
||||
ctx,
|
||||
organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint}),
|
||||
options.FindOne().SetProjection(bson.M{"quality": 1}),
|
||||
).Decode(&existing)
|
||||
if existingErr != nil && existingErr != mongo.ErrNoDocuments {
|
||||
return existingErr
|
||||
}
|
||||
if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
|
||||
return nil
|
||||
}
|
||||
now := nowISO()
|
||||
if item.ID == "" {
|
||||
item.ID = domain.NewID("dataset-version")
|
||||
}
|
||||
if item.CreatedAt == "" {
|
||||
item.CreatedAt = now
|
||||
}
|
||||
item.UpdatedAt = now
|
||||
filter := organizationScopedFilter(item.OrganizationID, bson.M{"fingerprint": item.Fingerprint})
|
||||
update := bson.M{
|
||||
"$set": bson.M{
|
||||
"datasetId": item.DatasetID, "taskId": item.TaskID, "projectId": item.ProjectID,
|
||||
"name": item.Name, "split": item.Split, "fingerprint": item.Fingerprint, "volume": item.Volume,
|
||||
"storage": item.Storage, "manifestPath": item.ManifestPath, "partitionDir": item.PartitionDir,
|
||||
"producerRunId": item.ProducerRunID, "producerNodeId": item.ProducerNodeID, "artifactId": item.ArtifactID,
|
||||
"quality": item.Quality, "updatedAt": now, "organizationId": normalizedOrganizationID(item.OrganizationID),
|
||||
},
|
||||
"$setOnInsert": bson.M{"id": item.ID, "createdAt": item.CreatedAt},
|
||||
}
|
||||
_, err := store.datasetVersions.UpdateOne(ctx, filter, update, options.Update().SetUpsert(true))
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListDatasetVersions(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error) {
|
||||
extra := bson.M{}
|
||||
if query.ProjectID != "" {
|
||||
extra["projectId"] = query.ProjectID
|
||||
}
|
||||
if query.TaskID != "" {
|
||||
extra["taskId"] = query.TaskID
|
||||
}
|
||||
timeRange := bson.M{}
|
||||
if query.From != "" {
|
||||
timeRange["$gte"] = query.From
|
||||
}
|
||||
if query.To != "" {
|
||||
timeRange["$lt"] = query.To
|
||||
}
|
||||
if len(timeRange) > 0 {
|
||||
extra["createdAt"] = timeRange
|
||||
}
|
||||
limit := int64(query.Limit)
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
cursor, err := store.datasetVersions.Find(
|
||||
ctx,
|
||||
organizationScopedFilter(organizationID, extra),
|
||||
options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}}).SetLimit(limit),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeMany[domain.DatasetVersionRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) UpsertDataMetricFact(ctx context.Context, item domain.DataMetricFactRecord) error {
|
||||
if strings.TrimSpace(item.SourceKey) == "" {
|
||||
return fmt.Errorf("metric source key is required")
|
||||
}
|
||||
if item.ID == "" {
|
||||
item.ID = domain.NewID("data-fact")
|
||||
}
|
||||
if item.RecordedAt == "" {
|
||||
item.RecordedAt = nowISO()
|
||||
}
|
||||
var existing domain.DataMetricFactRecord
|
||||
existingErr := store.dataMetricFacts.FindOne(
|
||||
ctx,
|
||||
bson.M{"sourceKey": item.SourceKey},
|
||||
options.FindOne().SetProjection(bson.M{"quality": 1}),
|
||||
).Decode(&existing)
|
||||
if existingErr != nil && existingErr != mongo.ErrNoDocuments {
|
||||
return existingErr
|
||||
}
|
||||
if existingErr == nil && dataMetricQualityRank(existing.Quality) > dataMetricQualityRank(item.Quality) {
|
||||
return nil
|
||||
}
|
||||
_, err := store.dataMetricFacts.ReplaceOne(ctx, bson.M{"sourceKey": item.SourceKey}, item, options.Replace().SetUpsert(true))
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *MongoStore) findDataMetricFacts(ctx context.Context, organizationID string, query domain.DataOperationsQuery) ([]domain.DataMetricFactRecord, error) {
|
||||
cursor, err := store.dataMetricFacts.Find(
|
||||
ctx,
|
||||
dataOperationsMongoFilter(organizationID, query),
|
||||
options.Find().SetSort(bson.D{{Key: "occurredAt", Value: -1}, {Key: "id", Value: -1}}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeMany[domain.DataMetricFactRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListDataMetricFactsPage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataMetricFactPage, error) {
|
||||
items, err := store.findDataMetricFacts(ctx, organizationID, query)
|
||||
if err != nil {
|
||||
return domain.DataMetricFactPage{}, err
|
||||
}
|
||||
return factsPage(items, query), nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) GetDataOperationsSummary(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.DataOperationsSummary, error) {
|
||||
facts, err := store.findDataMetricFacts(ctx, organizationID, query)
|
||||
if err != nil {
|
||||
return domain.DataOperationsSummary{}, err
|
||||
}
|
||||
tasks, err := store.ListCollectionTasks(ctx, organizationID)
|
||||
if err != nil {
|
||||
return domain.DataOperationsSummary{}, err
|
||||
}
|
||||
return BuildDataOperationsSummary(facts, tasks, query), nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListTrainingUsagePage(ctx context.Context, organizationID string, query domain.DataOperationsQuery) (domain.TrainingUsagePage, error) {
|
||||
query.Stage = domain.DataMetricStageTrain
|
||||
items, err := store.findDataMetricFacts(ctx, organizationID, query)
|
||||
if err != nil {
|
||||
return domain.TrainingUsagePage{}, err
|
||||
}
|
||||
return trainingUsageFromFacts(items, query), nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListDataOperationsBackfillRuns(
|
||||
ctx context.Context,
|
||||
organizationID string,
|
||||
from string,
|
||||
pageCursor *domain.RunSummaryCursor,
|
||||
limit int,
|
||||
) (domain.DataOperationsRunPage, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
if limit > 500 {
|
||||
limit = 500
|
||||
}
|
||||
filter := organizationScopedFilter(organizationID, bson.M{"createdAt": bson.M{"$gte": from}})
|
||||
filter = combineRunSummaryFilters(filter, runSummaryCursorFilter(pageCursor))
|
||||
cursor, err := store.runs.Find(
|
||||
ctx,
|
||||
filter,
|
||||
options.Find().
|
||||
SetProjection(bson.M{
|
||||
"id": 1, "organizationId": 1, "workflowId": 1, "workflowName": 1,
|
||||
"projectId": 1, "projectName": 1, "taskId": 1, "status": 1,
|
||||
"createdAt": 1, "updatedAt": 1, "endedAt": 1, "nodeExecutions": 1,
|
||||
}).
|
||||
SetSort(bson.D{{Key: "createdAt", Value: -1}, {Key: "id", Value: -1}}).
|
||||
SetLimit(int64(limit+1)),
|
||||
)
|
||||
if err != nil {
|
||||
return domain.DataOperationsRunPage{}, err
|
||||
}
|
||||
items, err := decodeMany[domain.RunRecord](ctx, cursor)
|
||||
if err != nil {
|
||||
return domain.DataOperationsRunPage{}, err
|
||||
}
|
||||
page := domain.DataOperationsRunPage{HasMore: len(items) > limit}
|
||||
if page.HasMore {
|
||||
items = items[:limit]
|
||||
}
|
||||
page.Items = items
|
||||
if page.HasMore && len(items) > 0 {
|
||||
last := items[len(items)-1]
|
||||
page.NextCursor = &domain.RunSummaryCursor{CreatedAt: last.CreatedAt, ID: last.ID}
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListRunsNeedingMetricMaterialization(ctx context.Context, limit int) ([]domain.RunRecord, error) {
|
||||
if limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
cursor, err := store.runs.Find(
|
||||
ctx,
|
||||
bson.M{"nodeExecutions": bson.M{"$elemMatch": bson.M{
|
||||
"dataMetrics": bson.M{"$exists": true},
|
||||
"$or": bson.A{
|
||||
bson.M{"metricsMaterializedAt": bson.M{"$exists": false}},
|
||||
bson.M{"metricsMaterializedAt": ""},
|
||||
},
|
||||
}}},
|
||||
options.Find().SetSort(bson.D{{Key: "updatedAt", Value: -1}}).SetLimit(int64(limit)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeMany[domain.RunRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) MarkNodeMetricsMaterialized(ctx context.Context, runID string, nodeID string, attempt int, at string) error {
|
||||
result, err := store.runs.UpdateOne(
|
||||
ctx,
|
||||
bson.M{"id": runID},
|
||||
bson.M{"$set": bson.M{"nodeExecutions.$[execution].metricsMaterializedAt": at}},
|
||||
options.Update().SetArrayFilters(options.ArrayFilters{Filters: []any{
|
||||
bson.M{"execution.nodeId": nodeID, "execution.attempt": attempt},
|
||||
}}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.MatchedCount == 0 {
|
||||
return fmt.Errorf("run %s not found", runID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
192
internal/store/data_operations_test.go
Normal file
192
internal/store/data_operations_test.go
Normal file
@ -0,0 +1,192 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/domain"
|
||||
)
|
||||
|
||||
func dataOperationsInt64(value int64) *int64 {
|
||||
return &value
|
||||
}
|
||||
|
||||
func dataOperationsFloat64(value float64) *float64 {
|
||||
return &value
|
||||
}
|
||||
|
||||
func TestMemoryDataOperationsSummarySeparatesConsumptionAndUniqueData(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := NewMemoryStore()
|
||||
task := domain.CollectionTaskRecord{
|
||||
ID: "task-y1",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: "project-y1",
|
||||
Name: "Y1 桌面整理",
|
||||
RobotModel: "Y1",
|
||||
Scene: "桌面整理",
|
||||
Status: domain.CollectionTaskStatusActive,
|
||||
Targets: domain.DataVolume{TrajectoryCount: dataOperationsInt64(100)},
|
||||
}
|
||||
if err := dataStore.UpsertCollectionTask(ctx, task); err != nil {
|
||||
t.Fatalf("upsert task: %v", err)
|
||||
}
|
||||
for _, item := range []domain.DataMetricFactRecord{
|
||||
{
|
||||
ID: "fact-collect",
|
||||
SourceKey: "run-collect/node/1/collect",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: "project-y1",
|
||||
TaskID: task.ID,
|
||||
TaskName: task.Name,
|
||||
RobotModel: "Y1",
|
||||
Scene: "桌面整理",
|
||||
RunID: "run-collect",
|
||||
NodeID: "collect",
|
||||
Attempt: 1,
|
||||
Stage: domain.DataMetricStageCollect,
|
||||
Output: domain.DataVolume{
|
||||
TrajectoryCount: dataOperationsInt64(40),
|
||||
DurationSeconds: dataOperationsFloat64(1200),
|
||||
},
|
||||
Quality: domain.DataMetricQualityExact,
|
||||
OccurredAt: "2026-07-23T01:00:00Z",
|
||||
},
|
||||
{
|
||||
ID: "fact-train-1",
|
||||
SourceKey: "run-train-1/node/1/train",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: "project-y1",
|
||||
TaskID: task.ID,
|
||||
RunID: "run-train-1",
|
||||
NodeID: "train",
|
||||
Attempt: 1,
|
||||
Stage: domain.DataMetricStageTrain,
|
||||
Input: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)},
|
||||
TrainingInputs: []domain.TrainingInputRecord{{
|
||||
DatasetVersionID: "dataset-version-y1",
|
||||
Split: "train",
|
||||
Volume: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)},
|
||||
}},
|
||||
Quality: domain.DataMetricQualityExact,
|
||||
OccurredAt: "2026-07-23T02:00:00Z",
|
||||
},
|
||||
{
|
||||
ID: "fact-train-2",
|
||||
SourceKey: "run-train-2/node/1/train",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
ProjectID: "project-y1",
|
||||
TaskID: task.ID,
|
||||
RunID: "run-train-2",
|
||||
NodeID: "train",
|
||||
Attempt: 1,
|
||||
Stage: domain.DataMetricStageTrain,
|
||||
Input: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)},
|
||||
TrainingInputs: []domain.TrainingInputRecord{{
|
||||
DatasetVersionID: "dataset-version-y1",
|
||||
Split: "train",
|
||||
Volume: domain.DataVolume{TrajectoryCount: dataOperationsInt64(40)},
|
||||
}},
|
||||
Quality: domain.DataMetricQualityDerived,
|
||||
OccurredAt: "2026-07-23T03:00:00Z",
|
||||
},
|
||||
} {
|
||||
if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil {
|
||||
t.Fatalf("upsert fact %s: %v", item.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
summary, err := dataStore.GetDataOperationsSummary(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{
|
||||
From: "2026-07-22T16:00:00Z",
|
||||
To: "2026-07-23T16:00:00Z",
|
||||
Timezone: "Asia/Shanghai",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("get summary: %v", err)
|
||||
}
|
||||
if summary.Training.Consumed.TrajectoryCount == nil || *summary.Training.Consumed.TrajectoryCount != 80 {
|
||||
t.Fatalf("consumed trajectories = %#v, want 80", summary.Training.Consumed.TrajectoryCount)
|
||||
}
|
||||
if summary.Training.Unique.TrajectoryCount == nil || *summary.Training.Unique.TrajectoryCount != 40 {
|
||||
t.Fatalf("unique trajectories = %#v, want 40", summary.Training.Unique.TrajectoryCount)
|
||||
}
|
||||
if summary.Training.RunCount != 2 {
|
||||
t.Fatalf("training run count = %d, want 2", summary.Training.RunCount)
|
||||
}
|
||||
if len(summary.Tasks) != 1 || summary.Tasks[0].Percent != 40 {
|
||||
t.Fatalf("task progress = %#v, want 40 percent", summary.Tasks)
|
||||
}
|
||||
if summary.ExactCount != 2 || summary.DerivedCount != 1 {
|
||||
t.Fatalf("quality counts = exact %d derived %d", summary.ExactCount, summary.DerivedCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryDataMetricUpsertIsIdempotent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := NewMemoryStore()
|
||||
item := domain.DataMetricFactRecord{
|
||||
ID: "fact-one",
|
||||
SourceKey: "run/node/1/collect",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
RunID: "run",
|
||||
NodeID: "node",
|
||||
Attempt: 1,
|
||||
Stage: domain.DataMetricStageCollect,
|
||||
Output: domain.DataVolume{TrajectoryCount: dataOperationsInt64(10)},
|
||||
Quality: domain.DataMetricQualityDerived,
|
||||
OccurredAt: "2026-07-23T01:00:00Z",
|
||||
}
|
||||
if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil {
|
||||
t.Fatalf("first upsert: %v", err)
|
||||
}
|
||||
item.ID = "fact-two"
|
||||
item.Output = domain.DataVolume{TrajectoryCount: dataOperationsInt64(12)}
|
||||
if err := dataStore.UpsertDataMetricFact(ctx, item); err != nil {
|
||||
t.Fatalf("second upsert: %v", err)
|
||||
}
|
||||
page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{
|
||||
From: "2026-07-23T00:00:00Z",
|
||||
To: "2026-07-24T00:00:00Z",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list facts: %v", err)
|
||||
}
|
||||
if len(page.Items) != 1 || page.Items[0].Output.TrajectoryCount == nil || *page.Items[0].Output.TrajectoryCount != 12 {
|
||||
t.Fatalf("idempotent items = %#v", page.Items)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemoryExactDataMetricIsNotDowngradedByBackfill(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := NewMemoryStore()
|
||||
exact := domain.DataMetricFactRecord{
|
||||
ID: "fact-exact",
|
||||
SourceKey: "run/node/1/collect",
|
||||
OrganizationID: domain.DefaultOrganizationID,
|
||||
RunID: "run",
|
||||
NodeID: "node",
|
||||
Attempt: 1,
|
||||
Stage: domain.DataMetricStageCollect,
|
||||
Output: domain.DataVolume{TrajectoryCount: dataOperationsInt64(10)},
|
||||
Quality: domain.DataMetricQualityExact,
|
||||
OccurredAt: "2026-07-23T01:00:00Z",
|
||||
}
|
||||
if err := dataStore.UpsertDataMetricFact(ctx, exact); err != nil {
|
||||
t.Fatalf("upsert exact fact: %v", err)
|
||||
}
|
||||
estimated := exact
|
||||
estimated.ID = "fact-estimated"
|
||||
estimated.Output = domain.DataVolume{TrajectoryCount: dataOperationsInt64(3)}
|
||||
estimated.Quality = domain.DataMetricQualityEstimated
|
||||
if err := dataStore.UpsertDataMetricFact(ctx, estimated); err != nil {
|
||||
t.Fatalf("upsert estimated fact: %v", err)
|
||||
}
|
||||
page, err := dataStore.ListDataMetricFactsPage(ctx, domain.DefaultOrganizationID, domain.DataOperationsQuery{})
|
||||
if err != nil {
|
||||
t.Fatalf("list facts: %v", err)
|
||||
}
|
||||
if len(page.Items) != 1 || page.Items[0].Quality != domain.DataMetricQualityExact ||
|
||||
page.Items[0].Output.TrajectoryCount == nil || *page.Items[0].Output.TrajectoryCount != 10 {
|
||||
t.Fatalf("facts = %#v", page.Items)
|
||||
}
|
||||
}
|
||||
@ -21,6 +21,9 @@ type MemoryStore struct {
|
||||
templates map[string]domain.TemplateRecord
|
||||
runs map[string]domain.RunRecord
|
||||
sourceStates map[string]domain.SourceStateRecord
|
||||
collectionTasks map[string]domain.CollectionTaskRecord
|
||||
datasetVersions map[string]domain.DatasetVersionRecord
|
||||
dataMetricFacts map[string]domain.DataMetricFactRecord
|
||||
auditTasks map[string]domain.AuditTaskRecord
|
||||
samplingTasks map[string]domain.SamplingTaskRecord
|
||||
users map[string]domain.UserRecord
|
||||
@ -42,6 +45,9 @@ func NewMemoryStore() *MemoryStore {
|
||||
templates: map[string]domain.TemplateRecord{},
|
||||
runs: map[string]domain.RunRecord{},
|
||||
sourceStates: map[string]domain.SourceStateRecord{},
|
||||
collectionTasks: map[string]domain.CollectionTaskRecord{},
|
||||
datasetVersions: map[string]domain.DatasetVersionRecord{},
|
||||
dataMetricFacts: map[string]domain.DataMetricFactRecord{},
|
||||
auditTasks: map[string]domain.AuditTaskRecord{},
|
||||
samplingTasks: map[string]domain.SamplingTaskRecord{},
|
||||
users: map[string]domain.UserRecord{},
|
||||
@ -276,6 +282,21 @@ func (store *MemoryStore) DeleteOrganizationCascade(_ context.Context, id string
|
||||
delete(store.sourceStates, key)
|
||||
}
|
||||
}
|
||||
for key, item := range store.collectionTasks {
|
||||
if organizationMatches(item.OrganizationID, organizationID) {
|
||||
delete(store.collectionTasks, key)
|
||||
}
|
||||
}
|
||||
for key, item := range store.datasetVersions {
|
||||
if organizationMatches(item.OrganizationID, organizationID) {
|
||||
delete(store.datasetVersions, key)
|
||||
}
|
||||
}
|
||||
for key, item := range store.dataMetricFacts {
|
||||
if organizationMatches(item.OrganizationID, organizationID) {
|
||||
delete(store.dataMetricFacts, key)
|
||||
}
|
||||
}
|
||||
for key, item := range store.auditTasks {
|
||||
if organizationMatches(item.OrganizationID, organizationID) {
|
||||
delete(store.auditTasks, key)
|
||||
@ -486,6 +507,7 @@ func (store *MemoryStore) CreateOperator(_ context.Context, input domain.CreateO
|
||||
SkillDefinition: input.SkillDefinition,
|
||||
Hyperparams: clone(input.Hyperparams),
|
||||
Resources: clone(input.Resources),
|
||||
BusinessStage: input.BusinessStage,
|
||||
PublicationStatus: input.PublicationStatus,
|
||||
GenerationSource: input.GenerationSource,
|
||||
PublicationEvidence: clone(input.PublicationEvidence),
|
||||
@ -842,6 +864,7 @@ func summarizeRun(run domain.RunRecord) domain.RunSummary {
|
||||
WorkflowName: run.WorkflowName,
|
||||
ProjectID: run.ProjectID,
|
||||
ProjectName: run.ProjectName,
|
||||
TaskID: run.TaskID,
|
||||
Trigger: run.Trigger,
|
||||
Attempt: run.Attempt,
|
||||
ParentRunID: run.ParentRunID,
|
||||
@ -1140,7 +1163,7 @@ func (store *MemoryStore) GetRun(_ context.Context, id string) (*domain.RunRecor
|
||||
return &cloned, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigger domain.RunTrigger) (*domain.RunRecord, error) {
|
||||
func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
workflow, ok := store.workflows[workflowID]
|
||||
@ -1187,6 +1210,9 @@ func (store *MemoryStore) CreateRun(_ context.Context, workflowID string, trigge
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if len(taskIDs) > 0 {
|
||||
run.TaskID = strings.TrimSpace(taskIDs[0])
|
||||
}
|
||||
store.runs[run.ID] = clone(run)
|
||||
|
||||
workflow.LastRunID = run.ID
|
||||
|
||||
@ -28,6 +28,9 @@ type MongoStore struct {
|
||||
templates *mongo.Collection
|
||||
runs *mongo.Collection
|
||||
sourceStates *mongo.Collection
|
||||
collectionTasks *mongo.Collection
|
||||
datasetVersions *mongo.Collection
|
||||
dataMetricFacts *mongo.Collection
|
||||
auditTasks *mongo.Collection
|
||||
samplingTasks *mongo.Collection
|
||||
users *mongo.Collection
|
||||
@ -49,6 +52,9 @@ func NewMongoStore(db *mongo.Database) *MongoStore {
|
||||
templates: db.Collection("templates"),
|
||||
runs: db.Collection("runs"),
|
||||
sourceStates: db.Collection("sourceStates"),
|
||||
collectionTasks: db.Collection("collectionTasks"),
|
||||
datasetVersions: db.Collection("datasetVersions"),
|
||||
dataMetricFacts: db.Collection("dataMetricFacts"),
|
||||
auditTasks: db.Collection("auditTasks"),
|
||||
samplingTasks: db.Collection("samplingTasks"),
|
||||
users: db.Collection("users"),
|
||||
@ -350,6 +356,7 @@ func (store *MongoStore) CreateOperator(ctx context.Context, input domain.Create
|
||||
SkillDefinition: input.SkillDefinition,
|
||||
Hyperparams: input.Hyperparams,
|
||||
Resources: input.Resources,
|
||||
BusinessStage: input.BusinessStage,
|
||||
PublicationStatus: input.PublicationStatus,
|
||||
GenerationSource: input.GenerationSource,
|
||||
PublicationEvidence: input.PublicationEvidence,
|
||||
@ -725,6 +732,7 @@ func runSummaryProjection() bson.M {
|
||||
"workflowName": 1,
|
||||
"projectId": 1,
|
||||
"projectName": 1,
|
||||
"taskId": 1,
|
||||
"trigger": 1,
|
||||
"attempt": 1,
|
||||
"parentRunId": 1,
|
||||
@ -1104,7 +1112,7 @@ func (store *MongoStore) GetRun(ctx context.Context, id string) (*domain.RunReco
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger) (*domain.RunRecord, error) {
|
||||
func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigger domain.RunTrigger, taskIDs ...string) (*domain.RunRecord, error) {
|
||||
workflow, err := store.GetWorkflow(ctx, workflowID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -1152,6 +1160,9 @@ func (store *MongoStore) CreateRun(ctx context.Context, workflowID string, trigg
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if len(taskIDs) > 0 {
|
||||
run.TaskID = strings.TrimSpace(taskIDs[0])
|
||||
}
|
||||
if err := store.UpsertRun(ctx, run); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -88,6 +88,23 @@ func (store *MongoStore) EnsureIndexes(ctx context.Context) error {
|
||||
index("organizationId_workflow_node", bson.D{{Key: "organizationId", Value: 1}, {Key: "workflowId", Value: 1}, {Key: "nodeId", Value: 1}}),
|
||||
index("workflowId_nodeId", bson.D{{Key: "workflowId", Value: 1}, {Key: "nodeId", Value: 1}}),
|
||||
},
|
||||
store.collectionTasks: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_status_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "updatedAt", Value: -1}}),
|
||||
index("organizationId_projectId_updatedAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "updatedAt", Value: -1}}),
|
||||
},
|
||||
store.datasetVersions: {
|
||||
uniqueIndex("organizationId_fingerprint_unique", bson.D{{Key: "organizationId", Value: 1}, {Key: "fingerprint", Value: 1}}),
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_projectId_taskId_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
},
|
||||
store.dataMetricFacts: {
|
||||
uniqueIndex("sourceKey_unique", bson.D{{Key: "sourceKey", Value: 1}}),
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_projectId_taskId_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "projectId", Value: 1}, {Key: "taskId", Value: 1}, {Key: "occurredAt", Value: -1}}),
|
||||
index("organizationId_stage_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "stage", Value: 1}, {Key: "occurredAt", Value: -1}}),
|
||||
index("organizationId_robotModel_scene_occurredAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "robotModel", Value: 1}, {Key: "scene", Value: 1}, {Key: "occurredAt", Value: -1}}),
|
||||
},
|
||||
store.auditTasks: {
|
||||
uniqueIndex("id_unique", bson.D{{Key: "id", Value: 1}}),
|
||||
index("organizationId_status_createdAt", bson.D{{Key: "organizationId", Value: 1}, {Key: "status", Value: 1}, {Key: "createdAt", Value: -1}}),
|
||||
|
||||
@ -160,6 +160,9 @@ func (store *MongoStore) normalizeLegacyDefaultOrganizationReferences(ctx contex
|
||||
store.templates,
|
||||
store.runs,
|
||||
store.sourceStates,
|
||||
store.collectionTasks,
|
||||
store.datasetVersions,
|
||||
store.dataMetricFacts,
|
||||
store.auditTasks,
|
||||
store.samplingTasks,
|
||||
store.users,
|
||||
@ -228,7 +231,9 @@ func (store *MongoStore) dropLegacyBuiltinOperatorDeletionIDIndex(ctx context.Co
|
||||
return nil
|
||||
}
|
||||
var commandError mongo.CommandError
|
||||
if errors.As(err, &commandError) && (commandError.Name == "IndexNotFound" || commandError.HasErrorCode(27)) {
|
||||
if errors.As(err, &commandError) &&
|
||||
(commandError.Name == "IndexNotFound" || commandError.Name == "NamespaceNotFound" ||
|
||||
commandError.HasErrorCode(27) || commandError.HasErrorCode(26)) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
|
||||
@ -80,7 +80,7 @@ type Store interface {
|
||||
CountWorkerRuns(context.Context) (WorkerRunCounts, error)
|
||||
HasActiveRun(context.Context, string) (bool, error)
|
||||
GetRun(context.Context, string) (*domain.RunRecord, error)
|
||||
CreateRun(context.Context, string, domain.RunTrigger) (*domain.RunRecord, error)
|
||||
CreateRun(context.Context, string, domain.RunTrigger, ...string) (*domain.RunRecord, error)
|
||||
ClaimNextQueuedRun(context.Context, string, time.Duration) (*domain.RunRecord, error)
|
||||
ClaimQueuedRun(context.Context, string, string, time.Duration) (*domain.RunRecord, error)
|
||||
AcquireRunLease(context.Context, string, string, time.Duration) (*domain.RunRecord, bool, error)
|
||||
@ -101,6 +101,19 @@ type Store interface {
|
||||
DeleteSourceState(context.Context, string, string, string) error
|
||||
DeleteSourceStatesByWorkflow(context.Context, string) error
|
||||
|
||||
ListCollectionTasks(context.Context, string) ([]domain.CollectionTaskRecord, error)
|
||||
GetCollectionTask(context.Context, string) (*domain.CollectionTaskRecord, error)
|
||||
UpsertCollectionTask(context.Context, domain.CollectionTaskRecord) error
|
||||
UpsertDatasetVersion(context.Context, domain.DatasetVersionRecord) error
|
||||
ListDatasetVersions(context.Context, string, domain.DataOperationsQuery) ([]domain.DatasetVersionRecord, error)
|
||||
UpsertDataMetricFact(context.Context, domain.DataMetricFactRecord) error
|
||||
ListDataMetricFactsPage(context.Context, string, domain.DataOperationsQuery) (domain.DataMetricFactPage, error)
|
||||
GetDataOperationsSummary(context.Context, string, domain.DataOperationsQuery) (domain.DataOperationsSummary, error)
|
||||
ListTrainingUsagePage(context.Context, string, domain.DataOperationsQuery) (domain.TrainingUsagePage, error)
|
||||
ListDataOperationsBackfillRuns(context.Context, string, string, *domain.RunSummaryCursor, int) (domain.DataOperationsRunPage, error)
|
||||
ListRunsNeedingMetricMaterialization(context.Context, int) ([]domain.RunRecord, error)
|
||||
MarkNodeMetricsMaterialized(context.Context, string, string, int, string) error
|
||||
|
||||
ListAuditTasks(context.Context, string) ([]domain.AuditTaskRecord, error)
|
||||
ListAuditTasksByOrganization(context.Context, string, string) ([]domain.AuditTaskRecord, error)
|
||||
GetAuditTask(context.Context, string) (*domain.AuditTaskRecord, error)
|
||||
|
||||
@ -33,12 +33,14 @@ type executionResult struct {
|
||||
Logs []domain.LogEntryRecord `json:"logs"`
|
||||
Artifacts []domain.ExecutionArtifact `json:"artifacts"`
|
||||
DataRefs []domain.DataRefRecord `json:"dataRefs,omitempty"`
|
||||
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
|
||||
StageManifestPath string `json:"stageManifestPath,omitempty"`
|
||||
ResourceSeries []domain.ResourceSample `json:"resourceSeries"`
|
||||
}
|
||||
|
||||
type stageManifestRecord struct {
|
||||
DataRefs []domain.DataRefRecord `json:"dataRefs"`
|
||||
DataRefs []domain.DataRefRecord `json:"dataRefs"`
|
||||
DataMetrics *domain.DataMetricsRecord `json:"dataMetrics,omitempty"`
|
||||
}
|
||||
|
||||
var protectedSystemEnv = map[string]struct{}{
|
||||
@ -593,7 +595,49 @@ func loadStageManifestDataRefs(path string) ([]domain.DataRefRecord, error) {
|
||||
return manifest.DataRefs, nil
|
||||
}
|
||||
|
||||
func loadStageManifestDataMetrics(path string) (*domain.DataMetricsRecord, error) {
|
||||
trimmed := strings.TrimSpace(path)
|
||||
if trimmed == "" {
|
||||
return nil, nil
|
||||
}
|
||||
payload, err := os.ReadFile(trimmed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var manifest stageManifestRecord
|
||||
if err := json.Unmarshal(payload, &manifest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return manifest.DataMetrics, nil
|
||||
}
|
||||
|
||||
func (runtime *Runtime) resolveNodeBusinessStage(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode) domain.WorkflowNode {
|
||||
if run != nil && strings.TrimSpace(run.TaskID) != "" {
|
||||
if task, err := runtime.store.GetCollectionTask(ctx, run.TaskID); err == nil && task != nil {
|
||||
for stage, nodeID := range task.StageBindings {
|
||||
if strings.TrimSpace(nodeID) == node.ID {
|
||||
node.Data.BusinessStage = stage
|
||||
node.Data.MetricBoundary = true
|
||||
return node
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if node.Data.BusinessStage != "" {
|
||||
node.Data.MetricBoundary = true
|
||||
return node
|
||||
}
|
||||
if strings.TrimSpace(node.Data.OperatorID) != "" {
|
||||
if operator, err := runtime.store.GetOperator(ctx, node.Data.OperatorID); err == nil && operator != nil && operator.BusinessStage != "" {
|
||||
node.Data.BusinessStage = operator.BusinessStage
|
||||
node.Data.MetricBoundary = true
|
||||
}
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord, node domain.WorkflowNode, inputArtifacts []domain.ExecutionArtifact) (executionResult, *k8s.JobReference, error) {
|
||||
node = runtime.resolveNodeBusinessStage(ctx, run, node)
|
||||
workspace, err := prepareNodeWorkspace(runtime.workspaceRoot, run.ID, node.ID)
|
||||
if err != nil {
|
||||
return executionResult{}, nil, err
|
||||
@ -675,6 +719,9 @@ func (runtime *Runtime) executeNode(ctx context.Context, run *domain.RunRecord,
|
||||
if err == nil {
|
||||
parsed.StageManifestPath = workspace.StageManifestPath
|
||||
parsed.DataRefs, err = loadStageManifestDataRefs(workspace.StageManifestPath)
|
||||
if err == nil && parsed.DataMetrics == nil {
|
||||
parsed.DataMetrics, err = loadStageManifestDataMetrics(workspace.StageManifestPath)
|
||||
}
|
||||
}
|
||||
// Custom-container operators bypass operator-runner's ResourceSampler, so they
|
||||
// emit no resourceSeries in result.json. The launcher samples their GPU usage
|
||||
|
||||
@ -15,6 +15,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/dataops"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/k8s"
|
||||
"droboticflow/internal/lightweight"
|
||||
@ -564,6 +565,12 @@ func applyExecution(run *domain.RunRecord, execution domain.NodeExecutionRecord)
|
||||
} else {
|
||||
run.NodeExecutions = append(run.NodeExecutions, execution)
|
||||
}
|
||||
if execution.DataMetrics != nil && execution.DataMetrics.Stage == domain.DataMetricStageTrain && len(execution.DataMetrics.TrainingInputs) > 0 {
|
||||
if run.Metadata == nil {
|
||||
run.Metadata = &domain.RunMetadata{}
|
||||
}
|
||||
run.Metadata.TrainingInputs = append([]domain.TrainingInputRecord(nil), execution.DataMetrics.TrainingInputs...)
|
||||
}
|
||||
run.Logs = []domain.LogEntryRecord{}
|
||||
run.ResourceSeries = []domain.ResourceSample{}
|
||||
run.Artifacts = []domain.ExecutionArtifact{}
|
||||
@ -901,6 +908,7 @@ func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord {
|
||||
WorkflowName: summary.WorkflowName,
|
||||
ProjectID: summary.ProjectID,
|
||||
ProjectName: summary.ProjectName,
|
||||
TaskID: summary.TaskID,
|
||||
Trigger: summary.Trigger,
|
||||
Attempt: summary.Attempt,
|
||||
ParentRunID: summary.ParentRunID,
|
||||
@ -923,6 +931,18 @@ func runRecordsFromSummaries(summaries []domain.RunSummary) []domain.RunRecord {
|
||||
return runs
|
||||
}
|
||||
|
||||
func (runtime *Runtime) materializeNodeMetricsBestEffort(ctx context.Context, run *domain.RunRecord, execution *domain.NodeExecutionRecord) {
|
||||
if run == nil || execution == nil || execution.DataMetrics == nil || execution.MetricsMaterializedAt != "" {
|
||||
return
|
||||
}
|
||||
if err := dataops.MaterializeNodeMetrics(ctx, runtime.store, run, execution); err != nil {
|
||||
fmt.Printf("[worker] materialize data metrics for run %s node %s failed: %v\n", run.ID, execution.NodeID, err)
|
||||
return
|
||||
}
|
||||
execution.MetricsMaterializedAt = nowISO()
|
||||
applyExecution(run, *execution)
|
||||
}
|
||||
|
||||
func (runtime *Runtime) processClaimedRun(ctx context.Context, claimed *domain.RunRecord) {
|
||||
if claimed == nil {
|
||||
return
|
||||
@ -1090,6 +1110,9 @@ func (runtime *Runtime) ProcessAvailable(ctx context.Context) error {
|
||||
return processErr
|
||||
}
|
||||
runtime.updateEffectivePollInterval(settings)
|
||||
if _, err := dataops.ReconcileRunMetrics(ctx, runtime.store, 20); err != nil {
|
||||
fmt.Printf("[worker] reconcile data metrics failed: %v\n", err)
|
||||
}
|
||||
reconcileRuns, err := runtime.store.ListReconcileRunSummaries(ctx, workerReconcileRunLimit)
|
||||
if err != nil {
|
||||
processErr = err
|
||||
@ -1391,10 +1414,16 @@ func (runtime *Runtime) recoverSucceededJob(ctx context.Context, run *domain.Run
|
||||
execution.Logs = append(execution.Logs, parsed.Logs...)
|
||||
execution.Artifacts = parsed.Artifacts
|
||||
execution.DataRefs = parsed.DataRefs
|
||||
execution.DataMetrics = parsed.DataMetrics
|
||||
execution.ResourceSeries = parsed.ResourceSeries
|
||||
execution.Error = ""
|
||||
executionMap[node.ID] = execution
|
||||
applyExecution(run, execution)
|
||||
if err := runtime.store.UpsertRun(ctx, *run); err != nil {
|
||||
return err
|
||||
}
|
||||
runtime.materializeNodeMetricsBestEffort(ctx, run, &execution)
|
||||
executionMap[node.ID] = execution
|
||||
if run.Status == domain.RunStatusStopping {
|
||||
_, err := runtime.finalizeCancelledRun(ctx, workflow, run, executionMap, "用户已停止工作流")
|
||||
return err
|
||||
@ -2124,7 +2153,7 @@ func (runtime *Runtime) enqueueRetryRun(ctx context.Context, workflow *domain.Wo
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry)
|
||||
retryRun, err := runtime.store.CreateRun(ctx, workflow.ID, domain.RunTriggerRetry, failedRun.TaskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -2375,6 +2404,27 @@ func enqueueReadyNodes(workflow domain.WorkflowRecord, executionMap map[string]d
|
||||
}
|
||||
}
|
||||
|
||||
func upstreamBusinessVolume(workflow domain.WorkflowRecord, executionMap map[string]domain.NodeExecutionRecord, nodeID string) domain.DataVolume {
|
||||
for _, edge := range domain.GetIncomingEdges(workflow.Edges, nodeID) {
|
||||
execution, ok := executionMap[edge.Source]
|
||||
if !ok || execution.DataMetrics == nil {
|
||||
continue
|
||||
}
|
||||
if execution.DataMetrics.TrainReady.TrajectoryCount != nil ||
|
||||
execution.DataMetrics.TrainReady.DurationSeconds != nil ||
|
||||
execution.DataMetrics.TrainReady.Bytes != nil {
|
||||
return execution.DataMetrics.TrainReady
|
||||
}
|
||||
if execution.DataMetrics.TechnicalValid.TrajectoryCount != nil ||
|
||||
execution.DataMetrics.TechnicalValid.DurationSeconds != nil ||
|
||||
execution.DataMetrics.TechnicalValid.Bytes != nil {
|
||||
return execution.DataMetrics.TechnicalValid
|
||||
}
|
||||
return execution.DataMetrics.Output
|
||||
}
|
||||
return domain.DataVolume{}
|
||||
}
|
||||
|
||||
func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecord) (*domain.RunRecord, error) {
|
||||
if _, ok, err := runtime.store.AcquireRunLease(ctx, claimed.ID, runtime.leaseOwner(), runtime.leaseTTL()); err != nil {
|
||||
return nil, err
|
||||
@ -2469,6 +2519,25 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
|
||||
execution.Status = domain.NodeExecutionStatusCompleted
|
||||
execution.EndedAt = nowISO()
|
||||
execution.Logs = append(execution.Logs, logEntry("INFO", fmt.Sprintf("Audit decision: %s", task.Status), node.Data.Label))
|
||||
resolvedNode := runtime.resolveNodeBusinessStage(ctx, currentRun, *node)
|
||||
if resolvedNode.Data.BusinessStage == domain.DataMetricStageAudit {
|
||||
volume := upstreamBusinessVolume(*workflow, executionMap, node.ID)
|
||||
execution.DataMetrics = &domain.DataMetricsRecord{
|
||||
SchemaVersion: 1,
|
||||
Stage: domain.DataMetricStageAudit,
|
||||
Input: volume,
|
||||
Output: volume,
|
||||
Quality: domain.DataMetricQualityExact,
|
||||
OccurredAt: execution.EndedAt,
|
||||
}
|
||||
if task.Status == domain.AuditStatusApproved {
|
||||
execution.DataMetrics.TrainReady = volume
|
||||
} else {
|
||||
execution.DataMetrics.Output = domain.DataVolume{}
|
||||
execution.DataMetrics.Rejected = volume
|
||||
execution.DataMetrics.RejectionReasons = map[string]int64{"人工审核拒绝": 1}
|
||||
}
|
||||
}
|
||||
executionMap[node.ID] = execution
|
||||
applyExecution(currentRun, execution)
|
||||
for _, target := range edgeTargets(domain.GetDecisionOutgoingEdges(workflow.Edges, node.ID, task.Status)) {
|
||||
@ -2480,6 +2549,8 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
|
||||
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution)
|
||||
executionMap[node.ID] = execution
|
||||
}
|
||||
break
|
||||
}
|
||||
@ -2868,6 +2939,7 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
|
||||
execution.Logs = append(execution.Logs, outcome.result.Logs...)
|
||||
execution.Artifacts = outcome.result.Artifacts
|
||||
execution.DataRefs = outcome.result.DataRefs
|
||||
execution.DataMetrics = outcome.result.DataMetrics
|
||||
execution.StageManifestPath = outcome.result.StageManifestPath
|
||||
execution.ResourceSeries = outcome.result.ResourceSeries
|
||||
executionMap[outcome.nodeID] = execution
|
||||
@ -2890,6 +2962,8 @@ func (runtime *Runtime) processRun(ctx context.Context, claimed *domain.RunRecor
|
||||
if err := runtime.store.UpsertRun(ctx, *currentRun); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
runtime.materializeNodeMetricsBestEffort(ctx, currentRun, &execution)
|
||||
executionMap[outcome.nodeID] = execution
|
||||
if forceStopRequested && len(runningNodes) == 0 {
|
||||
return runtime.finalizeCancelledRun(ctx, workflow, currentRun, executionMap, "用户已强制停止工作流")
|
||||
}
|
||||
|
||||
@ -86,6 +86,7 @@ const Templates = lazyRoute(() => import("./pages/Templates"));
|
||||
const Users = lazyRoute(() => import("./pages/Users"));
|
||||
const Audit = lazyRoute(() => import("./pages/Audit"));
|
||||
const DataAssets = lazyRoute(() => import("./pages/DataAssets"));
|
||||
const DataOperations = lazyRoute(() => import("./pages/DataOperations"));
|
||||
const DatabaseBrowser = lazyRoute(() => import("./pages/DatabaseBrowser"));
|
||||
const ObjectStorageBrowser = lazyRoute(() => import("./pages/ObjectStorageBrowser"));
|
||||
const LegacyDataAssetsRedirect = lazyRoute(() => import("./pages/LegacyDataAssetsRedirect"));
|
||||
@ -110,6 +111,7 @@ export default function App() {
|
||||
<Route path="/operators" element={<Operators />} />
|
||||
<Route path="/workflow/:id" element={<WorkflowEditor />} />
|
||||
<Route path="/data-assets" element={<DataAssets />} />
|
||||
<Route path="/data-operations" element={<DataOperations />} />
|
||||
<Route path="/database" element={<DatabaseBrowser />} />
|
||||
<Route path="/object-storage" element={<ObjectStorageBrowser />} />
|
||||
<Route path="/source-data" element={<LegacyDataAssetsRedirect />} />
|
||||
|
||||
@ -21,6 +21,7 @@ import {
|
||||
Database,
|
||||
Cloudy,
|
||||
DatabaseZap,
|
||||
BarChart3,
|
||||
LogOut,
|
||||
Loader2,
|
||||
} from "lucide-react";
|
||||
@ -41,6 +42,12 @@ const navItems = [
|
||||
path: "/projects",
|
||||
roles: ["admin", "pipeline_engineer", "data_operator", "operator"],
|
||||
},
|
||||
{
|
||||
icon: BarChart3,
|
||||
label: "数据运营",
|
||||
path: "/data-operations",
|
||||
roles: ["admin", "pipeline_engineer", "data_operator", "qa_reviewer", "compliance_auditor", "auditor", "operator"],
|
||||
},
|
||||
{
|
||||
icon: FileCode,
|
||||
label: "工作流模板",
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
const operatorTypes = new Set(["builtin", "custom", "lightweight"]);
|
||||
const portTypes = new Set(["json", "image", "video", "tensor", "file"]);
|
||||
const portTransmissions = new Set(["", "bus", "oss", "shm", "stream"]);
|
||||
const businessStages = new Set(["", "collect", "process", "clean", "validate", "annotate", "audit", "train"]);
|
||||
const positiveInteger = /^[1-9]\d*$/;
|
||||
const positiveResourceQuantity = /^(?:[1-9]\d*(?:\.\d+)?|0\.\d+)(?:(?:[eE][+-]?\d+)|(?:[numkKMGTP](?:i)?))?$/;
|
||||
|
||||
@ -73,6 +74,7 @@ export function buildOperatorPayload(source = {}, overrides = {}) {
|
||||
publicationEvidence: deepClone(overrides.publicationEvidence ?? source.publicationEvidence),
|
||||
hyperparams: deepClone(overrides.hyperparams ?? source.hyperparams ?? []),
|
||||
resources: normalizeResources(deepClone(overrides.resources ?? source.resources)),
|
||||
businessStage: trimString(overrides.businessStage ?? source.businessStage).toLowerCase(),
|
||||
};
|
||||
}
|
||||
|
||||
@ -133,6 +135,9 @@ export function getOperatorPayloadIssue(source = {}, overrides = {}) {
|
||||
if (payload.version && !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(payload.version)) {
|
||||
return "算子版本只能包含字母、数字、下划线、点和短横线,长度不超过 128";
|
||||
}
|
||||
if (!businessStages.has(payload.businessStage)) {
|
||||
return "业务阶段必须是采集、处理、清洗、校验、标注、审核或训练";
|
||||
}
|
||||
return (
|
||||
validatePorts("inputs", payload.inputs) ||
|
||||
validatePorts("outputs", payload.outputs) ||
|
||||
|
||||
1337
src/pages/DataOperations.tsx
Normal file
1337
src/pages/DataOperations.tsx
Normal file
File diff suppressed because it is too large
Load Diff
@ -39,7 +39,7 @@ import {
|
||||
import { buildOperatorPayload, getOperatorPayloadIssue } from "../lib/operators.js";
|
||||
import { cn } from "../lib/utils";
|
||||
import { canManagePipelineDesign } from "../lib/user-roles";
|
||||
import { Operator, PortDefinition, PortTransmission, PortType } from "../types";
|
||||
import { DataMetricStage, Operator, PortDefinition, PortTransmission, PortType } from "../types";
|
||||
import { generateOperatorFromDescription } from "../services/aiService";
|
||||
import type { GeneratedOperator } from "../services/api/ai";
|
||||
import {
|
||||
@ -76,6 +76,19 @@ const gpuTypeOptions = [
|
||||
"NVIDIA T4 16GB",
|
||||
"NVIDIA GPU",
|
||||
];
|
||||
const businessStageOptions: Array<{ value: DataMetricStage; label: string }> = [
|
||||
{ value: "collect", label: "采集" },
|
||||
{ value: "process", label: "处理 / 转换" },
|
||||
{ value: "clean", label: "清洗" },
|
||||
{ value: "validate", label: "技术校验" },
|
||||
{ value: "annotate", label: "标注" },
|
||||
{ value: "audit", label: "审核 / 训练可用" },
|
||||
{ value: "train", label: "训练" },
|
||||
];
|
||||
|
||||
function businessStageLabel(stage?: DataMetricStage) {
|
||||
return businessStageOptions.find((option) => option.value === stage)?.label || "";
|
||||
}
|
||||
|
||||
type GeneratedImplementation = NonNullable<GeneratedOperator["implementation"]>;
|
||||
|
||||
@ -1399,6 +1412,11 @@ export default function Operators() {
|
||||
SHM {trimResource(op.resources?.sharedMemorySize)}
|
||||
</span>
|
||||
)}
|
||||
{op.businessStage && (
|
||||
<span className="text-[10px] px-2 py-0.5 rounded-full font-mono uppercase tracking-wider bg-emerald-500/10 text-emerald-300 border border-emerald-500/30">
|
||||
阶段 {businessStageLabel(op.businessStage)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -2010,6 +2028,34 @@ export default function Operators() {
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className="grid gap-2 md:grid-cols-[minmax(0,280px)_minmax(0,1fr)] md:items-end">
|
||||
<div className="space-y-1.5">
|
||||
<label className="text-[10px] font-medium uppercase tracking-wider text-gray-500">
|
||||
默认业务阶段
|
||||
</label>
|
||||
<select
|
||||
className="w-full rounded-md border border-[#2a2b2f] bg-[#151619] px-3 py-2 text-xs focus:border-blue-500 focus:outline-none"
|
||||
value={newOp.businessStage || ""}
|
||||
onChange={(event) =>
|
||||
setNewOp((current) => ({
|
||||
...current,
|
||||
businessStage: (event.target.value || undefined) as DataMetricStage | undefined,
|
||||
}))
|
||||
}
|
||||
>
|
||||
<option value="">不指定(由任务或节点决定)</option>
|
||||
{businessStageOptions.map((option) => (
|
||||
<option key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<p className="text-[10px] leading-relaxed text-gray-600">
|
||||
用于算子独立运行时的指标归类;完整任务绑定和工作流节点覆盖值优先。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 gap-3 md:grid-cols-5">
|
||||
<div className="space-y-1.5">
|
||||
<label className="text-[10px] font-medium uppercase tracking-wider text-gray-500">
|
||||
|
||||
@ -145,6 +145,7 @@ import { workflowsApi } from "../services/api/workflows";
|
||||
import { templatesApi } from "../services/api/templates";
|
||||
import { projectsApi } from "../services/api/projects";
|
||||
import type {
|
||||
DataMetricStage,
|
||||
ExecutionData,
|
||||
ExecutionRun,
|
||||
ExecutionRunSummary,
|
||||
@ -219,9 +220,21 @@ interface OperatorNodeData {
|
||||
memory?: string;
|
||||
gpu?: string;
|
||||
sharedMemorySize?: string;
|
||||
businessStage?: DataMetricStage;
|
||||
metricBoundary?: boolean;
|
||||
executionState?: OperatorExecutionState;
|
||||
}
|
||||
|
||||
const nodeBusinessStageOptions: Array<{ value: DataMetricStage; label: string }> = [
|
||||
{ value: "collect", label: "采集" },
|
||||
{ value: "process", label: "处理 / 转换" },
|
||||
{ value: "clean", label: "清洗" },
|
||||
{ value: "validate", label: "技术校验" },
|
||||
{ value: "annotate", label: "标注" },
|
||||
{ value: "audit", label: "审核 / 训练可用" },
|
||||
{ value: "train", label: "训练" },
|
||||
];
|
||||
|
||||
function isOperatorCandidate(value: unknown): value is Operator {
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
@ -794,6 +807,10 @@ export default function WorkflowEditor() {
|
||||
}, [operators, selectedNode?.data]);
|
||||
const inheritedSharedMemorySize = String(selectedNodeOperator?.resources?.sharedMemorySize || "").trim();
|
||||
const sharedMemoryPlaceholder = inheritedSharedMemorySize ? `继承算子 ${inheritedSharedMemorySize}` : "继承平台默认";
|
||||
const inheritedBusinessStage = selectedNodeOperator?.businessStage;
|
||||
const businessStagePlaceholder = inheritedBusinessStage
|
||||
? `继承算子:${nodeBusinessStageOptions.find((option) => option.value === inheritedBusinessStage)?.label || inheritedBusinessStage}`
|
||||
: "不指定";
|
||||
const selectedNodeRequestsGPU = hasPositiveGPURequest((selectedNode?.data as OperatorNodeData | undefined)?.gpu);
|
||||
const selectedNodeGPUBlocked = selectedNodeRequestsGPU && workerCapacity?.gpu && !workerCapacity.gpu.available;
|
||||
const workflowGPUCapacityBlocker = useMemo(
|
||||
@ -2479,6 +2496,7 @@ export default function WorkflowEditor() {
|
||||
memory: op.resources?.memoryLimit || "",
|
||||
gpu: op.resources?.gpuLimit || "",
|
||||
sharedMemorySize: "",
|
||||
businessStage: undefined,
|
||||
},
|
||||
};
|
||||
|
||||
@ -6543,6 +6561,32 @@ export default function WorkflowEditor() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="grid gap-2 md:grid-cols-[minmax(0,260px)_minmax(0,1fr)] md:items-end">
|
||||
<div className="space-y-1">
|
||||
<label className="text-[10px] text-gray-500 uppercase font-mono">业务阶段</label>
|
||||
<select
|
||||
className="w-full rounded border border-[#2a2b2f] bg-[#0a0a0a] px-2 py-1 text-xs focus:border-blue-500 focus:outline-none disabled:opacity-50"
|
||||
value={selectedNode.data.businessStage || ""}
|
||||
onChange={(event) =>
|
||||
updateNodeData({
|
||||
businessStage: (event.target.value || undefined) as DataMetricStage | undefined,
|
||||
})
|
||||
}
|
||||
disabled={isReadOnly}
|
||||
>
|
||||
<option value="">{businessStagePlaceholder}</option>
|
||||
{nodeBusinessStageOptions.map((option) => (
|
||||
<option key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<p className="text-[10px] leading-snug text-gray-600">
|
||||
完整任务绑定优先,其次使用当前节点;留空时继承算子默认阶段。
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className={cn("grid gap-2", isConfigPanelExpanded ? "grid-cols-4" : "grid-cols-2")}>
|
||||
<div className="space-y-1">
|
||||
<label className="text-[10px] text-gray-500 uppercase font-mono">资源限制 (CPU)</label>
|
||||
|
||||
78
src/services/api/data-operations.ts
Normal file
78
src/services/api/data-operations.ts
Normal file
@ -0,0 +1,78 @@
|
||||
import type {
|
||||
CollectionTask,
|
||||
CollectionTaskInput,
|
||||
DataMetricFact,
|
||||
DataMetricStage,
|
||||
DataOperationsSummary,
|
||||
DatasetVersion,
|
||||
ExecutionRun,
|
||||
TrainingUsage,
|
||||
} from "../../types";
|
||||
import { apiRequest } from "./client";
|
||||
|
||||
export type DataOperationsFilters = {
|
||||
from?: string;
|
||||
to?: string;
|
||||
timezone?: string;
|
||||
projectId?: string;
|
||||
taskId?: string;
|
||||
robotModel?: string;
|
||||
scene?: string;
|
||||
stage?: DataMetricStage;
|
||||
limit?: number;
|
||||
cursor?: string;
|
||||
};
|
||||
|
||||
const queryString = (filters: DataOperationsFilters = {}) => {
|
||||
const params = new URLSearchParams();
|
||||
for (const [key, value] of Object.entries(filters)) {
|
||||
if (value !== undefined && value !== "") {
|
||||
params.set(key, String(value));
|
||||
}
|
||||
}
|
||||
const value = params.toString();
|
||||
return value ? `?${value}` : "";
|
||||
};
|
||||
|
||||
export const dataOperationsApi = {
|
||||
listTasks: async (filters: Pick<DataOperationsFilters, "projectId"> & { status?: string } = {}) =>
|
||||
(
|
||||
await apiRequest<{ items: CollectionTask[] }>(
|
||||
`/api/data-collection-tasks${queryString(filters as DataOperationsFilters)}`,
|
||||
)
|
||||
).items,
|
||||
getTask: (id: string) => apiRequest<CollectionTask>(`/api/data-collection-tasks/${id}`),
|
||||
createTask: (payload: CollectionTaskInput) =>
|
||||
apiRequest<CollectionTask>("/api/data-collection-tasks", { method: "POST" }, payload),
|
||||
updateTask: (id: string, payload: CollectionTaskInput) =>
|
||||
apiRequest<CollectionTask>(`/api/data-collection-tasks/${id}`, { method: "PATCH" }, payload),
|
||||
startTask: (id: string) =>
|
||||
apiRequest<{ task: CollectionTask; run: ExecutionRun }>(`/api/data-collection-tasks/${id}/start`, {
|
||||
method: "POST",
|
||||
}),
|
||||
completeTask: (id: string) =>
|
||||
apiRequest<CollectionTask>(`/api/data-collection-tasks/${id}/complete`, { method: "POST" }),
|
||||
getSummary: (filters: DataOperationsFilters = {}, signal?: AbortSignal) =>
|
||||
apiRequest<DataOperationsSummary>(`/api/data-operations/summary${queryString(filters)}`, { signal }),
|
||||
listStageItems: (filters: DataOperationsFilters = {}, signal?: AbortSignal) =>
|
||||
apiRequest<{ items: DataMetricFact[]; nextCursor?: string; hasMore: boolean }>(
|
||||
`/api/data-operations/stage-items${queryString(filters)}`,
|
||||
{ signal },
|
||||
),
|
||||
listTrainingUsage: (filters: DataOperationsFilters = {}, signal?: AbortSignal) =>
|
||||
apiRequest<{ items: TrainingUsage[]; nextCursor?: string; hasMore: boolean }>(
|
||||
`/api/data-operations/training-usage${queryString(filters)}`,
|
||||
{ signal },
|
||||
),
|
||||
listDatasets: async (filters: DataOperationsFilters = {}, signal?: AbortSignal) =>
|
||||
(
|
||||
await apiRequest<{ items: DatasetVersion[] }>(`/api/data-operations/datasets${queryString(filters)}`, {
|
||||
signal,
|
||||
})
|
||||
).items,
|
||||
backfill: () =>
|
||||
apiRequest<{ windowDays: number; processed: number; skipped: number; failures: number; completedAt: string }>(
|
||||
"/api/data-operations/backfill",
|
||||
{ method: "POST" },
|
||||
),
|
||||
};
|
||||
207
src/types.ts
207
src/types.ts
@ -261,6 +261,7 @@ export interface Operator {
|
||||
gpuLimit?: string;
|
||||
sharedMemorySize?: string;
|
||||
};
|
||||
businessStage?: DataMetricStage;
|
||||
}
|
||||
|
||||
export interface HyperParam {
|
||||
@ -609,6 +610,7 @@ export interface LightweightOperatorCodeManifest {
|
||||
|
||||
export interface ExecutionRunMetadata {
|
||||
lightweightOperatorCodeManifests?: Record<string, LightweightOperatorCodeManifest>;
|
||||
trainingInputs?: TrainingInput[];
|
||||
}
|
||||
|
||||
export interface ScanConfig {
|
||||
@ -817,6 +819,8 @@ export interface NodeExecution {
|
||||
logs: LogEntry[];
|
||||
artifacts: ExecutionData[];
|
||||
dataRefs?: DataRef[];
|
||||
dataMetrics?: DataMetrics;
|
||||
metricsMaterializedAt?: string;
|
||||
resourceSeries: ResourceUsage[];
|
||||
error?: string;
|
||||
}
|
||||
@ -839,6 +843,207 @@ export interface DataRef {
|
||||
partitionDir?: string;
|
||||
partitionCount?: number;
|
||||
itemCount?: number;
|
||||
datasetId?: string;
|
||||
datasetVersionId?: string;
|
||||
split?: string;
|
||||
fingerprint?: string;
|
||||
}
|
||||
|
||||
export type DataMetricStage = "collect" | "process" | "clean" | "validate" | "annotate" | "audit" | "train";
|
||||
export type DataMetricQuality = "exact" | "derived" | "estimated";
|
||||
export type CollectionTaskStatus = "draft" | "active" | "paused" | "completed" | "cancelled";
|
||||
export type CollectionTaskMode = "collection_only" | "full_training";
|
||||
|
||||
export interface DataVolume {
|
||||
trajectoryCount?: number;
|
||||
durationSeconds?: number;
|
||||
bytes?: number;
|
||||
fileCount?: number;
|
||||
}
|
||||
|
||||
export interface DatasetMetricRef {
|
||||
datasetId?: string;
|
||||
datasetVersionId?: string;
|
||||
name?: string;
|
||||
split?: string;
|
||||
fingerprint?: string;
|
||||
kind?: string;
|
||||
datasetType?: string;
|
||||
manifestPath?: string;
|
||||
payloadPath?: string;
|
||||
partitionDir?: string;
|
||||
partitionCount?: number;
|
||||
itemCount?: number;
|
||||
}
|
||||
|
||||
export interface TrainingInput {
|
||||
datasetId?: string;
|
||||
datasetVersionId?: string;
|
||||
name?: string;
|
||||
split?: string;
|
||||
volume?: DataVolume;
|
||||
manifestPath?: string;
|
||||
partitionDir?: string;
|
||||
external?: boolean;
|
||||
quality?: DataMetricQuality;
|
||||
}
|
||||
|
||||
export interface DataMetrics {
|
||||
schemaVersion: number;
|
||||
stage: DataMetricStage;
|
||||
input?: DataVolume;
|
||||
output?: DataVolume;
|
||||
rejected?: DataVolume;
|
||||
technicalValid?: DataVolume;
|
||||
trainReady?: DataVolume;
|
||||
datasetRefs?: DatasetMetricRef[];
|
||||
trainingInputs?: TrainingInput[];
|
||||
rejectionReasons?: Record<string, number>;
|
||||
quality?: DataMetricQuality;
|
||||
occurredAt?: string;
|
||||
}
|
||||
|
||||
export interface CollectionTaskRequirements {
|
||||
requireTechnicalValidation: boolean;
|
||||
requireQuality: boolean;
|
||||
requireAnnotation: boolean;
|
||||
requireAudit: boolean;
|
||||
}
|
||||
|
||||
export interface CollectionTask {
|
||||
id: string;
|
||||
organizationId?: string;
|
||||
projectId: string;
|
||||
projectName?: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
robotModel: string;
|
||||
robotVersion?: string;
|
||||
scene: string;
|
||||
modalities: string[];
|
||||
format?: string;
|
||||
mode: CollectionTaskMode;
|
||||
status: CollectionTaskStatus;
|
||||
workflowId?: string;
|
||||
workflowName?: string;
|
||||
stageBindings?: Partial<Record<DataMetricStage, string>>;
|
||||
targets: DataVolume;
|
||||
requirements: CollectionTaskRequirements;
|
||||
splitRules?: Partial<Record<"train" | "val" | "test", number>>;
|
||||
lastRunId?: string;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
}
|
||||
|
||||
export type CollectionTaskInput = Omit<
|
||||
CollectionTask,
|
||||
"id" | "organizationId" | "projectName" | "workflowName" | "lastRunId" | "createdAt" | "updatedAt"
|
||||
>;
|
||||
|
||||
export interface DatasetVersion {
|
||||
id: string;
|
||||
datasetId: string;
|
||||
taskId?: string;
|
||||
projectId?: string;
|
||||
name: string;
|
||||
split?: string;
|
||||
fingerprint: string;
|
||||
volume?: DataVolume;
|
||||
storage?: AssetStorageLocation;
|
||||
manifestPath?: string;
|
||||
partitionDir?: string;
|
||||
producerRunId?: string;
|
||||
producerNodeId?: string;
|
||||
artifactId?: string;
|
||||
quality: DataMetricQuality;
|
||||
createdAt: string;
|
||||
updatedAt?: string;
|
||||
}
|
||||
|
||||
export interface DataMetricFact {
|
||||
id: string;
|
||||
sourceKey: string;
|
||||
projectId?: string;
|
||||
projectName?: string;
|
||||
taskId?: string;
|
||||
taskName?: string;
|
||||
robotModel?: string;
|
||||
scene?: string;
|
||||
workflowId?: string;
|
||||
workflowName?: string;
|
||||
runId: string;
|
||||
nodeId: string;
|
||||
nodeLabel?: string;
|
||||
attempt: number;
|
||||
stage: DataMetricStage;
|
||||
input?: DataVolume;
|
||||
output?: DataVolume;
|
||||
rejected?: DataVolume;
|
||||
technicalValid?: DataVolume;
|
||||
trainReady?: DataVolume;
|
||||
datasetRefs?: DatasetMetricRef[];
|
||||
trainingInputs?: TrainingInput[];
|
||||
rejectionReasons?: Record<string, number>;
|
||||
quality: DataMetricQuality;
|
||||
occurredAt: string;
|
||||
recordedAt: string;
|
||||
}
|
||||
|
||||
export interface DataStageSummary {
|
||||
stage: DataMetricStage;
|
||||
input?: DataVolume;
|
||||
output?: DataVolume;
|
||||
rejected?: DataVolume;
|
||||
technicalValid?: DataVolume;
|
||||
trainReady?: DataVolume;
|
||||
runCount: number;
|
||||
}
|
||||
|
||||
export interface DataOperationsSummary {
|
||||
from: string;
|
||||
to: string;
|
||||
timezone: string;
|
||||
updatedAt: string;
|
||||
freshness: string;
|
||||
stages: DataStageSummary[];
|
||||
training: {
|
||||
consumed: DataVolume;
|
||||
unique: DataVolume;
|
||||
runCount: number;
|
||||
};
|
||||
hourly: Array<{ hour: string; stages: Partial<Record<DataMetricStage, DataVolume>> }>;
|
||||
rejectionReasons: Record<string, number>;
|
||||
tasks: Array<{
|
||||
taskId: string;
|
||||
taskName: string;
|
||||
status: CollectionTaskStatus;
|
||||
targets: DataVolume;
|
||||
collected: DataVolume;
|
||||
percent: number;
|
||||
}>;
|
||||
factCount: number;
|
||||
exactCount: number;
|
||||
derivedCount: number;
|
||||
estimatedCount: number;
|
||||
unknownCount: number;
|
||||
}
|
||||
|
||||
export interface TrainingUsage {
|
||||
id: string;
|
||||
taskId?: string;
|
||||
taskName?: string;
|
||||
projectId?: string;
|
||||
projectName?: string;
|
||||
workflowId?: string;
|
||||
workflowName?: string;
|
||||
runId: string;
|
||||
nodeId: string;
|
||||
nodeLabel?: string;
|
||||
model?: string;
|
||||
inputs: TrainingInput[];
|
||||
consumed: DataVolume;
|
||||
quality: DataMetricQuality;
|
||||
occurredAt: string;
|
||||
}
|
||||
|
||||
export interface InputLineage {
|
||||
@ -915,6 +1120,7 @@ export interface ExecutionRun {
|
||||
workflowName: string;
|
||||
projectId: string;
|
||||
projectName: string;
|
||||
taskId?: string;
|
||||
trigger: "manual" | "scheduled" | "retry" | "rerun_subtree";
|
||||
attempt?: number;
|
||||
parentRunId?: string;
|
||||
@ -968,6 +1174,7 @@ export interface ExecutionRunSummary {
|
||||
workflowName: string;
|
||||
projectId: string;
|
||||
projectName: string;
|
||||
taskId?: string;
|
||||
trigger: "manual" | "scheduled" | "retry" | "rerun_subtree";
|
||||
attempt?: number;
|
||||
parentRunId?: string;
|
||||
|
||||
37
tests/frontend/data-operations.test.mjs
Normal file
37
tests/frontend/data-operations.test.mjs
Normal file
@ -0,0 +1,37 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
|
||||
const pageURL = new URL("../../src/pages/DataOperations.tsx", import.meta.url);
|
||||
const serviceURL = new URL("../../src/services/api/data-operations.ts", import.meta.url);
|
||||
|
||||
test("data operations page exposes complete task and business metric surfaces", async () => {
|
||||
const source = await readFile(pageURL, "utf8");
|
||||
|
||||
assert.match(source, /数据总览/);
|
||||
assert.match(source, /完整任务/);
|
||||
assert.match(source, /数据版本/);
|
||||
assert.match(source, /训练使用/);
|
||||
assert.match(source, /技术有效/);
|
||||
assert.match(source, /训练可用/);
|
||||
assert.match(source, /训练消费/);
|
||||
});
|
||||
|
||||
test("data operations polling pauses while the page is hidden", async () => {
|
||||
const source = await readFile(pageURL, "utf8");
|
||||
|
||||
assert.match(source, /document\.hidden/);
|
||||
assert.match(source, /visibilitychange/);
|
||||
assert.match(source, /30_000/);
|
||||
assert.match(source, /controller\?\.abort\(\)/);
|
||||
});
|
||||
|
||||
test("data operations client uses additive compatibility endpoints", async () => {
|
||||
const source = await readFile(serviceURL, "utf8");
|
||||
|
||||
assert.match(source, /\/api\/data-collection-tasks/);
|
||||
assert.match(source, /\/api\/data-operations\/summary/);
|
||||
assert.match(source, /\/api\/data-operations\/stage-items/);
|
||||
assert.match(source, /\/api\/data-operations\/training-usage/);
|
||||
assert.match(source, /\/api\/data-operations\/datasets/);
|
||||
});
|
||||
@ -55,6 +55,22 @@ test("buildOperatorPayload deep clones mutable operator fields", () => {
|
||||
assert.equal(source.hyperparams[0].value, "fast");
|
||||
});
|
||||
|
||||
test("buildOperatorPayload normalizes and validates business stage", () => {
|
||||
const base = {
|
||||
name: "cleaner",
|
||||
type: "custom",
|
||||
category: "DATA",
|
||||
description: "clean data",
|
||||
inputs: [],
|
||||
outputs: [],
|
||||
businessStage: " CLEAN ",
|
||||
};
|
||||
const payload = buildOperatorPayload(base);
|
||||
assert.equal(payload.businessStage, "clean");
|
||||
assert.equal(getOperatorPayloadIssue(base), "");
|
||||
assert.match(getOperatorPayloadIssue({ ...base, businessStage: "publish" }), /业务阶段/);
|
||||
});
|
||||
|
||||
test("operator payload validation rejects missing metadata and invalid ports", () => {
|
||||
assert.equal(getOperatorPayloadIssue({ name: " ", description: "ok" }), "算子名称不能为空");
|
||||
assert.equal(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user