From da25d845fcc7d2c19d556e373d21de913b7fd5bb Mon Sep 17 00:00:00 2001 From: eust-w Date: Sun, 28 Jun 2026 19:47:06 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=AA=20Require=20live=20workflow=20line?= =?UTF-8?q?age=20metrics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/live-workflow-acceptance.sh | 29 ++++ .../live_workflow_acceptance_script_test.py | 132 +++++++++++++++++- 2 files changed, 160 insertions(+), 1 deletion(-) diff --git a/scripts/live-workflow-acceptance.sh b/scripts/live-workflow-acceptance.sh index 86c6ea67..828c78a7 100755 --- a/scripts/live-workflow-acceptance.sh +++ b/scripts/live-workflow-acceptance.sh @@ -249,6 +249,35 @@ if needle not in messages: raise SystemExit(f"expected log substring not found: fi if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then + FINAL_RUN_JSON="$final_run" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY' +import json +import os + +payload = json.loads(os.environ["FINAL_RUN_JSON"]) +executions = {item.get("nodeId"): item for item in payload.get("nodeExecutions") or [] if isinstance(item, dict)} +upload = executions.get("upload-1") +if not upload: + raise SystemExit("expected upload-1 node execution in completed workflow") + +lineage = upload.get("inputLineage") or [] +if not any(item.get("sourceNodeId") == "source-1" and item.get("targetNodeId") == "upload-1" and item.get("artifactId") for item in lineage): + raise SystemExit("expected source-1 to upload-1 inputLineage artifact") +if not upload.get("inputArtifacts"): + raise SystemExit("expected upload-1 inputArtifacts from source-1") + +gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip() +if gpu_limit: + samples = [] + for item in payload.get("resourceSeries") or []: + if isinstance(item, dict): + samples.append(item) + for node in payload.get("nodeExecutions") or []: + for item in node.get("resourceSeries") or []: + if isinstance(item, dict): + samples.append(item) + if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples): + raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries") +PY printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []); if not items: raise SystemExit("expected at least one artifact")' fi diff --git a/tests/backend/live_workflow_acceptance_script_test.py b/tests/backend/live_workflow_acceptance_script_test.py index 65449f87..413c0e9c 100644 --- a/tests/backend/live_workflow_acceptance_script_test.py +++ b/tests/backend/live_workflow_acceptance_script_test.py @@ -70,7 +70,31 @@ elif url.endswith("/api/runs/run-1/logs"): elif url.endswith("/api/runs/run-1/artifacts"): payload = {"items": [{"name": "source-manifest.json"}]} elif url.endswith("/api/runs/run-1"): - payload = {"status": "completed"} + payload = { + "status": "completed", + "nodeExecutions": [ + { + "nodeId": "source-1", + "status": "completed", + "resourceSeries": [], + }, + { + "nodeId": "upload-1", + "status": "completed", + "inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}], + "inputLineage": [ + { + "sourceNodeId": "source-1", + "sourcePortId": "out-1", + "targetNodeId": "upload-1", + "targetPortId": "in-1", + "artifactId": "artifact-1", + } + ], + "resourceSeries": [], + }, + ], + } else: raise SystemExit(f"unexpected URL: {url}") print(json.dumps(payload, separators=(",", ":"))) @@ -93,6 +117,112 @@ print(json.dumps(payload, separators=(",", ":"))) assert "run status: completed" in result.stdout +def test_acceptance_requires_completed_workflow_lineage(tmp_path): + fake_curl = tmp_path / "curl" + fake_curl.write_text( + """#!/usr/bin/env python3 +import json +import sys + +url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "") +if url.endswith("/api/health"): + payload = {"status": "ok"} +elif url.endswith("/api/projects"): + payload = {"id": "project-1"} +elif url.endswith("/api/projects/project-1/workflows"): + payload = {"id": "workflow-1"} +elif url.endswith("/api/workflows/workflow-1/run"): + payload = {"id": "run-1"} +elif url.endswith("/api/runs/run-1/logs"): + payload = {"items": [{"message": "ok"}]} +elif url.endswith("/api/runs/run-1/artifacts"): + payload = {"items": [{"name": "source-manifest.json"}]} +elif url.endswith("/api/runs/run-1"): + payload = {"status": "completed", "nodeExecutions": [{"nodeId": "upload-1", "status": "completed"}]} +else: + raise SystemExit(f"unexpected URL: {url}") +print(json.dumps(payload, separators=(",", ":"))) +""", + encoding="utf-8", + ) + fake_curl.chmod(0o755) + + result = run_acceptance( + { + "API_BASE_URL": "http://poc.example.test", + "PATH": f"{tmp_path}:{os.environ['PATH']}", + "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true", + "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", + "WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data", + } + ) + + assert result.returncode == 1 + assert "expected source-1 to upload-1 inputLineage artifact" in result.stderr + + +def test_acceptance_requires_gpu_memory_when_completed_gpu_workflow(tmp_path): + fake_curl = tmp_path / "curl" + fake_curl.write_text( + """#!/usr/bin/env python3 +import json +import sys + +url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "") +if url.endswith("/api/health"): + payload = {"status": "ok"} +elif url.endswith("/api/projects"): + payload = {"id": "project-1"} +elif url.endswith("/api/projects/project-1/workflows"): + payload = {"id": "workflow-1"} +elif url.endswith("/api/workflows/workflow-1/run"): + payload = {"id": "run-1"} +elif url.endswith("/api/runs/run-1/logs"): + payload = {"items": [{"message": "ok"}]} +elif url.endswith("/api/runs/run-1/artifacts"): + payload = {"items": [{"name": "source-manifest.json"}]} +elif url.endswith("/api/runs/run-1"): + payload = { + "status": "completed", + "nodeExecutions": [ + { + "nodeId": "upload-1", + "status": "completed", + "inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}], + "inputLineage": [ + { + "sourceNodeId": "source-1", + "targetNodeId": "upload-1", + "artifactId": "artifact-1", + } + ], + } + ], + } +else: + raise SystemExit(f"unexpected URL: {url}") +print(json.dumps(payload, separators=(",", ":"))) +""", + encoding="utf-8", + ) + fake_curl.chmod(0o755) + + result = run_acceptance( + { + "API_BASE_URL": "http://poc.example.test", + "PATH": f"{tmp_path}:{os.environ['PATH']}", + "WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "completed", + "WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1", + "WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true", + "WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1", + "WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data", + } + ) + + assert result.returncode == 1 + assert "expected GPU memory metrics in completed GPU workflow resourceSeries" in result.stderr + + def test_acceptance_can_expect_gpu_preflight_failure(tmp_path): captured_workflow = tmp_path / "workflow.json" fake_curl = tmp_path / "curl"