🧪 Require live workflow lineage metrics

This commit is contained in:
eust-w 2026-06-28 19:47:06 +08:00
parent f04c11f05e
commit da25d845fc
2 changed files with 160 additions and 1 deletions

View File

@ -249,6 +249,35 @@ if needle not in messages: raise SystemExit(f"expected log substring not found:
fi
if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then
FINAL_RUN_JSON="$final_run" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY'
import json
import os
payload = json.loads(os.environ["FINAL_RUN_JSON"])
executions = {item.get("nodeId"): item for item in payload.get("nodeExecutions") or [] if isinstance(item, dict)}
upload = executions.get("upload-1")
if not upload:
raise SystemExit("expected upload-1 node execution in completed workflow")
lineage = upload.get("inputLineage") or []
if not any(item.get("sourceNodeId") == "source-1" and item.get("targetNodeId") == "upload-1" and item.get("artifactId") for item in lineage):
raise SystemExit("expected source-1 to upload-1 inputLineage artifact")
if not upload.get("inputArtifacts"):
raise SystemExit("expected upload-1 inputArtifacts from source-1")
gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip()
if gpu_limit:
samples = []
for item in payload.get("resourceSeries") or []:
if isinstance(item, dict):
samples.append(item)
for node in payload.get("nodeExecutions") or []:
for item in node.get("resourceSeries") or []:
if isinstance(item, dict):
samples.append(item)
if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples):
raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries")
PY
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
if not items: raise SystemExit("expected at least one artifact")'
fi

View File

@ -70,7 +70,31 @@ elif url.endswith("/api/runs/run-1/logs"):
elif url.endswith("/api/runs/run-1/artifacts"):
payload = {"items": [{"name": "source-manifest.json"}]}
elif url.endswith("/api/runs/run-1"):
payload = {"status": "completed"}
payload = {
"status": "completed",
"nodeExecutions": [
{
"nodeId": "source-1",
"status": "completed",
"resourceSeries": [],
},
{
"nodeId": "upload-1",
"status": "completed",
"inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}],
"inputLineage": [
{
"sourceNodeId": "source-1",
"sourcePortId": "out-1",
"targetNodeId": "upload-1",
"targetPortId": "in-1",
"artifactId": "artifact-1",
}
],
"resourceSeries": [],
},
],
}
else:
raise SystemExit(f"unexpected URL: {url}")
print(json.dumps(payload, separators=(",", ":")))
@ -93,6 +117,112 @@ print(json.dumps(payload, separators=(",", ":")))
assert "run status: completed" in result.stdout
def test_acceptance_requires_completed_workflow_lineage(tmp_path):
fake_curl = tmp_path / "curl"
fake_curl.write_text(
"""#!/usr/bin/env python3
import json
import sys
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
if url.endswith("/api/health"):
payload = {"status": "ok"}
elif url.endswith("/api/projects"):
payload = {"id": "project-1"}
elif url.endswith("/api/projects/project-1/workflows"):
payload = {"id": "workflow-1"}
elif url.endswith("/api/workflows/workflow-1/run"):
payload = {"id": "run-1"}
elif url.endswith("/api/runs/run-1/logs"):
payload = {"items": [{"message": "ok"}]}
elif url.endswith("/api/runs/run-1/artifacts"):
payload = {"items": [{"name": "source-manifest.json"}]}
elif url.endswith("/api/runs/run-1"):
payload = {"status": "completed", "nodeExecutions": [{"nodeId": "upload-1", "status": "completed"}]}
else:
raise SystemExit(f"unexpected URL: {url}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
fake_curl.chmod(0o755)
result = run_acceptance(
{
"API_BASE_URL": "http://poc.example.test",
"PATH": f"{tmp_path}:{os.environ['PATH']}",
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
}
)
assert result.returncode == 1
assert "expected source-1 to upload-1 inputLineage artifact" in result.stderr
def test_acceptance_requires_gpu_memory_when_completed_gpu_workflow(tmp_path):
fake_curl = tmp_path / "curl"
fake_curl.write_text(
"""#!/usr/bin/env python3
import json
import sys
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
if url.endswith("/api/health"):
payload = {"status": "ok"}
elif url.endswith("/api/projects"):
payload = {"id": "project-1"}
elif url.endswith("/api/projects/project-1/workflows"):
payload = {"id": "workflow-1"}
elif url.endswith("/api/workflows/workflow-1/run"):
payload = {"id": "run-1"}
elif url.endswith("/api/runs/run-1/logs"):
payload = {"items": [{"message": "ok"}]}
elif url.endswith("/api/runs/run-1/artifacts"):
payload = {"items": [{"name": "source-manifest.json"}]}
elif url.endswith("/api/runs/run-1"):
payload = {
"status": "completed",
"nodeExecutions": [
{
"nodeId": "upload-1",
"status": "completed",
"inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}],
"inputLineage": [
{
"sourceNodeId": "source-1",
"targetNodeId": "upload-1",
"artifactId": "artifact-1",
}
],
}
],
}
else:
raise SystemExit(f"unexpected URL: {url}")
print(json.dumps(payload, separators=(",", ":")))
""",
encoding="utf-8",
)
fake_curl.chmod(0o755)
result = run_acceptance(
{
"API_BASE_URL": "http://poc.example.test",
"PATH": f"{tmp_path}:{os.environ['PATH']}",
"WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "completed",
"WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1",
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
}
)
assert result.returncode == 1
assert "expected GPU memory metrics in completed GPU workflow resourceSeries" in result.stderr
def test_acceptance_can_expect_gpu_preflight_failure(tmp_path):
captured_workflow = tmp_path / "workflow.json"
fake_curl = tmp_path / "curl"