forked from longtao.wu/droboticflow
🧪 Require live workflow lineage metrics
This commit is contained in:
parent
f04c11f05e
commit
da25d845fc
@ -249,6 +249,35 @@ if needle not in messages: raise SystemExit(f"expected log substring not found:
|
||||
fi
|
||||
|
||||
if [[ "$WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS" == "completed" ]]; then
|
||||
FINAL_RUN_JSON="$final_run" WORKFLOW_ACCEPTANCE_GPU_LIMIT="$WORKFLOW_ACCEPTANCE_GPU_LIMIT" python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
|
||||
payload = json.loads(os.environ["FINAL_RUN_JSON"])
|
||||
executions = {item.get("nodeId"): item for item in payload.get("nodeExecutions") or [] if isinstance(item, dict)}
|
||||
upload = executions.get("upload-1")
|
||||
if not upload:
|
||||
raise SystemExit("expected upload-1 node execution in completed workflow")
|
||||
|
||||
lineage = upload.get("inputLineage") or []
|
||||
if not any(item.get("sourceNodeId") == "source-1" and item.get("targetNodeId") == "upload-1" and item.get("artifactId") for item in lineage):
|
||||
raise SystemExit("expected source-1 to upload-1 inputLineage artifact")
|
||||
if not upload.get("inputArtifacts"):
|
||||
raise SystemExit("expected upload-1 inputArtifacts from source-1")
|
||||
|
||||
gpu_limit = os.environ.get("WORKFLOW_ACCEPTANCE_GPU_LIMIT", "").strip()
|
||||
if gpu_limit:
|
||||
samples = []
|
||||
for item in payload.get("resourceSeries") or []:
|
||||
if isinstance(item, dict):
|
||||
samples.append(item)
|
||||
for node in payload.get("nodeExecutions") or []:
|
||||
for item in node.get("resourceSeries") or []:
|
||||
if isinstance(item, dict):
|
||||
samples.append(item)
|
||||
if not any(int(sample.get("gpuMemoryTotalMiB") or 0) > 0 for sample in samples):
|
||||
raise SystemExit("expected GPU memory metrics in completed GPU workflow resourceSeries")
|
||||
PY
|
||||
printf '%s' "$artifacts_response" | python3 -c 'import json, sys; payload = json.load(sys.stdin); items = payload.get("items", []);
|
||||
if not items: raise SystemExit("expected at least one artifact")'
|
||||
fi
|
||||
|
||||
@ -70,7 +70,31 @@ elif url.endswith("/api/runs/run-1/logs"):
|
||||
elif url.endswith("/api/runs/run-1/artifacts"):
|
||||
payload = {"items": [{"name": "source-manifest.json"}]}
|
||||
elif url.endswith("/api/runs/run-1"):
|
||||
payload = {"status": "completed"}
|
||||
payload = {
|
||||
"status": "completed",
|
||||
"nodeExecutions": [
|
||||
{
|
||||
"nodeId": "source-1",
|
||||
"status": "completed",
|
||||
"resourceSeries": [],
|
||||
},
|
||||
{
|
||||
"nodeId": "upload-1",
|
||||
"status": "completed",
|
||||
"inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}],
|
||||
"inputLineage": [
|
||||
{
|
||||
"sourceNodeId": "source-1",
|
||||
"sourcePortId": "out-1",
|
||||
"targetNodeId": "upload-1",
|
||||
"targetPortId": "in-1",
|
||||
"artifactId": "artifact-1",
|
||||
}
|
||||
],
|
||||
"resourceSeries": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
else:
|
||||
raise SystemExit(f"unexpected URL: {url}")
|
||||
print(json.dumps(payload, separators=(",", ":")))
|
||||
@ -93,6 +117,112 @@ print(json.dumps(payload, separators=(",", ":")))
|
||||
assert "run status: completed" in result.stdout
|
||||
|
||||
|
||||
def test_acceptance_requires_completed_workflow_lineage(tmp_path):
|
||||
fake_curl = tmp_path / "curl"
|
||||
fake_curl.write_text(
|
||||
"""#!/usr/bin/env python3
|
||||
import json
|
||||
import sys
|
||||
|
||||
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
|
||||
if url.endswith("/api/health"):
|
||||
payload = {"status": "ok"}
|
||||
elif url.endswith("/api/projects"):
|
||||
payload = {"id": "project-1"}
|
||||
elif url.endswith("/api/projects/project-1/workflows"):
|
||||
payload = {"id": "workflow-1"}
|
||||
elif url.endswith("/api/workflows/workflow-1/run"):
|
||||
payload = {"id": "run-1"}
|
||||
elif url.endswith("/api/runs/run-1/logs"):
|
||||
payload = {"items": [{"message": "ok"}]}
|
||||
elif url.endswith("/api/runs/run-1/artifacts"):
|
||||
payload = {"items": [{"name": "source-manifest.json"}]}
|
||||
elif url.endswith("/api/runs/run-1"):
|
||||
payload = {"status": "completed", "nodeExecutions": [{"nodeId": "upload-1", "status": "completed"}]}
|
||||
else:
|
||||
raise SystemExit(f"unexpected URL: {url}")
|
||||
print(json.dumps(payload, separators=(",", ":")))
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_curl.chmod(0o755)
|
||||
|
||||
result = run_acceptance(
|
||||
{
|
||||
"API_BASE_URL": "http://poc.example.test",
|
||||
"PATH": f"{tmp_path}:{os.environ['PATH']}",
|
||||
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
|
||||
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
||||
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
|
||||
}
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "expected source-1 to upload-1 inputLineage artifact" in result.stderr
|
||||
|
||||
|
||||
def test_acceptance_requires_gpu_memory_when_completed_gpu_workflow(tmp_path):
|
||||
fake_curl = tmp_path / "curl"
|
||||
fake_curl.write_text(
|
||||
"""#!/usr/bin/env python3
|
||||
import json
|
||||
import sys
|
||||
|
||||
url = next((arg for arg in sys.argv[1:] if arg.startswith("http://") or arg.startswith("https://")), "")
|
||||
if url.endswith("/api/health"):
|
||||
payload = {"status": "ok"}
|
||||
elif url.endswith("/api/projects"):
|
||||
payload = {"id": "project-1"}
|
||||
elif url.endswith("/api/projects/project-1/workflows"):
|
||||
payload = {"id": "workflow-1"}
|
||||
elif url.endswith("/api/workflows/workflow-1/run"):
|
||||
payload = {"id": "run-1"}
|
||||
elif url.endswith("/api/runs/run-1/logs"):
|
||||
payload = {"items": [{"message": "ok"}]}
|
||||
elif url.endswith("/api/runs/run-1/artifacts"):
|
||||
payload = {"items": [{"name": "source-manifest.json"}]}
|
||||
elif url.endswith("/api/runs/run-1"):
|
||||
payload = {
|
||||
"status": "completed",
|
||||
"nodeExecutions": [
|
||||
{
|
||||
"nodeId": "upload-1",
|
||||
"status": "completed",
|
||||
"inputArtifacts": [{"id": "artifact-1", "nodeId": "source-1"}],
|
||||
"inputLineage": [
|
||||
{
|
||||
"sourceNodeId": "source-1",
|
||||
"targetNodeId": "upload-1",
|
||||
"artifactId": "artifact-1",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
else:
|
||||
raise SystemExit(f"unexpected URL: {url}")
|
||||
print(json.dumps(payload, separators=(",", ":")))
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_curl.chmod(0o755)
|
||||
|
||||
result = run_acceptance(
|
||||
{
|
||||
"API_BASE_URL": "http://poc.example.test",
|
||||
"PATH": f"{tmp_path}:{os.environ['PATH']}",
|
||||
"WORKFLOW_ACCEPTANCE_EXPECT_TERMINAL_STATUS": "completed",
|
||||
"WORKFLOW_ACCEPTANCE_GPU_LIMIT": "1",
|
||||
"WORKFLOW_ACCEPTANCE_REQUIRE_EXPLICIT_ENDPOINT": "true",
|
||||
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
||||
"WORKFLOW_ACCEPTANCE_SOURCE_PATH": "/workspace/real-data",
|
||||
}
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "expected GPU memory metrics in completed GPU workflow resourceSeries" in result.stderr
|
||||
|
||||
|
||||
def test_acceptance_can_expect_gpu_preflight_failure(tmp_path):
|
||||
captured_workflow = tmp_path / "workflow.json"
|
||||
fake_curl = tmp_path / "curl"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user