forked from longtao.wu/droboticflow
333 lines
12 KiB
Python
333 lines
12 KiB
Python
import json
|
|
import os
|
|
import pathlib
|
|
import subprocess
|
|
|
|
|
|
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "scripts" / "poc-gpu-workflow-blocked-acceptance.sh"
|
|
|
|
|
|
def test_poc_gpu_workflow_acceptance_uses_public_load_balancer_and_expected_failure(tmp_path):
|
|
captured_workflow = tmp_path / "workflow.json"
|
|
stub_bin = tmp_path / "bin"
|
|
stub_bin.mkdir()
|
|
|
|
kubectl = stub_bin / "kubectl"
|
|
kubectl.write_text(
|
|
"""#!/usr/bin/env python3
|
|
import json
|
|
import sys
|
|
|
|
args = sys.argv[1:]
|
|
if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]:
|
|
print(json.dumps({
|
|
"status": {
|
|
"loadBalancer": {
|
|
"ingress": [
|
|
{"ip": "192.168.0.74"},
|
|
{"ip": "120.48.164.19"}
|
|
]
|
|
}
|
|
}
|
|
}))
|
|
raise SystemExit(0)
|
|
print("unexpected kubectl call: " + " ".join(args), file=sys.stderr)
|
|
raise SystemExit(97)
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
kubectl.chmod(0o755)
|
|
|
|
curl = stub_bin / "curl"
|
|
curl.write_text(
|
|
f"""#!/usr/bin/env python3
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
args = sys.argv[1:]
|
|
url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "")
|
|
if url and not url.startswith("http://120.48.164.19"):
|
|
raise SystemExit(f"expected public load balancer URL, got {{url}}")
|
|
|
|
body = ""
|
|
for index, arg in enumerate(args):
|
|
if arg == "-d" and index + 1 < len(args):
|
|
body = args[index + 1]
|
|
|
|
if url.endswith("/api/health"):
|
|
payload = {{"status": "ok"}}
|
|
elif url.endswith("/api/auth/login"):
|
|
payload = {{"email": "operator@example.test"}}
|
|
elif url.endswith("/api/projects"):
|
|
payload = {{"id": "project-1"}}
|
|
elif url.endswith("/api/projects/project-1/workflows"):
|
|
pathlib.Path({str(captured_workflow)!r}).write_text(body, encoding="utf-8")
|
|
payload = {{"id": "workflow-1"}}
|
|
elif url.endswith("/api/workflows/workflow-1/run"):
|
|
payload = {{"id": "run-1"}}
|
|
elif url.endswith("/api/runs/run-1/logs"):
|
|
payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}}
|
|
elif url.endswith("/api/runs/run-1/artifacts"):
|
|
payload = {{"items": []}}
|
|
elif url.endswith("/api/runs/run-1"):
|
|
payload = {{"status": "failed"}}
|
|
else:
|
|
raise SystemExit(f"unexpected URL: {{url}}")
|
|
print(json.dumps(payload, separators=(",", ":")))
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
curl.chmod(0o755)
|
|
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"DEPLOY_ENV": "poc",
|
|
"DROBOTICFLOW_ACCEPTANCE_EMAIL": "operator@example.test",
|
|
"DROBOTICFLOW_ACCEPTANCE_PASSWORD": "secret",
|
|
"K8S_NAMESPACE": "droboticflow",
|
|
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
|
|
"POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "false",
|
|
"POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform",
|
|
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
|
}
|
|
)
|
|
env.pop("API_BASE_URL", None)
|
|
env.pop("BASE_URL", None)
|
|
|
|
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert "run status: failed" in result.stdout
|
|
workflow = json.loads(captured_workflow.read_text(encoding="utf-8"))
|
|
source_node = next(node for node in workflow["nodes"] if node["id"] == "source-1")
|
|
assert source_node["data"]["gpu"] == "1"
|
|
|
|
|
|
def test_poc_gpu_workflow_acceptance_uses_token_endpoint_and_worker_port_forward(tmp_path):
|
|
captured_acceptance = tmp_path / "acceptance.json"
|
|
run_status = tmp_path / "run-status.txt"
|
|
stub_bin = tmp_path / "bin"
|
|
stub_bin.mkdir()
|
|
|
|
kubectl = stub_bin / "kubectl"
|
|
kubectl.write_text(
|
|
"""#!/usr/bin/env python3
|
|
import json
|
|
import sys
|
|
import time
|
|
|
|
args = sys.argv[1:]
|
|
if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]:
|
|
print(json.dumps({
|
|
"status": {
|
|
"loadBalancer": {
|
|
"ingress": [
|
|
{"ip": "192.168.0.74"},
|
|
{"ip": "120.48.164.19"}
|
|
]
|
|
}
|
|
}
|
|
}))
|
|
raise SystemExit(0)
|
|
if args == ["-n", "droboticflow", "get", "secret", "droboticflow-auth-secret", "-o", "json"]:
|
|
print(json.dumps({
|
|
"data": {
|
|
"DROBOTICFLOW_API_TOKEN": "c2hhcmVkLXRva2Vu"
|
|
}
|
|
}))
|
|
raise SystemExit(0)
|
|
if len(args) >= 5 and args[:4] == ["-n", "droboticflow", "port-forward", "service/droboticflow-worker"]:
|
|
print("Forwarding from 127.0.0.1 -> 3002", flush=True)
|
|
time.sleep(30)
|
|
raise SystemExit(0)
|
|
print("unexpected kubectl call: " + " ".join(args), file=sys.stderr)
|
|
raise SystemExit(97)
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
kubectl.chmod(0o755)
|
|
|
|
curl = stub_bin / "curl"
|
|
curl.write_text(
|
|
f"""#!/usr/bin/env python3
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
status_file = pathlib.Path({str(run_status)!r})
|
|
captured_acceptance = pathlib.Path({str(captured_acceptance)!r})
|
|
args = sys.argv[1:]
|
|
url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "")
|
|
headers = [args[index + 1] for index, arg in enumerate(args[:-1]) if arg == "-H"]
|
|
body = ""
|
|
for index, arg in enumerate(args):
|
|
if arg == "-d" and index + 1 < len(args):
|
|
body = args[index + 1]
|
|
|
|
if url.startswith("http://120.48.164.19") and "Authorization: Bearer shared-token" not in headers:
|
|
raise SystemExit("expected bearer token for API call")
|
|
if url.startswith("http://") and "120.48.164.19" not in url and "127.0.0.1" not in url:
|
|
raise SystemExit(f"expected public load balancer or worker port-forward URL, got {{url}}")
|
|
|
|
if url.endswith("/api/health"):
|
|
payload = {{"status": "ok"}}
|
|
elif url.endswith("/api/acceptance/poc-gpu-workflow"):
|
|
captured_acceptance.write_text(body, encoding="utf-8")
|
|
status_file.write_text("queued", encoding="utf-8")
|
|
payload = {{
|
|
"project": {{"id": "project-token", "name": "POC GPU Workflow Acceptance"}},
|
|
"workflow": {{"id": "workflow-token", "name": "POC GPU Unschedulable Acceptance Workflow"}},
|
|
"run": {{"id": "run-token", "status": "queued"}}
|
|
}}
|
|
elif url.endswith("/worker-api/health"):
|
|
payload = {{"status": "ok", "service": "droboticflow-worker"}}
|
|
elif url.endswith("/worker-api/process-once"):
|
|
status_file.write_text("failed", encoding="utf-8")
|
|
payload = {{"run": {{"id": "run-token", "status": "failed"}}}}
|
|
elif url.endswith("/api/runs/run-token/logs"):
|
|
payload = {{"items": [{{"message": "gpu request cannot be scheduled: requested nvidia.com/gpu=1"}}]}}
|
|
elif url.endswith("/api/runs/run-token"):
|
|
payload = {{
|
|
"id": "run-token",
|
|
"status": status_file.read_text(encoding="utf-8").strip(),
|
|
"resourceSeries": [{{"timestamp": "2026-06-28T00:00:00Z", "cpu": 1, "memory": 64, "gpu": 3, "gpuMemoryUsedMiB": 1024, "gpuMemoryTotalMiB": 81920}}],
|
|
}}
|
|
else:
|
|
raise SystemExit(f"unexpected URL: {{url}}")
|
|
print(json.dumps(payload, separators=(",", ":")))
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
curl.chmod(0o755)
|
|
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"DEPLOY_ENV": "poc",
|
|
"K8S_NAMESPACE": "droboticflow",
|
|
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
|
|
"POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "false",
|
|
"POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform",
|
|
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
|
}
|
|
)
|
|
env.pop("API_BASE_URL", None)
|
|
env.pop("BASE_URL", None)
|
|
env.pop("DROBOTICFLOW_API_TOKEN", None)
|
|
env.pop("DROBOTICFLOW_ACCEPTANCE_EMAIL", None)
|
|
env.pop("DROBOTICFLOW_ACCEPTANCE_PASSWORD", None)
|
|
|
|
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert "created POC GPU acceptance project=project-token workflow=workflow-token run=run-token" in result.stdout
|
|
assert "worker port-forward:" in result.stdout
|
|
assert "run status: failed" in result.stdout
|
|
body = json.loads(captured_acceptance.read_text(encoding="utf-8"))
|
|
assert body == {"sourcePath": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform"}
|
|
|
|
|
|
def test_poc_gpu_workflow_acceptance_can_expect_completed_gpu_run(tmp_path):
|
|
captured_acceptance = tmp_path / "acceptance.json"
|
|
run_status = tmp_path / "run-status.txt"
|
|
stub_bin = tmp_path / "bin"
|
|
stub_bin.mkdir()
|
|
|
|
kubectl = stub_bin / "kubectl"
|
|
kubectl.write_text(
|
|
"""#!/usr/bin/env python3
|
|
import json
|
|
import sys
|
|
import time
|
|
|
|
args = sys.argv[1:]
|
|
if args == ["-n", "droboticflow", "get", "svc", "droboticflow-api-public", "-o", "json"]:
|
|
print(json.dumps({"status": {"loadBalancer": {"ingress": [{"ip": "120.48.164.19"}]}}}))
|
|
raise SystemExit(0)
|
|
if args == ["-n", "droboticflow", "get", "secret", "droboticflow-auth-secret", "-o", "json"]:
|
|
print(json.dumps({"data": {"DROBOTICFLOW_API_TOKEN": "c2hhcmVkLXRva2Vu"}}))
|
|
raise SystemExit(0)
|
|
if len(args) >= 5 and args[:4] == ["-n", "droboticflow", "port-forward", "service/droboticflow-worker"]:
|
|
print("Forwarding from 127.0.0.1 -> 3002", flush=True)
|
|
time.sleep(30)
|
|
raise SystemExit(0)
|
|
print("unexpected kubectl call: " + " ".join(args), file=sys.stderr)
|
|
raise SystemExit(97)
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
kubectl.chmod(0o755)
|
|
|
|
curl = stub_bin / "curl"
|
|
curl.write_text(
|
|
f"""#!/usr/bin/env python3
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
status_file = pathlib.Path({str(run_status)!r})
|
|
captured_acceptance = pathlib.Path({str(captured_acceptance)!r})
|
|
args = sys.argv[1:]
|
|
url = next((arg for arg in args if arg.startswith("http://") or arg.startswith("https://")), "")
|
|
body = ""
|
|
for index, arg in enumerate(args):
|
|
if arg == "-d" and index + 1 < len(args):
|
|
body = args[index + 1]
|
|
|
|
if url.endswith("/api/health"):
|
|
payload = {{"status": "ok"}}
|
|
elif url.endswith("/api/acceptance/poc-gpu-workflow"):
|
|
captured_acceptance.write_text(body, encoding="utf-8")
|
|
status_file.write_text("queued", encoding="utf-8")
|
|
payload = {{"project": {{"id": "project-token"}}, "workflow": {{"id": "workflow-token"}}, "run": {{"id": "run-token", "status": "queued"}}}}
|
|
elif url.endswith("/worker-api/health"):
|
|
payload = {{"status": "ok"}}
|
|
elif url.endswith("/worker-api/process-once"):
|
|
status_file.write_text("completed", encoding="utf-8")
|
|
payload = {{"run": {{"id": "run-token", "status": "completed"}}}}
|
|
elif url.endswith("/api/runs/run-token/logs"):
|
|
payload = {{"items": [{{"message": "GPU operator completed"}}]}}
|
|
elif url.endswith("/api/runs/run-token/artifacts"):
|
|
payload = {{"items": [{{"name": "manifest.json", "type": "json", "size": 128}}]}}
|
|
elif url.endswith("/api/runs/run-token"):
|
|
payload = {{
|
|
"id": "run-token",
|
|
"status": status_file.read_text(encoding="utf-8").strip(),
|
|
"resourceSeries": [{{"timestamp": "2026-06-28T00:00:00Z", "cpu": 1, "memory": 64, "gpu": 3, "gpuMemoryUsedMiB": 1024, "gpuMemoryTotalMiB": 81920}}],
|
|
}}
|
|
else:
|
|
raise SystemExit(f"unexpected URL: {{url}}")
|
|
print(json.dumps(payload, separators=(",", ":")))
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
curl.chmod(0o755)
|
|
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"DEPLOY_ENV": "poc",
|
|
"K8S_NAMESPACE": "droboticflow",
|
|
"PATH": f"{stub_bin}{os.pathsep}{env.get('PATH', '')}",
|
|
"POC_GPU_WORKFLOW_EXPECT_AVAILABLE": "true",
|
|
"POC_GPU_WORKFLOW_SOURCE_PATH": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform",
|
|
"WORKFLOW_ACCEPTANCE_RUN_TIMEOUT_SECONDS": "1",
|
|
}
|
|
)
|
|
env.pop("API_BASE_URL", None)
|
|
env.pop("BASE_URL", None)
|
|
env.pop("DROBOTICFLOW_API_TOKEN", None)
|
|
env.pop("DROBOTICFLOW_ACCEPTANCE_EMAIL", None)
|
|
env.pop("DROBOTICFLOW_ACCEPTANCE_PASSWORD", None)
|
|
|
|
result = subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
|
|
|
assert result.returncode == 0, result.stderr + result.stdout
|
|
assert "run status: completed" in result.stdout
|
|
assert "artifacts summary:" in result.stdout
|
|
body = json.loads(captured_acceptance.read_text(encoding="utf-8"))
|
|
assert body == {"sourcePath": "/workspace/datasets/fixtures/poc-operator-acceptance/xplatform"}
|