forked from longtao.wu/droboticflow
🚀 Harden production runtime and acceptance
This commit is contained in:
parent
0ef6b6be58
commit
20b36fe6a0
@ -6,6 +6,7 @@ MONGO_DB=droboticflow
|
||||
MONGO_CONNECT_RETRIES=30
|
||||
MONGO_CONNECT_RETRY_DELAY_MS=2000
|
||||
WORKER_POLL_INTERVAL_MS=2000
|
||||
WORKER_API_BASE_URL=http://droboticflow-worker:3002
|
||||
K8S_NAMESPACE=droboticflow
|
||||
K8S_IMAGE_PULL_SECRET_NAME=droboticflow-registry
|
||||
WORKSPACE_PVC_NAME=droboticflow-workspace
|
||||
@ -15,6 +16,7 @@ OPERATOR_RUNNER_IMAGE=
|
||||
OPERATOR_GPU_NODE_SELECTOR=
|
||||
OPERATOR_GPU_TOLERATIONS_JSON=
|
||||
OPERATOR_GPU_RUNTIME_CLASS=
|
||||
OPERATOR_JOB_TTL_SECONDS=86400
|
||||
OPERATOR_ALLOW_SHELL_COMMANDS=false
|
||||
OPERATOR_COMMAND_ALLOWLIST_REGEX=
|
||||
OPERATOR_COMMAND_DENYLIST_REGEX=
|
||||
|
||||
77
.github/workflows/acceptance.yml
vendored
Normal file
77
.github/workflows/acceptance.yml
vendored
Normal file
@ -0,0 +1,77 @@
|
||||
name: Acceptance
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
api_base_url:
|
||||
description: "Public API base URL, for example http://<node-ip>:30080"
|
||||
required: false
|
||||
type: string
|
||||
worker_base_url:
|
||||
description: "Optional direct worker base URL. Leave empty to use API proxy only."
|
||||
required: false
|
||||
type: string
|
||||
k8s_namespace:
|
||||
description: "Kubernetes namespace for GPU smoke"
|
||||
required: false
|
||||
default: droboticflow
|
||||
type: string
|
||||
run_api_smoke:
|
||||
description: "Run API/workflow smoke test"
|
||||
required: true
|
||||
default: true
|
||||
type: boolean
|
||||
run_gpu_smoke:
|
||||
description: "Run real GPU scheduling and nvidia-smi smoke test"
|
||||
required: true
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
acceptance:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
K8S_NAMESPACE: ${{ inputs.k8s_namespace }}
|
||||
KUBE_CONFIG_B64: ${{ secrets.KUBE_CONFIG_B64 }}
|
||||
DROBOTICFLOW_API_TOKEN: ${{ secrets.DROBOTICFLOW_API_TOKEN }}
|
||||
DROBOTICFLOW_SMOKE_EMAIL: ${{ secrets.DROBOTICFLOW_SMOKE_EMAIL }}
|
||||
DROBOTICFLOW_SMOKE_PASSWORD: ${{ secrets.DROBOTICFLOW_SMOKE_PASSWORD }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up kubectl
|
||||
uses: azure/setup-kubectl@v4
|
||||
with:
|
||||
version: v1.34.2
|
||||
|
||||
- name: Configure kubeconfig
|
||||
run: |
|
||||
if [[ -z "${KUBE_CONFIG_B64}" ]]; then
|
||||
echo "KUBE_CONFIG_B64 secret is not set; Kubernetes smoke steps require it."
|
||||
exit 0
|
||||
fi
|
||||
mkdir -p "$HOME/.kube"
|
||||
printf '%s' "$KUBE_CONFIG_B64" | base64 --decode > "$HOME/.kube/config"
|
||||
chmod 600 "$HOME/.kube/config"
|
||||
|
||||
- name: API smoke
|
||||
if: inputs.run_api_smoke
|
||||
env:
|
||||
API_BASE_URL: ${{ inputs.api_base_url }}
|
||||
WORKER_BASE_URL: ${{ inputs.worker_base_url }}
|
||||
run: |
|
||||
if [[ -z "$API_BASE_URL" ]]; then
|
||||
echo "api_base_url is required when run_api_smoke=true" >&2
|
||||
exit 1
|
||||
fi
|
||||
./scripts/smoke-test.sh "$API_BASE_URL" "$WORKER_BASE_URL"
|
||||
|
||||
- name: GPU smoke
|
||||
if: inputs.run_gpu_smoke
|
||||
run: |
|
||||
if [[ ! -s "$HOME/.kube/config" ]]; then
|
||||
echo "KUBE_CONFIG_B64 secret is required when run_gpu_smoke=true" >&2
|
||||
exit 1
|
||||
fi
|
||||
./scripts/gpu-smoke-test.sh
|
||||
@ -66,6 +66,8 @@ Drobodata is a workflow orchestration platform for embodied AI and vision pipeli
|
||||
- `npm run build`: build frontend assets and Go binaries
|
||||
- `npm run verify`: lint, Go tests, Python operator-runner tests, frontend node tests, build, dependency audit, and Kubernetes manifest dry-run validation
|
||||
- `npm run smoke`: local smoke test against the running API and worker
|
||||
- `npm run gpu:smoke`: real Kubernetes GPU scheduling and `nvidia-smi` smoke test
|
||||
- `npm run k8s:cleanup-jobs`: delete old finished operator-runner Jobs that predate TTL cleanup
|
||||
- `npm run reconcile:data -- <api-base-url>`: remove legacy demo data, reset settings, and bootstrap built-in operators through the real API
|
||||
- `python3 scripts/wjzh_operator_regression.py --preflight-only`: preflight the production-style WJZH real-data operator regression against a live API
|
||||
|
||||
@ -80,6 +82,7 @@ Key environment variables:
|
||||
- `MONGO_CONNECT_RETRIES`: startup retry count for API and worker Mongo connections
|
||||
- `MONGO_CONNECT_RETRY_DELAY_MS`: delay between Mongo connection retries
|
||||
- `WORKER_POLL_INTERVAL_MS`: worker polling interval
|
||||
- `WORKER_API_BASE_URL`: API-side upstream URL for `/worker-api/*`, default `http://droboticflow-worker:3002`
|
||||
- `K8S_NAMESPACE`: namespace used for Jobs
|
||||
- `K8S_IMAGE_PULL_SECRET_NAME`: image pull secret injected into operator-runner Jobs
|
||||
- `WORKSPACE_PVC_NAME`: shared PVC used by worker, API, and operator-runner Jobs
|
||||
@ -89,6 +92,7 @@ Key environment variables:
|
||||
- `OPERATOR_GPU_NODE_SELECTOR`: optional comma-separated node selector for GPU Jobs, for example `accelerator=nvidia`
|
||||
- `OPERATOR_GPU_TOLERATIONS_JSON`: optional Kubernetes tolerations JSON array for GPU Jobs
|
||||
- `OPERATOR_GPU_RUNTIME_CLASS`: optional runtimeClassName for GPU Jobs, for example `nvidia`
|
||||
- `OPERATOR_JOB_TTL_SECONDS`: Kubernetes TTL for finished operator-runner Jobs, default `86400`; set `0` to disable automatic TTL injection
|
||||
- `OPERATOR_ALLOW_SHELL_COMMANDS`: controls whether `COMMAND` / `SCRIPT` shell execution is allowed, default `false`
|
||||
- `OPERATOR_COMMAND_ALLOWLIST_REGEX`: optional regular expression that explicit shell commands must match
|
||||
- `OPERATOR_COMMAND_DENYLIST_REGEX`: optional regular expression that rejects matching explicit shell commands
|
||||
@ -107,7 +111,7 @@ The settings page can update K8s access details, the worker defaults, the global
|
||||
|
||||
The API now includes a cookie-session account foundation for organization-based access. Auth endpoints cover bootstrap, login, logout, current session, and super-admin organization/user management. Existing local deployments remain compatible by default; public LoadBalancer deploys require `DROBOTICFLOW_REQUIRE_SESSION_AUTH=true` so browser business APIs keep role checks. Session-authenticated unsafe requests use a double-submit `droboticflow_csrf` cookie plus `X-Droboticflow-CSRF-Token` header. Worker control endpoints under `/worker-api` require an admin/operator session, or an API token when session auth is not enforced. Do not embed `DROBOTICFLOW_API_TOKEN` into frontend builds; it is intended for machine-to-machine calls only.
|
||||
|
||||
The Kubernetes runtime now assumes a shared workspace PVC. Worker writes per-node `input-artifacts.json` files into the shared workspace, operator-runner Jobs read those files through `INPUT_ARTIFACTS_PATH`, then persist `result.json` and `stage-manifest.json` back into the same PVC. Large artifact lists no longer need to travel through giant env vars or stdout payloads. Node CPU and memory limits are applied to Kubernetes Job container resources; GPU requests use `nvidia.com/gpu` when the node or default resource config sets a positive GPU count. GPU scheduling requires a cluster with GPU nodes and the NVIDIA device plugin/runtime installed. Optional GPU node selector, tolerations, and runtime class settings are only injected into Jobs that request a positive GPU count. Invalid GPU selector or tolerations configuration fails worker startup instead of being ignored. Run `npm run gpu:smoke` against a real GPU cluster to validate allocatable GPU resources, scheduling, runtime, and `nvidia-smi` execution.
|
||||
The Kubernetes runtime now assumes a shared workspace PVC. Worker writes per-node `input-artifacts.json` files into the shared workspace, operator-runner Jobs read those files through `INPUT_ARTIFACTS_PATH`, then persist `result.json` and `stage-manifest.json` back into the same PVC. Large artifact lists no longer need to travel through giant env vars or stdout payloads. Node CPU and memory limits are applied to Kubernetes Job container resources; GPU requests use `nvidia.com/gpu` when the node or default resource config sets a positive GPU count. GPU scheduling requires a cluster with GPU nodes and the NVIDIA device plugin/runtime installed. Optional GPU node selector, tolerations, and runtime class settings are only injected into Jobs that request a positive GPU count. Invalid GPU selector or tolerations configuration fails worker startup instead of being ignored. Operator-runner Jobs also receive `ttlSecondsAfterFinished` from `OPERATOR_JOB_TTL_SECONDS`, and the cleanup script can remove old completed Jobs that were created before TTL was enabled. Run `npm run gpu:smoke` against a real GPU cluster to validate allocatable GPU resources, scheduling, runtime, and `nvidia-smi` execution. Runs that execute on NVIDIA hardware report GPU utilization, GPU memory, GPU count, and GPU model in the resource samples shown on the monitoring page.
|
||||
|
||||
Explicit `COMMAND` / `SCRIPT` execution is disabled by default. Deployments that need the shell operator must set `OPERATOR_ALLOW_SHELL_COMMANDS=true` and should constrain commands through allowlist and denylist regular expressions. These policy variables are protected system env vars: workflow and node hyperparams cannot override the worker-provided values.
|
||||
|
||||
@ -219,6 +223,8 @@ Run a smoke test against a deployed cluster:
|
||||
./scripts/smoke-test.sh http://<node-ip>:30080
|
||||
```
|
||||
|
||||
Run real-environment acceptance from GitHub Actions with the manual `Acceptance` workflow. It accepts the deployed API URL, optional direct worker URL, and can run GPU smoke when the repository secret `KUBE_CONFIG_B64` is configured.
|
||||
|
||||
See the detailed guides:
|
||||
|
||||
- [`docs/api.md`](docs/api.md)
|
||||
@ -230,6 +236,7 @@ See the detailed guides:
|
||||
- [`docs/operators/README.md`](docs/operators/README.md)
|
||||
- [`docs/operator-authoring.md`](docs/operator-authoring.md)
|
||||
- [`docs/operations.md`](docs/operations.md)
|
||||
- [`docs/production-readiness.md`](docs/production-readiness.md)
|
||||
- [`docs/wjzh-operator-regression.md`](docs/wjzh-operator-regression.md)
|
||||
- [`docs/feature-overview-playwright-validation-2026-04-15.md`](docs/feature-overview-playwright-validation-2026-04-15.md)
|
||||
- [`docs/feature-overview-playwright-validation-2026-04-15.html`](docs/feature-overview-playwright-validation-2026-04-15.html)
|
||||
|
||||
@ -18,15 +18,45 @@ def resource_snapshot() -> tuple[float, int]:
|
||||
return cpu_seconds, memory_mib
|
||||
|
||||
|
||||
def gpu_snapshot() -> int:
|
||||
def parse_gpu_snapshot_output(output: str) -> dict[str, Any]:
|
||||
samples: list[dict[str, Any]] = []
|
||||
for line in output.splitlines():
|
||||
parts = [part.strip() for part in line.split(",")]
|
||||
if len(parts) < 4:
|
||||
continue
|
||||
try:
|
||||
samples.append(
|
||||
{
|
||||
"gpu": int(parts[0]),
|
||||
"gpuMemoryUsedMiB": int(parts[1]),
|
||||
"gpuMemoryTotalMiB": int(parts[2]),
|
||||
"gpuName": parts[3],
|
||||
}
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
if not samples:
|
||||
return {"gpu": 0}
|
||||
|
||||
names = sorted({sample["gpuName"] for sample in samples if sample["gpuName"]})
|
||||
return {
|
||||
"gpu": max(0, min(100, max(sample["gpu"] for sample in samples))),
|
||||
"gpuMemoryUsedMiB": max(0, sum(sample["gpuMemoryUsedMiB"] for sample in samples)),
|
||||
"gpuMemoryTotalMiB": max(0, sum(sample["gpuMemoryTotalMiB"] for sample in samples)),
|
||||
"gpuName": ", ".join(names),
|
||||
"gpuCount": len(samples),
|
||||
}
|
||||
|
||||
|
||||
def gpu_snapshot() -> dict[str, Any]:
|
||||
nvidia_smi = shutil.which("nvidia-smi")
|
||||
if not nvidia_smi:
|
||||
return 0
|
||||
return {"gpu": 0}
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[
|
||||
nvidia_smi,
|
||||
"--query-gpu=utilization.gpu",
|
||||
"--query-gpu=utilization.gpu,memory.used,memory.total,name",
|
||||
"--format=csv,noheader,nounits",
|
||||
],
|
||||
check=False,
|
||||
@ -35,31 +65,20 @@ def gpu_snapshot() -> int:
|
||||
timeout=2,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return 0
|
||||
return {"gpu": 0}
|
||||
if completed.returncode != 0:
|
||||
return 0
|
||||
values: list[int] = []
|
||||
for line in completed.stdout.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
values.append(int(line))
|
||||
except ValueError:
|
||||
continue
|
||||
if not values:
|
||||
return 0
|
||||
return max(0, min(100, max(values)))
|
||||
return {"gpu": 0}
|
||||
return parse_gpu_snapshot_output(completed.stdout)
|
||||
|
||||
|
||||
def resource_series(start_cpu: float, start_memory: int, started_at: str, ended_at: str) -> list[dict[str, Any]]:
|
||||
end_cpu, end_memory = resource_snapshot()
|
||||
gpu_percent = gpu_snapshot()
|
||||
gpu_metrics = gpu_snapshot()
|
||||
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
|
||||
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
|
||||
elapsed = max((ended - started).total_seconds(), 0.001)
|
||||
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
|
||||
return [
|
||||
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
|
||||
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, "gpu": gpu_percent},
|
||||
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, **gpu_metrics},
|
||||
]
|
||||
|
||||
@ -15,6 +15,12 @@ assert SPEC is not None and SPEC.loader is not None
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
RESOURCE_MONITOR_PATH = pathlib.Path(__file__).with_name("resource_monitor.py")
|
||||
RESOURCE_MONITOR_SPEC = importlib.util.spec_from_file_location("operator_runner_resource_monitor", RESOURCE_MONITOR_PATH)
|
||||
assert RESOURCE_MONITOR_SPEC is not None and RESOURCE_MONITOR_SPEC.loader is not None
|
||||
RESOURCE_MONITOR = importlib.util.module_from_spec(RESOURCE_MONITOR_SPEC)
|
||||
RESOURCE_MONITOR_SPEC.loader.exec_module(RESOURCE_MONITOR)
|
||||
|
||||
|
||||
class _FakeHTTPResponse:
|
||||
def __init__(self, payload: bytes):
|
||||
@ -119,6 +125,28 @@ class CommandPolicyTests(unittest.TestCase):
|
||||
MODULE.validate_command_policy("bash script.sh")
|
||||
|
||||
|
||||
class ResourceMonitorTests(unittest.TestCase):
|
||||
def test_parse_gpu_snapshot_output_summarizes_real_nvidia_smi_columns(self):
|
||||
snapshot = RESOURCE_MONITOR.parse_gpu_snapshot_output(
|
||||
"\n".join(
|
||||
[
|
||||
"12, 4096, 24576, NVIDIA L4",
|
||||
"83, 8192, 24576, NVIDIA L4",
|
||||
"not-a-number, 1, 1, broken",
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(snapshot["gpu"], 83)
|
||||
self.assertEqual(snapshot["gpuMemoryUsedMiB"], 12288)
|
||||
self.assertEqual(snapshot["gpuMemoryTotalMiB"], 49152)
|
||||
self.assertEqual(snapshot["gpuName"], "NVIDIA L4")
|
||||
self.assertEqual(snapshot["gpuCount"], 2)
|
||||
|
||||
def test_parse_gpu_snapshot_output_defaults_when_nvidia_smi_has_no_valid_rows(self):
|
||||
self.assertEqual(RESOURCE_MONITOR.parse_gpu_snapshot_output(""), {"gpu": 0})
|
||||
|
||||
|
||||
class SourcePathFromItemTests(unittest.TestCase):
|
||||
def test_prefers_workspace_bos_cache_for_bos_source_paths(self):
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
|
||||
@ -73,6 +73,7 @@ func main() {
|
||||
cfg.OperatorGPUNodeSelector,
|
||||
cfg.OperatorGPUTolerations,
|
||||
cfg.OperatorGPURuntimeClass,
|
||||
cfg.OperatorJobTTLSeconds,
|
||||
)
|
||||
if err != nil {
|
||||
fmt.Printf("[worker] failed to initialize runtime: %v\n", err)
|
||||
|
||||
14
docs/api.md
14
docs/api.md
@ -228,6 +228,20 @@ The built-in catalog includes the original workflow primitives plus:
|
||||
- `GET /api/runs/:id/resources`
|
||||
- `GET /api/runs/:id/artifacts`
|
||||
|
||||
`GET /api/runs/:id/logs`, `GET /api/runs/:id/resources`, and `GET /api/runs/:id/artifacts` accept optional `limit` and `offset` query parameters. `skip` is accepted as an alias for `offset`. Logs and resource samples default to the latest 1000 items when the run contains a large history; artifacts default from the start. The response includes:
|
||||
|
||||
```json
|
||||
{
|
||||
"items": [],
|
||||
"total": 1200,
|
||||
"limit": 1000,
|
||||
"offset": 200,
|
||||
"truncated": true
|
||||
}
|
||||
```
|
||||
|
||||
Resource samples include `timestamp`, `cpu`, and `memory`. When NVIDIA GPU metrics are available, samples may also include `gpu`, `gpuMemoryUsedMiB`, `gpuMemoryTotalMiB`, `gpuName`, and `gpuCount`.
|
||||
|
||||
`GET /api/runs/:id/artifacts`
|
||||
|
||||
Each artifact item can now include:
|
||||
|
||||
@ -162,6 +162,7 @@ http://<node-ip>:30080
|
||||
- `MONGO_CONNECT_RETRIES`
|
||||
- `MONGO_CONNECT_RETRY_DELAY_MS`
|
||||
- `WORKER_POLL_INTERVAL_MS`
|
||||
- `WORKER_API_BASE_URL`
|
||||
- `API_NODE_PORT`
|
||||
- `API_REPLICAS`
|
||||
- `WORKER_REPLICAS`
|
||||
@ -178,6 +179,7 @@ http://<node-ip>:30080
|
||||
- `OPERATOR_GPU_NODE_SELECTOR`
|
||||
- `OPERATOR_GPU_TOLERATIONS_JSON`
|
||||
- `OPERATOR_GPU_RUNTIME_CLASS`
|
||||
- `OPERATOR_JOB_TTL_SECONDS`
|
||||
- `OPERATOR_ALLOW_SHELL_COMMANDS`
|
||||
- `OPERATOR_COMMAND_ALLOWLIST_REGEX`
|
||||
- `OPERATOR_COMMAND_DENYLIST_REGEX`
|
||||
@ -189,10 +191,12 @@ http://<node-ip>:30080
|
||||
- The main manifest only deploys the application components. It does not create MongoDB resources in-cluster.
|
||||
- `MONGO_URI` must reference an external MongoDB instance; localhost, `mongodb:27017`, and in-cluster `.svc` addresses are rejected by `scripts/deploy-k8s.sh`.
|
||||
- API and worker entrypoints are Go binaries with Mongo startup retry, so they tolerate the database becoming ready after the Deployments start.
|
||||
- The API proxies `/worker-api/*` to `WORKER_API_BASE_URL`, defaulting to the in-cluster `http://droboticflow-worker:3002` Service. Override it only when the worker Service name, namespace, or exposure model differs.
|
||||
- The worker runs with Kubernetes RBAC that allows creating and watching Jobs in the namespace.
|
||||
- The operator-runner Job template inherits the private registry pull secret name from `K8S_IMAGE_PULL_SECRET_NAME`.
|
||||
- Operator-runner Jobs apply CPU and memory requests/limits from system defaults and node overrides. A positive GPU value is emitted as `nvidia.com/gpu` in both requests and limits; the target cluster must have GPU nodes and the NVIDIA device plugin/runtime configured.
|
||||
- GPU-only scheduling settings are optional: `OPERATOR_GPU_NODE_SELECTOR` accepts comma-separated `key=value` pairs, `OPERATOR_GPU_TOLERATIONS_JSON` accepts a Kubernetes tolerations JSON array, and `OPERATOR_GPU_RUNTIME_CLASS` sets `runtimeClassName`. These settings are only injected into Jobs that request a positive GPU count. Invalid selector or tolerations values fail worker startup.
|
||||
- Finished operator-runner Jobs receive `ttlSecondsAfterFinished` from `OPERATOR_JOB_TTL_SECONDS`, default `86400`. Use `npm run k8s:cleanup-jobs` to delete old finished Jobs that predate TTL, or set `DRY_RUN=true` first to list candidates.
|
||||
- Explicit `COMMAND` / `SCRIPT` execution is disabled by default. Set `OPERATOR_ALLOW_SHELL_COMMANDS=true` only for deployments that need shell operators, and constrain it with `OPERATOR_COMMAND_ALLOWLIST_REGEX` / `OPERATOR_COMMAND_DENYLIST_REGEX` where possible. These values are injected by the worker and are not overridable from workflow hyperparams.
|
||||
- The operator-runner is not a standalone Deployment. It is launched on demand as a Python Job container.
|
||||
- Failed operator-runner Jobs report Job conditions, Pod phase, container waiting or termination reasons, Pod events, and available logs back through the run error path.
|
||||
@ -206,5 +210,6 @@ http://<node-ip>:30080
|
||||
- `IMAGE_TAG` defaults to the current git short SHA in build, deploy, and render scripts. Set it explicitly for release names or rollback-friendly tags.
|
||||
- `scripts/smoke-test.sh` can exercise session-authenticated deployments when `DROBOTICFLOW_SMOKE_EMAIL` and `DROBOTICFLOW_SMOKE_PASSWORD` are set; it logs in, persists cookies, and sends the CSRF header on write requests.
|
||||
- `npm run gpu:smoke` creates a temporary GPU Job in the target namespace and runs `nvidia-smi`. Override `GPU_SMOKE_IMAGE`, `GPU_RESOURCE_NAME`, `GPU_RESOURCE_COUNT`, and the `OPERATOR_GPU_*` scheduling variables when the cluster uses custom GPU images or labels.
|
||||
- GitHub Actions includes a manual `Acceptance` workflow for deployed-environment checks. Configure `KUBE_CONFIG_B64` for Kubernetes access and optionally `DROBOTICFLOW_API_TOKEN`, `DROBOTICFLOW_SMOKE_EMAIL`, and `DROBOTICFLOW_SMOKE_PASSWORD` for authenticated smoke tests.
|
||||
- If you use DashScope-compatible models, prefer a backend-allowed API key. Do not deploy tool-only Coding Plan keys into the cluster.
|
||||
- Production hardening still depends on the target cluster policy: TLS ingress, network policies, backup strategy, and secret rotation.
|
||||
|
||||
@ -68,6 +68,19 @@ This includes TypeScript/Go linting, frontend Prettier formatting checks, Go tes
|
||||
|
||||
The worker exposes Prometheus text metrics at `/worker-api/metrics`. The endpoint includes run status gauges, ProcessAvailable counters, dispatch counters, node execution counters, cumulative duration counters, duration histograms, and the most recent dispatch queue depth. Kubernetes deployments annotate the worker Service for Prometheus scraping at this path.
|
||||
|
||||
Runs that execute on NVIDIA hardware also persist GPU utilization, GPU memory used/total in MiB, GPU count, and GPU model into `run.resourceSeries`. The monitoring page shows these fields in the resource modal. A missing GPU memory value means the operator-runner could not execute `nvidia-smi`, not that the workflow requested no GPU.
|
||||
|
||||
## Job Retention
|
||||
|
||||
New operator-runner Jobs use `OPERATOR_JOB_TTL_SECONDS` as `ttlSecondsAfterFinished`, defaulting to one day. For clusters upgraded from older manifests, clean pre-existing finished Jobs with:
|
||||
|
||||
```bash
|
||||
DRY_RUN=true npm run k8s:cleanup-jobs
|
||||
npm run k8s:cleanup-jobs
|
||||
```
|
||||
|
||||
Set `OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS` and `OPERATOR_JOB_CLEANUP_STATUSES=succeeded,failed` to adjust the retention window and terminal states.
|
||||
|
||||
## Account Security
|
||||
|
||||
The API supports HttpOnly cookie-session auth through `/api/auth/*` and super-admin organization/user management under `/api/admin/*`. Existing deployments keep their current behavior unless `DROBOTICFLOW_REQUIRE_SESSION_AUTH=true` is set. Before enabling it, bootstrap the first super admin and verify login through `/api/auth/login`.
|
||||
@ -76,6 +89,10 @@ The API supports HttpOnly cookie-session auth through `/api/auth/*` and super-ad
|
||||
|
||||
Explicit `COMMAND` / `SCRIPT` execution is disabled by default. Set `OPERATOR_ALLOW_SHELL_COMMANDS=true` only for deployments that need shell operators, and constrain commands with `OPERATOR_COMMAND_ALLOWLIST_REGEX` and `OPERATOR_COMMAND_DENYLIST_REGEX`. These are platform-controlled variables passed from the worker to operator-runner Jobs, and workflow hyperparams cannot override them.
|
||||
|
||||
## Acceptance Workflow
|
||||
|
||||
The GitHub Actions `Acceptance` workflow is manual. Use it after a deployment by entering the deployed API URL and enabling API smoke, GPU smoke, or both. GPU smoke requires `KUBE_CONFIG_B64` to contain a base64-encoded kubeconfig with permission to create a temporary Job in `K8S_NAMESPACE`.
|
||||
|
||||
## Generated Artifacts
|
||||
|
||||
Build output, local Playwright screenshots/videos, pytest caches, and zip bundles should stay out of git. Regenerate validation artifacts locally or publish them as CI/release artifacts when they need to be shared.
|
||||
|
||||
64
docs/production-readiness.md
Normal file
64
docs/production-readiness.md
Normal file
@ -0,0 +1,64 @@
|
||||
# Production Readiness Notes
|
||||
|
||||
This note tracks the production-facing safeguards that must stay true for Drobodata deployments.
|
||||
|
||||
## Runtime Boundaries
|
||||
|
||||
- Production execution uses the Go worker and Python operator-runner Jobs. The API does not execute workflow nodes in-process.
|
||||
- The API proxies `/worker-api/*` through `WORKER_API_BASE_URL`; the default is the in-cluster worker Service.
|
||||
- Worker proxy health, stats, metrics, and retry calls use short upstream timeouts. Manual `process-once` uses a long timeout because it can execute real queued work.
|
||||
- Explicit shell operators are disabled by default through `OPERATOR_ALLOW_SHELL_COMMANDS=false`; allowlist and denylist regexes are platform-controlled and cannot be overridden by workflow hyperparams.
|
||||
|
||||
## Kubernetes Jobs
|
||||
|
||||
- Operator-runner Jobs receive CPU and memory requests/limits from system defaults and node overrides.
|
||||
- A positive GPU value emits `nvidia.com/gpu` in both requests and limits.
|
||||
- GPU node selector, tolerations, and runtime class are only injected for Jobs that request GPU.
|
||||
- Finished Jobs receive `ttlSecondsAfterFinished` from `OPERATOR_JOB_TTL_SECONDS`, default `86400`.
|
||||
- Use `DRY_RUN=true npm run k8s:cleanup-jobs` before deleting old finished Jobs that predate TTL.
|
||||
|
||||
## Data Plane Limits
|
||||
|
||||
- Object-storage JSON object creation is capped at 4 MiB.
|
||||
- Browser multipart uploads stream to BOS and are capped at 2 GiB.
|
||||
- Large inline object reads use preview or presigned URL behavior instead of loading full objects into API memory.
|
||||
- Run logs, resource samples, and artifacts support `limit` and `offset`. Logs and resources default to the latest 1000 records for large runs.
|
||||
- Data-asset aggregation uses artifact-focused run projections so list endpoints do not read complete logs and execution traces.
|
||||
|
||||
## GPU Observability
|
||||
|
||||
- Operator-runner resource samples include CPU seconds, process memory, GPU utilization, GPU memory used/total, GPU count, and GPU model when `nvidia-smi` is available.
|
||||
- The monitoring page displays GPU utilization, GPU memory, and GPU device information.
|
||||
- `npm run gpu:smoke` remains the real acceptance check for allocatable GPU resources, scheduling, runtime, and `nvidia-smi`.
|
||||
|
||||
## Acceptance
|
||||
|
||||
Run local verification before merging:
|
||||
|
||||
```bash
|
||||
npm run verify
|
||||
```
|
||||
|
||||
Run deployed smoke after rollout:
|
||||
|
||||
```bash
|
||||
./scripts/smoke-test.sh http://<node-ip>:30080
|
||||
```
|
||||
|
||||
Run GPU smoke on a real GPU cluster:
|
||||
|
||||
```bash
|
||||
npm run gpu:smoke
|
||||
```
|
||||
|
||||
The manual GitHub Actions `Acceptance` workflow can run API smoke and GPU smoke against a deployed environment. Configure `KUBE_CONFIG_B64` when GPU smoke needs Kubernetes access.
|
||||
|
||||
## Remaining Cluster Responsibilities
|
||||
|
||||
Drobodata now exposes the necessary application controls, but production readiness still depends on the target cluster:
|
||||
|
||||
- TLS ingress or trusted private network exposure
|
||||
- network policies around MongoDB, API, worker, and operator-runner Jobs
|
||||
- MongoDB backup, restore, and credential rotation
|
||||
- registry credential rotation
|
||||
- Prometheus scraping and alerting for worker metrics and failed Jobs
|
||||
@ -1067,7 +1067,7 @@ func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]d
|
||||
for _, workflow := range workflows {
|
||||
workflowByID[workflow.ID] = workflow
|
||||
}
|
||||
runs, err := server.listRunsForContext(ctx, string(domain.RunStatusCompleted))
|
||||
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@ -2437,7 +2437,7 @@ func (server *server) packageDetail(ctx context.Context, runID string, packageNa
|
||||
}
|
||||
|
||||
func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) {
|
||||
runs, err := server.listRunsForRequest(request.Context(), request, string(domain.RunStatusCompleted))
|
||||
runs, err := server.listRunArtifactsForRequest(request.Context(), request, string(domain.RunStatusCompleted))
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
@ -2564,7 +2564,7 @@ func (server *server) syncSamplingTasks(ctx context.Context) error {
|
||||
for _, task := range existing {
|
||||
existingByRun[task.RunID] = task
|
||||
}
|
||||
runs, err := server.listRunsForContext(ctx, string(domain.RunStatusCompleted))
|
||||
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -25,6 +25,7 @@ type objectStorageBrowser interface {
|
||||
CreateBucket(context.Context, string) (domain.ObjectStorageBucketSummary, error)
|
||||
DeleteBucket(context.Context, string) error
|
||||
PutObject(context.Context, string, string, []byte, string) (domain.ObjectStorageObjectSummary, error)
|
||||
PutObjectStream(context.Context, string, string, io.Reader, string) (domain.ObjectStorageObjectSummary, error)
|
||||
RenameObject(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error)
|
||||
DeleteObject(context.Context, string, string) error
|
||||
DeletePrefix(context.Context, string, string) error
|
||||
@ -42,6 +43,8 @@ type bosObjectStorageBrowser struct {
|
||||
const (
|
||||
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
|
||||
objectStoragePartialPreviewBytes int64 = 64 * 1024
|
||||
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
|
||||
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
|
||||
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
|
||||
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
|
||||
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
|
||||
@ -273,6 +276,15 @@ func (browser *bosObjectStorageBrowser) PutObject(ctx context.Context, bucket st
|
||||
return browser.GetObjectSummary(ctx, bucket, objectKey)
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
|
||||
objectKey := strings.Trim(strings.TrimSpace(key), "/")
|
||||
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, nil, contentType)}
|
||||
if _, err := browser.client.PutObjectFromStreamWithContext(ctx, strings.TrimSpace(bucket), objectKey, reader, args); err != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, err
|
||||
}
|
||||
return browser.GetObjectSummary(ctx, bucket, objectKey)
|
||||
}
|
||||
|
||||
func (browser *bosObjectStorageBrowser) RenameObject(ctx context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
|
||||
objectKey := exactObjectStorageKey(key)
|
||||
destinationKey := exactObjectStorageKey(newKey)
|
||||
@ -445,6 +457,10 @@ func renderObjectStorageWriteError(writer http.ResponseWriter, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if isRequestTooLarge(err) {
|
||||
renderJSON(writer, http.StatusRequestEntityTooLarge, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
|
||||
renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()})
|
||||
return
|
||||
@ -452,6 +468,14 @@ func renderObjectStorageWriteError(writer http.ResponseWriter, err error) {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
}
|
||||
|
||||
func isRequestTooLarge(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
return errors.As(err, &maxBytesErr)
|
||||
}
|
||||
|
||||
func exactObjectStorageKey(key string) string {
|
||||
return strings.TrimSpace(key)
|
||||
}
|
||||
@ -555,8 +579,13 @@ func (server *server) handleObjectStorageBucketDelete(writer http.ResponseWriter
|
||||
}
|
||||
|
||||
func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter, request *http.Request) {
|
||||
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageJSONPutMaxBytes)
|
||||
input, err := decodeObjectStorageObjectPutInput(request)
|
||||
if err != nil {
|
||||
if isRequestTooLarge(err) {
|
||||
renderObjectStorageWriteError(writer, err)
|
||||
return
|
||||
}
|
||||
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
|
||||
return
|
||||
}
|
||||
@ -593,6 +622,7 @@ func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter
|
||||
}
|
||||
|
||||
func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter, request *http.Request) {
|
||||
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageMultipartUploadMaxBytes)
|
||||
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
|
||||
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
|
||||
if err != nil {
|
||||
@ -620,6 +650,10 @@ func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter
|
||||
return
|
||||
}
|
||||
if err := request.ParseMultipartForm(32 << 20); err != nil {
|
||||
if isRequestTooLarge(err) {
|
||||
renderObjectStorageWriteError(writer, err)
|
||||
return
|
||||
}
|
||||
renderValidationError(writer, validationIssue{Field: "file", Message: "invalid multipart payload"})
|
||||
return
|
||||
}
|
||||
@ -629,16 +663,11 @@ func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
payload, err := io.ReadAll(file)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
contentType := strings.TrimSpace(request.URL.Query().Get("contentType"))
|
||||
if contentType == "" && header != nil {
|
||||
contentType = strings.TrimSpace(header.Header.Get("Content-Type"))
|
||||
}
|
||||
item, err := browser.PutObject(request.Context(), bucket, key, payload, contentType)
|
||||
item, err := browser.PutObjectStream(request.Context(), bucket, key, file, contentType)
|
||||
if err != nil {
|
||||
renderObjectStorageWriteError(writer, err)
|
||||
return
|
||||
|
||||
@ -6,6 +6,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"droboticflow/internal/store"
|
||||
|
||||
"github.com/baidubce/bce-sdk-go/bce"
|
||||
)
|
||||
|
||||
@ -35,3 +37,14 @@ func TestRenderObjectStorageWriteErrorTreatsBOSNoSuchKeyAsNotFound(t *testing.T)
|
||||
t.Fatalf("expected response to include NoSuchKey, got %s", recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestObjectStorageJSONPutRejectsOversizedBody(t *testing.T) {
|
||||
body := `{"key":"big.txt","content":"` + strings.Repeat("x", int(objectStorageJSONPutMaxBytes)) + `"}`
|
||||
handler := NewHandler(store.NewMemoryStore(), nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/object-storage/objects", strings.NewReader(body)))
|
||||
|
||||
if recorder.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("expected oversized JSON put to return 413, got %d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
@ -49,10 +49,10 @@ type runJobTerminator interface {
|
||||
var storageConnectionBucketsResolver = listStorageBucketsFromConnection
|
||||
var storageConnectionObjectsResolver = listStorageObjectsFromConnection
|
||||
var runJobTerminatorFactory = func(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string) (runJobTerminator, error) {
|
||||
return k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, "", "", "")
|
||||
return k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, "", "", "", 0)
|
||||
}
|
||||
var workerAPIBaseURLResolver = func() string {
|
||||
return "http://droboticflow-worker:3002"
|
||||
return config.Load().WorkerAPIBaseURL
|
||||
}
|
||||
|
||||
const maxStoragePathFavoritesPerScope = 12
|
||||
@ -1953,7 +1953,7 @@ func (server *server) handleWorkflowRunsList(writer http.ResponseWriter, request
|
||||
if !server.ensureTenantAccess(writer, request, workflow.OrganizationID, "Workflow") {
|
||||
return
|
||||
}
|
||||
items, err := server.listRunsForRequest(request.Context(), request, "")
|
||||
items, err := server.listRunSummariesForRequest(request.Context(), request, "")
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusInternalServerError, err)
|
||||
return
|
||||
@ -1966,7 +1966,7 @@ func (server *server) handleWorkflowRunsList(writer http.ResponseWriter, request
|
||||
if !server.filterByTenant(request, item.OrganizationID) {
|
||||
continue
|
||||
}
|
||||
summaries = append(summaries, runSummaryFromRecord(item))
|
||||
summaries = append(summaries, item)
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": summaries})
|
||||
}
|
||||
@ -2603,7 +2603,7 @@ func (server *server) handleRunsLogs(writer http.ResponseWriter, request *http.R
|
||||
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") {
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": item.Logs})
|
||||
renderJSON(writer, http.StatusOK, pagedResponse(request, item.Logs, 1000, 5000, true))
|
||||
}
|
||||
|
||||
func (server *server) handleRunsResources(writer http.ResponseWriter, request *http.Request) {
|
||||
@ -2619,7 +2619,7 @@ func (server *server) handleRunsResources(writer http.ResponseWriter, request *h
|
||||
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") {
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": item.ResourceSeries})
|
||||
renderJSON(writer, http.StatusOK, pagedResponse(request, item.ResourceSeries, 1000, 5000, true))
|
||||
}
|
||||
|
||||
func (server *server) handleRunsArtifacts(writer http.ResponseWriter, request *http.Request) {
|
||||
@ -2635,7 +2635,7 @@ func (server *server) handleRunsArtifacts(writer http.ResponseWriter, request *h
|
||||
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") {
|
||||
return
|
||||
}
|
||||
renderJSON(writer, http.StatusOK, map[string]any{"items": item.Artifacts})
|
||||
renderJSON(writer, http.StatusOK, pagedResponse(request, item.Artifacts, 1000, 5000, false))
|
||||
}
|
||||
|
||||
func (server *server) handleAuditTasksList(writer http.ResponseWriter, request *http.Request) {
|
||||
@ -3516,6 +3516,43 @@ func decodeJSON(writer http.ResponseWriter, request *http.Request, target any) b
|
||||
return true
|
||||
}
|
||||
|
||||
func pagedResponse[T any](request *http.Request, items []T, defaultLimit int, maxLimit int, defaultTail bool) map[string]any {
|
||||
total := len(items)
|
||||
limit := parsePositiveIntQuery(request.URL.Query().Get("limit"), defaultLimit, maxLimit)
|
||||
if limit <= 0 {
|
||||
limit = defaultLimit
|
||||
if maxLimit > 0 && limit > maxLimit {
|
||||
limit = maxLimit
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = total
|
||||
}
|
||||
}
|
||||
offsetRaw := request.URL.Query().Get("offset")
|
||||
if strings.TrimSpace(offsetRaw) == "" {
|
||||
offsetRaw = request.URL.Query().Get("skip")
|
||||
}
|
||||
offset := parsePositiveIntQuery(offsetRaw, 0, 0)
|
||||
tailRequested := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("tail")), "true")
|
||||
if (defaultTail || tailRequested) && strings.TrimSpace(offsetRaw) == "" && limit < total {
|
||||
offset = total - limit
|
||||
}
|
||||
if offset > total {
|
||||
offset = total
|
||||
}
|
||||
end := offset + limit
|
||||
if end > total {
|
||||
end = total
|
||||
}
|
||||
return map[string]any{
|
||||
"items": items[offset:end],
|
||||
"total": total,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"truncated": offset > 0 || end < total,
|
||||
}
|
||||
}
|
||||
|
||||
func renderValidationError(writer http.ResponseWriter, issues ...validationIssue) {
|
||||
renderJSON(writer, http.StatusBadRequest, map[string]any{"error": "Validation failed", "issues": issues})
|
||||
}
|
||||
|
||||
@ -15,7 +15,9 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"droboticflow/internal/config"
|
||||
"droboticflow/internal/domain"
|
||||
"droboticflow/internal/store"
|
||||
|
||||
@ -479,6 +481,60 @@ func TestWorkerAPIHealthIsProxiedThroughAPI(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerAPIBaseURLUsesConfiguredEnv(t *testing.T) {
|
||||
worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path != "/worker-api/health" {
|
||||
t.Fatalf("unexpected upstream path: %s", request.URL.Path)
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/json")
|
||||
_, _ = writer.Write([]byte(`{"status":"configured"}`))
|
||||
}))
|
||||
defer worker.Close()
|
||||
|
||||
originalResolver := workerAPIBaseURLResolver
|
||||
workerAPIBaseURLResolver = func() string { return config.Load().WorkerAPIBaseURL }
|
||||
t.Cleanup(func() {
|
||||
workerAPIBaseURLResolver = originalResolver
|
||||
})
|
||||
t.Setenv("WORKER_API_BASE_URL", worker.URL)
|
||||
|
||||
handler := NewHandler(store.NewMemoryStore(), nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil))
|
||||
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
if got := strings.TrimSpace(recorder.Body.String()); got != `{"status":"configured"}` {
|
||||
t.Fatalf("unexpected configured worker payload: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerAPIProxyTimesOutSlowHealthChecks(t *testing.T) {
|
||||
worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
writer.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer worker.Close()
|
||||
|
||||
originalResolver := workerAPIBaseURLResolver
|
||||
originalTimeout := workerAPIShortProxyTimeout
|
||||
workerAPIBaseURLResolver = func() string { return worker.URL }
|
||||
workerAPIShortProxyTimeout = 10 * time.Millisecond
|
||||
t.Cleanup(func() {
|
||||
workerAPIBaseURLResolver = originalResolver
|
||||
workerAPIShortProxyTimeout = originalTimeout
|
||||
})
|
||||
|
||||
handler := NewHandler(store.NewMemoryStore(), nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil))
|
||||
|
||||
if recorder.Code != http.StatusBadGateway {
|
||||
t.Fatalf("expected proxy timeout to return 502, got %d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerAPIProcessOnceIsProxiedThroughAPI(t *testing.T) {
|
||||
worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.Method != http.MethodPost {
|
||||
@ -1433,6 +1489,90 @@ func TestRunsListReturnsSummaryPayload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunLogsEndpointPaginatesAndDefaultsToTail(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
logs := make([]domain.LogEntryRecord, 0, 1100)
|
||||
for index := 0; index < 1100; index++ {
|
||||
logs = append(logs, domain.LogEntryRecord{Timestamp: "2026-04-13T01:00:00Z", Level: "INFO", Message: fmt.Sprintf("line-%04d", index)})
|
||||
}
|
||||
if err := dataStore.UpsertRun(ctx, domain.RunRecord{
|
||||
ID: "run-logs-page",
|
||||
WorkflowID: "workflow-logs",
|
||||
WorkflowName: "Logs",
|
||||
ProjectID: "project-logs",
|
||||
ProjectName: "Logs",
|
||||
Trigger: domain.RunTriggerManual,
|
||||
Status: domain.RunStatusCompleted,
|
||||
Logs: logs,
|
||||
NodeExecutions: []domain.NodeExecutionRecord{},
|
||||
ResourceSeries: []domain.ResourceSample{},
|
||||
Artifacts: []domain.ExecutionArtifact{},
|
||||
K8sRefs: []domain.K8sRef{},
|
||||
CreatedAt: "2026-04-13T01:00:00Z",
|
||||
UpdatedAt: "2026-04-13T01:00:00Z",
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert run: %v", err)
|
||||
}
|
||||
|
||||
handler := NewHandler(dataStore, nil)
|
||||
recorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/runs/run-logs-page/logs", nil))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("logs status: %d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
var response struct {
|
||||
Items []domain.LogEntryRecord `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
Truncated bool `json:"truncated"`
|
||||
}
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode logs response: %v", err)
|
||||
}
|
||||
if response.Total != 1100 || response.Limit != 1000 || response.Offset != 100 || !response.Truncated {
|
||||
t.Fatalf("unexpected default page metadata: %+v", response)
|
||||
}
|
||||
if len(response.Items) != 1000 || response.Items[0].Message != "line-0100" {
|
||||
t.Fatalf("expected tail page starting at line-0100, got len=%d first=%q", len(response.Items), response.Items[0].Message)
|
||||
}
|
||||
|
||||
pageRecorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(pageRecorder, httptest.NewRequest(http.MethodGet, "/api/runs/run-logs-page/logs?limit=2&offset=3", nil))
|
||||
if pageRecorder.Code != http.StatusOK {
|
||||
t.Fatalf("explicit page status: %d body=%s", pageRecorder.Code, pageRecorder.Body.String())
|
||||
}
|
||||
var explicitPage struct {
|
||||
Items []domain.LogEntryRecord `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
if err := json.Unmarshal(pageRecorder.Body.Bytes(), &explicitPage); err != nil {
|
||||
t.Fatalf("decode explicit page: %v", err)
|
||||
}
|
||||
if explicitPage.Total != 1100 || explicitPage.Limit != 2 || explicitPage.Offset != 3 || len(explicitPage.Items) != 2 || explicitPage.Items[0].Message != "line-0003" {
|
||||
t.Fatalf("unexpected explicit page: %+v", explicitPage)
|
||||
}
|
||||
|
||||
zeroLimitRecorder := httptest.NewRecorder()
|
||||
handler.ServeHTTP(zeroLimitRecorder, httptest.NewRequest(http.MethodGet, "/api/runs/run-logs-page/logs?limit=0", nil))
|
||||
if zeroLimitRecorder.Code != http.StatusOK {
|
||||
t.Fatalf("zero limit status: %d body=%s", zeroLimitRecorder.Code, zeroLimitRecorder.Body.String())
|
||||
}
|
||||
var zeroLimitPage struct {
|
||||
Items []domain.LogEntryRecord `json:"items"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
if err := json.Unmarshal(zeroLimitRecorder.Body.Bytes(), &zeroLimitPage); err != nil {
|
||||
t.Fatalf("decode zero limit page: %v", err)
|
||||
}
|
||||
if zeroLimitPage.Limit != 1000 || len(zeroLimitPage.Items) != 1000 {
|
||||
t.Fatalf("expected zero limit to fall back to default page size, got %+v", zeroLimitPage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkflowRunsListReturnsWorkflowOnlySummaries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dataStore := store.NewMemoryStore()
|
||||
@ -3980,6 +4120,14 @@ func (browser *fakeObjectStorageBrowser) PutObject(_ context.Context, bucket str
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (browser *fakeObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
|
||||
payload, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return domain.ObjectStorageObjectSummary{}, err
|
||||
}
|
||||
return browser.PutObject(ctx, bucket, key, payload, contentType)
|
||||
}
|
||||
|
||||
func (browser *fakeObjectStorageBrowser) RenameObject(_ context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
|
||||
item := browser.objects[bucket+"|"+key]
|
||||
delete(browser.objects, bucket+"|"+key)
|
||||
|
||||
@ -166,6 +166,20 @@ func (server *server) listRunsForContext(ctx context.Context, status string) ([]
|
||||
return server.store.ListRuns(ctx, status)
|
||||
}
|
||||
|
||||
func (server *server) listRunArtifactsForRequest(ctx context.Context, request *http.Request, status string) ([]domain.RunRecord, error) {
|
||||
if server.tenantEnforced(request) {
|
||||
return server.store.ListRunArtifactsByOrganization(ctx, server.requestOrganizationID(request), status)
|
||||
}
|
||||
return server.store.ListRunArtifacts(ctx, status)
|
||||
}
|
||||
|
||||
func (server *server) listRunArtifactsForContext(ctx context.Context, status string) ([]domain.RunRecord, error) {
|
||||
if server.tenantEnforcedContext(ctx) {
|
||||
return server.store.ListRunArtifactsByOrganization(ctx, server.contextOrganizationID(ctx), status)
|
||||
}
|
||||
return server.store.ListRunArtifacts(ctx, status)
|
||||
}
|
||||
|
||||
func (server *server) listRunSummariesForRequest(ctx context.Context, request *http.Request, status string) ([]domain.RunSummary, error) {
|
||||
if server.tenantEnforced(request) {
|
||||
return server.store.ListRunSummariesByOrganization(ctx, server.requestOrganizationID(request), status)
|
||||
|
||||
@ -1,33 +1,49 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
var (
|
||||
workerAPIShortProxyTimeout = 15 * time.Second
|
||||
workerAPILongProxyTimeout = 4 * time.Hour
|
||||
)
|
||||
|
||||
var workerAPIProxyClient = &http.Client{}
|
||||
|
||||
func workerAPIBaseURL() string {
|
||||
return strings.TrimRight(strings.TrimSpace(workerAPIBaseURLResolver()), "/")
|
||||
}
|
||||
|
||||
func proxyWorkerAPI(writer http.ResponseWriter, request *http.Request, path string) {
|
||||
func proxyWorkerAPI(writer http.ResponseWriter, request *http.Request, path string, timeout time.Duration) {
|
||||
baseURL := workerAPIBaseURL()
|
||||
if baseURL == "" {
|
||||
renderJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "worker API is not configured"})
|
||||
return
|
||||
}
|
||||
|
||||
requestCtx := request.Context()
|
||||
cancel := func() {}
|
||||
if timeout > 0 {
|
||||
requestCtx, cancel = context.WithTimeout(requestCtx, timeout)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
upstreamURL := baseURL + path
|
||||
upstreamRequest, err := http.NewRequestWithContext(request.Context(), request.Method, upstreamURL, request.Body)
|
||||
upstreamRequest, err := http.NewRequestWithContext(requestCtx, request.Method, upstreamURL, request.Body)
|
||||
if err != nil {
|
||||
renderError(writer, http.StatusBadGateway, err)
|
||||
return
|
||||
}
|
||||
upstreamRequest.Header = request.Header.Clone()
|
||||
|
||||
response, err := http.DefaultClient.Do(upstreamRequest)
|
||||
response, err := workerAPIProxyClient.Do(upstreamRequest)
|
||||
if err != nil {
|
||||
renderJSON(writer, http.StatusBadGateway, map[string]string{
|
||||
"error": "worker API request failed",
|
||||
@ -47,21 +63,21 @@ func proxyWorkerAPI(writer http.ResponseWriter, request *http.Request, path stri
|
||||
}
|
||||
|
||||
func (server *server) handleWorkerAPIHealth(writer http.ResponseWriter, request *http.Request) {
|
||||
proxyWorkerAPI(writer, request, "/worker-api/health")
|
||||
proxyWorkerAPI(writer, request, "/worker-api/health", workerAPIShortProxyTimeout)
|
||||
}
|
||||
|
||||
func (server *server) handleWorkerAPIStats(writer http.ResponseWriter, request *http.Request) {
|
||||
proxyWorkerAPI(writer, request, "/worker-api/stats")
|
||||
proxyWorkerAPI(writer, request, "/worker-api/stats", workerAPIShortProxyTimeout)
|
||||
}
|
||||
|
||||
func (server *server) handleWorkerAPIMetrics(writer http.ResponseWriter, request *http.Request) {
|
||||
proxyWorkerAPI(writer, request, "/worker-api/metrics")
|
||||
proxyWorkerAPI(writer, request, "/worker-api/metrics", workerAPIShortProxyTimeout)
|
||||
}
|
||||
|
||||
func (server *server) handleWorkerAPIProcessOnce(writer http.ResponseWriter, request *http.Request) {
|
||||
proxyWorkerAPI(writer, request, "/worker-api/process-once")
|
||||
proxyWorkerAPI(writer, request, "/worker-api/process-once", workerAPILongProxyTimeout)
|
||||
}
|
||||
|
||||
func (server *server) handleWorkerAPIRetry(writer http.ResponseWriter, request *http.Request) {
|
||||
proxyWorkerAPI(writer, request, "/worker-api/runs/"+chi.URLParam(request, "id")+"/retry")
|
||||
proxyWorkerAPI(writer, request, "/worker-api/runs/"+chi.URLParam(request, "id")+"/retry", workerAPIShortProxyTimeout)
|
||||
}
|
||||
|
||||
@ -15,6 +15,7 @@ type AppConfig struct {
|
||||
MongoConnectRetries int
|
||||
MongoConnectRetryDelayMS int
|
||||
PollIntervalMS int
|
||||
WorkerAPIBaseURL string
|
||||
K8sNamespace string
|
||||
K8sImagePullSecretName string
|
||||
WorkspacePVCName string
|
||||
@ -31,6 +32,7 @@ type AppConfig struct {
|
||||
OperatorGPUNodeSelector string
|
||||
OperatorGPUTolerations string
|
||||
OperatorGPURuntimeClass string
|
||||
OperatorJobTTLSeconds int
|
||||
}
|
||||
|
||||
func parseInt(value string, fallback int) int {
|
||||
@ -51,6 +53,7 @@ func Load() AppConfig {
|
||||
MongoConnectRetries: parseInt(os.Getenv("MONGO_CONNECT_RETRIES"), 30),
|
||||
MongoConnectRetryDelayMS: parseInt(os.Getenv("MONGO_CONNECT_RETRY_DELAY_MS"), 2000),
|
||||
PollIntervalMS: parseInt(os.Getenv("WORKER_POLL_INTERVAL_MS"), 2000),
|
||||
WorkerAPIBaseURL: getenv("WORKER_API_BASE_URL", "http://droboticflow-worker:3002"),
|
||||
K8sNamespace: getenv("K8S_NAMESPACE", "droboticflow"),
|
||||
K8sImagePullSecretName: getenv("K8S_IMAGE_PULL_SECRET_NAME", "droboticflow-registry"),
|
||||
WorkspacePVCName: strings.TrimSpace(os.Getenv("WORKSPACE_PVC_NAME")),
|
||||
@ -67,6 +70,7 @@ func Load() AppConfig {
|
||||
OperatorGPUNodeSelector: strings.TrimSpace(os.Getenv("OPERATOR_GPU_NODE_SELECTOR")),
|
||||
OperatorGPUTolerations: strings.TrimSpace(os.Getenv("OPERATOR_GPU_TOLERATIONS_JSON")),
|
||||
OperatorGPURuntimeClass: strings.TrimSpace(os.Getenv("OPERATOR_GPU_RUNTIME_CLASS")),
|
||||
OperatorJobTTLSeconds: parseInt(os.Getenv("OPERATOR_JOB_TTL_SECONDS"), 86400),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -278,10 +278,14 @@ type LogEntryRecord struct {
|
||||
}
|
||||
|
||||
type ResourceSample struct {
|
||||
Timestamp string `json:"timestamp" bson:"timestamp"`
|
||||
CPU int `json:"cpu" bson:"cpu"`
|
||||
Memory int `json:"memory" bson:"memory"`
|
||||
GPU int `json:"gpu,omitempty" bson:"gpu,omitempty"`
|
||||
Timestamp string `json:"timestamp" bson:"timestamp"`
|
||||
CPU int `json:"cpu" bson:"cpu"`
|
||||
Memory int `json:"memory" bson:"memory"`
|
||||
GPU int `json:"gpu,omitempty" bson:"gpu,omitempty"`
|
||||
GPUMemoryUsedMiB int `json:"gpuMemoryUsedMiB,omitempty" bson:"gpuMemoryUsedMiB,omitempty"`
|
||||
GPUMemoryTotalMiB int `json:"gpuMemoryTotalMiB,omitempty" bson:"gpuMemoryTotalMiB,omitempty"`
|
||||
GPUName string `json:"gpuName,omitempty" bson:"gpuName,omitempty"`
|
||||
GPUCount int `json:"gpuCount,omitempty" bson:"gpuCount,omitempty"`
|
||||
}
|
||||
|
||||
type ArtifactMetadata struct {
|
||||
|
||||
@ -41,6 +41,7 @@ type LaunchInput struct {
|
||||
GPUNodeSelector map[string]string
|
||||
GPUTolerations []corev1.Toleration
|
||||
GPURuntimeClass string
|
||||
JobTTLSeconds int32
|
||||
}
|
||||
|
||||
type JobReference struct {
|
||||
@ -77,6 +78,7 @@ type Launcher struct {
|
||||
gpuNodeSelector map[string]string
|
||||
gpuTolerations []corev1.Toleration
|
||||
gpuRuntimeClass string
|
||||
jobTTLSeconds int32
|
||||
clientset kubernetes.Interface
|
||||
}
|
||||
|
||||
@ -308,6 +310,12 @@ func CreateJobManifest(namespace string, imagePullSecretName string, workspacePV
|
||||
},
|
||||
},
|
||||
}
|
||||
if input.JobTTLSeconds > 0 {
|
||||
job.Spec.TTLSecondsAfterFinished = func() *int32 {
|
||||
value := input.JobTTLSeconds
|
||||
return &value
|
||||
}()
|
||||
}
|
||||
if input.TimeoutSeconds > 0 {
|
||||
job.Spec.ActiveDeadlineSeconds = func() *int64 {
|
||||
value := int64(input.TimeoutSeconds)
|
||||
@ -350,7 +358,7 @@ func loadRestConfig() (*rest.Config, error) {
|
||||
return clientcmd.BuildConfigFromFlags("", kubeconfig)
|
||||
}
|
||||
|
||||
func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string) (*Launcher, error) {
|
||||
func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int) (*Launcher, error) {
|
||||
trimmedPVCName := strings.TrimSpace(workspacePVCName)
|
||||
if trimmedPVCName == "" {
|
||||
return nil, fmt.Errorf("workspace pvc name is required")
|
||||
@ -370,6 +378,9 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if jobTTLSeconds < 0 {
|
||||
jobTTLSeconds = 0
|
||||
}
|
||||
launcher := &Launcher{
|
||||
namespace: namespace,
|
||||
imagePullSecretName: imagePullSecretName,
|
||||
@ -378,6 +389,7 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
|
||||
gpuNodeSelector: parsedGPUNodeSelector,
|
||||
gpuTolerations: parsedGPUTolerations,
|
||||
gpuRuntimeClass: strings.TrimSpace(gpuRuntimeClass),
|
||||
jobTTLSeconds: int32(jobTTLSeconds),
|
||||
}
|
||||
config, err := loadRestConfig()
|
||||
if err != nil {
|
||||
@ -572,6 +584,9 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc
|
||||
if strings.TrimSpace(input.GPURuntimeClass) == "" {
|
||||
input.GPURuntimeClass = launcher.gpuRuntimeClass
|
||||
}
|
||||
if input.JobTTLSeconds == 0 {
|
||||
input.JobTTLSeconds = launcher.jobTTLSeconds
|
||||
}
|
||||
job := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, launcher.workspacePVCName, launcher.workspaceMountPath, input)
|
||||
created, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Create(ctx, job, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
|
||||
@ -33,6 +33,7 @@ func TestCreateJobManifest(t *testing.T) {
|
||||
Effect: corev1.TaintEffectNoSchedule,
|
||||
}},
|
||||
GPURuntimeClass: "nvidia",
|
||||
JobTTLSeconds: 3600,
|
||||
Env: map[string]string{
|
||||
"CPU_LIMIT": "500m",
|
||||
"MEMORY_LIMIT": "1Gi",
|
||||
@ -87,6 +88,9 @@ func TestCreateJobManifest(t *testing.T) {
|
||||
if manifest.Spec.Template.Spec.SecurityContext == nil || manifest.Spec.Template.Spec.SecurityContext.SeccompProfile == nil || manifest.Spec.Template.Spec.SecurityContext.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault {
|
||||
t.Fatalf("expected RuntimeDefault seccomp profile, got %#v", manifest.Spec.Template.Spec.SecurityContext)
|
||||
}
|
||||
if manifest.Spec.TTLSecondsAfterFinished == nil || *manifest.Spec.TTLSecondsAfterFinished != 3600 {
|
||||
t.Fatalf("expected ttlSecondsAfterFinished 3600, got %#v", manifest.Spec.TTLSecondsAfterFinished)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateJobManifestSkipsGPUSchedulingWithoutGPURequest(t *testing.T) {
|
||||
@ -110,17 +114,29 @@ func TestCreateJobManifestSkipsGPUSchedulingWithoutGPURequest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateJobManifestOmitsTTLWhenDisabled(t *testing.T) {
|
||||
manifest := CreateJobManifest("droboticflow", "", "droboticflow-workspace", "/workspace", LaunchInput{
|
||||
Image: "runner:latest",
|
||||
RunID: "run-1",
|
||||
NodeID: "node-1",
|
||||
})
|
||||
|
||||
if manifest.Spec.TTLSecondsAfterFinished != nil {
|
||||
t.Fatalf("did not expect ttlSecondsAfterFinished when disabled, got %#v", manifest.Spec.TTLSecondsAfterFinished)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLauncherRequiresWorkspacePVCName(t *testing.T) {
|
||||
if _, err := NewLauncher("droboticflow", "registry", "", "/workspace", "", "", ""); err == nil {
|
||||
if _, err := NewLauncher("droboticflow", "registry", "", "/workspace", "", "", "", 86400); err == nil {
|
||||
t.Fatalf("expected workspace pvc validation error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewLauncherRejectsInvalidGPUSchedulingConfig(t *testing.T) {
|
||||
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "accelerator", "", ""); err == nil || !strings.Contains(err.Error(), "invalid gpu node selector") {
|
||||
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "accelerator", "", "", 86400); err == nil || !strings.Contains(err.Error(), "invalid gpu node selector") {
|
||||
t.Fatalf("expected invalid gpu node selector error, got %v", err)
|
||||
}
|
||||
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "", "not-json", ""); err == nil || !strings.Contains(err.Error(), "invalid gpu tolerations json") {
|
||||
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "", "not-json", "", 86400); err == nil || !strings.Contains(err.Error(), "invalid gpu tolerations json") {
|
||||
t.Fatalf("expected invalid gpu tolerations error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -703,6 +703,14 @@ func (store *MemoryStore) ListRunsByOrganization(_ context.Context, organization
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListRunArtifacts(ctx context.Context, status string) ([]domain.RunRecord, error) {
|
||||
return store.ListRuns(ctx, status)
|
||||
}
|
||||
|
||||
func (store *MemoryStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
|
||||
return store.ListRunsByOrganization(ctx, organizationID, status)
|
||||
}
|
||||
|
||||
func (store *MemoryStore) CountRunsByStatus(_ context.Context) (map[domain.RunStatus]int, error) {
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
|
||||
@ -563,6 +563,57 @@ func (store *MongoStore) ListRunsByOrganization(ctx context.Context, organizatio
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func runArtifactProjection() bson.M {
|
||||
return bson.M{
|
||||
"id": 1,
|
||||
"organizationId": 1,
|
||||
"workflowId": 1,
|
||||
"workflowName": 1,
|
||||
"projectId": 1,
|
||||
"projectName": 1,
|
||||
"status": 1,
|
||||
"artifacts": 1,
|
||||
"createdAt": 1,
|
||||
"updatedAt": 1,
|
||||
}
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListRunArtifacts(ctx context.Context, status string) ([]domain.RunRecord, error) {
|
||||
filter := bson.M{}
|
||||
if status != "" {
|
||||
filter["status"] = status
|
||||
}
|
||||
cursor, err := store.runs.Find(
|
||||
ctx,
|
||||
filter,
|
||||
options.Find().
|
||||
SetProjection(runArtifactProjection()).
|
||||
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeMany[domain.RunRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
|
||||
filter := bson.M{}
|
||||
if status != "" {
|
||||
filter["status"] = status
|
||||
}
|
||||
cursor, err := store.runs.Find(
|
||||
ctx,
|
||||
organizationScopedFilter(organizationID, filter),
|
||||
options.Find().
|
||||
SetProjection(runArtifactProjection()).
|
||||
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decodeMany[domain.RunRecord](ctx, cursor)
|
||||
}
|
||||
|
||||
func (store *MongoStore) CountRunsByStatus(ctx context.Context) (map[domain.RunStatus]int, error) {
|
||||
cursor, err := store.runs.Aggregate(ctx, mongo.Pipeline{
|
||||
{{Key: "$group", Value: bson.M{
|
||||
|
||||
@ -46,6 +46,8 @@ type Store interface {
|
||||
|
||||
ListRuns(context.Context, string) ([]domain.RunRecord, error)
|
||||
ListRunsByOrganization(context.Context, string, string) ([]domain.RunRecord, error)
|
||||
ListRunArtifacts(context.Context, string) ([]domain.RunRecord, error)
|
||||
ListRunArtifactsByOrganization(context.Context, string, string) ([]domain.RunRecord, error)
|
||||
CountRunsByStatus(context.Context) (map[domain.RunStatus]int, error)
|
||||
ListRunSummaries(context.Context, string) ([]domain.RunSummary, error)
|
||||
ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error)
|
||||
|
||||
@ -43,8 +43,8 @@ var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{
|
||||
domain.RunStatusWaitingForAudit: {},
|
||||
}
|
||||
|
||||
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string) (*Runtime, error) {
|
||||
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass)
|
||||
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int) (*Runtime, error) {
|
||||
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -81,6 +81,7 @@ data:
|
||||
MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__"
|
||||
MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__"
|
||||
WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__"
|
||||
WORKER_API_BASE_URL: "__WORKER_API_BASE_URL__"
|
||||
K8S_NAMESPACE: "__NAMESPACE__"
|
||||
K8S_IMAGE_PULL_SECRET_NAME: "__K8S_IMAGE_PULL_SECRET_NAME__"
|
||||
WORKSPACE_PVC_NAME: "__WORKSPACE_PVC_NAME__"
|
||||
@ -90,6 +91,7 @@ data:
|
||||
OPERATOR_GPU_TOLERATIONS_JSON: |-
|
||||
__OPERATOR_GPU_TOLERATIONS_JSON__
|
||||
OPERATOR_GPU_RUNTIME_CLASS: "__OPERATOR_GPU_RUNTIME_CLASS__"
|
||||
OPERATOR_JOB_TTL_SECONDS: "__OPERATOR_JOB_TTL_SECONDS__"
|
||||
OPERATOR_ALLOW_SHELL_COMMANDS: "__OPERATOR_ALLOW_SHELL_COMMANDS__"
|
||||
OPERATOR_COMMAND_ALLOWLIST_REGEX: |-
|
||||
__OPERATOR_COMMAND_ALLOWLIST_REGEX__
|
||||
@ -208,6 +210,11 @@ spec:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: WORKSPACE_MOUNT_PATH
|
||||
- name: WORKER_API_BASE_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: WORKER_API_BASE_URL
|
||||
- name: AI_API_BASE_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
@ -404,6 +411,11 @@ spec:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: OPERATOR_GPU_RUNTIME_CLASS
|
||||
- name: OPERATOR_JOB_TTL_SECONDS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: droboticflow-config
|
||||
key: OPERATOR_JOB_TTL_SECONDS
|
||||
- name: OPERATOR_ALLOW_SHELL_COMMANDS
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
|
||||
@ -34,6 +34,7 @@
|
||||
"k8s:render": "./scripts/render-manifests.sh",
|
||||
"k8s:validate": "./scripts/render-manifests.sh | kubectl apply --dry-run=client --validate=false -f -",
|
||||
"k8s:deploy": "./scripts/deploy-k8s.sh",
|
||||
"k8s:cleanup-jobs": "./scripts/cleanup-operator-jobs.sh",
|
||||
"gpu:smoke": "./scripts/gpu-smoke-test.sh",
|
||||
"smoke": "./scripts/smoke-test.sh",
|
||||
"reconcile:data": "./scripts/reconcile-system-data.sh"
|
||||
|
||||
90
scripts/cleanup-operator-jobs.sh
Executable file
90
scripts/cleanup-operator-jobs.sh
Executable file
@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
||||
|
||||
if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then
|
||||
DEPLOY_ENV="$1"
|
||||
shift
|
||||
fi
|
||||
|
||||
export DEPLOY_ENV
|
||||
# shellcheck source=scripts/load-deploy-env.sh
|
||||
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
||||
|
||||
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
||||
OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS="${OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS:-86400}"
|
||||
OPERATOR_JOB_CLEANUP_STATUSES="${OPERATOR_JOB_CLEANUP_STATUSES:-succeeded,failed}"
|
||||
DRY_RUN="${DRY_RUN:-false}"
|
||||
|
||||
job_names="$(
|
||||
kubectl -n "$K8S_NAMESPACE" get jobs \
|
||||
-l app=droboticflow,component=operator-runner \
|
||||
-o json |
|
||||
OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS="$OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS" \
|
||||
OPERATOR_JOB_CLEANUP_STATUSES="$OPERATOR_JOB_CLEANUP_STATUSES" \
|
||||
python3 -c '
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
min_age_seconds = int(os.environ["OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS"])
|
||||
statuses = {item.strip().lower() for item in os.environ["OPERATOR_JOB_CLEANUP_STATUSES"].split(",") if item.strip()}
|
||||
payload = json.load(sys.stdin)
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
|
||||
def parse_time(value):
|
||||
if not value:
|
||||
return None
|
||||
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
|
||||
def condition_time(status, condition_type):
|
||||
for condition in status.get("conditions", []) or []:
|
||||
if str(condition.get("type", "")).lower() != condition_type:
|
||||
continue
|
||||
if str(condition.get("status", "")).lower() != "true":
|
||||
continue
|
||||
parsed = parse_time(condition.get("lastTransitionTime"))
|
||||
if parsed:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
for item in payload.get("items", []):
|
||||
status = item.get("status", {})
|
||||
succeeded = int(status.get("succeeded") or 0)
|
||||
failed = int(status.get("failed") or 0)
|
||||
state = ""
|
||||
if succeeded > 0:
|
||||
state = "succeeded"
|
||||
finished_at = parse_time(status.get("completionTime")) or condition_time(status, "complete")
|
||||
elif failed > 0:
|
||||
state = "failed"
|
||||
finished_at = parse_time(status.get("completionTime")) or condition_time(status, "failed")
|
||||
else:
|
||||
finished_at = None
|
||||
if not state or state not in statuses:
|
||||
continue
|
||||
if finished_at is None or (now - finished_at).total_seconds() < min_age_seconds:
|
||||
continue
|
||||
name = item.get("metadata", {}).get("name", "")
|
||||
if name:
|
||||
print(name)
|
||||
'
|
||||
)"
|
||||
|
||||
if [[ -z "$job_names" ]]; then
|
||||
echo "No finished operator-runner Jobs matched cleanup filters in namespace $K8S_NAMESPACE."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == "true" ]]; then
|
||||
printf '%s\n' "$job_names"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while IFS= read -r job_name; do
|
||||
[[ -z "$job_name" ]] && continue
|
||||
kubectl -n "$K8S_NAMESPACE" delete job "$job_name"
|
||||
done <<<"$job_names"
|
||||
@ -14,6 +14,7 @@ export MONGO_DB="${MONGO_DB:-poc}"
|
||||
export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
|
||||
export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
|
||||
export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
|
||||
export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
|
||||
|
||||
export K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
||||
export WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
||||
@ -27,6 +28,7 @@ export API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-true}"
|
||||
export API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
|
||||
export API_REPLICAS="${API_REPLICAS:-1}"
|
||||
export WORKER_REPLICAS="${WORKER_REPLICAS:-1}"
|
||||
export OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
|
||||
|
||||
export AI_API_BASE_URL="${AI_API_BASE_URL:-https://coding.dashscope.aliyuncs.com/v1}"
|
||||
export AI_MODEL="${AI_MODEL:-glm-5}"
|
||||
|
||||
@ -14,6 +14,7 @@ export MONGO_DB="${MONGO_DB:-droboticflow}"
|
||||
export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
|
||||
export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
|
||||
export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
|
||||
export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
|
||||
|
||||
export K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
||||
export WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
||||
@ -23,6 +24,7 @@ export API_NODE_PORT="${API_NODE_PORT:-30080}"
|
||||
export API_SERVICE_TYPE="${API_SERVICE_TYPE:-NodePort}"
|
||||
export API_REPLICAS="${API_REPLICAS:-1}"
|
||||
export WORKER_REPLICAS="${WORKER_REPLICAS:-1}"
|
||||
export OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
|
||||
|
||||
export AI_API_BASE_URL="${AI_API_BASE_URL:-https://coding.dashscope.aliyuncs.com/v1}"
|
||||
export AI_MODEL="${AI_MODEL:-glm-5}"
|
||||
|
||||
@ -28,6 +28,7 @@ MONGO_URI="${MONGO_URI:-}"
|
||||
MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
|
||||
MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
|
||||
WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
|
||||
WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
|
||||
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
||||
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
||||
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
|
||||
@ -45,6 +46,7 @@ OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPA
|
||||
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
|
||||
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
|
||||
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
|
||||
OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
|
||||
OPERATOR_ALLOW_SHELL_COMMANDS="${OPERATOR_ALLOW_SHELL_COMMANDS:-false}"
|
||||
OPERATOR_COMMAND_ALLOWLIST_REGEX="${OPERATOR_COMMAND_ALLOWLIST_REGEX:-}"
|
||||
OPERATOR_COMMAND_DENYLIST_REGEX="${OPERATOR_COMMAND_DENYLIST_REGEX:-}"
|
||||
@ -156,6 +158,7 @@ env \
|
||||
MONGO_CONNECT_RETRIES="$MONGO_CONNECT_RETRIES" \
|
||||
MONGO_CONNECT_RETRY_DELAY_MS="$MONGO_CONNECT_RETRY_DELAY_MS" \
|
||||
WORKER_POLL_INTERVAL_MS="$WORKER_POLL_INTERVAL_MS" \
|
||||
WORKER_API_BASE_URL="$WORKER_API_BASE_URL" \
|
||||
K8S_IMAGE_PULL_SECRET_NAME="$K8S_IMAGE_PULL_SECRET_NAME" \
|
||||
WORKSPACE_PVC_NAME="$WORKSPACE_PVC_NAME" \
|
||||
WORKSPACE_MOUNT_PATH="$WORKSPACE_MOUNT_PATH" \
|
||||
@ -173,6 +176,7 @@ env \
|
||||
OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" \
|
||||
OPERATOR_GPU_TOLERATIONS_JSON="$OPERATOR_GPU_TOLERATIONS_JSON" \
|
||||
OPERATOR_GPU_RUNTIME_CLASS="$OPERATOR_GPU_RUNTIME_CLASS" \
|
||||
OPERATOR_JOB_TTL_SECONDS="$OPERATOR_JOB_TTL_SECONDS" \
|
||||
OPERATOR_ALLOW_SHELL_COMMANDS="$OPERATOR_ALLOW_SHELL_COMMANDS" \
|
||||
OPERATOR_COMMAND_ALLOWLIST_REGEX="$OPERATOR_COMMAND_ALLOWLIST_REGEX" \
|
||||
OPERATOR_COMMAND_DENYLIST_REGEX="$OPERATOR_COMMAND_DENYLIST_REGEX" \
|
||||
|
||||
@ -20,6 +20,7 @@ MONGO_DB="${MONGO_DB:-droboticflow}"
|
||||
MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
|
||||
MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
|
||||
WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
|
||||
WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
|
||||
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
||||
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
||||
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
|
||||
@ -33,6 +34,7 @@ AI_MODEL="${AI_MODEL:-glm-5}"
|
||||
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
|
||||
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
|
||||
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
|
||||
OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
|
||||
OPERATOR_ALLOW_SHELL_COMMANDS="${OPERATOR_ALLOW_SHELL_COMMANDS:-false}"
|
||||
OPERATOR_COMMAND_ALLOWLIST_REGEX="${OPERATOR_COMMAND_ALLOWLIST_REGEX:-}"
|
||||
OPERATOR_COMMAND_DENYLIST_REGEX="${OPERATOR_COMMAND_DENYLIST_REGEX:-}"
|
||||
@ -68,6 +70,7 @@ rendered="${rendered//__MONGO_DB__/$MONGO_DB}"
|
||||
rendered="${rendered//__MONGO_CONNECT_RETRIES__/$MONGO_CONNECT_RETRIES}"
|
||||
rendered="${rendered//__MONGO_CONNECT_RETRY_DELAY_MS__/$MONGO_CONNECT_RETRY_DELAY_MS}"
|
||||
rendered="${rendered//__WORKER_POLL_INTERVAL_MS__/$WORKER_POLL_INTERVAL_MS}"
|
||||
rendered="${rendered//__WORKER_API_BASE_URL__/$WORKER_API_BASE_URL}"
|
||||
rendered="${rendered//__K8S_IMAGE_PULL_SECRET_NAME__/$K8S_IMAGE_PULL_SECRET_NAME}"
|
||||
rendered="${rendered//__WORKSPACE_PVC_NAME__/$WORKSPACE_PVC_NAME}"
|
||||
rendered="${rendered//__WORKSPACE_MOUNT_PATH__/$WORKSPACE_MOUNT_PATH}"
|
||||
@ -83,6 +86,7 @@ rendered="${rendered//__OPERATOR_RUNNER_IMAGE__/$OPERATOR_RUNNER_IMAGE}"
|
||||
rendered="${rendered//__OPERATOR_GPU_NODE_SELECTOR__/$OPERATOR_GPU_NODE_SELECTOR}"
|
||||
rendered="${rendered//__OPERATOR_GPU_TOLERATIONS_JSON__/$OPERATOR_GPU_TOLERATIONS_JSON}"
|
||||
rendered="${rendered//__OPERATOR_GPU_RUNTIME_CLASS__/$OPERATOR_GPU_RUNTIME_CLASS}"
|
||||
rendered="${rendered//__OPERATOR_JOB_TTL_SECONDS__/$OPERATOR_JOB_TTL_SECONDS}"
|
||||
rendered="${rendered//__OPERATOR_ALLOW_SHELL_COMMANDS__/$OPERATOR_ALLOW_SHELL_COMMANDS}"
|
||||
rendered="${rendered//__OPERATOR_COMMAND_ALLOWLIST_REGEX__/$OPERATOR_COMMAND_ALLOWLIST_REGEX}"
|
||||
rendered="${rendered//__OPERATOR_COMMAND_DENYLIST_REGEX__/$OPERATOR_COMMAND_DENYLIST_REGEX}"
|
||||
|
||||
@ -23,7 +23,8 @@ import {
|
||||
FileText,
|
||||
} from "lucide-react";
|
||||
import { cn } from "../lib/utils";
|
||||
import { monitoringApi, LogEntry, ResourceUsage, ExecutionData } from "../services/api/monitoring";
|
||||
import { monitoringApi, type LogEntry, type ResourceUsage, type ExecutionData } from "../services/api/monitoring";
|
||||
import type { ExecutionRunSummary } from "../types";
|
||||
import { motion, AnimatePresence } from "motion/react";
|
||||
import { LineChart, Line, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, AreaChart, Area } from "recharts";
|
||||
|
||||
@ -43,9 +44,25 @@ const artifactExtension = (item: ExecutionData) => {
|
||||
return parts.length > 1 ? parts[parts.length - 1].toLowerCase() : "";
|
||||
};
|
||||
|
||||
type ModalType = "logs" | "resources" | "data";
|
||||
|
||||
type ArtifactPreview = {
|
||||
kind?: string;
|
||||
format?: string;
|
||||
url?: string;
|
||||
rows?: Record<string, unknown>[];
|
||||
columns?: string[];
|
||||
rowCount?: number;
|
||||
lineCount?: number;
|
||||
truncated?: boolean;
|
||||
text?: string;
|
||||
value?: unknown;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
|
||||
const rawPreviewPayload = (item: ExecutionData) => {
|
||||
const preview = item.data?.preview;
|
||||
return preview && typeof preview === "object" ? (preview as Record<string, any>) : null;
|
||||
return preview && typeof preview === "object" ? (preview as ArtifactPreview) : null;
|
||||
};
|
||||
|
||||
const derivedPreview = (item: ExecutionData) => {
|
||||
@ -144,11 +161,11 @@ export default function Monitoring() {
|
||||
const { timeZone } = useTimeZone();
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [statusFilter, setStatusFilter] = useState("all");
|
||||
const [runs, setRuns] = useState<any[]>([]);
|
||||
const [runs, setRuns] = useState<ExecutionRunSummary[]>([]);
|
||||
|
||||
// Modal states
|
||||
const [activeModal, setActiveModal] = useState<"logs" | "resources" | "data" | null>(null);
|
||||
const [selectedRun, setSelectedRun] = useState<any>(null);
|
||||
const [activeModal, setActiveModal] = useState<ModalType | null>(null);
|
||||
const [selectedRun, setSelectedRun] = useState<ExecutionRunSummary | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [modalData, setModalData] = useState<{
|
||||
logs?: LogEntry[];
|
||||
@ -195,7 +212,7 @@ export default function Monitoring() {
|
||||
});
|
||||
}, [runs, searchQuery, statusFilter]);
|
||||
|
||||
const openModal = async (type: "logs" | "resources" | "data", run: any) => {
|
||||
const openModal = async (type: ModalType, run: ExecutionRunSummary) => {
|
||||
setSelectedRun(run);
|
||||
setActiveModal(type);
|
||||
setLoading(true);
|
||||
@ -243,7 +260,7 @@ export default function Monitoring() {
|
||||
setRunActionPendingId(runId);
|
||||
try {
|
||||
const updated = await monitoringApi.stopRun(runId);
|
||||
setRuns((previous) => previous.map((item) => (item.id === runId ? updated : item)));
|
||||
setRuns((previous) => previous.map((item) => (item.id === runId ? { ...item, ...updated } : item)));
|
||||
} catch (error) {
|
||||
console.error("Failed to stop run", error);
|
||||
} finally {
|
||||
@ -258,7 +275,7 @@ export default function Monitoring() {
|
||||
setRunActionPendingId(runId);
|
||||
try {
|
||||
const updated = await monitoringApi.forceStopRun(runId);
|
||||
setRuns((previous) => previous.map((item) => (item.id === runId ? updated : item)));
|
||||
setRuns((previous) => previous.map((item) => (item.id === runId ? { ...item, ...updated } : item)));
|
||||
} catch (error) {
|
||||
console.error("Failed to force stop run", error);
|
||||
} finally {
|
||||
@ -329,7 +346,7 @@ export default function Monitoring() {
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{preview.rows.map((row: Record<string, any>, index: number) => (
|
||||
{preview.rows.map((row: Record<string, unknown>, index: number) => (
|
||||
<tr key={index} className="border-b border-[#1f2024] last:border-b-0">
|
||||
{columns.map((column) => (
|
||||
<td key={`${index}-${column}`} className="px-3 py-2 text-gray-200 align-top">
|
||||
@ -484,7 +501,7 @@ export default function Monitoring() {
|
||||
|
||||
{activeModal === "resources" && (
|
||||
<div className="space-y-8">
|
||||
<div className="grid grid-cols-3 gap-4">
|
||||
<div className="grid grid-cols-2 lg:grid-cols-4 gap-4">
|
||||
{[
|
||||
{
|
||||
label: "CPU Usage",
|
||||
@ -501,6 +518,13 @@ export default function Monitoring() {
|
||||
value: `${Math.round(modalData.resources?.[modalData.resources.length - 1]?.gpu || 0)}%`,
|
||||
color: "text-purple-500",
|
||||
},
|
||||
{
|
||||
label: "GPU MEM",
|
||||
value: modalData.resources?.[modalData.resources.length - 1]?.gpuMemoryTotalMiB
|
||||
? `${Math.round(modalData.resources[modalData.resources.length - 1].gpuMemoryUsedMiB || 0)} / ${Math.round(modalData.resources[modalData.resources.length - 1].gpuMemoryTotalMiB || 0)} MiB`
|
||||
: "N/A",
|
||||
color: "text-cyan-400",
|
||||
},
|
||||
].map((stat) => (
|
||||
<div key={stat.label} className="bg-[#0a0a0a] border border-[#2a2b2f] p-4 rounded-xl">
|
||||
<div className="text-[10px] text-gray-500 uppercase font-mono mb-1">{stat.label}</div>
|
||||
@ -508,6 +532,15 @@ export default function Monitoring() {
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
{modalData.resources?.[modalData.resources.length - 1]?.gpuName && (
|
||||
<div className="rounded-xl border border-[#2a2b2f] bg-[#0a0a0a] px-4 py-3 text-xs text-gray-300">
|
||||
<span className="text-gray-500 font-mono uppercase mr-2">GPU Device</span>
|
||||
{modalData.resources[modalData.resources.length - 1].gpuName}
|
||||
{modalData.resources[modalData.resources.length - 1].gpuCount
|
||||
? ` x ${modalData.resources[modalData.resources.length - 1].gpuCount}`
|
||||
: ""}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="h-[300px] w-full bg-[#0a0a0a] rounded-xl border border-[#2a2b2f] p-4">
|
||||
<ResponsiveContainer width="100%" height="100%">
|
||||
|
||||
@ -8,6 +8,14 @@ type StorageBrowseTarget = {
|
||||
connectionId?: string;
|
||||
};
|
||||
|
||||
type PagedResponse<T> = {
|
||||
items: T[];
|
||||
total?: number;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
truncated?: boolean;
|
||||
};
|
||||
|
||||
export const monitoringApi = {
|
||||
listRuns: async () => (await apiRequest<{ items: ExecutionRunSummary[] }>("/api/runs")).items,
|
||||
getRun: (runId: string) => apiRequest<ExecutionRun>(`/api/runs/${runId}`),
|
||||
@ -16,11 +24,11 @@ export const monitoringApi = {
|
||||
stopRun: (runId: string) => apiRequest<ExecutionRun>(`/api/runs/${runId}/stop`, { method: "POST" }),
|
||||
forceStopRun: (runId: string) => apiRequest<ExecutionRun>(`/api/runs/${runId}/force-stop`, { method: "POST" }),
|
||||
getLogs: async (runId: string): Promise<LogEntry[]> =>
|
||||
(await apiRequest<{ items: LogEntry[] }>(`/api/runs/${runId}/logs`)).items,
|
||||
(await apiRequest<PagedResponse<LogEntry>>(`/api/runs/${runId}/logs`)).items,
|
||||
getResourceUsage: async (runId: string): Promise<ResourceUsage[]> =>
|
||||
(await apiRequest<{ items: ResourceUsage[] }>(`/api/runs/${runId}/resources`)).items,
|
||||
(await apiRequest<PagedResponse<ResourceUsage>>(`/api/runs/${runId}/resources`)).items,
|
||||
getExecutionData: async (runId: string): Promise<ExecutionData[]> =>
|
||||
(await apiRequest<{ items: ExecutionData[] }>(`/api/runs/${runId}/artifacts`)).items,
|
||||
(await apiRequest<PagedResponse<ExecutionData>>(`/api/runs/${runId}/artifacts`)).items,
|
||||
listOssBuckets: async ({ sourcePath = "", connectionId = "" }: StorageBrowseTarget): Promise<string[]> => {
|
||||
const query = new URLSearchParams();
|
||||
if (connectionId.trim()) {
|
||||
|
||||
@ -414,6 +414,10 @@ export interface ResourceUsage {
|
||||
cpu: number;
|
||||
memory: number;
|
||||
gpu?: number;
|
||||
gpuMemoryUsedMiB?: number;
|
||||
gpuMemoryTotalMiB?: number;
|
||||
gpuName?: string;
|
||||
gpuCount?: number;
|
||||
}
|
||||
|
||||
export interface ExecutionData {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user