🚀 Harden production runtime and acceptance

This commit is contained in:
eust-w 2026-06-16 14:28:49 +08:00
parent 0ef6b6be58
commit 20b36fe6a0
35 changed files with 818 additions and 67 deletions

View File

@ -6,6 +6,7 @@ MONGO_DB=droboticflow
MONGO_CONNECT_RETRIES=30
MONGO_CONNECT_RETRY_DELAY_MS=2000
WORKER_POLL_INTERVAL_MS=2000
WORKER_API_BASE_URL=http://droboticflow-worker:3002
K8S_NAMESPACE=droboticflow
K8S_IMAGE_PULL_SECRET_NAME=droboticflow-registry
WORKSPACE_PVC_NAME=droboticflow-workspace
@ -15,6 +16,7 @@ OPERATOR_RUNNER_IMAGE=
OPERATOR_GPU_NODE_SELECTOR=
OPERATOR_GPU_TOLERATIONS_JSON=
OPERATOR_GPU_RUNTIME_CLASS=
OPERATOR_JOB_TTL_SECONDS=86400
OPERATOR_ALLOW_SHELL_COMMANDS=false
OPERATOR_COMMAND_ALLOWLIST_REGEX=
OPERATOR_COMMAND_DENYLIST_REGEX=

77
.github/workflows/acceptance.yml vendored Normal file
View File

@ -0,0 +1,77 @@
name: Acceptance
on:
workflow_dispatch:
inputs:
api_base_url:
description: "Public API base URL, for example http://<node-ip>:30080"
required: false
type: string
worker_base_url:
description: "Optional direct worker base URL. Leave empty to use API proxy only."
required: false
type: string
k8s_namespace:
description: "Kubernetes namespace for GPU smoke"
required: false
default: droboticflow
type: string
run_api_smoke:
description: "Run API/workflow smoke test"
required: true
default: true
type: boolean
run_gpu_smoke:
description: "Run real GPU scheduling and nvidia-smi smoke test"
required: true
default: false
type: boolean
jobs:
acceptance:
runs-on: ubuntu-latest
env:
K8S_NAMESPACE: ${{ inputs.k8s_namespace }}
KUBE_CONFIG_B64: ${{ secrets.KUBE_CONFIG_B64 }}
DROBOTICFLOW_API_TOKEN: ${{ secrets.DROBOTICFLOW_API_TOKEN }}
DROBOTICFLOW_SMOKE_EMAIL: ${{ secrets.DROBOTICFLOW_SMOKE_EMAIL }}
DROBOTICFLOW_SMOKE_PASSWORD: ${{ secrets.DROBOTICFLOW_SMOKE_PASSWORD }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up kubectl
uses: azure/setup-kubectl@v4
with:
version: v1.34.2
- name: Configure kubeconfig
run: |
if [[ -z "${KUBE_CONFIG_B64}" ]]; then
echo "KUBE_CONFIG_B64 secret is not set; Kubernetes smoke steps require it."
exit 0
fi
mkdir -p "$HOME/.kube"
printf '%s' "$KUBE_CONFIG_B64" | base64 --decode > "$HOME/.kube/config"
chmod 600 "$HOME/.kube/config"
- name: API smoke
if: inputs.run_api_smoke
env:
API_BASE_URL: ${{ inputs.api_base_url }}
WORKER_BASE_URL: ${{ inputs.worker_base_url }}
run: |
if [[ -z "$API_BASE_URL" ]]; then
echo "api_base_url is required when run_api_smoke=true" >&2
exit 1
fi
./scripts/smoke-test.sh "$API_BASE_URL" "$WORKER_BASE_URL"
- name: GPU smoke
if: inputs.run_gpu_smoke
run: |
if [[ ! -s "$HOME/.kube/config" ]]; then
echo "KUBE_CONFIG_B64 secret is required when run_gpu_smoke=true" >&2
exit 1
fi
./scripts/gpu-smoke-test.sh

View File

@ -66,6 +66,8 @@ Drobodata is a workflow orchestration platform for embodied AI and vision pipeli
- `npm run build`: build frontend assets and Go binaries
- `npm run verify`: lint, Go tests, Python operator-runner tests, frontend node tests, build, dependency audit, and Kubernetes manifest dry-run validation
- `npm run smoke`: local smoke test against the running API and worker
- `npm run gpu:smoke`: real Kubernetes GPU scheduling and `nvidia-smi` smoke test
- `npm run k8s:cleanup-jobs`: delete old finished operator-runner Jobs that predate TTL cleanup
- `npm run reconcile:data -- <api-base-url>`: remove legacy demo data, reset settings, and bootstrap built-in operators through the real API
- `python3 scripts/wjzh_operator_regression.py --preflight-only`: preflight the production-style WJZH real-data operator regression against a live API
@ -80,6 +82,7 @@ Key environment variables:
- `MONGO_CONNECT_RETRIES`: startup retry count for API and worker Mongo connections
- `MONGO_CONNECT_RETRY_DELAY_MS`: delay between Mongo connection retries
- `WORKER_POLL_INTERVAL_MS`: worker polling interval
- `WORKER_API_BASE_URL`: API-side upstream URL for `/worker-api/*`, default `http://droboticflow-worker:3002`
- `K8S_NAMESPACE`: namespace used for Jobs
- `K8S_IMAGE_PULL_SECRET_NAME`: image pull secret injected into operator-runner Jobs
- `WORKSPACE_PVC_NAME`: shared PVC used by worker, API, and operator-runner Jobs
@ -89,6 +92,7 @@ Key environment variables:
- `OPERATOR_GPU_NODE_SELECTOR`: optional comma-separated node selector for GPU Jobs, for example `accelerator=nvidia`
- `OPERATOR_GPU_TOLERATIONS_JSON`: optional Kubernetes tolerations JSON array for GPU Jobs
- `OPERATOR_GPU_RUNTIME_CLASS`: optional runtimeClassName for GPU Jobs, for example `nvidia`
- `OPERATOR_JOB_TTL_SECONDS`: Kubernetes TTL for finished operator-runner Jobs, default `86400`; set `0` to disable automatic TTL injection
- `OPERATOR_ALLOW_SHELL_COMMANDS`: controls whether `COMMAND` / `SCRIPT` shell execution is allowed, default `false`
- `OPERATOR_COMMAND_ALLOWLIST_REGEX`: optional regular expression that explicit shell commands must match
- `OPERATOR_COMMAND_DENYLIST_REGEX`: optional regular expression that rejects matching explicit shell commands
@ -107,7 +111,7 @@ The settings page can update K8s access details, the worker defaults, the global
The API now includes a cookie-session account foundation for organization-based access. Auth endpoints cover bootstrap, login, logout, current session, and super-admin organization/user management. Existing local deployments remain compatible by default; public LoadBalancer deploys require `DROBOTICFLOW_REQUIRE_SESSION_AUTH=true` so browser business APIs keep role checks. Session-authenticated unsafe requests use a double-submit `droboticflow_csrf` cookie plus `X-Droboticflow-CSRF-Token` header. Worker control endpoints under `/worker-api` require an admin/operator session, or an API token when session auth is not enforced. Do not embed `DROBOTICFLOW_API_TOKEN` into frontend builds; it is intended for machine-to-machine calls only.
The Kubernetes runtime now assumes a shared workspace PVC. Worker writes per-node `input-artifacts.json` files into the shared workspace, operator-runner Jobs read those files through `INPUT_ARTIFACTS_PATH`, then persist `result.json` and `stage-manifest.json` back into the same PVC. Large artifact lists no longer need to travel through giant env vars or stdout payloads. Node CPU and memory limits are applied to Kubernetes Job container resources; GPU requests use `nvidia.com/gpu` when the node or default resource config sets a positive GPU count. GPU scheduling requires a cluster with GPU nodes and the NVIDIA device plugin/runtime installed. Optional GPU node selector, tolerations, and runtime class settings are only injected into Jobs that request a positive GPU count. Invalid GPU selector or tolerations configuration fails worker startup instead of being ignored. Run `npm run gpu:smoke` against a real GPU cluster to validate allocatable GPU resources, scheduling, runtime, and `nvidia-smi` execution.
The Kubernetes runtime now assumes a shared workspace PVC. Worker writes per-node `input-artifacts.json` files into the shared workspace, operator-runner Jobs read those files through `INPUT_ARTIFACTS_PATH`, then persist `result.json` and `stage-manifest.json` back into the same PVC. Large artifact lists no longer need to travel through giant env vars or stdout payloads. Node CPU and memory limits are applied to Kubernetes Job container resources; GPU requests use `nvidia.com/gpu` when the node or default resource config sets a positive GPU count. GPU scheduling requires a cluster with GPU nodes and the NVIDIA device plugin/runtime installed. Optional GPU node selector, tolerations, and runtime class settings are only injected into Jobs that request a positive GPU count. Invalid GPU selector or tolerations configuration fails worker startup instead of being ignored. Operator-runner Jobs also receive `ttlSecondsAfterFinished` from `OPERATOR_JOB_TTL_SECONDS`, and the cleanup script can remove old completed Jobs that were created before TTL was enabled. Run `npm run gpu:smoke` against a real GPU cluster to validate allocatable GPU resources, scheduling, runtime, and `nvidia-smi` execution. Runs that execute on NVIDIA hardware report GPU utilization, GPU memory, GPU count, and GPU model in the resource samples shown on the monitoring page.
Explicit `COMMAND` / `SCRIPT` execution is disabled by default. Deployments that need the shell operator must set `OPERATOR_ALLOW_SHELL_COMMANDS=true` and should constrain commands through allowlist and denylist regular expressions. These policy variables are protected system env vars: workflow and node hyperparams cannot override the worker-provided values.
@ -219,6 +223,8 @@ Run a smoke test against a deployed cluster:
./scripts/smoke-test.sh http://<node-ip>:30080
```
Run real-environment acceptance from GitHub Actions with the manual `Acceptance` workflow. It accepts the deployed API URL, optional direct worker URL, and can run GPU smoke when the repository secret `KUBE_CONFIG_B64` is configured.
See the detailed guides:
- [`docs/api.md`](docs/api.md)
@ -230,6 +236,7 @@ See the detailed guides:
- [`docs/operators/README.md`](docs/operators/README.md)
- [`docs/operator-authoring.md`](docs/operator-authoring.md)
- [`docs/operations.md`](docs/operations.md)
- [`docs/production-readiness.md`](docs/production-readiness.md)
- [`docs/wjzh-operator-regression.md`](docs/wjzh-operator-regression.md)
- [`docs/feature-overview-playwright-validation-2026-04-15.md`](docs/feature-overview-playwright-validation-2026-04-15.md)
- [`docs/feature-overview-playwright-validation-2026-04-15.html`](docs/feature-overview-playwright-validation-2026-04-15.html)

View File

@ -18,15 +18,45 @@ def resource_snapshot() -> tuple[float, int]:
return cpu_seconds, memory_mib
def gpu_snapshot() -> int:
def parse_gpu_snapshot_output(output: str) -> dict[str, Any]:
samples: list[dict[str, Any]] = []
for line in output.splitlines():
parts = [part.strip() for part in line.split(",")]
if len(parts) < 4:
continue
try:
samples.append(
{
"gpu": int(parts[0]),
"gpuMemoryUsedMiB": int(parts[1]),
"gpuMemoryTotalMiB": int(parts[2]),
"gpuName": parts[3],
}
)
except ValueError:
continue
if not samples:
return {"gpu": 0}
names = sorted({sample["gpuName"] for sample in samples if sample["gpuName"]})
return {
"gpu": max(0, min(100, max(sample["gpu"] for sample in samples))),
"gpuMemoryUsedMiB": max(0, sum(sample["gpuMemoryUsedMiB"] for sample in samples)),
"gpuMemoryTotalMiB": max(0, sum(sample["gpuMemoryTotalMiB"] for sample in samples)),
"gpuName": ", ".join(names),
"gpuCount": len(samples),
}
def gpu_snapshot() -> dict[str, Any]:
nvidia_smi = shutil.which("nvidia-smi")
if not nvidia_smi:
return 0
return {"gpu": 0}
try:
completed = subprocess.run(
[
nvidia_smi,
"--query-gpu=utilization.gpu",
"--query-gpu=utilization.gpu,memory.used,memory.total,name",
"--format=csv,noheader,nounits",
],
check=False,
@ -35,31 +65,20 @@ def gpu_snapshot() -> int:
timeout=2,
)
except (OSError, subprocess.SubprocessError):
return 0
return {"gpu": 0}
if completed.returncode != 0:
return 0
values: list[int] = []
for line in completed.stdout.splitlines():
line = line.strip()
if not line:
continue
try:
values.append(int(line))
except ValueError:
continue
if not values:
return 0
return max(0, min(100, max(values)))
return {"gpu": 0}
return parse_gpu_snapshot_output(completed.stdout)
def resource_series(start_cpu: float, start_memory: int, started_at: str, ended_at: str) -> list[dict[str, Any]]:
end_cpu, end_memory = resource_snapshot()
gpu_percent = gpu_snapshot()
gpu_metrics = gpu_snapshot()
started = dt.datetime.fromisoformat(started_at.replace("Z", "+00:00"))
ended = dt.datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
elapsed = max((ended - started).total_seconds(), 0.001)
cpu_percent = max(0, min(100, int(((end_cpu - start_cpu) / elapsed) * 100)))
return [
{"timestamp": started_at, "cpu": 0, "memory": start_memory, "gpu": 0},
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, "gpu": gpu_percent},
{"timestamp": ended_at, "cpu": cpu_percent, "memory": end_memory, **gpu_metrics},
]

View File

@ -15,6 +15,12 @@ assert SPEC is not None and SPEC.loader is not None
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
RESOURCE_MONITOR_PATH = pathlib.Path(__file__).with_name("resource_monitor.py")
RESOURCE_MONITOR_SPEC = importlib.util.spec_from_file_location("operator_runner_resource_monitor", RESOURCE_MONITOR_PATH)
assert RESOURCE_MONITOR_SPEC is not None and RESOURCE_MONITOR_SPEC.loader is not None
RESOURCE_MONITOR = importlib.util.module_from_spec(RESOURCE_MONITOR_SPEC)
RESOURCE_MONITOR_SPEC.loader.exec_module(RESOURCE_MONITOR)
class _FakeHTTPResponse:
def __init__(self, payload: bytes):
@ -119,6 +125,28 @@ class CommandPolicyTests(unittest.TestCase):
MODULE.validate_command_policy("bash script.sh")
class ResourceMonitorTests(unittest.TestCase):
def test_parse_gpu_snapshot_output_summarizes_real_nvidia_smi_columns(self):
snapshot = RESOURCE_MONITOR.parse_gpu_snapshot_output(
"\n".join(
[
"12, 4096, 24576, NVIDIA L4",
"83, 8192, 24576, NVIDIA L4",
"not-a-number, 1, 1, broken",
]
)
)
self.assertEqual(snapshot["gpu"], 83)
self.assertEqual(snapshot["gpuMemoryUsedMiB"], 12288)
self.assertEqual(snapshot["gpuMemoryTotalMiB"], 49152)
self.assertEqual(snapshot["gpuName"], "NVIDIA L4")
self.assertEqual(snapshot["gpuCount"], 2)
def test_parse_gpu_snapshot_output_defaults_when_nvidia_smi_has_no_valid_rows(self):
self.assertEqual(RESOURCE_MONITOR.parse_gpu_snapshot_output(""), {"gpu": 0})
class SourcePathFromItemTests(unittest.TestCase):
def test_prefers_workspace_bos_cache_for_bos_source_paths(self):
with tempfile.TemporaryDirectory() as temp_dir:

View File

@ -73,6 +73,7 @@ func main() {
cfg.OperatorGPUNodeSelector,
cfg.OperatorGPUTolerations,
cfg.OperatorGPURuntimeClass,
cfg.OperatorJobTTLSeconds,
)
if err != nil {
fmt.Printf("[worker] failed to initialize runtime: %v\n", err)

View File

@ -228,6 +228,20 @@ The built-in catalog includes the original workflow primitives plus:
- `GET /api/runs/:id/resources`
- `GET /api/runs/:id/artifacts`
`GET /api/runs/:id/logs`, `GET /api/runs/:id/resources`, and `GET /api/runs/:id/artifacts` accept optional `limit` and `offset` query parameters. `skip` is accepted as an alias for `offset`. Logs and resource samples default to the latest 1000 items when the run contains a large history; artifacts default from the start. The response includes:
```json
{
"items": [],
"total": 1200,
"limit": 1000,
"offset": 200,
"truncated": true
}
```
Resource samples include `timestamp`, `cpu`, and `memory`. When NVIDIA GPU metrics are available, samples may also include `gpu`, `gpuMemoryUsedMiB`, `gpuMemoryTotalMiB`, `gpuName`, and `gpuCount`.
`GET /api/runs/:id/artifacts`
Each artifact item can now include:

View File

@ -162,6 +162,7 @@ http://<node-ip>:30080
- `MONGO_CONNECT_RETRIES`
- `MONGO_CONNECT_RETRY_DELAY_MS`
- `WORKER_POLL_INTERVAL_MS`
- `WORKER_API_BASE_URL`
- `API_NODE_PORT`
- `API_REPLICAS`
- `WORKER_REPLICAS`
@ -178,6 +179,7 @@ http://<node-ip>:30080
- `OPERATOR_GPU_NODE_SELECTOR`
- `OPERATOR_GPU_TOLERATIONS_JSON`
- `OPERATOR_GPU_RUNTIME_CLASS`
- `OPERATOR_JOB_TTL_SECONDS`
- `OPERATOR_ALLOW_SHELL_COMMANDS`
- `OPERATOR_COMMAND_ALLOWLIST_REGEX`
- `OPERATOR_COMMAND_DENYLIST_REGEX`
@ -189,10 +191,12 @@ http://<node-ip>:30080
- The main manifest only deploys the application components. It does not create MongoDB resources in-cluster.
- `MONGO_URI` must reference an external MongoDB instance; localhost, `mongodb:27017`, and in-cluster `.svc` addresses are rejected by `scripts/deploy-k8s.sh`.
- API and worker entrypoints are Go binaries with Mongo startup retry, so they tolerate the database becoming ready after the Deployments start.
- The API proxies `/worker-api/*` to `WORKER_API_BASE_URL`, defaulting to the in-cluster `http://droboticflow-worker:3002` Service. Override it only when the worker Service name, namespace, or exposure model differs.
- The worker runs with Kubernetes RBAC that allows creating and watching Jobs in the namespace.
- The operator-runner Job template inherits the private registry pull secret name from `K8S_IMAGE_PULL_SECRET_NAME`.
- Operator-runner Jobs apply CPU and memory requests/limits from system defaults and node overrides. A positive GPU value is emitted as `nvidia.com/gpu` in both requests and limits; the target cluster must have GPU nodes and the NVIDIA device plugin/runtime configured.
- GPU-only scheduling settings are optional: `OPERATOR_GPU_NODE_SELECTOR` accepts comma-separated `key=value` pairs, `OPERATOR_GPU_TOLERATIONS_JSON` accepts a Kubernetes tolerations JSON array, and `OPERATOR_GPU_RUNTIME_CLASS` sets `runtimeClassName`. These settings are only injected into Jobs that request a positive GPU count. Invalid selector or tolerations values fail worker startup.
- Finished operator-runner Jobs receive `ttlSecondsAfterFinished` from `OPERATOR_JOB_TTL_SECONDS`, default `86400`. Use `npm run k8s:cleanup-jobs` to delete old finished Jobs that predate TTL, or set `DRY_RUN=true` first to list candidates.
- Explicit `COMMAND` / `SCRIPT` execution is disabled by default. Set `OPERATOR_ALLOW_SHELL_COMMANDS=true` only for deployments that need shell operators, and constrain it with `OPERATOR_COMMAND_ALLOWLIST_REGEX` / `OPERATOR_COMMAND_DENYLIST_REGEX` where possible. These values are injected by the worker and are not overridable from workflow hyperparams.
- The operator-runner is not a standalone Deployment. It is launched on demand as a Python Job container.
- Failed operator-runner Jobs report Job conditions, Pod phase, container waiting or termination reasons, Pod events, and available logs back through the run error path.
@ -206,5 +210,6 @@ http://<node-ip>:30080
- `IMAGE_TAG` defaults to the current git short SHA in build, deploy, and render scripts. Set it explicitly for release names or rollback-friendly tags.
- `scripts/smoke-test.sh` can exercise session-authenticated deployments when `DROBOTICFLOW_SMOKE_EMAIL` and `DROBOTICFLOW_SMOKE_PASSWORD` are set; it logs in, persists cookies, and sends the CSRF header on write requests.
- `npm run gpu:smoke` creates a temporary GPU Job in the target namespace and runs `nvidia-smi`. Override `GPU_SMOKE_IMAGE`, `GPU_RESOURCE_NAME`, `GPU_RESOURCE_COUNT`, and the `OPERATOR_GPU_*` scheduling variables when the cluster uses custom GPU images or labels.
- GitHub Actions includes a manual `Acceptance` workflow for deployed-environment checks. Configure `KUBE_CONFIG_B64` for Kubernetes access and optionally `DROBOTICFLOW_API_TOKEN`, `DROBOTICFLOW_SMOKE_EMAIL`, and `DROBOTICFLOW_SMOKE_PASSWORD` for authenticated smoke tests.
- If you use DashScope-compatible models, prefer a backend-allowed API key. Do not deploy tool-only Coding Plan keys into the cluster.
- Production hardening still depends on the target cluster policy: TLS ingress, network policies, backup strategy, and secret rotation.

View File

@ -68,6 +68,19 @@ This includes TypeScript/Go linting, frontend Prettier formatting checks, Go tes
The worker exposes Prometheus text metrics at `/worker-api/metrics`. The endpoint includes run status gauges, ProcessAvailable counters, dispatch counters, node execution counters, cumulative duration counters, duration histograms, and the most recent dispatch queue depth. Kubernetes deployments annotate the worker Service for Prometheus scraping at this path.
Runs that execute on NVIDIA hardware also persist GPU utilization, GPU memory used/total in MiB, GPU count, and GPU model into `run.resourceSeries`. The monitoring page shows these fields in the resource modal. A missing GPU memory value means the operator-runner could not execute `nvidia-smi`, not that the workflow requested no GPU.
## Job Retention
New operator-runner Jobs use `OPERATOR_JOB_TTL_SECONDS` as `ttlSecondsAfterFinished`, defaulting to one day. For clusters upgraded from older manifests, clean pre-existing finished Jobs with:
```bash
DRY_RUN=true npm run k8s:cleanup-jobs
npm run k8s:cleanup-jobs
```
Set `OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS` and `OPERATOR_JOB_CLEANUP_STATUSES=succeeded,failed` to adjust the retention window and terminal states.
## Account Security
The API supports HttpOnly cookie-session auth through `/api/auth/*` and super-admin organization/user management under `/api/admin/*`. Existing deployments keep their current behavior unless `DROBOTICFLOW_REQUIRE_SESSION_AUTH=true` is set. Before enabling it, bootstrap the first super admin and verify login through `/api/auth/login`.
@ -76,6 +89,10 @@ The API supports HttpOnly cookie-session auth through `/api/auth/*` and super-ad
Explicit `COMMAND` / `SCRIPT` execution is disabled by default. Set `OPERATOR_ALLOW_SHELL_COMMANDS=true` only for deployments that need shell operators, and constrain commands with `OPERATOR_COMMAND_ALLOWLIST_REGEX` and `OPERATOR_COMMAND_DENYLIST_REGEX`. These are platform-controlled variables passed from the worker to operator-runner Jobs, and workflow hyperparams cannot override them.
## Acceptance Workflow
The GitHub Actions `Acceptance` workflow is manual. Use it after a deployment by entering the deployed API URL and enabling API smoke, GPU smoke, or both. GPU smoke requires `KUBE_CONFIG_B64` to contain a base64-encoded kubeconfig with permission to create a temporary Job in `K8S_NAMESPACE`.
## Generated Artifacts
Build output, local Playwright screenshots/videos, pytest caches, and zip bundles should stay out of git. Regenerate validation artifacts locally or publish them as CI/release artifacts when they need to be shared.

View File

@ -0,0 +1,64 @@
# Production Readiness Notes
This note tracks the production-facing safeguards that must stay true for Drobodata deployments.
## Runtime Boundaries
- Production execution uses the Go worker and Python operator-runner Jobs. The API does not execute workflow nodes in-process.
- The API proxies `/worker-api/*` through `WORKER_API_BASE_URL`; the default is the in-cluster worker Service.
- Worker proxy health, stats, metrics, and retry calls use short upstream timeouts. Manual `process-once` uses a long timeout because it can execute real queued work.
- Explicit shell operators are disabled by default through `OPERATOR_ALLOW_SHELL_COMMANDS=false`; allowlist and denylist regexes are platform-controlled and cannot be overridden by workflow hyperparams.
## Kubernetes Jobs
- Operator-runner Jobs receive CPU and memory requests/limits from system defaults and node overrides.
- A positive GPU value emits `nvidia.com/gpu` in both requests and limits.
- GPU node selector, tolerations, and runtime class are only injected for Jobs that request GPU.
- Finished Jobs receive `ttlSecondsAfterFinished` from `OPERATOR_JOB_TTL_SECONDS`, default `86400`.
- Use `DRY_RUN=true npm run k8s:cleanup-jobs` before deleting old finished Jobs that predate TTL.
## Data Plane Limits
- Object-storage JSON object creation is capped at 4 MiB.
- Browser multipart uploads stream to BOS and are capped at 2 GiB.
- Large inline object reads use preview or presigned URL behavior instead of loading full objects into API memory.
- Run logs, resource samples, and artifacts support `limit` and `offset`. Logs and resources default to the latest 1000 records for large runs.
- Data-asset aggregation uses artifact-focused run projections so list endpoints do not read complete logs and execution traces.
## GPU Observability
- Operator-runner resource samples include CPU seconds, process memory, GPU utilization, GPU memory used/total, GPU count, and GPU model when `nvidia-smi` is available.
- The monitoring page displays GPU utilization, GPU memory, and GPU device information.
- `npm run gpu:smoke` remains the real acceptance check for allocatable GPU resources, scheduling, runtime, and `nvidia-smi`.
## Acceptance
Run local verification before merging:
```bash
npm run verify
```
Run deployed smoke after rollout:
```bash
./scripts/smoke-test.sh http://<node-ip>:30080
```
Run GPU smoke on a real GPU cluster:
```bash
npm run gpu:smoke
```
The manual GitHub Actions `Acceptance` workflow can run API smoke and GPU smoke against a deployed environment. Configure `KUBE_CONFIG_B64` when GPU smoke needs Kubernetes access.
## Remaining Cluster Responsibilities
Drobodata now exposes the necessary application controls, but production readiness still depends on the target cluster:
- TLS ingress or trusted private network exposure
- network policies around MongoDB, API, worker, and operator-runner Jobs
- MongoDB backup, restore, and credential rotation
- registry credential rotation
- Prometheus scraping and alerting for worker metrics and failed Jobs

View File

@ -1067,7 +1067,7 @@ func (server *server) latestCompletedScanRuns(ctx context.Context) (map[string]d
for _, workflow := range workflows {
workflowByID[workflow.ID] = workflow
}
runs, err := server.listRunsForContext(ctx, string(domain.RunStatusCompleted))
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
if err != nil {
return nil, nil, err
}
@ -2437,7 +2437,7 @@ func (server *server) packageDetail(ctx context.Context, runID string, packageNa
}
func (server *server) handlePackagesList(writer http.ResponseWriter, request *http.Request) {
runs, err := server.listRunsForRequest(request.Context(), request, string(domain.RunStatusCompleted))
runs, err := server.listRunArtifactsForRequest(request.Context(), request, string(domain.RunStatusCompleted))
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
@ -2564,7 +2564,7 @@ func (server *server) syncSamplingTasks(ctx context.Context) error {
for _, task := range existing {
existingByRun[task.RunID] = task
}
runs, err := server.listRunsForContext(ctx, string(domain.RunStatusCompleted))
runs, err := server.listRunArtifactsForContext(ctx, string(domain.RunStatusCompleted))
if err != nil {
return err
}

View File

@ -25,6 +25,7 @@ type objectStorageBrowser interface {
CreateBucket(context.Context, string) (domain.ObjectStorageBucketSummary, error)
DeleteBucket(context.Context, string) error
PutObject(context.Context, string, string, []byte, string) (domain.ObjectStorageObjectSummary, error)
PutObjectStream(context.Context, string, string, io.Reader, string) (domain.ObjectStorageObjectSummary, error)
RenameObject(context.Context, string, string, string) (domain.ObjectStorageObjectSummary, error)
DeleteObject(context.Context, string, string) error
DeletePrefix(context.Context, string, string) error
@ -42,6 +43,8 @@ type bosObjectStorageBrowser struct {
const (
objectStorageInlinePreviewMaxBytes int64 = 256 * 1024
objectStoragePartialPreviewBytes int64 = 64 * 1024
objectStorageJSONPutMaxBytes int64 = 4 * 1024 * 1024
objectStorageMultipartUploadMaxBytes int64 = 2 * 1024 * 1024 * 1024
objectStoragePreviewTruncatedHeader = "X-Drobodata-Preview-Truncated"
objectStoragePreviewOriginalSizeHeader = "X-Drobodata-Preview-Original-Bytes"
objectStoragePreviewReturnedSizeHeader = "X-Drobodata-Preview-Returned-Bytes"
@ -273,6 +276,15 @@ func (browser *bosObjectStorageBrowser) PutObject(ctx context.Context, bucket st
return browser.GetObjectSummary(ctx, bucket, objectKey)
}
func (browser *bosObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
objectKey := strings.Trim(strings.TrimSpace(key), "/")
args := &bosapi.PutObjectArgs{ContentType: detectObjectStorageContentType(objectKey, nil, contentType)}
if _, err := browser.client.PutObjectFromStreamWithContext(ctx, strings.TrimSpace(bucket), objectKey, reader, args); err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.GetObjectSummary(ctx, bucket, objectKey)
}
func (browser *bosObjectStorageBrowser) RenameObject(ctx context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
objectKey := exactObjectStorageKey(key)
destinationKey := exactObjectStorageKey(newKey)
@ -445,6 +457,10 @@ func renderObjectStorageWriteError(writer http.ResponseWriter, err error) {
if err == nil {
return
}
if isRequestTooLarge(err) {
renderJSON(writer, http.StatusRequestEntityTooLarge, map[string]string{"error": err.Error()})
return
}
if os.IsNotExist(err) || isBOSNoSuchKey(err) {
renderJSON(writer, http.StatusNotFound, map[string]string{"error": err.Error()})
return
@ -452,6 +468,14 @@ func renderObjectStorageWriteError(writer http.ResponseWriter, err error) {
renderError(writer, http.StatusInternalServerError, err)
}
func isRequestTooLarge(err error) bool {
if err == nil {
return false
}
var maxBytesErr *http.MaxBytesError
return errors.As(err, &maxBytesErr)
}
func exactObjectStorageKey(key string) string {
return strings.TrimSpace(key)
}
@ -555,8 +579,13 @@ func (server *server) handleObjectStorageBucketDelete(writer http.ResponseWriter
}
func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter, request *http.Request) {
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageJSONPutMaxBytes)
input, err := decodeObjectStorageObjectPutInput(request)
if err != nil {
if isRequestTooLarge(err) {
renderObjectStorageWriteError(writer, err)
return
}
renderValidationError(writer, validationIssue{Field: "body", Message: "invalid JSON payload"})
return
}
@ -593,6 +622,7 @@ func (server *server) handleObjectStorageObjectCreate(writer http.ResponseWriter
}
func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter, request *http.Request) {
request.Body = http.MaxBytesReader(writer, request.Body, objectStorageMultipartUploadMaxBytes)
connectionID := strings.TrimSpace(request.URL.Query().Get("connectionId"))
browser, connection, cleanup, err := server.resolveObjectStorageBrowser(request.Context(), connectionID)
if err != nil {
@ -620,6 +650,10 @@ func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter
return
}
if err := request.ParseMultipartForm(32 << 20); err != nil {
if isRequestTooLarge(err) {
renderObjectStorageWriteError(writer, err)
return
}
renderValidationError(writer, validationIssue{Field: "file", Message: "invalid multipart payload"})
return
}
@ -629,16 +663,11 @@ func (server *server) handleObjectStorageObjectUpload(writer http.ResponseWriter
return
}
defer file.Close()
payload, err := io.ReadAll(file)
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
}
contentType := strings.TrimSpace(request.URL.Query().Get("contentType"))
if contentType == "" && header != nil {
contentType = strings.TrimSpace(header.Header.Get("Content-Type"))
}
item, err := browser.PutObject(request.Context(), bucket, key, payload, contentType)
item, err := browser.PutObjectStream(request.Context(), bucket, key, file, contentType)
if err != nil {
renderObjectStorageWriteError(writer, err)
return

View File

@ -6,6 +6,8 @@ import (
"strings"
"testing"
"droboticflow/internal/store"
"github.com/baidubce/bce-sdk-go/bce"
)
@ -35,3 +37,14 @@ func TestRenderObjectStorageWriteErrorTreatsBOSNoSuchKeyAsNotFound(t *testing.T)
t.Fatalf("expected response to include NoSuchKey, got %s", recorder.Body.String())
}
}
func TestObjectStorageJSONPutRejectsOversizedBody(t *testing.T) {
body := `{"key":"big.txt","content":"` + strings.Repeat("x", int(objectStorageJSONPutMaxBytes)) + `"}`
handler := NewHandler(store.NewMemoryStore(), nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/api/object-storage/objects", strings.NewReader(body)))
if recorder.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("expected oversized JSON put to return 413, got %d body=%s", recorder.Code, recorder.Body.String())
}
}

View File

@ -49,10 +49,10 @@ type runJobTerminator interface {
var storageConnectionBucketsResolver = listStorageBucketsFromConnection
var storageConnectionObjectsResolver = listStorageObjectsFromConnection
var runJobTerminatorFactory = func(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string) (runJobTerminator, error) {
return k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, "", "", "")
return k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, "", "", "", 0)
}
var workerAPIBaseURLResolver = func() string {
return "http://droboticflow-worker:3002"
return config.Load().WorkerAPIBaseURL
}
const maxStoragePathFavoritesPerScope = 12
@ -1953,7 +1953,7 @@ func (server *server) handleWorkflowRunsList(writer http.ResponseWriter, request
if !server.ensureTenantAccess(writer, request, workflow.OrganizationID, "Workflow") {
return
}
items, err := server.listRunsForRequest(request.Context(), request, "")
items, err := server.listRunSummariesForRequest(request.Context(), request, "")
if err != nil {
renderError(writer, http.StatusInternalServerError, err)
return
@ -1966,7 +1966,7 @@ func (server *server) handleWorkflowRunsList(writer http.ResponseWriter, request
if !server.filterByTenant(request, item.OrganizationID) {
continue
}
summaries = append(summaries, runSummaryFromRecord(item))
summaries = append(summaries, item)
}
renderJSON(writer, http.StatusOK, map[string]any{"items": summaries})
}
@ -2603,7 +2603,7 @@ func (server *server) handleRunsLogs(writer http.ResponseWriter, request *http.R
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") {
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": item.Logs})
renderJSON(writer, http.StatusOK, pagedResponse(request, item.Logs, 1000, 5000, true))
}
func (server *server) handleRunsResources(writer http.ResponseWriter, request *http.Request) {
@ -2619,7 +2619,7 @@ func (server *server) handleRunsResources(writer http.ResponseWriter, request *h
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") {
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": item.ResourceSeries})
renderJSON(writer, http.StatusOK, pagedResponse(request, item.ResourceSeries, 1000, 5000, true))
}
func (server *server) handleRunsArtifacts(writer http.ResponseWriter, request *http.Request) {
@ -2635,7 +2635,7 @@ func (server *server) handleRunsArtifacts(writer http.ResponseWriter, request *h
if !server.ensureTenantAccess(writer, request, item.OrganizationID, "Run") {
return
}
renderJSON(writer, http.StatusOK, map[string]any{"items": item.Artifacts})
renderJSON(writer, http.StatusOK, pagedResponse(request, item.Artifacts, 1000, 5000, false))
}
func (server *server) handleAuditTasksList(writer http.ResponseWriter, request *http.Request) {
@ -3516,6 +3516,43 @@ func decodeJSON(writer http.ResponseWriter, request *http.Request, target any) b
return true
}
func pagedResponse[T any](request *http.Request, items []T, defaultLimit int, maxLimit int, defaultTail bool) map[string]any {
total := len(items)
limit := parsePositiveIntQuery(request.URL.Query().Get("limit"), defaultLimit, maxLimit)
if limit <= 0 {
limit = defaultLimit
if maxLimit > 0 && limit > maxLimit {
limit = maxLimit
}
if limit <= 0 {
limit = total
}
}
offsetRaw := request.URL.Query().Get("offset")
if strings.TrimSpace(offsetRaw) == "" {
offsetRaw = request.URL.Query().Get("skip")
}
offset := parsePositiveIntQuery(offsetRaw, 0, 0)
tailRequested := strings.EqualFold(strings.TrimSpace(request.URL.Query().Get("tail")), "true")
if (defaultTail || tailRequested) && strings.TrimSpace(offsetRaw) == "" && limit < total {
offset = total - limit
}
if offset > total {
offset = total
}
end := offset + limit
if end > total {
end = total
}
return map[string]any{
"items": items[offset:end],
"total": total,
"limit": limit,
"offset": offset,
"truncated": offset > 0 || end < total,
}
}
func renderValidationError(writer http.ResponseWriter, issues ...validationIssue) {
renderJSON(writer, http.StatusBadRequest, map[string]any{"error": "Validation failed", "issues": issues})
}

View File

@ -15,7 +15,9 @@ import (
"sort"
"strings"
"testing"
"time"
"droboticflow/internal/config"
"droboticflow/internal/domain"
"droboticflow/internal/store"
@ -479,6 +481,60 @@ func TestWorkerAPIHealthIsProxiedThroughAPI(t *testing.T) {
}
}
func TestWorkerAPIBaseURLUsesConfiguredEnv(t *testing.T) {
worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/worker-api/health" {
t.Fatalf("unexpected upstream path: %s", request.URL.Path)
}
writer.Header().Set("Content-Type", "application/json")
_, _ = writer.Write([]byte(`{"status":"configured"}`))
}))
defer worker.Close()
originalResolver := workerAPIBaseURLResolver
workerAPIBaseURLResolver = func() string { return config.Load().WorkerAPIBaseURL }
t.Cleanup(func() {
workerAPIBaseURLResolver = originalResolver
})
t.Setenv("WORKER_API_BASE_URL", worker.URL)
handler := NewHandler(store.NewMemoryStore(), nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", recorder.Code, recorder.Body.String())
}
if got := strings.TrimSpace(recorder.Body.String()); got != `{"status":"configured"}` {
t.Fatalf("unexpected configured worker payload: %s", got)
}
}
func TestWorkerAPIProxyTimesOutSlowHealthChecks(t *testing.T) {
worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
time.Sleep(100 * time.Millisecond)
writer.WriteHeader(http.StatusOK)
}))
defer worker.Close()
originalResolver := workerAPIBaseURLResolver
originalTimeout := workerAPIShortProxyTimeout
workerAPIBaseURLResolver = func() string { return worker.URL }
workerAPIShortProxyTimeout = 10 * time.Millisecond
t.Cleanup(func() {
workerAPIBaseURLResolver = originalResolver
workerAPIShortProxyTimeout = originalTimeout
})
handler := NewHandler(store.NewMemoryStore(), nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/worker-api/health", nil))
if recorder.Code != http.StatusBadGateway {
t.Fatalf("expected proxy timeout to return 502, got %d body=%s", recorder.Code, recorder.Body.String())
}
}
func TestWorkerAPIProcessOnceIsProxiedThroughAPI(t *testing.T) {
worker := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.Method != http.MethodPost {
@ -1433,6 +1489,90 @@ func TestRunsListReturnsSummaryPayload(t *testing.T) {
}
}
func TestRunLogsEndpointPaginatesAndDefaultsToTail(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
logs := make([]domain.LogEntryRecord, 0, 1100)
for index := 0; index < 1100; index++ {
logs = append(logs, domain.LogEntryRecord{Timestamp: "2026-04-13T01:00:00Z", Level: "INFO", Message: fmt.Sprintf("line-%04d", index)})
}
if err := dataStore.UpsertRun(ctx, domain.RunRecord{
ID: "run-logs-page",
WorkflowID: "workflow-logs",
WorkflowName: "Logs",
ProjectID: "project-logs",
ProjectName: "Logs",
Trigger: domain.RunTriggerManual,
Status: domain.RunStatusCompleted,
Logs: logs,
NodeExecutions: []domain.NodeExecutionRecord{},
ResourceSeries: []domain.ResourceSample{},
Artifacts: []domain.ExecutionArtifact{},
K8sRefs: []domain.K8sRef{},
CreatedAt: "2026-04-13T01:00:00Z",
UpdatedAt: "2026-04-13T01:00:00Z",
}); err != nil {
t.Fatalf("upsert run: %v", err)
}
handler := NewHandler(dataStore, nil)
recorder := httptest.NewRecorder()
handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/runs/run-logs-page/logs", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("logs status: %d body=%s", recorder.Code, recorder.Body.String())
}
var response struct {
Items []domain.LogEntryRecord `json:"items"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Truncated bool `json:"truncated"`
}
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode logs response: %v", err)
}
if response.Total != 1100 || response.Limit != 1000 || response.Offset != 100 || !response.Truncated {
t.Fatalf("unexpected default page metadata: %+v", response)
}
if len(response.Items) != 1000 || response.Items[0].Message != "line-0100" {
t.Fatalf("expected tail page starting at line-0100, got len=%d first=%q", len(response.Items), response.Items[0].Message)
}
pageRecorder := httptest.NewRecorder()
handler.ServeHTTP(pageRecorder, httptest.NewRequest(http.MethodGet, "/api/runs/run-logs-page/logs?limit=2&offset=3", nil))
if pageRecorder.Code != http.StatusOK {
t.Fatalf("explicit page status: %d body=%s", pageRecorder.Code, pageRecorder.Body.String())
}
var explicitPage struct {
Items []domain.LogEntryRecord `json:"items"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
if err := json.Unmarshal(pageRecorder.Body.Bytes(), &explicitPage); err != nil {
t.Fatalf("decode explicit page: %v", err)
}
if explicitPage.Total != 1100 || explicitPage.Limit != 2 || explicitPage.Offset != 3 || len(explicitPage.Items) != 2 || explicitPage.Items[0].Message != "line-0003" {
t.Fatalf("unexpected explicit page: %+v", explicitPage)
}
zeroLimitRecorder := httptest.NewRecorder()
handler.ServeHTTP(zeroLimitRecorder, httptest.NewRequest(http.MethodGet, "/api/runs/run-logs-page/logs?limit=0", nil))
if zeroLimitRecorder.Code != http.StatusOK {
t.Fatalf("zero limit status: %d body=%s", zeroLimitRecorder.Code, zeroLimitRecorder.Body.String())
}
var zeroLimitPage struct {
Items []domain.LogEntryRecord `json:"items"`
Limit int `json:"limit"`
}
if err := json.Unmarshal(zeroLimitRecorder.Body.Bytes(), &zeroLimitPage); err != nil {
t.Fatalf("decode zero limit page: %v", err)
}
if zeroLimitPage.Limit != 1000 || len(zeroLimitPage.Items) != 1000 {
t.Fatalf("expected zero limit to fall back to default page size, got %+v", zeroLimitPage)
}
}
func TestWorkflowRunsListReturnsWorkflowOnlySummaries(t *testing.T) {
ctx := context.Background()
dataStore := store.NewMemoryStore()
@ -3980,6 +4120,14 @@ func (browser *fakeObjectStorageBrowser) PutObject(_ context.Context, bucket str
return item, nil
}
func (browser *fakeObjectStorageBrowser) PutObjectStream(ctx context.Context, bucket string, key string, reader io.Reader, contentType string) (domain.ObjectStorageObjectSummary, error) {
payload, err := io.ReadAll(reader)
if err != nil {
return domain.ObjectStorageObjectSummary{}, err
}
return browser.PutObject(ctx, bucket, key, payload, contentType)
}
func (browser *fakeObjectStorageBrowser) RenameObject(_ context.Context, bucket string, key string, newKey string) (domain.ObjectStorageObjectSummary, error) {
item := browser.objects[bucket+"|"+key]
delete(browser.objects, bucket+"|"+key)

View File

@ -166,6 +166,20 @@ func (server *server) listRunsForContext(ctx context.Context, status string) ([]
return server.store.ListRuns(ctx, status)
}
func (server *server) listRunArtifactsForRequest(ctx context.Context, request *http.Request, status string) ([]domain.RunRecord, error) {
if server.tenantEnforced(request) {
return server.store.ListRunArtifactsByOrganization(ctx, server.requestOrganizationID(request), status)
}
return server.store.ListRunArtifacts(ctx, status)
}
func (server *server) listRunArtifactsForContext(ctx context.Context, status string) ([]domain.RunRecord, error) {
if server.tenantEnforcedContext(ctx) {
return server.store.ListRunArtifactsByOrganization(ctx, server.contextOrganizationID(ctx), status)
}
return server.store.ListRunArtifacts(ctx, status)
}
func (server *server) listRunSummariesForRequest(ctx context.Context, request *http.Request, status string) ([]domain.RunSummary, error) {
if server.tenantEnforced(request) {
return server.store.ListRunSummariesByOrganization(ctx, server.requestOrganizationID(request), status)

View File

@ -1,33 +1,49 @@
package api
import (
"context"
"io"
"net/http"
"strings"
"time"
"github.com/go-chi/chi/v5"
)
var (
workerAPIShortProxyTimeout = 15 * time.Second
workerAPILongProxyTimeout = 4 * time.Hour
)
var workerAPIProxyClient = &http.Client{}
func workerAPIBaseURL() string {
return strings.TrimRight(strings.TrimSpace(workerAPIBaseURLResolver()), "/")
}
func proxyWorkerAPI(writer http.ResponseWriter, request *http.Request, path string) {
func proxyWorkerAPI(writer http.ResponseWriter, request *http.Request, path string, timeout time.Duration) {
baseURL := workerAPIBaseURL()
if baseURL == "" {
renderJSON(writer, http.StatusServiceUnavailable, map[string]string{"error": "worker API is not configured"})
return
}
requestCtx := request.Context()
cancel := func() {}
if timeout > 0 {
requestCtx, cancel = context.WithTimeout(requestCtx, timeout)
}
defer cancel()
upstreamURL := baseURL + path
upstreamRequest, err := http.NewRequestWithContext(request.Context(), request.Method, upstreamURL, request.Body)
upstreamRequest, err := http.NewRequestWithContext(requestCtx, request.Method, upstreamURL, request.Body)
if err != nil {
renderError(writer, http.StatusBadGateway, err)
return
}
upstreamRequest.Header = request.Header.Clone()
response, err := http.DefaultClient.Do(upstreamRequest)
response, err := workerAPIProxyClient.Do(upstreamRequest)
if err != nil {
renderJSON(writer, http.StatusBadGateway, map[string]string{
"error": "worker API request failed",
@ -47,21 +63,21 @@ func proxyWorkerAPI(writer http.ResponseWriter, request *http.Request, path stri
}
func (server *server) handleWorkerAPIHealth(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/health")
proxyWorkerAPI(writer, request, "/worker-api/health", workerAPIShortProxyTimeout)
}
func (server *server) handleWorkerAPIStats(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/stats")
proxyWorkerAPI(writer, request, "/worker-api/stats", workerAPIShortProxyTimeout)
}
func (server *server) handleWorkerAPIMetrics(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/metrics")
proxyWorkerAPI(writer, request, "/worker-api/metrics", workerAPIShortProxyTimeout)
}
func (server *server) handleWorkerAPIProcessOnce(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/process-once")
proxyWorkerAPI(writer, request, "/worker-api/process-once", workerAPILongProxyTimeout)
}
func (server *server) handleWorkerAPIRetry(writer http.ResponseWriter, request *http.Request) {
proxyWorkerAPI(writer, request, "/worker-api/runs/"+chi.URLParam(request, "id")+"/retry")
proxyWorkerAPI(writer, request, "/worker-api/runs/"+chi.URLParam(request, "id")+"/retry", workerAPIShortProxyTimeout)
}

View File

@ -15,6 +15,7 @@ type AppConfig struct {
MongoConnectRetries int
MongoConnectRetryDelayMS int
PollIntervalMS int
WorkerAPIBaseURL string
K8sNamespace string
K8sImagePullSecretName string
WorkspacePVCName string
@ -31,6 +32,7 @@ type AppConfig struct {
OperatorGPUNodeSelector string
OperatorGPUTolerations string
OperatorGPURuntimeClass string
OperatorJobTTLSeconds int
}
func parseInt(value string, fallback int) int {
@ -51,6 +53,7 @@ func Load() AppConfig {
MongoConnectRetries: parseInt(os.Getenv("MONGO_CONNECT_RETRIES"), 30),
MongoConnectRetryDelayMS: parseInt(os.Getenv("MONGO_CONNECT_RETRY_DELAY_MS"), 2000),
PollIntervalMS: parseInt(os.Getenv("WORKER_POLL_INTERVAL_MS"), 2000),
WorkerAPIBaseURL: getenv("WORKER_API_BASE_URL", "http://droboticflow-worker:3002"),
K8sNamespace: getenv("K8S_NAMESPACE", "droboticflow"),
K8sImagePullSecretName: getenv("K8S_IMAGE_PULL_SECRET_NAME", "droboticflow-registry"),
WorkspacePVCName: strings.TrimSpace(os.Getenv("WORKSPACE_PVC_NAME")),
@ -67,6 +70,7 @@ func Load() AppConfig {
OperatorGPUNodeSelector: strings.TrimSpace(os.Getenv("OPERATOR_GPU_NODE_SELECTOR")),
OperatorGPUTolerations: strings.TrimSpace(os.Getenv("OPERATOR_GPU_TOLERATIONS_JSON")),
OperatorGPURuntimeClass: strings.TrimSpace(os.Getenv("OPERATOR_GPU_RUNTIME_CLASS")),
OperatorJobTTLSeconds: parseInt(os.Getenv("OPERATOR_JOB_TTL_SECONDS"), 86400),
}
}

View File

@ -278,10 +278,14 @@ type LogEntryRecord struct {
}
type ResourceSample struct {
Timestamp string `json:"timestamp" bson:"timestamp"`
CPU int `json:"cpu" bson:"cpu"`
Memory int `json:"memory" bson:"memory"`
GPU int `json:"gpu,omitempty" bson:"gpu,omitempty"`
Timestamp string `json:"timestamp" bson:"timestamp"`
CPU int `json:"cpu" bson:"cpu"`
Memory int `json:"memory" bson:"memory"`
GPU int `json:"gpu,omitempty" bson:"gpu,omitempty"`
GPUMemoryUsedMiB int `json:"gpuMemoryUsedMiB,omitempty" bson:"gpuMemoryUsedMiB,omitempty"`
GPUMemoryTotalMiB int `json:"gpuMemoryTotalMiB,omitempty" bson:"gpuMemoryTotalMiB,omitempty"`
GPUName string `json:"gpuName,omitempty" bson:"gpuName,omitempty"`
GPUCount int `json:"gpuCount,omitempty" bson:"gpuCount,omitempty"`
}
type ArtifactMetadata struct {

View File

@ -41,6 +41,7 @@ type LaunchInput struct {
GPUNodeSelector map[string]string
GPUTolerations []corev1.Toleration
GPURuntimeClass string
JobTTLSeconds int32
}
type JobReference struct {
@ -77,6 +78,7 @@ type Launcher struct {
gpuNodeSelector map[string]string
gpuTolerations []corev1.Toleration
gpuRuntimeClass string
jobTTLSeconds int32
clientset kubernetes.Interface
}
@ -308,6 +310,12 @@ func CreateJobManifest(namespace string, imagePullSecretName string, workspacePV
},
},
}
if input.JobTTLSeconds > 0 {
job.Spec.TTLSecondsAfterFinished = func() *int32 {
value := input.JobTTLSeconds
return &value
}()
}
if input.TimeoutSeconds > 0 {
job.Spec.ActiveDeadlineSeconds = func() *int64 {
value := int64(input.TimeoutSeconds)
@ -350,7 +358,7 @@ func loadRestConfig() (*rest.Config, error) {
return clientcmd.BuildConfigFromFlags("", kubeconfig)
}
func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string) (*Launcher, error) {
func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int) (*Launcher, error) {
trimmedPVCName := strings.TrimSpace(workspacePVCName)
if trimmedPVCName == "" {
return nil, fmt.Errorf("workspace pvc name is required")
@ -370,6 +378,9 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
if err != nil {
return nil, err
}
if jobTTLSeconds < 0 {
jobTTLSeconds = 0
}
launcher := &Launcher{
namespace: namespace,
imagePullSecretName: imagePullSecretName,
@ -378,6 +389,7 @@ func NewLauncher(namespace string, imagePullSecretName string, workspacePVCName
gpuNodeSelector: parsedGPUNodeSelector,
gpuTolerations: parsedGPUTolerations,
gpuRuntimeClass: strings.TrimSpace(gpuRuntimeClass),
jobTTLSeconds: int32(jobTTLSeconds),
}
config, err := loadRestConfig()
if err != nil {
@ -572,6 +584,9 @@ func (launcher *Launcher) Launch(ctx context.Context, input LaunchInput) (*Launc
if strings.TrimSpace(input.GPURuntimeClass) == "" {
input.GPURuntimeClass = launcher.gpuRuntimeClass
}
if input.JobTTLSeconds == 0 {
input.JobTTLSeconds = launcher.jobTTLSeconds
}
job := CreateJobManifest(launcher.namespace, launcher.imagePullSecretName, launcher.workspacePVCName, launcher.workspaceMountPath, input)
created, err := launcher.clientset.BatchV1().Jobs(launcher.namespace).Create(ctx, job, metav1.CreateOptions{})
if err != nil {

View File

@ -33,6 +33,7 @@ func TestCreateJobManifest(t *testing.T) {
Effect: corev1.TaintEffectNoSchedule,
}},
GPURuntimeClass: "nvidia",
JobTTLSeconds: 3600,
Env: map[string]string{
"CPU_LIMIT": "500m",
"MEMORY_LIMIT": "1Gi",
@ -87,6 +88,9 @@ func TestCreateJobManifest(t *testing.T) {
if manifest.Spec.Template.Spec.SecurityContext == nil || manifest.Spec.Template.Spec.SecurityContext.SeccompProfile == nil || manifest.Spec.Template.Spec.SecurityContext.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault {
t.Fatalf("expected RuntimeDefault seccomp profile, got %#v", manifest.Spec.Template.Spec.SecurityContext)
}
if manifest.Spec.TTLSecondsAfterFinished == nil || *manifest.Spec.TTLSecondsAfterFinished != 3600 {
t.Fatalf("expected ttlSecondsAfterFinished 3600, got %#v", manifest.Spec.TTLSecondsAfterFinished)
}
}
func TestCreateJobManifestSkipsGPUSchedulingWithoutGPURequest(t *testing.T) {
@ -110,17 +114,29 @@ func TestCreateJobManifestSkipsGPUSchedulingWithoutGPURequest(t *testing.T) {
}
}
func TestCreateJobManifestOmitsTTLWhenDisabled(t *testing.T) {
manifest := CreateJobManifest("droboticflow", "", "droboticflow-workspace", "/workspace", LaunchInput{
Image: "runner:latest",
RunID: "run-1",
NodeID: "node-1",
})
if manifest.Spec.TTLSecondsAfterFinished != nil {
t.Fatalf("did not expect ttlSecondsAfterFinished when disabled, got %#v", manifest.Spec.TTLSecondsAfterFinished)
}
}
func TestNewLauncherRequiresWorkspacePVCName(t *testing.T) {
if _, err := NewLauncher("droboticflow", "registry", "", "/workspace", "", "", ""); err == nil {
if _, err := NewLauncher("droboticflow", "registry", "", "/workspace", "", "", "", 86400); err == nil {
t.Fatalf("expected workspace pvc validation error")
}
}
func TestNewLauncherRejectsInvalidGPUSchedulingConfig(t *testing.T) {
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "accelerator", "", ""); err == nil || !strings.Contains(err.Error(), "invalid gpu node selector") {
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "accelerator", "", "", 86400); err == nil || !strings.Contains(err.Error(), "invalid gpu node selector") {
t.Fatalf("expected invalid gpu node selector error, got %v", err)
}
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "", "not-json", ""); err == nil || !strings.Contains(err.Error(), "invalid gpu tolerations json") {
if _, err := NewLauncher("droboticflow", "registry", "workspace", "/workspace", "", "not-json", "", 86400); err == nil || !strings.Contains(err.Error(), "invalid gpu tolerations json") {
t.Fatalf("expected invalid gpu tolerations error, got %v", err)
}
}

View File

@ -703,6 +703,14 @@ func (store *MemoryStore) ListRunsByOrganization(_ context.Context, organization
return items, nil
}
func (store *MemoryStore) ListRunArtifacts(ctx context.Context, status string) ([]domain.RunRecord, error) {
return store.ListRuns(ctx, status)
}
func (store *MemoryStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
return store.ListRunsByOrganization(ctx, organizationID, status)
}
func (store *MemoryStore) CountRunsByStatus(_ context.Context) (map[domain.RunStatus]int, error) {
store.mu.Lock()
defer store.mu.Unlock()

View File

@ -563,6 +563,57 @@ func (store *MongoStore) ListRunsByOrganization(ctx context.Context, organizatio
return items, nil
}
func runArtifactProjection() bson.M {
return bson.M{
"id": 1,
"organizationId": 1,
"workflowId": 1,
"workflowName": 1,
"projectId": 1,
"projectName": 1,
"status": 1,
"artifacts": 1,
"createdAt": 1,
"updatedAt": 1,
}
}
func (store *MongoStore) ListRunArtifacts(ctx context.Context, status string) ([]domain.RunRecord, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.runs.Find(
ctx,
filter,
options.Find().
SetProjection(runArtifactProjection()).
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
)
if err != nil {
return nil, err
}
return decodeMany[domain.RunRecord](ctx, cursor)
}
func (store *MongoStore) ListRunArtifactsByOrganization(ctx context.Context, organizationID string, status string) ([]domain.RunRecord, error) {
filter := bson.M{}
if status != "" {
filter["status"] = status
}
cursor, err := store.runs.Find(
ctx,
organizationScopedFilter(organizationID, filter),
options.Find().
SetProjection(runArtifactProjection()).
SetSort(bson.D{{Key: "createdAt", Value: -1}}),
)
if err != nil {
return nil, err
}
return decodeMany[domain.RunRecord](ctx, cursor)
}
func (store *MongoStore) CountRunsByStatus(ctx context.Context) (map[domain.RunStatus]int, error) {
cursor, err := store.runs.Aggregate(ctx, mongo.Pipeline{
{{Key: "$group", Value: bson.M{

View File

@ -46,6 +46,8 @@ type Store interface {
ListRuns(context.Context, string) ([]domain.RunRecord, error)
ListRunsByOrganization(context.Context, string, string) ([]domain.RunRecord, error)
ListRunArtifacts(context.Context, string) ([]domain.RunRecord, error)
ListRunArtifactsByOrganization(context.Context, string, string) ([]domain.RunRecord, error)
CountRunsByStatus(context.Context) (map[domain.RunStatus]int, error)
ListRunSummaries(context.Context, string) ([]domain.RunSummary, error)
ListRunSummariesByOrganization(context.Context, string, string) ([]domain.RunSummary, error)

View File

@ -43,8 +43,8 @@ var queuedOrActiveStatuses = map[domain.RunStatus]struct{}{
domain.RunStatusWaitingForAudit: {},
}
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string) (*Runtime, error) {
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass)
func NewRuntime(dataStore store.Store, namespace string, imagePullSecretName string, operatorRunnerImage string, workspacePVCName string, workspaceMountPath string, gpuNodeSelector string, gpuTolerationsJSON string, gpuRuntimeClass string, jobTTLSeconds int) (*Runtime, error) {
launcher, err := k8s.NewLauncher(namespace, imagePullSecretName, workspacePVCName, workspaceMountPath, gpuNodeSelector, gpuTolerationsJSON, gpuRuntimeClass, jobTTLSeconds)
if err != nil {
return nil, err
}

View File

@ -81,6 +81,7 @@ data:
MONGO_CONNECT_RETRIES: "__MONGO_CONNECT_RETRIES__"
MONGO_CONNECT_RETRY_DELAY_MS: "__MONGO_CONNECT_RETRY_DELAY_MS__"
WORKER_POLL_INTERVAL_MS: "__WORKER_POLL_INTERVAL_MS__"
WORKER_API_BASE_URL: "__WORKER_API_BASE_URL__"
K8S_NAMESPACE: "__NAMESPACE__"
K8S_IMAGE_PULL_SECRET_NAME: "__K8S_IMAGE_PULL_SECRET_NAME__"
WORKSPACE_PVC_NAME: "__WORKSPACE_PVC_NAME__"
@ -90,6 +91,7 @@ data:
OPERATOR_GPU_TOLERATIONS_JSON: |-
__OPERATOR_GPU_TOLERATIONS_JSON__
OPERATOR_GPU_RUNTIME_CLASS: "__OPERATOR_GPU_RUNTIME_CLASS__"
OPERATOR_JOB_TTL_SECONDS: "__OPERATOR_JOB_TTL_SECONDS__"
OPERATOR_ALLOW_SHELL_COMMANDS: "__OPERATOR_ALLOW_SHELL_COMMANDS__"
OPERATOR_COMMAND_ALLOWLIST_REGEX: |-
__OPERATOR_COMMAND_ALLOWLIST_REGEX__
@ -208,6 +210,11 @@ spec:
configMapKeyRef:
name: droboticflow-config
key: WORKSPACE_MOUNT_PATH
- name: WORKER_API_BASE_URL
valueFrom:
configMapKeyRef:
name: droboticflow-config
key: WORKER_API_BASE_URL
- name: AI_API_BASE_URL
valueFrom:
configMapKeyRef:
@ -404,6 +411,11 @@ spec:
configMapKeyRef:
name: droboticflow-config
key: OPERATOR_GPU_RUNTIME_CLASS
- name: OPERATOR_JOB_TTL_SECONDS
valueFrom:
configMapKeyRef:
name: droboticflow-config
key: OPERATOR_JOB_TTL_SECONDS
- name: OPERATOR_ALLOW_SHELL_COMMANDS
valueFrom:
configMapKeyRef:

View File

@ -34,6 +34,7 @@
"k8s:render": "./scripts/render-manifests.sh",
"k8s:validate": "./scripts/render-manifests.sh | kubectl apply --dry-run=client --validate=false -f -",
"k8s:deploy": "./scripts/deploy-k8s.sh",
"k8s:cleanup-jobs": "./scripts/cleanup-operator-jobs.sh",
"gpu:smoke": "./scripts/gpu-smoke-test.sh",
"smoke": "./scripts/smoke-test.sh",
"reconcile:data": "./scripts/reconcile-system-data.sh"

View File

@ -0,0 +1,90 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then
DEPLOY_ENV="$1"
shift
fi
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS="${OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS:-86400}"
OPERATOR_JOB_CLEANUP_STATUSES="${OPERATOR_JOB_CLEANUP_STATUSES:-succeeded,failed}"
DRY_RUN="${DRY_RUN:-false}"
job_names="$(
kubectl -n "$K8S_NAMESPACE" get jobs \
-l app=droboticflow,component=operator-runner \
-o json |
OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS="$OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS" \
OPERATOR_JOB_CLEANUP_STATUSES="$OPERATOR_JOB_CLEANUP_STATUSES" \
python3 -c '
import datetime as dt
import json
import os
import sys
min_age_seconds = int(os.environ["OPERATOR_JOB_CLEANUP_MIN_AGE_SECONDS"])
statuses = {item.strip().lower() for item in os.environ["OPERATOR_JOB_CLEANUP_STATUSES"].split(",") if item.strip()}
payload = json.load(sys.stdin)
now = dt.datetime.now(dt.timezone.utc)
def parse_time(value):
if not value:
return None
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
def condition_time(status, condition_type):
for condition in status.get("conditions", []) or []:
if str(condition.get("type", "")).lower() != condition_type:
continue
if str(condition.get("status", "")).lower() != "true":
continue
parsed = parse_time(condition.get("lastTransitionTime"))
if parsed:
return parsed
return None
for item in payload.get("items", []):
status = item.get("status", {})
succeeded = int(status.get("succeeded") or 0)
failed = int(status.get("failed") or 0)
state = ""
if succeeded > 0:
state = "succeeded"
finished_at = parse_time(status.get("completionTime")) or condition_time(status, "complete")
elif failed > 0:
state = "failed"
finished_at = parse_time(status.get("completionTime")) or condition_time(status, "failed")
else:
finished_at = None
if not state or state not in statuses:
continue
if finished_at is None or (now - finished_at).total_seconds() < min_age_seconds:
continue
name = item.get("metadata", {}).get("name", "")
if name:
print(name)
'
)"
if [[ -z "$job_names" ]]; then
echo "No finished operator-runner Jobs matched cleanup filters in namespace $K8S_NAMESPACE."
exit 0
fi
if [[ "$DRY_RUN" == "true" ]]; then
printf '%s\n' "$job_names"
exit 0
fi
while IFS= read -r job_name; do
[[ -z "$job_name" ]] && continue
kubectl -n "$K8S_NAMESPACE" delete job "$job_name"
done <<<"$job_names"

View File

@ -14,6 +14,7 @@ export MONGO_DB="${MONGO_DB:-poc}"
export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
export K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
export WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
@ -27,6 +28,7 @@ export API_PUBLIC_LOAD_BALANCER="${API_PUBLIC_LOAD_BALANCER:-true}"
export API_PUBLIC_SERVICE_NAME="${API_PUBLIC_SERVICE_NAME:-droboticflow-api-public}"
export API_REPLICAS="${API_REPLICAS:-1}"
export WORKER_REPLICAS="${WORKER_REPLICAS:-1}"
export OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
export AI_API_BASE_URL="${AI_API_BASE_URL:-https://coding.dashscope.aliyuncs.com/v1}"
export AI_MODEL="${AI_MODEL:-glm-5}"

View File

@ -14,6 +14,7 @@ export MONGO_DB="${MONGO_DB:-droboticflow}"
export MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
export MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
export WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
export WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
export K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
export WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
@ -23,6 +24,7 @@ export API_NODE_PORT="${API_NODE_PORT:-30080}"
export API_SERVICE_TYPE="${API_SERVICE_TYPE:-NodePort}"
export API_REPLICAS="${API_REPLICAS:-1}"
export WORKER_REPLICAS="${WORKER_REPLICAS:-1}"
export OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
export AI_API_BASE_URL="${AI_API_BASE_URL:-https://coding.dashscope.aliyuncs.com/v1}"
export AI_MODEL="${AI_MODEL:-glm-5}"

View File

@ -28,6 +28,7 @@ MONGO_URI="${MONGO_URI:-}"
MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
@ -45,6 +46,7 @@ OPERATOR_RUNNER_IMAGE="${OPERATOR_RUNNER_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPA
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
OPERATOR_ALLOW_SHELL_COMMANDS="${OPERATOR_ALLOW_SHELL_COMMANDS:-false}"
OPERATOR_COMMAND_ALLOWLIST_REGEX="${OPERATOR_COMMAND_ALLOWLIST_REGEX:-}"
OPERATOR_COMMAND_DENYLIST_REGEX="${OPERATOR_COMMAND_DENYLIST_REGEX:-}"
@ -156,6 +158,7 @@ env \
MONGO_CONNECT_RETRIES="$MONGO_CONNECT_RETRIES" \
MONGO_CONNECT_RETRY_DELAY_MS="$MONGO_CONNECT_RETRY_DELAY_MS" \
WORKER_POLL_INTERVAL_MS="$WORKER_POLL_INTERVAL_MS" \
WORKER_API_BASE_URL="$WORKER_API_BASE_URL" \
K8S_IMAGE_PULL_SECRET_NAME="$K8S_IMAGE_PULL_SECRET_NAME" \
WORKSPACE_PVC_NAME="$WORKSPACE_PVC_NAME" \
WORKSPACE_MOUNT_PATH="$WORKSPACE_MOUNT_PATH" \
@ -173,6 +176,7 @@ env \
OPERATOR_GPU_NODE_SELECTOR="$OPERATOR_GPU_NODE_SELECTOR" \
OPERATOR_GPU_TOLERATIONS_JSON="$OPERATOR_GPU_TOLERATIONS_JSON" \
OPERATOR_GPU_RUNTIME_CLASS="$OPERATOR_GPU_RUNTIME_CLASS" \
OPERATOR_JOB_TTL_SECONDS="$OPERATOR_JOB_TTL_SECONDS" \
OPERATOR_ALLOW_SHELL_COMMANDS="$OPERATOR_ALLOW_SHELL_COMMANDS" \
OPERATOR_COMMAND_ALLOWLIST_REGEX="$OPERATOR_COMMAND_ALLOWLIST_REGEX" \
OPERATOR_COMMAND_DENYLIST_REGEX="$OPERATOR_COMMAND_DENYLIST_REGEX" \

View File

@ -20,6 +20,7 @@ MONGO_DB="${MONGO_DB:-droboticflow}"
MONGO_CONNECT_RETRIES="${MONGO_CONNECT_RETRIES:-30}"
MONGO_CONNECT_RETRY_DELAY_MS="${MONGO_CONNECT_RETRY_DELAY_MS:-2000}"
WORKER_POLL_INTERVAL_MS="${WORKER_POLL_INTERVAL_MS:-2000}"
WORKER_API_BASE_URL="${WORKER_API_BASE_URL:-http://droboticflow-worker:3002}"
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
@ -33,6 +34,7 @@ AI_MODEL="${AI_MODEL:-glm-5}"
OPERATOR_GPU_NODE_SELECTOR="${OPERATOR_GPU_NODE_SELECTOR:-}"
OPERATOR_GPU_TOLERATIONS_JSON="${OPERATOR_GPU_TOLERATIONS_JSON:-}"
OPERATOR_GPU_RUNTIME_CLASS="${OPERATOR_GPU_RUNTIME_CLASS:-}"
OPERATOR_JOB_TTL_SECONDS="${OPERATOR_JOB_TTL_SECONDS:-86400}"
OPERATOR_ALLOW_SHELL_COMMANDS="${OPERATOR_ALLOW_SHELL_COMMANDS:-false}"
OPERATOR_COMMAND_ALLOWLIST_REGEX="${OPERATOR_COMMAND_ALLOWLIST_REGEX:-}"
OPERATOR_COMMAND_DENYLIST_REGEX="${OPERATOR_COMMAND_DENYLIST_REGEX:-}"
@ -68,6 +70,7 @@ rendered="${rendered//__MONGO_DB__/$MONGO_DB}"
rendered="${rendered//__MONGO_CONNECT_RETRIES__/$MONGO_CONNECT_RETRIES}"
rendered="${rendered//__MONGO_CONNECT_RETRY_DELAY_MS__/$MONGO_CONNECT_RETRY_DELAY_MS}"
rendered="${rendered//__WORKER_POLL_INTERVAL_MS__/$WORKER_POLL_INTERVAL_MS}"
rendered="${rendered//__WORKER_API_BASE_URL__/$WORKER_API_BASE_URL}"
rendered="${rendered//__K8S_IMAGE_PULL_SECRET_NAME__/$K8S_IMAGE_PULL_SECRET_NAME}"
rendered="${rendered//__WORKSPACE_PVC_NAME__/$WORKSPACE_PVC_NAME}"
rendered="${rendered//__WORKSPACE_MOUNT_PATH__/$WORKSPACE_MOUNT_PATH}"
@ -83,6 +86,7 @@ rendered="${rendered//__OPERATOR_RUNNER_IMAGE__/$OPERATOR_RUNNER_IMAGE}"
rendered="${rendered//__OPERATOR_GPU_NODE_SELECTOR__/$OPERATOR_GPU_NODE_SELECTOR}"
rendered="${rendered//__OPERATOR_GPU_TOLERATIONS_JSON__/$OPERATOR_GPU_TOLERATIONS_JSON}"
rendered="${rendered//__OPERATOR_GPU_RUNTIME_CLASS__/$OPERATOR_GPU_RUNTIME_CLASS}"
rendered="${rendered//__OPERATOR_JOB_TTL_SECONDS__/$OPERATOR_JOB_TTL_SECONDS}"
rendered="${rendered//__OPERATOR_ALLOW_SHELL_COMMANDS__/$OPERATOR_ALLOW_SHELL_COMMANDS}"
rendered="${rendered//__OPERATOR_COMMAND_ALLOWLIST_REGEX__/$OPERATOR_COMMAND_ALLOWLIST_REGEX}"
rendered="${rendered//__OPERATOR_COMMAND_DENYLIST_REGEX__/$OPERATOR_COMMAND_DENYLIST_REGEX}"

View File

@ -23,7 +23,8 @@ import {
FileText,
} from "lucide-react";
import { cn } from "../lib/utils";
import { monitoringApi, LogEntry, ResourceUsage, ExecutionData } from "../services/api/monitoring";
import { monitoringApi, type LogEntry, type ResourceUsage, type ExecutionData } from "../services/api/monitoring";
import type { ExecutionRunSummary } from "../types";
import { motion, AnimatePresence } from "motion/react";
import { LineChart, Line, XAxis, YAxis, CartesianGrid, Tooltip, ResponsiveContainer, AreaChart, Area } from "recharts";
@ -43,9 +44,25 @@ const artifactExtension = (item: ExecutionData) => {
return parts.length > 1 ? parts[parts.length - 1].toLowerCase() : "";
};
type ModalType = "logs" | "resources" | "data";
type ArtifactPreview = {
kind?: string;
format?: string;
url?: string;
rows?: Record<string, unknown>[];
columns?: string[];
rowCount?: number;
lineCount?: number;
truncated?: boolean;
text?: string;
value?: unknown;
[key: string]: unknown;
};
const rawPreviewPayload = (item: ExecutionData) => {
const preview = item.data?.preview;
return preview && typeof preview === "object" ? (preview as Record<string, any>) : null;
return preview && typeof preview === "object" ? (preview as ArtifactPreview) : null;
};
const derivedPreview = (item: ExecutionData) => {
@ -144,11 +161,11 @@ export default function Monitoring() {
const { timeZone } = useTimeZone();
const [searchQuery, setSearchQuery] = useState("");
const [statusFilter, setStatusFilter] = useState("all");
const [runs, setRuns] = useState<any[]>([]);
const [runs, setRuns] = useState<ExecutionRunSummary[]>([]);
// Modal states
const [activeModal, setActiveModal] = useState<"logs" | "resources" | "data" | null>(null);
const [selectedRun, setSelectedRun] = useState<any>(null);
const [activeModal, setActiveModal] = useState<ModalType | null>(null);
const [selectedRun, setSelectedRun] = useState<ExecutionRunSummary | null>(null);
const [loading, setLoading] = useState(false);
const [modalData, setModalData] = useState<{
logs?: LogEntry[];
@ -195,7 +212,7 @@ export default function Monitoring() {
});
}, [runs, searchQuery, statusFilter]);
const openModal = async (type: "logs" | "resources" | "data", run: any) => {
const openModal = async (type: ModalType, run: ExecutionRunSummary) => {
setSelectedRun(run);
setActiveModal(type);
setLoading(true);
@ -243,7 +260,7 @@ export default function Monitoring() {
setRunActionPendingId(runId);
try {
const updated = await monitoringApi.stopRun(runId);
setRuns((previous) => previous.map((item) => (item.id === runId ? updated : item)));
setRuns((previous) => previous.map((item) => (item.id === runId ? { ...item, ...updated } : item)));
} catch (error) {
console.error("Failed to stop run", error);
} finally {
@ -258,7 +275,7 @@ export default function Monitoring() {
setRunActionPendingId(runId);
try {
const updated = await monitoringApi.forceStopRun(runId);
setRuns((previous) => previous.map((item) => (item.id === runId ? updated : item)));
setRuns((previous) => previous.map((item) => (item.id === runId ? { ...item, ...updated } : item)));
} catch (error) {
console.error("Failed to force stop run", error);
} finally {
@ -329,7 +346,7 @@ export default function Monitoring() {
</tr>
</thead>
<tbody>
{preview.rows.map((row: Record<string, any>, index: number) => (
{preview.rows.map((row: Record<string, unknown>, index: number) => (
<tr key={index} className="border-b border-[#1f2024] last:border-b-0">
{columns.map((column) => (
<td key={`${index}-${column}`} className="px-3 py-2 text-gray-200 align-top">
@ -484,7 +501,7 @@ export default function Monitoring() {
{activeModal === "resources" && (
<div className="space-y-8">
<div className="grid grid-cols-3 gap-4">
<div className="grid grid-cols-2 lg:grid-cols-4 gap-4">
{[
{
label: "CPU Usage",
@ -501,6 +518,13 @@ export default function Monitoring() {
value: `${Math.round(modalData.resources?.[modalData.resources.length - 1]?.gpu || 0)}%`,
color: "text-purple-500",
},
{
label: "GPU MEM",
value: modalData.resources?.[modalData.resources.length - 1]?.gpuMemoryTotalMiB
? `${Math.round(modalData.resources[modalData.resources.length - 1].gpuMemoryUsedMiB || 0)} / ${Math.round(modalData.resources[modalData.resources.length - 1].gpuMemoryTotalMiB || 0)} MiB`
: "N/A",
color: "text-cyan-400",
},
].map((stat) => (
<div key={stat.label} className="bg-[#0a0a0a] border border-[#2a2b2f] p-4 rounded-xl">
<div className="text-[10px] text-gray-500 uppercase font-mono mb-1">{stat.label}</div>
@ -508,6 +532,15 @@ export default function Monitoring() {
</div>
))}
</div>
{modalData.resources?.[modalData.resources.length - 1]?.gpuName && (
<div className="rounded-xl border border-[#2a2b2f] bg-[#0a0a0a] px-4 py-3 text-xs text-gray-300">
<span className="text-gray-500 font-mono uppercase mr-2">GPU Device</span>
{modalData.resources[modalData.resources.length - 1].gpuName}
{modalData.resources[modalData.resources.length - 1].gpuCount
? ` x ${modalData.resources[modalData.resources.length - 1].gpuCount}`
: ""}
</div>
)}
<div className="h-[300px] w-full bg-[#0a0a0a] rounded-xl border border-[#2a2b2f] p-4">
<ResponsiveContainer width="100%" height="100%">

View File

@ -8,6 +8,14 @@ type StorageBrowseTarget = {
connectionId?: string;
};
type PagedResponse<T> = {
items: T[];
total?: number;
limit?: number;
offset?: number;
truncated?: boolean;
};
export const monitoringApi = {
listRuns: async () => (await apiRequest<{ items: ExecutionRunSummary[] }>("/api/runs")).items,
getRun: (runId: string) => apiRequest<ExecutionRun>(`/api/runs/${runId}`),
@ -16,11 +24,11 @@ export const monitoringApi = {
stopRun: (runId: string) => apiRequest<ExecutionRun>(`/api/runs/${runId}/stop`, { method: "POST" }),
forceStopRun: (runId: string) => apiRequest<ExecutionRun>(`/api/runs/${runId}/force-stop`, { method: "POST" }),
getLogs: async (runId: string): Promise<LogEntry[]> =>
(await apiRequest<{ items: LogEntry[] }>(`/api/runs/${runId}/logs`)).items,
(await apiRequest<PagedResponse<LogEntry>>(`/api/runs/${runId}/logs`)).items,
getResourceUsage: async (runId: string): Promise<ResourceUsage[]> =>
(await apiRequest<{ items: ResourceUsage[] }>(`/api/runs/${runId}/resources`)).items,
(await apiRequest<PagedResponse<ResourceUsage>>(`/api/runs/${runId}/resources`)).items,
getExecutionData: async (runId: string): Promise<ExecutionData[]> =>
(await apiRequest<{ items: ExecutionData[] }>(`/api/runs/${runId}/artifacts`)).items,
(await apiRequest<PagedResponse<ExecutionData>>(`/api/runs/${runId}/artifacts`)).items,
listOssBuckets: async ({ sourcePath = "", connectionId = "" }: StorageBrowseTarget): Promise<string[]> => {
const query = new URLSearchParams();
if (connectionId.trim()) {

View File

@ -414,6 +414,10 @@ export interface ResourceUsage {
cpu: number;
memory: number;
gpu?: number;
gpuMemoryUsedMiB?: number;
gpuMemoryTotalMiB?: number;
gpuName?: string;
gpuCount?: number;
}
export interface ExecutionData {