diff --git a/.env.example b/.env.example index 33352778..70779079 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,7 @@ OPERATOR_COMMAND_DENYLIST_REGEX= VITE_API_BASE_URL= VITE_DEV_API_BASE_URL=http://127.0.0.1:3001 VITE_DEV_WORKER_BASE_URL=http://127.0.0.1:3002 +SMOKE_REQUIRE_EXPLICIT_ENDPOINT=false # Optional AI generation. Leave all three empty to disable AI generation. # When AI_API_KEY is set, AI_API_BASE_URL and AI_MODEL must both be explicit. AI_API_BASE_URL= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d74f2620..c1c0276a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,8 +41,11 @@ jobs: - name: Install Python test dependencies run: python -m pip install --upgrade pip pytest bce-python-sdk numpy opencv-python-headless pymongo - - name: Verify - run: npm run verify + - name: Verify local checks + run: npm run verify:local + + - name: Security audit + run: npm run verify:security docker-build: runs-on: ubuntu-latest diff --git a/README.md b/README.md index d5fe2288..8e72f00d 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,9 @@ Drobodata is a workflow orchestration platform for embodied AI and vision pipeli - `npm run format:check`: check frontend formatting without writing files - `npm test`: Go unit and integration tests - `npm run build`: build frontend assets and Go binaries -- `npm run verify`: web/Go/script lint, Go tests, Python operator-runner tests, frontend node tests, build, dependency audit, and Kubernetes manifest dry-run validation +- `npm run verify:local`: web/Go/script lint, Go tests, Python operator-runner tests, frontend node tests, build, and Kubernetes manifest dry-run validation +- `npm run verify:security`: dependency audit with retry +- `npm run verify`: run local verification first, then dependency audit - `npm run smoke`: local smoke test against the running API and worker - `npm run gpu:smoke`: real Kubernetes GPU scheduling and `nvidia-smi` smoke test - `npm run k8s:cleanup-jobs`: delete old finished operator-runner Jobs that predate TTL cleanup @@ -116,8 +118,9 @@ Key environment variables: - `VITE_API_BASE_URL`: optional frontend API base URL override; leave empty for same-origin `/api`, or set an absolute `http://` / `https://` URL - `VITE_DEV_API_BASE_URL`: optional Vite dev proxy target for `/api`, default `http://127.0.0.1:3001` - `VITE_DEV_WORKER_BASE_URL`: optional Vite dev proxy target for `/worker-api`, default `http://127.0.0.1:3002` +- `SMOKE_REQUIRE_EXPLICIT_ENDPOINT`: set to `true` for deployed-environment smoke tests so omitted API endpoints fail instead of falling back to localhost -Explicit startup values are validated before API/worker startup, before Kubernetes manifest rendering, and before `scripts/deploy-k8s.sh` performs cluster writes. `MONGO_URI` is required at runtime and must be a `mongodb://` or `mongodb+srv://` URI without whitespace or unreplaced placeholder markers. `PORT`/`WORKER_PORT` must be valid TCP ports, Mongo retry count and retry delay must be positive, worker poll interval must be positive, replica counts must be non-negative in manifests, NodePort must be in the Kubernetes `30000..32767` range, and `OPERATOR_JOB_TTL_SECONDS` must be non-negative (`0` disables TTL injection). Manifest, build, and deploy wrappers no longer carry a repository-default image registry: callers must set `API_IMAGE`, `WORKER_IMAGE`, and `OPERATOR_RUNNER_IMAGE` explicitly, or set `REGISTRY_HOST` and `REGISTRY_NAMESPACE` so those image references can be derived from `IMAGE_TAG`. Manifest and deploy wrappers also reject invalid Kubernetes namespace/Secret/PVC/Service names, non-absolute workspace mount paths, malformed worker/BOS service URLs, malformed non-empty AI service URLs, missing AI base/model when an AI key is configured, and container image references with whitespace. Multi-line ConfigMap block values such as GPU tolerations JSON and shell command allow/deny regexes are indented before manifest output so readable multi-line input stays valid YAML. Explicit boolean env values accept `true/false`, `1/0`, `yes/no`, or `on/off`; other spellings fail fast instead of silently falling back, and accepted true aliases drive deploy-time branches such as public LoadBalancer creation. +Explicit startup values are validated before API/worker startup, before Kubernetes manifest rendering, and before `scripts/deploy-k8s.sh` performs cluster writes. `MONGO_URI` is required at runtime and must be a `mongodb://` or `mongodb+srv://` URI without whitespace or unreplaced placeholder markers. `PORT`/`WORKER_PORT` must be valid TCP ports, Mongo retry count and retry delay must be positive, worker poll interval must be positive, replica counts must be non-negative in manifests, NodePort must be in the Kubernetes `30000..32767` range, and `OPERATOR_JOB_TTL_SECONDS` must be non-negative (`0` disables TTL injection). Manifest, build, and deploy wrappers no longer carry a repository-default image registry: callers must set `API_IMAGE`, `WORKER_IMAGE`, and `OPERATOR_RUNNER_IMAGE` explicitly, or set `REGISTRY_HOST` and `REGISTRY_NAMESPACE` so those image references can be derived from `IMAGE_TAG`; all three wrappers validate generated or explicit image references before Docker or Kubernetes calls. Manifest and deploy wrappers also reject invalid Kubernetes namespace/Secret/PVC/Service names, non-absolute workspace mount paths, malformed worker/BOS service URLs, malformed non-empty AI service URLs, missing AI base/model when an AI key is configured, and container image references with whitespace. Multi-line ConfigMap block values such as GPU tolerations JSON and shell command allow/deny regexes are indented before manifest output so readable multi-line input stays valid YAML. Explicit boolean env values accept `true/false`, `1/0`, `yes/no`, or `on/off`; other spellings fail fast instead of silently falling back, and accepted true aliases drive deploy-time branches such as public LoadBalancer creation. Development and acceptance entrypoints resolve API endpoints through shared helpers instead of each script carrying its own localhost default. Node/Vite entrypoints use `scripts/lib/service-endpoints.mjs`, Python operational scripts use `scripts/lib/droboticflow_env.py`, and shell wrappers use `scripts/lib/service-endpoints.sh`. `BASE_URL` is accepted by browser-oriented acceptance scripts, while deployment/reconcile wrappers preserve `API_BASE_URL` as their primary environment variable. Explicit endpoint values must be absolute `http://` or `https://` URLs so malformed values fail before Vite, curl, Playwright, or urllib requests are attempted. Browser-side `VITE_API_BASE_URL` follows the same rule, with the empty value reserved for same-origin API requests. @@ -244,11 +247,12 @@ npm run smoke Run a smoke test against a deployed cluster: ```bash +SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true \ SMOKE_SOURCE_PATH= \ ./scripts/smoke-test.sh http://:30080 ``` -`SMOKE_SOURCE_PATH` can be a filesystem/PVC path or a `bos://bucket/prefix` path. The real-source operator now infers `LOCAL` or `BOS` from that path and fails fast when no real source is configured, so an empty source configuration cannot pass smoke by producing an empty manifest. +`SMOKE_SOURCE_PATH` can be a filesystem/PVC path or a `bos://bucket/prefix` path. The real-source operator now infers `LOCAL` or `BOS` from that path and fails fast when no real source is configured, so an empty source configuration cannot pass smoke by producing an empty manifest. Set `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` for deployed-environment smoke so the script fails instead of falling back to localhost when `BASE_URL` / `API_BASE_URL` is omitted. Run browser object-storage acceptance against a deployed cluster: diff --git a/backend/operator-runner/main.py b/backend/operator-runner/main.py index 465dfe28..761186b4 100644 --- a/backend/operator-runner/main.py +++ b/backend/operator-runner/main.py @@ -1078,16 +1078,6 @@ def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any } -def parse_float_param(params: dict[str, str], key: str, default: float) -> float: - raw = params.get(key, "").strip() - if not raw: - return default - try: - return float(raw) - except Exception: - return default - - def parse_positive_float_param(params: dict[str, str], key: str, default: float) -> float: raw = params.get(key, "").strip() if not raw: @@ -1129,16 +1119,6 @@ def float_value(value: Any, default: float) -> float: return default -def parse_int_param(params: dict[str, str], key: str, default: int) -> int: - raw = params.get(key, "").strip() - if not raw: - return default - try: - return int(raw) - except Exception: - return default - - def parse_positive_int_param(params: dict[str, str], key: str, default: int) -> int: raw = params.get(key, "").strip() if not raw: diff --git a/cmd/worker/main.go b/cmd/worker/main.go index db017bf8..eba1e2d7 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -64,7 +64,7 @@ func main() { os.Exit(1) } if cfg.OperatorRunnerImage == "" { - fmt.Println("[worker] OPERATOR_RUNNER_IMAGE is required; deploy scripts default it to the current git image tag") + fmt.Println("[worker] OPERATOR_RUNNER_IMAGE is required; set it explicitly or configure REGISTRY_HOST and REGISTRY_NAMESPACE for deploy/render scripts") os.Exit(1) } runtime, err := worker.NewRuntime( diff --git a/docs/deployment.md b/docs/deployment.md index 9db43bc8..3131e653 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -211,7 +211,7 @@ http://:30080 - The main manifest only deploys the application components. It does not create MongoDB resources in-cluster. - `MONGO_URI` is required by API and worker startup and must be a concrete `mongodb://` or `mongodb+srv://` URI without whitespace or unreplaced placeholder markers. Kubernetes deployment must reference an external MongoDB instance; localhost, `mongodb:27017`, and in-cluster `.svc` addresses are rejected by `scripts/deploy-k8s.sh`. -- Startup configuration is validated before manifest output and before `scripts/deploy-k8s.sh` performs any cluster writes. `MONGO_CONNECT_RETRIES`, `MONGO_CONNECT_RETRY_DELAY_MS`, and `WORKER_POLL_INTERVAL_MS` must be positive integers; `API_REPLICAS`, `WORKER_REPLICAS`, and `OPERATOR_JOB_TTL_SECONDS` must be non-negative integers; `API_NODE_PORT` must be in `30000..32767` when `API_SERVICE_TYPE=NodePort`; boolean env values must be one of `true/false`, `1/0`, `yes/no`, or `on/off`. Kubernetes namespace, image-pull-secret, workspace PVC, and public LoadBalancer Service names must be lowercase DNS-compatible values; workspace mount paths must be absolute container paths; worker service URLs must be absolute `http://` or `https://` URLs; non-empty BOS and AI endpoint values must also be absolute `http://` or `https://` URLs; if `AI_API_KEY` is configured, both `AI_API_BASE_URL` and `AI_MODEL` are required and there is no provider/model hardcoded fallback; container image references must be non-empty and whitespace-free. `scripts/render-manifests.sh`, `scripts/deploy-k8s.sh`, and `scripts/build-and-push-images.sh` require explicit `API_IMAGE`, `WORKER_IMAGE`, and `OPERATOR_RUNNER_IMAGE` values unless `REGISTRY_HOST` and `REGISTRY_NAMESPACE` are both set. Multi-line ConfigMap block values, including `OPERATOR_GPU_TOLERATIONS_JSON`, `OPERATOR_COMMAND_ALLOWLIST_REGEX`, and `OPERATOR_COMMAND_DENYLIST_REGEX`, are indented before manifest output so formatted JSON and multi-line regex lists remain valid YAML. +- Startup configuration is validated before manifest output and before `scripts/deploy-k8s.sh` performs any cluster writes. `MONGO_CONNECT_RETRIES`, `MONGO_CONNECT_RETRY_DELAY_MS`, and `WORKER_POLL_INTERVAL_MS` must be positive integers; `API_REPLICAS`, `WORKER_REPLICAS`, and `OPERATOR_JOB_TTL_SECONDS` must be non-negative integers; `API_NODE_PORT` must be in `30000..32767` when `API_SERVICE_TYPE=NodePort`; boolean env values must be one of `true/false`, `1/0`, `yes/no`, or `on/off`. Kubernetes namespace, image-pull-secret, workspace PVC, and public LoadBalancer Service names must be lowercase DNS-compatible values; workspace mount paths must be absolute container paths; worker service URLs must be absolute `http://` or `https://` URLs; non-empty BOS and AI endpoint values must also be absolute `http://` or `https://` URLs; if `AI_API_KEY` is configured, both `AI_API_BASE_URL` and `AI_MODEL` are required and there is no provider/model hardcoded fallback; container image references must be non-empty and whitespace-free. `scripts/render-manifests.sh`, `scripts/deploy-k8s.sh`, and `scripts/build-and-push-images.sh` require explicit `API_IMAGE`, `WORKER_IMAGE`, and `OPERATOR_RUNNER_IMAGE` values unless `REGISTRY_HOST` and `REGISTRY_NAMESPACE` are both set, and all three scripts reject malformed image references before Docker or Kubernetes calls. Multi-line ConfigMap block values, including `OPERATOR_GPU_TOLERATIONS_JSON`, `OPERATOR_COMMAND_ALLOWLIST_REGEX`, and `OPERATOR_COMMAND_DENYLIST_REGEX`, are indented before manifest output so formatted JSON and multi-line regex lists remain valid YAML. - `BOS_ENDPOINT` is rendered into the API and worker ConfigMap as an empty value when unset. Leave it unset only when every BOS scan config or storage connection provides its own endpoint; if both the workflow/connection endpoint and global `BOS_ENDPOINT` are empty, real BOS operations fail with an explicit missing-endpoint error instead of falling back to a hardcoded region. - API and worker entrypoints are Go binaries with Mongo startup retry, so they tolerate the database becoming ready after the Deployments start. - The API proxies `/worker-api/*` to `WORKER_API_BASE_URL`, defaulting to the in-cluster `http://droboticflow-worker:3002` Service. Override it only when the worker Service name, namespace, or exposure model differs. @@ -231,7 +231,7 @@ http://:30080 - `DROBOTICFLOW_TRUSTED_PROXY_CIDRS` controls which reverse proxies may supply `X-Forwarded-For` / `X-Real-IP` for rate limiting. Leave it empty unless ingress or load balancer source ranges are known. - Session-authenticated unsafe requests require `X-Droboticflow-CSRF-Token` to match the readable `droboticflow_csrf` cookie. The bundled frontend sets this header automatically. - `IMAGE_TAG` defaults to the current git short SHA in build, deploy, and render scripts only when explicit registry coordinates are configured. Set it explicitly for release names or rollback-friendly tags. -- `scripts/smoke-test.sh` requires `SMOKE_SOURCE_PATH`, a real path visible to operator-runner Jobs. The `真实数据源` runner infers filesystem/PVC paths as `LOCAL` and `bos://bucket/prefix` paths as `BOS`; if no real source is configured, the source node fails instead of emitting an empty manifest. It can exercise session-authenticated deployments when `DROBOTICFLOW_SMOKE_EMAIL` and `DROBOTICFLOW_SMOKE_PASSWORD` are set; it logs in, persists cookies, and sends the CSRF header on write requests. +- `scripts/smoke-test.sh` requires `SMOKE_SOURCE_PATH`, a real path visible to operator-runner Jobs. The `真实数据源` runner infers filesystem/PVC paths as `LOCAL` and `bos://bucket/prefix` paths as `BOS`; if no real source is configured, the source node fails instead of emitting an empty manifest. Set `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` for deployed-environment smoke so missing `BASE_URL` / `API_BASE_URL` fails instead of falling back to localhost. The script can exercise session-authenticated deployments when `DROBOTICFLOW_SMOKE_EMAIL` and `DROBOTICFLOW_SMOKE_PASSWORD` are set; it logs in, persists cookies, and sends the CSRF header on write requests. - `scripts/reconcile-system-data.sh` delegates to `scripts/reconcile_system_data.py` and uses the same protected-client behavior. Set `DROBOTICFLOW_API_TOKEN` for token-protected APIs when session auth is not enforced. For public session-auth deployments, set `DROBOTICFLOW_SMOKE_EMAIL` and `DROBOTICFLOW_SMOKE_PASSWORD` to an organization admin account because the reconcile flow updates settings, users, templates, and operators. Builtin operators are read from the live `/api/operators` catalog and refreshed to `OPERATOR_RUNNER_IMAGE`; the script does not carry a separate builtin catalog. - `scripts/object-storage-acceptance.mjs` runs browser create, preview, rename, upload, delete, and optional video playback checks against a live storage connection. It never rewrites storage connection metadata; provide `OBJECT_STORAGE_CONNECTION_ID`, `OBJECT_STORAGE_CONNECTION_NAME`, or `OBJECT_STORAGE_BUCKET` explicitly. `OBJECT_STORAGE_ACCEPTANCE_PAYLOAD_FILE` is required and must point to a non-empty real UTF-8 text or JSON payload used for create/upload/preview checks; set `OBJECT_STORAGE_ACCEPTANCE_CONTENT_TYPE` when auto-detection is not enough. Set `OBJECT_STORAGE_VIDEO_KEY` only when an existing playable object should be validated. `PLAYWRIGHT_IGNORE_HTTPS_ERRORS` must be an explicit boolean spelling when set. The script supports `DROBOTICFLOW_API_TOKEN` for token-protected non-session APIs and `DROBOTICFLOW_SMOKE_EMAIL` / `DROBOTICFLOW_SMOKE_PASSWORD` for session-authenticated browser access. - `scripts/playwright-preview-operators-check.mjs` runs browser/operator preview acceptance with caller-supplied payloads and real source paths. Provide `PREVIEW_CHECK_CSV_PAYLOAD_FILE`, `PREVIEW_CHECK_TEXT_PAYLOAD_FILE`, `PREVIEW_CHECK_JSON_PAYLOAD_FILE`, `PREVIEW_CHECK_JSON_EXTRACT_FIELD`, `PREVIEW_CHECK_JSON_FILTER_FIELD`, `PREVIEW_CHECK_JSON_FIELD_MAP_JSON`, `PREVIEW_CHECK_SCAN_SOURCE_PATH`, `PREVIEW_CHECK_ROSBAG_SOURCE_PATH`, and `PREVIEW_CHECK_LEROBOT_SOURCE_PATH`. The script no longer generates built-in CSV/text/JSON/ROSBag/LeRobot data. Optional `PREVIEW_CHECK_SCAN_MIN_FILES` must be an exact safe positive integer. diff --git a/docs/operations.md b/docs/operations.md index d229d1e4..b1edba37 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -61,13 +61,14 @@ Data-asset APIs apply the same stale-data rule for local external payloads. If ` ## Local Verification -Run the full verification loop: +Run local verification and the security audit as separate gates: ```bash -npm run verify +npm run verify:local +npm run verify:security ``` -This includes TypeScript/Go linting, frontend Prettier formatting checks, Go tests, Python operator-runner tests, frontend node tests, production build, `npm audit --audit-level=moderate`, and Kubernetes manifest dry-run validation. +`verify:local` includes TypeScript/Go/script linting, frontend Prettier formatting checks, Go tests, Python operator-runner tests, frontend node tests, production build, and Kubernetes manifest dry-run validation. `verify:security` runs `npm audit --audit-level=moderate` through a retry wrapper so transient registry failures do not prevent local validation from running first. `npm run verify` runs both in that order. ## Runtime Metrics @@ -117,6 +118,8 @@ SMOKE_SOURCE_PATH= \ `SMOKE_SOURCE_PATH` may be a filesystem/PVC path or a `bos://bucket/prefix` path. The `真实数据源` runner infers `LOCAL` or `BOS` from the path and rejects a run that has no `SOURCE_TEXT`, `SOURCE_JSON`, `SOURCE_PATH`, configured BOS source, local `STORAGE_PREFIX`, or upstream artifact input. This prevents an unconfigured source node from completing with an empty manifest. +For deployed smoke, pass the deployed URL or set `BASE_URL` / `API_BASE_URL`, and set `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` so localhost defaults are disabled. + Run the production-style WJZH real-data regression: ```bash diff --git a/docs/production-readiness.md b/docs/production-readiness.md index a10e190b..700bb044 100644 --- a/docs/production-readiness.md +++ b/docs/production-readiness.md @@ -9,8 +9,8 @@ This note tracks the production-facing safeguards that must stay true for Drobod - Worker proxy health, stats, metrics, and retry calls use short upstream timeouts. Manual `process-once` uses a long timeout because it can execute real queued work. - Explicit shell operators are disabled by default through `OPERATOR_ALLOW_SHELL_COMMANDS=false`; allowlist and denylist regexes are platform-controlled and cannot be overridden by workflow hyperparams. - Environment-specific values such as `K8S_ENDPOINT`, registry host/namespace, registry username, deploy image references, and workspace BOS bucket have no private repository defaults; provide them from local deploy secrets or CI secrets. -- Manifest rendering and deployment wrappers reject invalid Kubernetes namespace/Secret/PVC/Service names, non-absolute workspace mount paths, malformed worker/AI/BOS service URLs, and image references with whitespace before emitting manifests or calling `kubectl`. Multi-line ConfigMap block values are indented before manifest output so formatted GPU tolerations JSON and multi-line shell command policy regexes remain valid YAML. -- Local development and acceptance endpoints are resolved through shared helpers instead of per-script localhost literals. Vite dev proxy targets can be overridden with `VITE_DEV_API_BASE_URL` and `VITE_DEV_WORKER_BASE_URL`; browser-oriented acceptance scripts accept `BASE_URL` or `API_BASE_URL`, while reconcile/deploy wrappers keep `API_BASE_URL` as the primary operational variable. Explicit endpoint values must be absolute `http://` or `https://` URLs and fail before Vite, curl, Playwright, or urllib requests are attempted. Browser-side `VITE_API_BASE_URL` follows the same absolute http(s) rule, with an empty value reserved for same-origin API calls. +- Manifest rendering, image build, and deployment wrappers reject invalid Kubernetes namespace/Secret/PVC/Service names, non-absolute workspace mount paths, malformed worker/AI/BOS service URLs, and malformed image references before emitting manifests, building images, or calling `kubectl`. Multi-line ConfigMap block values are indented before manifest output so formatted GPU tolerations JSON and multi-line shell command policy regexes remain valid YAML. +- Local development and acceptance endpoints are resolved through shared helpers instead of per-script localhost literals. Vite dev proxy targets can be overridden with `VITE_DEV_API_BASE_URL` and `VITE_DEV_WORKER_BASE_URL`; browser-oriented acceptance scripts accept `BASE_URL` or `API_BASE_URL`, while reconcile/deploy wrappers keep `API_BASE_URL` as the primary operational variable. Explicit endpoint values must be absolute `http://` or `https://` URLs and fail before Vite, curl, Playwright, or urllib requests are attempted. Browser-side `VITE_API_BASE_URL` follows the same absolute http(s) rule, with an empty value reserved for same-origin API calls. Deployed smoke runs should set `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` so an omitted endpoint cannot fall back to localhost. - Data reconciliation uses real API calls and supports protected deployments through `DROBOTICFLOW_API_TOKEN` or session login credentials plus CSRF headers. Public session-auth environments must use an organization admin account for reconcile because it updates settings and management resources. - Python live-acceptance and WJZH regression scripts use the shared `DroboticflowClient` helper for API token headers, session-login cookies, CSRF propagation, JSON calls, and binary object-storage calls. WJZH quality thresholds are caller-supplied through `QUALITY_MIN_CLARITY`, `QUALITY_MAX_SHAKE`, and `QUALITY_MAX_HAND_OCCLUSION`; window, slicing, and annotation timing are caller-supplied through the documented `*_SECONDS` values; sampling thresholds are caller-supplied through `SAMPLING_RULES_JSON`. The scripts do not carry built-in audit rules. New Python acceptance scripts should reuse that helper instead of carrying their own auth/request stack. - Python WJZH and X-platform acceptance scripts validate quality thresholds through the shared normalized-threshold gate. `QUALITY_MIN_CLARITY`, `QUALITY_MAX_SHAKE`, `QUALITY_MAX_HAND_OCCLUSION`, `XPLATFORM_MIN_CLARITY`, `XPLATFORM_MAX_SHAKE`, and `XPLATFORM_MAX_HAND_OCCLUSION` must be finite values in `[0, 1]`; invalid values fail before live workflows are created. @@ -57,19 +57,21 @@ This note tracks the production-facing safeguards that must stay true for Drobod Run local verification before merging: ```bash -npm run verify +npm run verify:local +npm run verify:security ``` -This gate includes shell syntax checks for operational scripts, so GPU smoke, deployment, cleanup, and reconciliation wrappers are parsed before a change is accepted. +The local gate includes shell syntax checks for operational scripts, so GPU smoke, deployment, cleanup, and reconciliation wrappers are parsed before a change is accepted. The security gate runs `npm audit --audit-level=moderate` through the retry wrapper; `npm run verify` runs both gates in order. Run deployed smoke after rollout: ```bash +SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true \ SMOKE_SOURCE_PATH= \ ./scripts/smoke-test.sh http://:30080 ``` -The smoke source path may be local/PVC-backed or `bos://bucket/prefix`. The source node now fails fast when no real input source is configured, so a misconfigured workflow cannot pass by producing an empty source manifest. +The smoke source path may be local/PVC-backed or `bos://bucket/prefix`. The source node now fails fast when no real input source is configured, so a misconfigured workflow cannot pass by producing an empty source manifest. `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` keeps deployed smoke from silently using local defaults. Run browser object-storage acceptance after rollout: diff --git a/package.json b/package.json index 1267aead..703a7edf 100644 --- a/package.json +++ b/package.json @@ -27,8 +27,10 @@ "test": "go test ./...", "test:python": "python3 -m pytest backend/operator-runner tests/backend", "test:frontend": "node --test tests/frontend/*.test.mjs", - "audit:moderate": "npm audit --audit-level=moderate", - "verify": "npm run lint && npm run format:check && npm test && npm run test:python && npm run test:frontend && npm run build && npm run audit:moderate && npm run k8s:validate", + "audit:moderate": "./scripts/npm-audit-retry.sh --audit-level=moderate", + "verify:local": "npm run lint && npm run format:check && npm test && npm run test:python && npm run test:frontend && npm run build && npm run k8s:validate", + "verify:security": "npm run audit:moderate", + "verify": "npm run verify:local && npm run verify:security", "docker:build:api": "docker build -f Dockerfile.api -t droboticflow-api:local .", "docker:build:worker": "docker build -f Dockerfile.worker -t droboticflow-worker:local .", "docker:build:operator-runner": "docker build -f Dockerfile.operator-runner -t droboticflow-operator-runner:local .", diff --git a/scripts/build-and-push-images.sh b/scripts/build-and-push-images.sh index a22d8bef..6d724292 100755 --- a/scripts/build-and-push-images.sh +++ b/scripts/build-and-push-images.sh @@ -22,6 +22,11 @@ PUSH="${PUSH:-true}" # shellcheck source=scripts/lib/image-config.sh source "$ROOT_DIR/scripts/lib/image-config.sh" droboticflow_resolve_image_configuration +# shellcheck source=scripts/lib/deploy-validation.sh +source "$ROOT_DIR/scripts/lib/deploy-validation.sh" +validate_image_reference API_IMAGE "$API_IMAGE" +validate_image_reference WORKER_IMAGE "$WORKER_IMAGE" +validate_image_reference OPERATOR_RUNNER_IMAGE "$OPERATOR_RUNNER_IMAGE" docker buildx inspect >/dev/null 2>&1 || docker buildx create --use >/dev/null diff --git a/scripts/npm-audit-retry.sh b/scripts/npm-audit-retry.sh new file mode 100755 index 00000000..66b3ab24 --- /dev/null +++ b/scripts/npm-audit-retry.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +attempts="${NPM_AUDIT_ATTEMPTS:-3}" +delay_seconds="${NPM_AUDIT_RETRY_DELAY_SECONDS:-5}" + +if ! [[ "$attempts" =~ ^[0-9]+$ ]] || (( attempts < 1 )); then + echo "ERROR: NPM_AUDIT_ATTEMPTS must be a positive integer." >&2 + exit 1 +fi + +if ! [[ "$delay_seconds" =~ ^[0-9]+$ ]]; then + echo "ERROR: NPM_AUDIT_RETRY_DELAY_SECONDS must be a non-negative integer." >&2 + exit 1 +fi + +for ((attempt = 1; attempt <= attempts; attempt += 1)); do + if npm audit "$@"; then + exit 0 + fi + status=$? + if (( attempt == attempts )); then + exit "$status" + fi + echo "npm audit failed on attempt $attempt/$attempts; retrying in ${delay_seconds}s..." >&2 + sleep "$delay_seconds" +done diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 1089bebb..05f14185 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -5,9 +5,30 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck disable=SC1091 source "$SCRIPT_DIR/lib/service-endpoints.sh" +SMOKE_REQUIRE_EXPLICIT_ENDPOINT="${SMOKE_REQUIRE_EXPLICIT_ENDPOINT:-false}" +case "$(printf '%s' "$SMOKE_REQUIRE_EXPLICIT_ENDPOINT" | tr '[:upper:]' '[:lower:]')" in + 1|true|yes|on) + SMOKE_REQUIRE_EXPLICIT_ENDPOINT="true" + ;; + 0|false|no|off) + SMOKE_REQUIRE_EXPLICIT_ENDPOINT="false" + ;; + *) + echo "ERROR: SMOKE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean." >&2 + exit 1 + ;; +esac + if [[ $# -gt 0 ]]; then API_BASE_URL="$(droboticflow_first_service_url "$1")" WORKER_BASE_URL="$(droboticflow_first_service_url "${2:-}" "${WORKER_BASE_URL:-}")" +elif [[ "$SMOKE_REQUIRE_EXPLICIT_ENDPOINT" == "true" ]]; then + API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")" + WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}")" + if [[ -z "$API_BASE_URL" ]]; then + echo "ERROR: API_BASE_URL or BASE_URL is required when SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true." >&2 + exit 1 + fi else API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}" "$(droboticflow_default_api_base_url)")" WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}" "$(droboticflow_default_worker_base_url)")" diff --git a/tests/backend/build_push_images_script_test.py b/tests/backend/build_push_images_script_test.py new file mode 100644 index 00000000..2a0f8240 --- /dev/null +++ b/tests/backend/build_push_images_script_test.py @@ -0,0 +1,46 @@ +import os +import pathlib +import subprocess + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "build-and-push-images.sh" + + +def run_build_script(extra_env: dict[str, str]) -> subprocess.CompletedProcess: + env = os.environ.copy() + for key in ( + "API_IMAGE", + "DEPLOY_ENV", + "DROBOTICFLOW_DEPLOY_ENV_LOADED", + "IMAGE_TAG", + "OPERATOR_RUNNER_IMAGE", + "PUSH", + "REGISTRY_HOST", + "REGISTRY_NAMESPACE", + "WORKER_IMAGE", + ): + env.pop(key, None) + env["DROBOTICFLOW_DEPLOY_ENV_LOADED"] = "test" + env.update(extra_env) + return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) + + +def test_build_and_push_requires_explicit_images_or_registry_configuration(): + result = run_build_script({}) + + assert result.returncode == 1 + assert "API_IMAGE is required unless REGISTRY_HOST and REGISTRY_NAMESPACE are both set" in result.stderr + + +def test_build_and_push_rejects_invalid_explicit_image_before_docker(): + result = run_build_script( + { + "API_IMAGE": "registry.example.com/dataflow/droboticflow-api:test bad", + "WORKER_IMAGE": "registry.example.com/dataflow/droboticflow-worker:test", + "OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test", + } + ) + + assert result.returncode == 1 + assert "API_IMAGE must be a non-empty container image reference" in result.stderr diff --git a/tests/backend/npm_audit_retry_script_test.py b/tests/backend/npm_audit_retry_script_test.py new file mode 100644 index 00000000..0cc75c76 --- /dev/null +++ b/tests/backend/npm_audit_retry_script_test.py @@ -0,0 +1,79 @@ +import os +import pathlib +import stat +import subprocess + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "npm-audit-retry.sh" + + +def run_audit_retry(tmp_path: pathlib.Path, extra_env: dict[str, str]) -> subprocess.CompletedProcess: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + counter_file = tmp_path / "npm-calls" + fake_npm = bin_dir / "npm" + fake_npm.write_text( + f"""#!/usr/bin/env bash +set -euo pipefail +if [[ "${{1:-}}" != "audit" ]]; then + echo "unexpected npm command: $*" >&2 + exit 2 +fi +count=0 +if [[ -f "{counter_file}" ]]; then + count="$(cat "{counter_file}")" +fi +count=$((count + 1)) +printf '%s' "$count" > "{counter_file}" +if (( count < 3 )); then + exit 42 +fi +exit 0 +""", + encoding="utf-8", + ) + fake_npm.chmod(fake_npm.stat().st_mode | stat.S_IXUSR) + + env = os.environ.copy() + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + for key in ("NPM_AUDIT_ATTEMPTS", "NPM_AUDIT_RETRY_DELAY_SECONDS"): + env.pop(key, None) + env.update(extra_env) + return subprocess.run( + ["bash", str(SCRIPT), "--audit-level=moderate"], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + check=False, + ) + + +def test_npm_audit_retry_retries_transient_failures(tmp_path): + result = run_audit_retry( + tmp_path, + { + "NPM_AUDIT_ATTEMPTS": "3", + "NPM_AUDIT_RETRY_DELAY_SECONDS": "0", + }, + ) + + assert result.returncode == 0 + assert "npm audit failed on attempt 1/3" in result.stderr + assert "npm audit failed on attempt 2/3" in result.stderr + assert (tmp_path / "npm-calls").read_text(encoding="utf-8") == "3" + + +def test_npm_audit_retry_rejects_invalid_attempts(tmp_path): + result = run_audit_retry( + tmp_path, + { + "NPM_AUDIT_ATTEMPTS": "0", + "NPM_AUDIT_RETRY_DELAY_SECONDS": "0", + }, + ) + + assert result.returncode == 1 + assert "NPM_AUDIT_ATTEMPTS must be a positive integer" in result.stderr + assert not (tmp_path / "npm-calls").exists() diff --git a/tests/backend/smoke_script_test.py b/tests/backend/smoke_script_test.py new file mode 100644 index 00000000..13a776a9 --- /dev/null +++ b/tests/backend/smoke_script_test.py @@ -0,0 +1,37 @@ +import os +import pathlib +import subprocess + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "smoke-test.sh" + + +def run_smoke(extra_env: dict[str, str]) -> subprocess.CompletedProcess: + env = os.environ.copy() + for key in ( + "API_BASE_URL", + "BASE_URL", + "SMOKE_REQUIRE_EXPLICIT_ENDPOINT", + "SMOKE_RUN_TIMEOUT_SECONDS", + "SMOKE_SOURCE_PATH", + "WORKER_API_BASE_URL", + "WORKER_BASE_URL", + ): + env.pop(key, None) + env.update(extra_env) + return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False) + + +def test_smoke_requires_explicit_endpoint_when_deployed_mode_is_enabled(): + result = run_smoke({"SMOKE_REQUIRE_EXPLICIT_ENDPOINT": "true", "SMOKE_SOURCE_PATH": "/workspace/real-data"}) + + assert result.returncode == 1 + assert "API_BASE_URL or BASE_URL is required when SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true" in result.stderr + + +def test_smoke_rejects_invalid_explicit_endpoint_mode_boolean(): + result = run_smoke({"SMOKE_REQUIRE_EXPLICIT_ENDPOINT": "maybe", "SMOKE_SOURCE_PATH": "/workspace/real-data"}) + + assert result.returncode == 1 + assert "SMOKE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean" in result.stderr