forked from longtao.wu/droboticflow
🔧 Harden verification and smoke guardrails
This commit is contained in:
parent
28435bb746
commit
dd990b63bf
@ -29,6 +29,7 @@ OPERATOR_COMMAND_DENYLIST_REGEX=
|
||||
VITE_API_BASE_URL=
|
||||
VITE_DEV_API_BASE_URL=http://127.0.0.1:3001
|
||||
VITE_DEV_WORKER_BASE_URL=http://127.0.0.1:3002
|
||||
SMOKE_REQUIRE_EXPLICIT_ENDPOINT=false
|
||||
# Optional AI generation. Leave all three empty to disable AI generation.
|
||||
# When AI_API_KEY is set, AI_API_BASE_URL and AI_MODEL must both be explicit.
|
||||
AI_API_BASE_URL=
|
||||
|
||||
7
.github/workflows/ci.yml
vendored
7
.github/workflows/ci.yml
vendored
@ -41,8 +41,11 @@ jobs:
|
||||
- name: Install Python test dependencies
|
||||
run: python -m pip install --upgrade pip pytest bce-python-sdk numpy opencv-python-headless pymongo
|
||||
|
||||
- name: Verify
|
||||
run: npm run verify
|
||||
- name: Verify local checks
|
||||
run: npm run verify:local
|
||||
|
||||
- name: Security audit
|
||||
run: npm run verify:security
|
||||
|
||||
docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
10
README.md
10
README.md
@ -64,7 +64,9 @@ Drobodata is a workflow orchestration platform for embodied AI and vision pipeli
|
||||
- `npm run format:check`: check frontend formatting without writing files
|
||||
- `npm test`: Go unit and integration tests
|
||||
- `npm run build`: build frontend assets and Go binaries
|
||||
- `npm run verify`: web/Go/script lint, Go tests, Python operator-runner tests, frontend node tests, build, dependency audit, and Kubernetes manifest dry-run validation
|
||||
- `npm run verify:local`: web/Go/script lint, Go tests, Python operator-runner tests, frontend node tests, build, and Kubernetes manifest dry-run validation
|
||||
- `npm run verify:security`: dependency audit with retry
|
||||
- `npm run verify`: run local verification first, then dependency audit
|
||||
- `npm run smoke`: local smoke test against the running API and worker
|
||||
- `npm run gpu:smoke`: real Kubernetes GPU scheduling and `nvidia-smi` smoke test
|
||||
- `npm run k8s:cleanup-jobs`: delete old finished operator-runner Jobs that predate TTL cleanup
|
||||
@ -116,8 +118,9 @@ Key environment variables:
|
||||
- `VITE_API_BASE_URL`: optional frontend API base URL override; leave empty for same-origin `/api`, or set an absolute `http://` / `https://` URL
|
||||
- `VITE_DEV_API_BASE_URL`: optional Vite dev proxy target for `/api`, default `http://127.0.0.1:3001`
|
||||
- `VITE_DEV_WORKER_BASE_URL`: optional Vite dev proxy target for `/worker-api`, default `http://127.0.0.1:3002`
|
||||
- `SMOKE_REQUIRE_EXPLICIT_ENDPOINT`: set to `true` for deployed-environment smoke tests so omitted API endpoints fail instead of falling back to localhost
|
||||
|
||||
Explicit startup values are validated before API/worker startup, before Kubernetes manifest rendering, and before `scripts/deploy-k8s.sh` performs cluster writes. `MONGO_URI` is required at runtime and must be a `mongodb://` or `mongodb+srv://` URI without whitespace or unreplaced placeholder markers. `PORT`/`WORKER_PORT` must be valid TCP ports, Mongo retry count and retry delay must be positive, worker poll interval must be positive, replica counts must be non-negative in manifests, NodePort must be in the Kubernetes `30000..32767` range, and `OPERATOR_JOB_TTL_SECONDS` must be non-negative (`0` disables TTL injection). Manifest, build, and deploy wrappers no longer carry a repository-default image registry: callers must set `API_IMAGE`, `WORKER_IMAGE`, and `OPERATOR_RUNNER_IMAGE` explicitly, or set `REGISTRY_HOST` and `REGISTRY_NAMESPACE` so those image references can be derived from `IMAGE_TAG`. Manifest and deploy wrappers also reject invalid Kubernetes namespace/Secret/PVC/Service names, non-absolute workspace mount paths, malformed worker/BOS service URLs, malformed non-empty AI service URLs, missing AI base/model when an AI key is configured, and container image references with whitespace. Multi-line ConfigMap block values such as GPU tolerations JSON and shell command allow/deny regexes are indented before manifest output so readable multi-line input stays valid YAML. Explicit boolean env values accept `true/false`, `1/0`, `yes/no`, or `on/off`; other spellings fail fast instead of silently falling back, and accepted true aliases drive deploy-time branches such as public LoadBalancer creation.
|
||||
Explicit startup values are validated before API/worker startup, before Kubernetes manifest rendering, and before `scripts/deploy-k8s.sh` performs cluster writes. `MONGO_URI` is required at runtime and must be a `mongodb://` or `mongodb+srv://` URI without whitespace or unreplaced placeholder markers. `PORT`/`WORKER_PORT` must be valid TCP ports, Mongo retry count and retry delay must be positive, worker poll interval must be positive, replica counts must be non-negative in manifests, NodePort must be in the Kubernetes `30000..32767` range, and `OPERATOR_JOB_TTL_SECONDS` must be non-negative (`0` disables TTL injection). Manifest, build, and deploy wrappers no longer carry a repository-default image registry: callers must set `API_IMAGE`, `WORKER_IMAGE`, and `OPERATOR_RUNNER_IMAGE` explicitly, or set `REGISTRY_HOST` and `REGISTRY_NAMESPACE` so those image references can be derived from `IMAGE_TAG`; all three wrappers validate generated or explicit image references before Docker or Kubernetes calls. Manifest and deploy wrappers also reject invalid Kubernetes namespace/Secret/PVC/Service names, non-absolute workspace mount paths, malformed worker/BOS service URLs, malformed non-empty AI service URLs, missing AI base/model when an AI key is configured, and container image references with whitespace. Multi-line ConfigMap block values such as GPU tolerations JSON and shell command allow/deny regexes are indented before manifest output so readable multi-line input stays valid YAML. Explicit boolean env values accept `true/false`, `1/0`, `yes/no`, or `on/off`; other spellings fail fast instead of silently falling back, and accepted true aliases drive deploy-time branches such as public LoadBalancer creation.
|
||||
|
||||
Development and acceptance entrypoints resolve API endpoints through shared helpers instead of each script carrying its own localhost default. Node/Vite entrypoints use `scripts/lib/service-endpoints.mjs`, Python operational scripts use `scripts/lib/droboticflow_env.py`, and shell wrappers use `scripts/lib/service-endpoints.sh`. `BASE_URL` is accepted by browser-oriented acceptance scripts, while deployment/reconcile wrappers preserve `API_BASE_URL` as their primary environment variable. Explicit endpoint values must be absolute `http://` or `https://` URLs so malformed values fail before Vite, curl, Playwright, or urllib requests are attempted. Browser-side `VITE_API_BASE_URL` follows the same rule, with the empty value reserved for same-origin API requests.
|
||||
|
||||
@ -244,11 +247,12 @@ npm run smoke
|
||||
Run a smoke test against a deployed cluster:
|
||||
|
||||
```bash
|
||||
SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true \
|
||||
SMOKE_SOURCE_PATH=<path-visible-to-operator-runner> \
|
||||
./scripts/smoke-test.sh http://<node-ip>:30080
|
||||
```
|
||||
|
||||
`SMOKE_SOURCE_PATH` can be a filesystem/PVC path or a `bos://bucket/prefix` path. The real-source operator now infers `LOCAL` or `BOS` from that path and fails fast when no real source is configured, so an empty source configuration cannot pass smoke by producing an empty manifest.
|
||||
`SMOKE_SOURCE_PATH` can be a filesystem/PVC path or a `bos://bucket/prefix` path. The real-source operator now infers `LOCAL` or `BOS` from that path and fails fast when no real source is configured, so an empty source configuration cannot pass smoke by producing an empty manifest. Set `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` for deployed-environment smoke so the script fails instead of falling back to localhost when `BASE_URL` / `API_BASE_URL` is omitted.
|
||||
|
||||
Run browser object-storage acceptance against a deployed cluster:
|
||||
|
||||
|
||||
@ -1078,16 +1078,6 @@ def summarize_lerobot_from_inputs(inputs: list[dict[str, Any]]) -> dict[str, Any
|
||||
}
|
||||
|
||||
|
||||
def parse_float_param(params: dict[str, str], key: str, default: float) -> float:
|
||||
raw = params.get(key, "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
return float(raw)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def parse_positive_float_param(params: dict[str, str], key: str, default: float) -> float:
|
||||
raw = params.get(key, "").strip()
|
||||
if not raw:
|
||||
@ -1129,16 +1119,6 @@ def float_value(value: Any, default: float) -> float:
|
||||
return default
|
||||
|
||||
|
||||
def parse_int_param(params: dict[str, str], key: str, default: int) -> int:
|
||||
raw = params.get(key, "").strip()
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
return int(raw)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def parse_positive_int_param(params: dict[str, str], key: str, default: int) -> int:
|
||||
raw = params.get(key, "").strip()
|
||||
if not raw:
|
||||
|
||||
@ -64,7 +64,7 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
if cfg.OperatorRunnerImage == "" {
|
||||
fmt.Println("[worker] OPERATOR_RUNNER_IMAGE is required; deploy scripts default it to the current git image tag")
|
||||
fmt.Println("[worker] OPERATOR_RUNNER_IMAGE is required; set it explicitly or configure REGISTRY_HOST and REGISTRY_NAMESPACE for deploy/render scripts")
|
||||
os.Exit(1)
|
||||
}
|
||||
runtime, err := worker.NewRuntime(
|
||||
|
||||
@ -211,7 +211,7 @@ http://<node-ip>:30080
|
||||
|
||||
- The main manifest only deploys the application components. It does not create MongoDB resources in-cluster.
|
||||
- `MONGO_URI` is required by API and worker startup and must be a concrete `mongodb://` or `mongodb+srv://` URI without whitespace or unreplaced placeholder markers. Kubernetes deployment must reference an external MongoDB instance; localhost, `mongodb:27017`, and in-cluster `.svc` addresses are rejected by `scripts/deploy-k8s.sh`.
|
||||
- Startup configuration is validated before manifest output and before `scripts/deploy-k8s.sh` performs any cluster writes. `MONGO_CONNECT_RETRIES`, `MONGO_CONNECT_RETRY_DELAY_MS`, and `WORKER_POLL_INTERVAL_MS` must be positive integers; `API_REPLICAS`, `WORKER_REPLICAS`, and `OPERATOR_JOB_TTL_SECONDS` must be non-negative integers; `API_NODE_PORT` must be in `30000..32767` when `API_SERVICE_TYPE=NodePort`; boolean env values must be one of `true/false`, `1/0`, `yes/no`, or `on/off`. Kubernetes namespace, image-pull-secret, workspace PVC, and public LoadBalancer Service names must be lowercase DNS-compatible values; workspace mount paths must be absolute container paths; worker service URLs must be absolute `http://` or `https://` URLs; non-empty BOS and AI endpoint values must also be absolute `http://` or `https://` URLs; if `AI_API_KEY` is configured, both `AI_API_BASE_URL` and `AI_MODEL` are required and there is no provider/model hardcoded fallback; container image references must be non-empty and whitespace-free. `scripts/render-manifests.sh`, `scripts/deploy-k8s.sh`, and `scripts/build-and-push-images.sh` require explicit `API_IMAGE`, `WORKER_IMAGE`, and `OPERATOR_RUNNER_IMAGE` values unless `REGISTRY_HOST` and `REGISTRY_NAMESPACE` are both set. Multi-line ConfigMap block values, including `OPERATOR_GPU_TOLERATIONS_JSON`, `OPERATOR_COMMAND_ALLOWLIST_REGEX`, and `OPERATOR_COMMAND_DENYLIST_REGEX`, are indented before manifest output so formatted JSON and multi-line regex lists remain valid YAML.
|
||||
- Startup configuration is validated before manifest output and before `scripts/deploy-k8s.sh` performs any cluster writes. `MONGO_CONNECT_RETRIES`, `MONGO_CONNECT_RETRY_DELAY_MS`, and `WORKER_POLL_INTERVAL_MS` must be positive integers; `API_REPLICAS`, `WORKER_REPLICAS`, and `OPERATOR_JOB_TTL_SECONDS` must be non-negative integers; `API_NODE_PORT` must be in `30000..32767` when `API_SERVICE_TYPE=NodePort`; boolean env values must be one of `true/false`, `1/0`, `yes/no`, or `on/off`. Kubernetes namespace, image-pull-secret, workspace PVC, and public LoadBalancer Service names must be lowercase DNS-compatible values; workspace mount paths must be absolute container paths; worker service URLs must be absolute `http://` or `https://` URLs; non-empty BOS and AI endpoint values must also be absolute `http://` or `https://` URLs; if `AI_API_KEY` is configured, both `AI_API_BASE_URL` and `AI_MODEL` are required and there is no provider/model hardcoded fallback; container image references must be non-empty and whitespace-free. `scripts/render-manifests.sh`, `scripts/deploy-k8s.sh`, and `scripts/build-and-push-images.sh` require explicit `API_IMAGE`, `WORKER_IMAGE`, and `OPERATOR_RUNNER_IMAGE` values unless `REGISTRY_HOST` and `REGISTRY_NAMESPACE` are both set, and all three scripts reject malformed image references before Docker or Kubernetes calls. Multi-line ConfigMap block values, including `OPERATOR_GPU_TOLERATIONS_JSON`, `OPERATOR_COMMAND_ALLOWLIST_REGEX`, and `OPERATOR_COMMAND_DENYLIST_REGEX`, are indented before manifest output so formatted JSON and multi-line regex lists remain valid YAML.
|
||||
- `BOS_ENDPOINT` is rendered into the API and worker ConfigMap as an empty value when unset. Leave it unset only when every BOS scan config or storage connection provides its own endpoint; if both the workflow/connection endpoint and global `BOS_ENDPOINT` are empty, real BOS operations fail with an explicit missing-endpoint error instead of falling back to a hardcoded region.
|
||||
- API and worker entrypoints are Go binaries with Mongo startup retry, so they tolerate the database becoming ready after the Deployments start.
|
||||
- The API proxies `/worker-api/*` to `WORKER_API_BASE_URL`, defaulting to the in-cluster `http://droboticflow-worker:3002` Service. Override it only when the worker Service name, namespace, or exposure model differs.
|
||||
@ -231,7 +231,7 @@ http://<node-ip>:30080
|
||||
- `DROBOTICFLOW_TRUSTED_PROXY_CIDRS` controls which reverse proxies may supply `X-Forwarded-For` / `X-Real-IP` for rate limiting. Leave it empty unless ingress or load balancer source ranges are known.
|
||||
- Session-authenticated unsafe requests require `X-Droboticflow-CSRF-Token` to match the readable `droboticflow_csrf` cookie. The bundled frontend sets this header automatically.
|
||||
- `IMAGE_TAG` defaults to the current git short SHA in build, deploy, and render scripts only when explicit registry coordinates are configured. Set it explicitly for release names or rollback-friendly tags.
|
||||
- `scripts/smoke-test.sh` requires `SMOKE_SOURCE_PATH`, a real path visible to operator-runner Jobs. The `真实数据源` runner infers filesystem/PVC paths as `LOCAL` and `bos://bucket/prefix` paths as `BOS`; if no real source is configured, the source node fails instead of emitting an empty manifest. It can exercise session-authenticated deployments when `DROBOTICFLOW_SMOKE_EMAIL` and `DROBOTICFLOW_SMOKE_PASSWORD` are set; it logs in, persists cookies, and sends the CSRF header on write requests.
|
||||
- `scripts/smoke-test.sh` requires `SMOKE_SOURCE_PATH`, a real path visible to operator-runner Jobs. The `真实数据源` runner infers filesystem/PVC paths as `LOCAL` and `bos://bucket/prefix` paths as `BOS`; if no real source is configured, the source node fails instead of emitting an empty manifest. Set `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` for deployed-environment smoke so missing `BASE_URL` / `API_BASE_URL` fails instead of falling back to localhost. The script can exercise session-authenticated deployments when `DROBOTICFLOW_SMOKE_EMAIL` and `DROBOTICFLOW_SMOKE_PASSWORD` are set; it logs in, persists cookies, and sends the CSRF header on write requests.
|
||||
- `scripts/reconcile-system-data.sh` delegates to `scripts/reconcile_system_data.py` and uses the same protected-client behavior. Set `DROBOTICFLOW_API_TOKEN` for token-protected APIs when session auth is not enforced. For public session-auth deployments, set `DROBOTICFLOW_SMOKE_EMAIL` and `DROBOTICFLOW_SMOKE_PASSWORD` to an organization admin account because the reconcile flow updates settings, users, templates, and operators. Builtin operators are read from the live `/api/operators` catalog and refreshed to `OPERATOR_RUNNER_IMAGE`; the script does not carry a separate builtin catalog.
|
||||
- `scripts/object-storage-acceptance.mjs` runs browser create, preview, rename, upload, delete, and optional video playback checks against a live storage connection. It never rewrites storage connection metadata; provide `OBJECT_STORAGE_CONNECTION_ID`, `OBJECT_STORAGE_CONNECTION_NAME`, or `OBJECT_STORAGE_BUCKET` explicitly. `OBJECT_STORAGE_ACCEPTANCE_PAYLOAD_FILE` is required and must point to a non-empty real UTF-8 text or JSON payload used for create/upload/preview checks; set `OBJECT_STORAGE_ACCEPTANCE_CONTENT_TYPE` when auto-detection is not enough. Set `OBJECT_STORAGE_VIDEO_KEY` only when an existing playable object should be validated. `PLAYWRIGHT_IGNORE_HTTPS_ERRORS` must be an explicit boolean spelling when set. The script supports `DROBOTICFLOW_API_TOKEN` for token-protected non-session APIs and `DROBOTICFLOW_SMOKE_EMAIL` / `DROBOTICFLOW_SMOKE_PASSWORD` for session-authenticated browser access.
|
||||
- `scripts/playwright-preview-operators-check.mjs` runs browser/operator preview acceptance with caller-supplied payloads and real source paths. Provide `PREVIEW_CHECK_CSV_PAYLOAD_FILE`, `PREVIEW_CHECK_TEXT_PAYLOAD_FILE`, `PREVIEW_CHECK_JSON_PAYLOAD_FILE`, `PREVIEW_CHECK_JSON_EXTRACT_FIELD`, `PREVIEW_CHECK_JSON_FILTER_FIELD`, `PREVIEW_CHECK_JSON_FIELD_MAP_JSON`, `PREVIEW_CHECK_SCAN_SOURCE_PATH`, `PREVIEW_CHECK_ROSBAG_SOURCE_PATH`, and `PREVIEW_CHECK_LEROBOT_SOURCE_PATH`. The script no longer generates built-in CSV/text/JSON/ROSBag/LeRobot data. Optional `PREVIEW_CHECK_SCAN_MIN_FILES` must be an exact safe positive integer.
|
||||
|
||||
@ -61,13 +61,14 @@ Data-asset APIs apply the same stale-data rule for local external payloads. If `
|
||||
|
||||
## Local Verification
|
||||
|
||||
Run the full verification loop:
|
||||
Run local verification and the security audit as separate gates:
|
||||
|
||||
```bash
|
||||
npm run verify
|
||||
npm run verify:local
|
||||
npm run verify:security
|
||||
```
|
||||
|
||||
This includes TypeScript/Go linting, frontend Prettier formatting checks, Go tests, Python operator-runner tests, frontend node tests, production build, `npm audit --audit-level=moderate`, and Kubernetes manifest dry-run validation.
|
||||
`verify:local` includes TypeScript/Go/script linting, frontend Prettier formatting checks, Go tests, Python operator-runner tests, frontend node tests, production build, and Kubernetes manifest dry-run validation. `verify:security` runs `npm audit --audit-level=moderate` through a retry wrapper so transient registry failures do not prevent local validation from running first. `npm run verify` runs both in that order.
|
||||
|
||||
## Runtime Metrics
|
||||
|
||||
@ -117,6 +118,8 @@ SMOKE_SOURCE_PATH=<path-visible-to-operator-runner> \
|
||||
|
||||
`SMOKE_SOURCE_PATH` may be a filesystem/PVC path or a `bos://bucket/prefix` path. The `真实数据源` runner infers `LOCAL` or `BOS` from the path and rejects a run that has no `SOURCE_TEXT`, `SOURCE_JSON`, `SOURCE_PATH`, configured BOS source, local `STORAGE_PREFIX`, or upstream artifact input. This prevents an unconfigured source node from completing with an empty manifest.
|
||||
|
||||
For deployed smoke, pass the deployed URL or set `BASE_URL` / `API_BASE_URL`, and set `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` so localhost defaults are disabled.
|
||||
|
||||
Run the production-style WJZH real-data regression:
|
||||
|
||||
```bash
|
||||
|
||||
@ -9,8 +9,8 @@ This note tracks the production-facing safeguards that must stay true for Drobod
|
||||
- Worker proxy health, stats, metrics, and retry calls use short upstream timeouts. Manual `process-once` uses a long timeout because it can execute real queued work.
|
||||
- Explicit shell operators are disabled by default through `OPERATOR_ALLOW_SHELL_COMMANDS=false`; allowlist and denylist regexes are platform-controlled and cannot be overridden by workflow hyperparams.
|
||||
- Environment-specific values such as `K8S_ENDPOINT`, registry host/namespace, registry username, deploy image references, and workspace BOS bucket have no private repository defaults; provide them from local deploy secrets or CI secrets.
|
||||
- Manifest rendering and deployment wrappers reject invalid Kubernetes namespace/Secret/PVC/Service names, non-absolute workspace mount paths, malformed worker/AI/BOS service URLs, and image references with whitespace before emitting manifests or calling `kubectl`. Multi-line ConfigMap block values are indented before manifest output so formatted GPU tolerations JSON and multi-line shell command policy regexes remain valid YAML.
|
||||
- Local development and acceptance endpoints are resolved through shared helpers instead of per-script localhost literals. Vite dev proxy targets can be overridden with `VITE_DEV_API_BASE_URL` and `VITE_DEV_WORKER_BASE_URL`; browser-oriented acceptance scripts accept `BASE_URL` or `API_BASE_URL`, while reconcile/deploy wrappers keep `API_BASE_URL` as the primary operational variable. Explicit endpoint values must be absolute `http://` or `https://` URLs and fail before Vite, curl, Playwright, or urllib requests are attempted. Browser-side `VITE_API_BASE_URL` follows the same absolute http(s) rule, with an empty value reserved for same-origin API calls.
|
||||
- Manifest rendering, image build, and deployment wrappers reject invalid Kubernetes namespace/Secret/PVC/Service names, non-absolute workspace mount paths, malformed worker/AI/BOS service URLs, and malformed image references before emitting manifests, building images, or calling `kubectl`. Multi-line ConfigMap block values are indented before manifest output so formatted GPU tolerations JSON and multi-line shell command policy regexes remain valid YAML.
|
||||
- Local development and acceptance endpoints are resolved through shared helpers instead of per-script localhost literals. Vite dev proxy targets can be overridden with `VITE_DEV_API_BASE_URL` and `VITE_DEV_WORKER_BASE_URL`; browser-oriented acceptance scripts accept `BASE_URL` or `API_BASE_URL`, while reconcile/deploy wrappers keep `API_BASE_URL` as the primary operational variable. Explicit endpoint values must be absolute `http://` or `https://` URLs and fail before Vite, curl, Playwright, or urllib requests are attempted. Browser-side `VITE_API_BASE_URL` follows the same absolute http(s) rule, with an empty value reserved for same-origin API calls. Deployed smoke runs should set `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` so an omitted endpoint cannot fall back to localhost.
|
||||
- Data reconciliation uses real API calls and supports protected deployments through `DROBOTICFLOW_API_TOKEN` or session login credentials plus CSRF headers. Public session-auth environments must use an organization admin account for reconcile because it updates settings and management resources.
|
||||
- Python live-acceptance and WJZH regression scripts use the shared `DroboticflowClient` helper for API token headers, session-login cookies, CSRF propagation, JSON calls, and binary object-storage calls. WJZH quality thresholds are caller-supplied through `QUALITY_MIN_CLARITY`, `QUALITY_MAX_SHAKE`, and `QUALITY_MAX_HAND_OCCLUSION`; window, slicing, and annotation timing are caller-supplied through the documented `*_SECONDS` values; sampling thresholds are caller-supplied through `SAMPLING_RULES_JSON`. The scripts do not carry built-in audit rules. New Python acceptance scripts should reuse that helper instead of carrying their own auth/request stack.
|
||||
- Python WJZH and X-platform acceptance scripts validate quality thresholds through the shared normalized-threshold gate. `QUALITY_MIN_CLARITY`, `QUALITY_MAX_SHAKE`, `QUALITY_MAX_HAND_OCCLUSION`, `XPLATFORM_MIN_CLARITY`, `XPLATFORM_MAX_SHAKE`, and `XPLATFORM_MAX_HAND_OCCLUSION` must be finite values in `[0, 1]`; invalid values fail before live workflows are created.
|
||||
@ -57,19 +57,21 @@ This note tracks the production-facing safeguards that must stay true for Drobod
|
||||
Run local verification before merging:
|
||||
|
||||
```bash
|
||||
npm run verify
|
||||
npm run verify:local
|
||||
npm run verify:security
|
||||
```
|
||||
|
||||
This gate includes shell syntax checks for operational scripts, so GPU smoke, deployment, cleanup, and reconciliation wrappers are parsed before a change is accepted.
|
||||
The local gate includes shell syntax checks for operational scripts, so GPU smoke, deployment, cleanup, and reconciliation wrappers are parsed before a change is accepted. The security gate runs `npm audit --audit-level=moderate` through the retry wrapper; `npm run verify` runs both gates in order.
|
||||
|
||||
Run deployed smoke after rollout:
|
||||
|
||||
```bash
|
||||
SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true \
|
||||
SMOKE_SOURCE_PATH=<path-visible-to-operator-runner> \
|
||||
./scripts/smoke-test.sh http://<node-ip>:30080
|
||||
```
|
||||
|
||||
The smoke source path may be local/PVC-backed or `bos://bucket/prefix`. The source node now fails fast when no real input source is configured, so a misconfigured workflow cannot pass by producing an empty source manifest.
|
||||
The smoke source path may be local/PVC-backed or `bos://bucket/prefix`. The source node now fails fast when no real input source is configured, so a misconfigured workflow cannot pass by producing an empty source manifest. `SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true` keeps deployed smoke from silently using local defaults.
|
||||
|
||||
Run browser object-storage acceptance after rollout:
|
||||
|
||||
|
||||
@ -27,8 +27,10 @@
|
||||
"test": "go test ./...",
|
||||
"test:python": "python3 -m pytest backend/operator-runner tests/backend",
|
||||
"test:frontend": "node --test tests/frontend/*.test.mjs",
|
||||
"audit:moderate": "npm audit --audit-level=moderate",
|
||||
"verify": "npm run lint && npm run format:check && npm test && npm run test:python && npm run test:frontend && npm run build && npm run audit:moderate && npm run k8s:validate",
|
||||
"audit:moderate": "./scripts/npm-audit-retry.sh --audit-level=moderate",
|
||||
"verify:local": "npm run lint && npm run format:check && npm test && npm run test:python && npm run test:frontend && npm run build && npm run k8s:validate",
|
||||
"verify:security": "npm run audit:moderate",
|
||||
"verify": "npm run verify:local && npm run verify:security",
|
||||
"docker:build:api": "docker build -f Dockerfile.api -t droboticflow-api:local .",
|
||||
"docker:build:worker": "docker build -f Dockerfile.worker -t droboticflow-worker:local .",
|
||||
"docker:build:operator-runner": "docker build -f Dockerfile.operator-runner -t droboticflow-operator-runner:local .",
|
||||
|
||||
@ -22,6 +22,11 @@ PUSH="${PUSH:-true}"
|
||||
# shellcheck source=scripts/lib/image-config.sh
|
||||
source "$ROOT_DIR/scripts/lib/image-config.sh"
|
||||
droboticflow_resolve_image_configuration
|
||||
# shellcheck source=scripts/lib/deploy-validation.sh
|
||||
source "$ROOT_DIR/scripts/lib/deploy-validation.sh"
|
||||
validate_image_reference API_IMAGE "$API_IMAGE"
|
||||
validate_image_reference WORKER_IMAGE "$WORKER_IMAGE"
|
||||
validate_image_reference OPERATOR_RUNNER_IMAGE "$OPERATOR_RUNNER_IMAGE"
|
||||
|
||||
docker buildx inspect >/dev/null 2>&1 || docker buildx create --use >/dev/null
|
||||
|
||||
|
||||
27
scripts/npm-audit-retry.sh
Executable file
27
scripts/npm-audit-retry.sh
Executable file
@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
attempts="${NPM_AUDIT_ATTEMPTS:-3}"
|
||||
delay_seconds="${NPM_AUDIT_RETRY_DELAY_SECONDS:-5}"
|
||||
|
||||
if ! [[ "$attempts" =~ ^[0-9]+$ ]] || (( attempts < 1 )); then
|
||||
echo "ERROR: NPM_AUDIT_ATTEMPTS must be a positive integer." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [[ "$delay_seconds" =~ ^[0-9]+$ ]]; then
|
||||
echo "ERROR: NPM_AUDIT_RETRY_DELAY_SECONDS must be a non-negative integer." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for ((attempt = 1; attempt <= attempts; attempt += 1)); do
|
||||
if npm audit "$@"; then
|
||||
exit 0
|
||||
fi
|
||||
status=$?
|
||||
if (( attempt == attempts )); then
|
||||
exit "$status"
|
||||
fi
|
||||
echo "npm audit failed on attempt $attempt/$attempts; retrying in ${delay_seconds}s..." >&2
|
||||
sleep "$delay_seconds"
|
||||
done
|
||||
@ -5,9 +5,30 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
source "$SCRIPT_DIR/lib/service-endpoints.sh"
|
||||
|
||||
SMOKE_REQUIRE_EXPLICIT_ENDPOINT="${SMOKE_REQUIRE_EXPLICIT_ENDPOINT:-false}"
|
||||
case "$(printf '%s' "$SMOKE_REQUIRE_EXPLICIT_ENDPOINT" | tr '[:upper:]' '[:lower:]')" in
|
||||
1|true|yes|on)
|
||||
SMOKE_REQUIRE_EXPLICIT_ENDPOINT="true"
|
||||
;;
|
||||
0|false|no|off)
|
||||
SMOKE_REQUIRE_EXPLICIT_ENDPOINT="false"
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: SMOKE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ $# -gt 0 ]]; then
|
||||
API_BASE_URL="$(droboticflow_first_service_url "$1")"
|
||||
WORKER_BASE_URL="$(droboticflow_first_service_url "${2:-}" "${WORKER_BASE_URL:-}")"
|
||||
elif [[ "$SMOKE_REQUIRE_EXPLICIT_ENDPOINT" == "true" ]]; then
|
||||
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}")"
|
||||
WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}")"
|
||||
if [[ -z "$API_BASE_URL" ]]; then
|
||||
echo "ERROR: API_BASE_URL or BASE_URL is required when SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true." >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
API_BASE_URL="$(droboticflow_first_service_url "${API_BASE_URL:-}" "${BASE_URL:-}" "$(droboticflow_default_api_base_url)")"
|
||||
WORKER_BASE_URL="$(droboticflow_first_service_url "${WORKER_BASE_URL:-}" "${WORKER_API_BASE_URL:-}" "$(droboticflow_default_worker_base_url)")"
|
||||
|
||||
46
tests/backend/build_push_images_script_test.py
Normal file
46
tests/backend/build_push_images_script_test.py
Normal file
@ -0,0 +1,46 @@
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "build-and-push-images.sh"
|
||||
|
||||
|
||||
def run_build_script(extra_env: dict[str, str]) -> subprocess.CompletedProcess:
|
||||
env = os.environ.copy()
|
||||
for key in (
|
||||
"API_IMAGE",
|
||||
"DEPLOY_ENV",
|
||||
"DROBOTICFLOW_DEPLOY_ENV_LOADED",
|
||||
"IMAGE_TAG",
|
||||
"OPERATOR_RUNNER_IMAGE",
|
||||
"PUSH",
|
||||
"REGISTRY_HOST",
|
||||
"REGISTRY_NAMESPACE",
|
||||
"WORKER_IMAGE",
|
||||
):
|
||||
env.pop(key, None)
|
||||
env["DROBOTICFLOW_DEPLOY_ENV_LOADED"] = "test"
|
||||
env.update(extra_env)
|
||||
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
||||
|
||||
|
||||
def test_build_and_push_requires_explicit_images_or_registry_configuration():
|
||||
result = run_build_script({})
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "API_IMAGE is required unless REGISTRY_HOST and REGISTRY_NAMESPACE are both set" in result.stderr
|
||||
|
||||
|
||||
def test_build_and_push_rejects_invalid_explicit_image_before_docker():
|
||||
result = run_build_script(
|
||||
{
|
||||
"API_IMAGE": "registry.example.com/dataflow/droboticflow-api:test bad",
|
||||
"WORKER_IMAGE": "registry.example.com/dataflow/droboticflow-worker:test",
|
||||
"OPERATOR_RUNNER_IMAGE": "registry.example.com/dataflow/droboticflow-operator-runner:test",
|
||||
}
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "API_IMAGE must be a non-empty container image reference" in result.stderr
|
||||
79
tests/backend/npm_audit_retry_script_test.py
Normal file
79
tests/backend/npm_audit_retry_script_test.py
Normal file
@ -0,0 +1,79 @@
|
||||
import os
|
||||
import pathlib
|
||||
import stat
|
||||
import subprocess
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "npm-audit-retry.sh"
|
||||
|
||||
|
||||
def run_audit_retry(tmp_path: pathlib.Path, extra_env: dict[str, str]) -> subprocess.CompletedProcess:
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
counter_file = tmp_path / "npm-calls"
|
||||
fake_npm = bin_dir / "npm"
|
||||
fake_npm.write_text(
|
||||
f"""#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
if [[ "${{1:-}}" != "audit" ]]; then
|
||||
echo "unexpected npm command: $*" >&2
|
||||
exit 2
|
||||
fi
|
||||
count=0
|
||||
if [[ -f "{counter_file}" ]]; then
|
||||
count="$(cat "{counter_file}")"
|
||||
fi
|
||||
count=$((count + 1))
|
||||
printf '%s' "$count" > "{counter_file}"
|
||||
if (( count < 3 )); then
|
||||
exit 42
|
||||
fi
|
||||
exit 0
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_npm.chmod(fake_npm.stat().st_mode | stat.S_IXUSR)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}"
|
||||
for key in ("NPM_AUDIT_ATTEMPTS", "NPM_AUDIT_RETRY_DELAY_SECONDS"):
|
||||
env.pop(key, None)
|
||||
env.update(extra_env)
|
||||
return subprocess.run(
|
||||
["bash", str(SCRIPT), "--audit-level=moderate"],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def test_npm_audit_retry_retries_transient_failures(tmp_path):
|
||||
result = run_audit_retry(
|
||||
tmp_path,
|
||||
{
|
||||
"NPM_AUDIT_ATTEMPTS": "3",
|
||||
"NPM_AUDIT_RETRY_DELAY_SECONDS": "0",
|
||||
},
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "npm audit failed on attempt 1/3" in result.stderr
|
||||
assert "npm audit failed on attempt 2/3" in result.stderr
|
||||
assert (tmp_path / "npm-calls").read_text(encoding="utf-8") == "3"
|
||||
|
||||
|
||||
def test_npm_audit_retry_rejects_invalid_attempts(tmp_path):
|
||||
result = run_audit_retry(
|
||||
tmp_path,
|
||||
{
|
||||
"NPM_AUDIT_ATTEMPTS": "0",
|
||||
"NPM_AUDIT_RETRY_DELAY_SECONDS": "0",
|
||||
},
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "NPM_AUDIT_ATTEMPTS must be a positive integer" in result.stderr
|
||||
assert not (tmp_path / "npm-calls").exists()
|
||||
37
tests/backend/smoke_script_test.py
Normal file
37
tests/backend/smoke_script_test.py
Normal file
@ -0,0 +1,37 @@
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "smoke-test.sh"
|
||||
|
||||
|
||||
def run_smoke(extra_env: dict[str, str]) -> subprocess.CompletedProcess:
|
||||
env = os.environ.copy()
|
||||
for key in (
|
||||
"API_BASE_URL",
|
||||
"BASE_URL",
|
||||
"SMOKE_REQUIRE_EXPLICIT_ENDPOINT",
|
||||
"SMOKE_RUN_TIMEOUT_SECONDS",
|
||||
"SMOKE_SOURCE_PATH",
|
||||
"WORKER_API_BASE_URL",
|
||||
"WORKER_BASE_URL",
|
||||
):
|
||||
env.pop(key, None)
|
||||
env.update(extra_env)
|
||||
return subprocess.run(["bash", str(SCRIPT)], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
|
||||
|
||||
|
||||
def test_smoke_requires_explicit_endpoint_when_deployed_mode_is_enabled():
|
||||
result = run_smoke({"SMOKE_REQUIRE_EXPLICIT_ENDPOINT": "true", "SMOKE_SOURCE_PATH": "/workspace/real-data"})
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "API_BASE_URL or BASE_URL is required when SMOKE_REQUIRE_EXPLICIT_ENDPOINT=true" in result.stderr
|
||||
|
||||
|
||||
def test_smoke_rejects_invalid_explicit_endpoint_mode_boolean():
|
||||
result = run_smoke({"SMOKE_REQUIRE_EXPLICIT_ENDPOINT": "maybe", "SMOKE_SOURCE_PATH": "/workspace/real-data"})
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "SMOKE_REQUIRE_EXPLICIT_ENDPOINT must be a boolean" in result.stderr
|
||||
Loading…
x
Reference in New Issue
Block a user