From d960fab4d0a4a0bcc1914c6657db78cfbb95adc3 Mon Sep 17 00:00:00 2001 From: eust-w Date: Sat, 22 Aug 2026 07:29:25 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=92=20Publish=20Qwen=20cache=20release?= =?UTF-8?q?=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../qwen_behavior_rough_cut/repository.json | 4 +- docs/qwen-node-local-cache.md | 4 + internal/api/builtin_operators_catalog.json | 4 +- k8s/qwen-node-cache.local-pv.template.yaml | 45 +++++++ k8s/qwen-node-cache.prepare-dir.template.yaml | 41 +++++++ operator-images/builtin/operator-images.json | 4 +- .../qwen_behavior_rough_cut/release.json | 32 +++-- .../qwen_behavior_rough_cut/repository.json | 6 +- scripts/build-poc-images-kaniko.sh | 81 ++++++++++++- scripts/render-qwen-node-cache-local-pv.sh | 64 ++++++++++ scripts/validate_kaniko_operator_bases.py | 71 +++++++++++ scripts/verify_registry_manifest.py | 111 ++++++++++++++++++ .../kaniko_operator_base_preflight_test.py | 90 ++++++++++++++ .../qwen_node_local_pv_template_test.py | 47 ++++++++ 14 files changed, 581 insertions(+), 23 deletions(-) create mode 100644 k8s/qwen-node-cache.local-pv.template.yaml create mode 100644 k8s/qwen-node-cache.prepare-dir.template.yaml create mode 100755 scripts/render-qwen-node-cache-local-pv.sh create mode 100644 scripts/validate_kaniko_operator_bases.py create mode 100644 scripts/verify_registry_manifest.py create mode 100644 tests/backend/kaniko_operator_base_preflight_test.py create mode 100644 tests/backend/qwen_node_local_pv_template_test.py diff --git a/backend/operators/builtin/qwen_behavior_rough_cut/repository.json b/backend/operators/builtin/qwen_behavior_rough_cut/repository.json index 41dfefd4..a4297e15 100644 --- a/backend/operators/builtin/qwen_behavior_rough_cut/repository.json +++ b/backend/operators/builtin/qwen_behavior_rough_cut/repository.json @@ -16,8 +16,8 @@ ], "containerEntrypoint": "backend/operator-runner/main.py", "entrypoint": "operator.py", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7", "metadata": "operator.json", "name": "Qwen3-VL 语义行为粗切(PTS)", "operatorId": "operator-builtin-qwen-behavior-rough-cut", diff --git a/docs/qwen-node-local-cache.md b/docs/qwen-node-local-cache.md index 5fbd43af..b59679e6 100644 --- a/docs/qwen-node-local-cache.md +++ b/docs/qwen-node-local-cache.md @@ -53,6 +53,10 @@ manifest SHA、Qwen repository/revision、config/processor/index 和完整布局 1. 保持 feature disabled 部署并验证 Worker 健康。 2. 在 GPU node 上只创建 `/home/cce/droboticflow-qwen-cache`,权限 `10001:10001`、`0750`; 不执行递归 chmod,也不格式化或挂载任何盘。 + 使用 `scripts/render-qwen-node-cache-local-pv.sh --prepare-job` 只渲染受限的一次性 + 准备 Job;该脚本不会执行 apply。默认 PV/PVC 同样通过 + `scripts/render-qwen-node-cache-local-pv.sh` 仅渲染,固定为 80Gi、Retain 与 hostname + affinity,模板不属于 `deploy-k8s.sh`。 3. 创建一个带 hostname nodeAffinity 的 `ReadWriteOnce` Local PV/PVC,再在 Worker 配置中 填入相同 PVC、mount path 与 hostname selector。 4. 先用真实固定 manifest 预热,检查 receipt、17.5GB 级别文件和 SHA 证据;再启用开关。 diff --git a/internal/api/builtin_operators_catalog.json b/internal/api/builtin_operators_catalog.json index 0a2af52c..2f9af2df 100644 --- a/internal/api/builtin_operators_catalog.json +++ b/internal/api/builtin_operators_catalog.json @@ -13102,8 +13102,8 @@ } ], "id": "operator-builtin-qwen-behavior-rough-cut", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7", "inputs": [ { "description": "视频人脸脱敏输出:blurred-manifest 与完整脱敏视频 artifact", diff --git a/k8s/qwen-node-cache.local-pv.template.yaml b/k8s/qwen-node-cache.local-pv.template.yaml new file mode 100644 index 00000000..dcf21bc2 --- /dev/null +++ b/k8s/qwen-node-cache.local-pv.template.yaml @@ -0,0 +1,45 @@ +# Render-only template. It is deliberately not included by deploy-k8s.sh. +# First run the node disk preflight and the separate, explicit directory +# preparation Job. Do not point this PV at a root-disk or kubelet path. +apiVersion: v1 +kind: PersistentVolume +metadata: + name: __QWEN_NODE_CACHE_PV_NAME__ + labels: + app: droboticflow + component: qwen-node-cache +spec: + capacity: + storage: __QWEN_NODE_CACHE_SIZE__ + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: droboticflow-qwen-node-local + local: + path: __QWEN_NODE_CACHE_HOST_PATH__ + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - __QWEN_NODE_CACHE_NODE_NAME__ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: __QWEN_NODE_CACHE_PVC_NAME__ + namespace: __QWEN_NODE_CACHE_NAMESPACE__ + labels: + app: droboticflow + component: qwen-node-cache +spec: + accessModes: + - ReadWriteOnce + storageClassName: droboticflow-qwen-node-local + volumeName: __QWEN_NODE_CACHE_PV_NAME__ + resources: + requests: + storage: __QWEN_NODE_CACHE_SIZE__ diff --git a/k8s/qwen-node-cache.prepare-dir.template.yaml b/k8s/qwen-node-cache.prepare-dir.template.yaml new file mode 100644 index 00000000..cffa6c6b --- /dev/null +++ b/k8s/qwen-node-cache.prepare-dir.template.yaml @@ -0,0 +1,41 @@ +# Render-only, one-shot directory preparation Job. It is never applied by the +# platform deployment. Its only mutation is creating/chowning the exact cache +# directory under the independently verified /home/cce NVMe mount. +apiVersion: batch/v1 +kind: Job +metadata: + name: __QWEN_NODE_CACHE_PREPARE_JOB_NAME__ + namespace: __QWEN_NODE_CACHE_NAMESPACE__ + labels: + app: droboticflow + component: qwen-node-cache-prepare +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 3600 + template: + spec: + nodeName: __QWEN_NODE_CACHE_NODE_NAME__ + restartPolicy: Never + containers: + - name: prepare + image: __QWEN_NODE_CACHE_PREPARE_IMAGE__ + command: ["/bin/sh", "-lc"] + args: + - | + set -eu + install -d -m 0750 -o 10001 -g 10001 /host-cache/__QWEN_NODE_CACHE_DIR_NAME__ + stat -c 'mode=%a owner=%u:%g path=%n' /host-cache/__QWEN_NODE_CACHE_DIR_NAME__ + securityContext: + runAsUser: 0 + runAsGroup: 0 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: host-cache-parent + mountPath: /host-cache + volumes: + - name: host-cache-parent + hostPath: + path: /home/cce + type: Directory diff --git a/operator-images/builtin/operator-images.json b/operator-images/builtin/operator-images.json index 236a3b50..214febf6 100644 --- a/operator-images/builtin/operator-images.json +++ b/operator-images/builtin/operator-images.json @@ -1469,8 +1469,8 @@ "orbbec-pts-csv" ], "dockerfile": "operator-repos/builtin/qwen_behavior_rough_cut/Dockerfile", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7", "operatorId": "operator-builtin-qwen-behavior-rough-cut", "operatorSlug": "qwen_behavior_rough_cut", "requiresGpu": true, diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/release.json b/operator-repos/builtin/qwen_behavior_rough_cut/release.json index cec18226..33f1da1f 100644 --- a/operator-repos/builtin/qwen_behavior_rough_cut/release.json +++ b/operator-repos/builtin/qwen_behavior_rough_cut/release.json @@ -1,24 +1,30 @@ { - "acceptance": {}, - "builtAt": "2026-08-21T09:11:39Z", - "digest": "sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc", + "acceptance": { + "artifactId": "fanyu-qwen-cache-disabled-compat-20260822-fanyu-v5-rough-cut-clips-manifest-clips-manifest.json", + "projectId": "project-271c97276a31d97dd3eff93e3f1ff3b0", + "runId": "fanyu-qwen-cache-disabled-compat-r2-20260822", + "verifiedAt": "2026-08-21T23:26:03Z", + "workflowId": "workflow-20b87364482c94d9df485d64453c9fac" + }, + "builtAt": "2026-08-21T23:26:03Z", + "digest": "sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7", "environmentReleases": { "poc": { - "digest": "sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc", + "digest": "sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7", "registry": "ccr-29eug8s3-pub.cnc.bj.baidubce.com", - "tag": "c859766-poc-fanyu-roughcut-20260821" + "tag": "b0734fe-poc-qwen-node-cache-r3-20260822063300" } }, - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc", - "kanikoJob": "droboticflow-kaniko-op-qwen-behavior-rough-cut-c859766-poc-fany", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7", + "kanikoJob": "droboticflow-kaniko-op-qwen-behavior-rough-cut-b0734fe-poc-qwen", "operatorId": "operator-builtin-qwen-behavior-rough-cut", "registry": "ccr-29eug8s3-pub.cnc.bj.baidubce.com", "schemaVersion": "droboticflow.operator.release/v1", "slug": "qwen_behavior_rough_cut", - "sourceCommit": "0f67e13fc5866353ce1f00152043e4b6bb6bf27b", - "sourceTreeSha256": "b33dd4212aace5bcfabb83ff55f1f52146c90e87840c09ae9b377f89cab4c6d2", - "tag": "c859766-poc-fanyu-roughcut-20260821" + "sourceCommit": "b0734fec382eac0f5a367064a378887dca7f2326", + "sourceTreeSha256": "c76621f817fe66bc11508a33cf28e168b1d7036b7e3f69855d40c254960261b8", + "tag": "b0734fe-poc-qwen-node-cache-r3-20260822063300" } diff --git a/operator-repos/builtin/qwen_behavior_rough_cut/repository.json b/operator-repos/builtin/qwen_behavior_rough_cut/repository.json index 57dd040f..3294a634 100644 --- a/operator-repos/builtin/qwen_behavior_rough_cut/repository.json +++ b/operator-repos/builtin/qwen_behavior_rough_cut/repository.json @@ -19,8 +19,8 @@ ], "containerEntrypoint": "backend/operator-runner/main.py", "entrypoint": "src/operator.py", - "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821", - "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc", + "image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300", + "imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7", "metadata": "operator.json", "name": "Qwen3-VL 语义行为粗切(PTS)", "operatorId": "operator-builtin-qwen-behavior-rough-cut", @@ -90,7 +90,7 @@ "runtimeSource": "operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py", "schemaVersion": "droboticflow.operator.repository/v1", "sourceRoot": "operator-repos/builtin/qwen_behavior_rough_cut", - "sourceTreeSha256": "b33dd4212aace5bcfabb83ff55f1f52146c90e87840c09ae9b377f89cab4c6d2", + "sourceTreeSha256": "c76621f817fe66bc11508a33cf28e168b1d7036b7e3f69855d40c254960261b8", "strictPocAcceptanceExternals": [], "type": "builtin" } diff --git a/scripts/build-poc-images-kaniko.sh b/scripts/build-poc-images-kaniko.sh index 972f7dec..5439f205 100755 --- a/scripts/build-poc-images-kaniko.sh +++ b/scripts/build-poc-images-kaniko.sh @@ -35,7 +35,9 @@ POC_KANIKO_NODE_NAME="${POC_KANIKO_NODE_NAME:-}" POC_KANIKO_BUILD_OPERATORS="${POC_KANIKO_BUILD_OPERATORS:-false}" POC_KANIKO_BUILD_OPERATOR_BASES="${POC_KANIKO_BUILD_OPERATOR_BASES:-true}" POC_KANIKO_OPERATOR_BASE_FILTER="${POC_KANIKO_OPERATOR_BASE_FILTER:-}" -POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS:-api,worker,drosync-coordinator,runner,headpose-service}" +# An explicit empty value is meaningful for an operator-only build. Use '-' so +# it is not silently expanded back to every platform component. +POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS-api,worker,drosync-coordinator,runner,headpose-service}" POC_KANIKO_OPERATOR_MANIFEST="${POC_KANIKO_OPERATOR_MANIFEST:-operator-images/builtin/operator-images.json}" POC_KANIKO_OPERATOR_FILTER="${POC_KANIKO_OPERATOR_FILTER:-}" POC_KANIKO_OPERATOR_CONCURRENCY="${POC_KANIKO_OPERATOR_CONCURRENCY:-4}" @@ -45,6 +47,9 @@ POC_KANIKO_OPERATOR_GPU_BASE_IMAGE="${POC_KANIKO_OPERATOR_GPU_BASE_IMAGE:-}" POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE="${POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE:-}" POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}" POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}" +POC_KANIKO_PRECHECK_ONLY="${POC_KANIKO_PRECHECK_ONLY:-false}" + +trap 'status=$?; printf "ERROR: build-poc-images-kaniko.sh failed at line %s (exit %s)\n" "$LINENO" "$status" >&2; exit "$status"' ERR if [[ -z "${IMAGE_TAG:-}" ]]; then IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)" @@ -88,6 +93,7 @@ validate_boolean POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK "$POC_KANIKO_SKIP_DEF validate_boolean POC_KANIKO_CLEAN_CONTEXT "$POC_KANIKO_CLEAN_CONTEXT" validate_boolean POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS" validate_boolean POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES" +validate_boolean POC_KANIKO_PRECHECK_ONLY "$POC_KANIKO_PRECHECK_ONLY" case "$POC_KANIKO_CONTEXT_SOURCE" in worktree|head|git|existing) ;; @@ -418,6 +424,65 @@ for component, dockerfile, image, aliases in specs: PY } +operator_dockerfile_uses_base_arg() { + local dockerfile="$1" + local base_arg="$2" + grep -Eq "^[[:space:]]*ARG[[:space:]]+${base_arg}([=[:space:]]|$)" "$ROOT_DIR/$dockerfile" +} + +operator_base_value() { + case "$1" in + OPERATOR_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE" ;; + OPERATOR_GPU_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" ;; + OPERATOR_HOLOBRAIN_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" ;; + OPERATOR_YOLO_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" ;; + *) echo "ERROR: unsupported operator base arg $1" >&2; exit 1 ;; + esac +} + +verify_registry_manifest_exists() { + local image="$1" + kubectl -n "$K8S_NAMESPACE" get secret "$K8S_IMAGE_PULL_SECRET_NAME" -o jsonpath='{.data.\.dockerconfigjson}' \ + | base64 --decode \ + | python3 "$ROOT_DIR/scripts/verify_registry_manifest.py" --image "$image" >&2 +} + +validate_prebuilt_operator_bases() { + local -a checked_images=() + local spec component dockerfile image base_arg base_image checked + for spec in "${operator_specs[@]}"; do + IFS=$'\t' read -r component dockerfile image <<<"$spec" + python3 "$ROOT_DIR/scripts/validate_kaniko_operator_bases.py" \ + --dockerfile "$ROOT_DIR/$dockerfile" \ + --operator-base "$POC_KANIKO_OPERATOR_BASE_IMAGE" \ + --operator-gpu-base "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" \ + --operator-holobrain-base "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" \ + --operator-yolo-base "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" \ + --require-immutable >&2 + for base_arg in OPERATOR_BASE_IMAGE OPERATOR_GPU_BASE_IMAGE OPERATOR_HOLOBRAIN_BASE_IMAGE OPERATOR_YOLO_BASE_IMAGE; do + if ! operator_dockerfile_uses_base_arg "$dockerfile" "$base_arg"; then + continue + fi + base_image="$(operator_base_value "$base_arg")" + # Bash 3 treats an empty array as unset under `set -u`; the fallback + # expansion keeps the first immutable-base preflight reachable. + for checked in "${checked_images[@]:-}"; do + if [[ "$checked" == "$base_image" ]]; then + base_image="" + break + fi + done + if [[ -z "$base_image" ]]; then + continue + fi + # This intentionally happens before any Kaniko Job is created. A base + # tag that has been GC'd must fail fast rather than consume build time. + verify_registry_manifest_exists "$base_image" + checked_images+=("$base_image") + done + done +} + upload_build_context() { case "$POC_KANIKO_CONTEXT_SOURCE" in head) @@ -504,6 +569,20 @@ if boolean_true POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS"; then operator_components+=("$component:$image") operator_component_count=$((operator_component_count + 1)) done < <(operator_image_specs) + if (( operator_component_count == 0 )); then + echo "ERROR: POC_KANIKO_OPERATOR_FILTER selected zero operator images." >&2 + exit 1 + fi +fi + +if (( operator_component_count > 0 )) && ! boolean_true POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"; then + validate_prebuilt_operator_bases +fi +if boolean_true POC_KANIKO_PRECHECK_ONLY "$POC_KANIKO_PRECHECK_ONLY"; then + printf 'KANIKO_BUILD_PLAN=contextSource=%s platformComponents=%s buildOperators=%s operatorFilter=%s buildOperatorBases=%s\n' \ + "$POC_KANIKO_CONTEXT_SOURCE" "${POC_KANIKO_COMPONENTS:-}" "$POC_KANIKO_BUILD_OPERATORS" "$POC_KANIKO_OPERATOR_FILTER" "$POC_KANIKO_BUILD_OPERATOR_BASES" + printf 'KANIKO_PRECHECK_ONLY=true\n' + exit 0 fi platform_specs=() diff --git a/scripts/render-qwen-node-cache-local-pv.sh b/scripts/render-qwen-node-cache-local-pv.sh new file mode 100755 index 00000000..0f340b5f --- /dev/null +++ b/scripts/render-qwen-node-cache-local-pv.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This helper only renders YAML to stdout. Applying the rendered PV/PVC or the +# explicit directory-preparation Job remains a separate operator action. +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MODE="pv" +if [[ "${1:-}" == "--prepare-job" ]]; then + MODE="prepare" + shift +fi +if [[ $# -gt 0 ]]; then + echo "usage: $0 [--prepare-job]" >&2 + exit 2 +fi + +QWEN_NODE_CACHE_NAMESPACE="${QWEN_NODE_CACHE_NAMESPACE:-droboticflow}" +QWEN_NODE_CACHE_NODE_NAME="${QWEN_NODE_CACHE_NODE_NAME:-192.168.80.136}" +QWEN_NODE_CACHE_HOST_PATH="${QWEN_NODE_CACHE_HOST_PATH:-/home/cce/droboticflow-qwen-cache}" +QWEN_NODE_CACHE_SIZE="${QWEN_NODE_CACHE_SIZE:-80Gi}" +QWEN_NODE_CACHE_PV_NAME="${QWEN_NODE_CACHE_PV_NAME:-droboticflow-qwen-node-cache-pv}" +QWEN_NODE_CACHE_PVC_NAME="${QWEN_NODE_CACHE_PVC_NAME:-droboticflow-qwen-node-cache}" +QWEN_NODE_CACHE_PREPARE_JOB_NAME="${QWEN_NODE_CACHE_PREPARE_JOB_NAME:-droboticflow-qwen-node-cache-prepare}" +QWEN_NODE_CACHE_PREPARE_IMAGE="${QWEN_NODE_CACHE_PREPARE_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-api@sha256:23fa92a39016d7c31f88b63974453d67a6da9bcc02783d4b24bff68566f86828}" +export QWEN_NODE_CACHE_NAMESPACE QWEN_NODE_CACHE_NODE_NAME QWEN_NODE_CACHE_HOST_PATH QWEN_NODE_CACHE_SIZE \ + QWEN_NODE_CACHE_PV_NAME QWEN_NODE_CACHE_PVC_NAME QWEN_NODE_CACHE_PREPARE_JOB_NAME QWEN_NODE_CACHE_PREPARE_IMAGE + +python3 - "$ROOT_DIR" "$MODE" <<'PY' +import os +import pathlib +import re +import sys + +root = pathlib.Path(sys.argv[1]) +mode = sys.argv[2] +values = {key: os.environ[key] for key in ( + "QWEN_NODE_CACHE_NAMESPACE", "QWEN_NODE_CACHE_NODE_NAME", "QWEN_NODE_CACHE_HOST_PATH", + "QWEN_NODE_CACHE_SIZE", "QWEN_NODE_CACHE_PV_NAME", "QWEN_NODE_CACHE_PVC_NAME", + "QWEN_NODE_CACHE_PREPARE_JOB_NAME", "QWEN_NODE_CACHE_PREPARE_IMAGE", +)} +label = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$") +for key in ("QWEN_NODE_CACHE_NAMESPACE", "QWEN_NODE_CACHE_PV_NAME", "QWEN_NODE_CACHE_PVC_NAME", "QWEN_NODE_CACHE_PREPARE_JOB_NAME"): + if len(values[key]) > 63 or not label.fullmatch(values[key]): + raise SystemExit(f"{key} must be a lowercase DNS label") +if not re.fullmatch(r"[a-z0-9]([a-z0-9.-]*[a-z0-9])?", values["QWEN_NODE_CACHE_NODE_NAME"]): + raise SystemExit("QWEN_NODE_CACHE_NODE_NAME must be a hostname or node IP") +path = pathlib.PurePosixPath(values["QWEN_NODE_CACHE_HOST_PATH"]) +if str(path) != values["QWEN_NODE_CACHE_HOST_PATH"] or not str(path).startswith("/home/cce/") or ".." in path.parts or str(path) == "/home/cce": + raise SystemExit("QWEN_NODE_CACHE_HOST_PATH must be a clean unique directory under /home/cce") +if not re.fullmatch(r"[1-9][0-9]*(Mi|Gi|Ti)", values["QWEN_NODE_CACHE_SIZE"]): + raise SystemExit("QWEN_NODE_CACHE_SIZE must be a positive Mi/Gi/Ti quantity") +if "@sha256:" not in values["QWEN_NODE_CACHE_PREPARE_IMAGE"]: + raise SystemExit("QWEN_NODE_CACHE_PREPARE_IMAGE must be digest-pinned") +template = root / "k8s" / ("qwen-node-cache.prepare-dir.template.yaml" if mode == "prepare" else "qwen-node-cache.local-pv.template.yaml") +rendered = template.read_text(encoding="utf-8") +replacements = {"__" + key + "__": value for key, value in values.items()} +replacements["__QWEN_NODE_CACHE_DIR_NAME__"] = path.name +for marker, value in replacements.items(): + rendered = rendered.replace(marker, value) +remaining = sorted(set(re.findall(r"__[A-Z0-9_]+__", rendered))) +if remaining: + raise SystemExit("unresolved placeholders: " + ", ".join(remaining)) +print(rendered, end="" if rendered.endswith("\n") else "\n") +PY diff --git a/scripts/validate_kaniko_operator_bases.py b/scripts/validate_kaniko_operator_bases.py new file mode 100644 index 00000000..1b2fc458 --- /dev/null +++ b/scripts/validate_kaniko_operator_bases.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Validate immutable base arguments for selected Kaniko operator builds.""" +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import sys + + +BASE_ARGS = { + "OPERATOR_BASE_IMAGE": "operator_base", + "OPERATOR_GPU_BASE_IMAGE": "operator_gpu_base", + "OPERATOR_HOLOBRAIN_BASE_IMAGE": "operator_holobrain_base", + "OPERATOR_YOLO_BASE_IMAGE": "operator_yolo_base", +} +_ARG = re.compile(r"^\s*ARG\s+(OPERATOR(?:_GPU|_HOLOBRAIN|_YOLO)?_BASE_IMAGE)(?:=|\s|$)", re.MULTILINE) +_DIGEST = re.compile(r"^.+@sha256:[a-f0-9]{64}$") + + +def required_base_args(dockerfile: pathlib.Path) -> list[str]: + try: + text = dockerfile.read_text(encoding="utf-8") + except OSError as exc: + raise RuntimeError(f"cannot read operator Dockerfile {dockerfile}: {exc}") from exc + return sorted(set(_ARG.findall(text))) + + +def validate(dockerfiles: list[pathlib.Path], values: dict[str, str], require_immutable: bool) -> list[dict[str, object]]: + result: list[dict[str, object]] = [] + for dockerfile in dockerfiles: + base_args = required_base_args(dockerfile) + selected: dict[str, str] = {} + for arg in base_args: + value = values[BASE_ARGS[arg]].strip() + if not value: + raise RuntimeError(f"{dockerfile}: {arg} is required") + if require_immutable and not _DIGEST.fullmatch(value): + raise RuntimeError(f"{dockerfile}: {arg} must be digest-pinned when operator base builds are disabled") + selected[arg] = value + result.append({"dockerfile": str(dockerfile), "baseArgs": selected}) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dockerfile", action="append", default=[], required=True) + parser.add_argument("--operator-base", default="") + parser.add_argument("--operator-gpu-base", default="") + parser.add_argument("--operator-holobrain-base", default="") + parser.add_argument("--operator-yolo-base", default="") + parser.add_argument("--require-immutable", action="store_true") + args = parser.parse_args() + values = { + "operator_base": args.operator_base, + "operator_gpu_base": args.operator_gpu_base, + "operator_holobrain_base": args.operator_holobrain_base, + "operator_yolo_base": args.operator_yolo_base, + } + try: + result = validate([pathlib.Path(value) for value in args.dockerfile], values, args.require_immutable) + except RuntimeError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_registry_manifest.py b/scripts/verify_registry_manifest.py new file mode 100644 index 00000000..cbc3229f --- /dev/null +++ b/scripts/verify_registry_manifest.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Fail closed when an immutable registry image manifest is unavailable. + +Docker auth configuration is read from stdin so credentials never need to be +placed in command arguments, temporary files, or output. +""" +from __future__ import annotations + +import argparse +import json +import re +import sys +from typing import Any, Callable +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + + +_IMAGE = re.compile(r"^(?P[^/@\s]+)/(?P[^@\s]+)@(?Psha256:[a-f0-9]{64})$") +_ACCEPT = "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json" + + +def parse_image(image: str) -> tuple[str, str, str]: + match = _IMAGE.fullmatch(image.strip()) + if match is None: + raise RuntimeError("registry preflight image must be a digest-pinned host/repository@sha256 reference") + return match.group("host"), match.group("repo"), match.group("digest") + + +def auth_value(config: dict[str, Any], host: str) -> str: + auths = config.get("auths") + if not isinstance(auths, dict): + raise RuntimeError("registry preflight docker config has no auths object") + entry = auths.get(host) or auths.get("https://" + host) + if not isinstance(entry, dict): + entry = next((value for key, value in auths.items() if host in str(key) and isinstance(value, dict)), None) + value = str((entry or {}).get("auth") or "").strip() + if not value: + raise RuntimeError("registry preflight docker auth is unavailable") + return value + + +def bearer_challenge(value: str) -> dict[str, str]: + match = re.match(r"^Bearer\s+(.+)$", value.strip(), re.IGNORECASE) + if match is None: + return {} + fields = {key.lower(): field for key, field in re.findall(r'([A-Za-z]+)="([^"]*)"', match.group(1))} + return fields if fields.get("realm") else {} + + +def read_manifest(request: Request, basic_auth: str, opener: Callable[..., Any]) -> tuple[str, dict[str, Any]]: + try: + with opener(request, timeout=30) as response: + return str(response.headers.get("Docker-Content-Digest") or ""), json.loads(response.read().decode("utf-8")) + except HTTPError as exc: + challenge = bearer_challenge(str(exc.headers.get("WWW-Authenticate") or "")) if exc.code == 401 else {} + if not challenge: + raise RuntimeError(f"registry manifest is unavailable for {request.full_url.rsplit('/manifests/', 1)[0]}: HTTP {exc.code}") from exc + token_query = {key: value for key, value in challenge.items() if key in {"service", "scope"}} + token_request = Request( + challenge["realm"] + (("?" + urlencode(token_query)) if token_query else ""), + headers={"Authorization": "Basic " + basic_auth}, + ) + try: + with opener(token_request, timeout=30) as token_response: + token_payload = json.loads(token_response.read().decode("utf-8")) + except (HTTPError, URLError, json.JSONDecodeError) as token_error: + raise RuntimeError("registry bearer token lookup failed") from token_error + token = str(token_payload.get("token") or token_payload.get("access_token") or "").strip() + if not token: + raise RuntimeError("registry bearer token response is missing a token") + retry = Request(request.full_url, headers={"Authorization": "Bearer " + token, "Accept": _ACCEPT}) + try: + with opener(retry, timeout=30) as response: + return str(response.headers.get("Docker-Content-Digest") or ""), json.loads(response.read().decode("utf-8")) + except HTTPError as retry_error: + raise RuntimeError(f"registry manifest is unavailable after bearer auth: HTTP {retry_error.code}") from retry_error + + +def verify(image: str, docker_config: dict[str, Any], opener: Callable[..., Any] = urlopen) -> dict[str, Any]: + host, repository, digest = parse_image(image) + request = Request( + f"https://{host}/v2/{repository}/manifests/{digest}", + headers={"Authorization": "Basic " + auth_value(docker_config, host), "Accept": _ACCEPT}, + ) + basic_auth = auth_value(docker_config, host) + try: + observed, payload = read_manifest(request, basic_auth, opener) + except URLError as exc: + raise RuntimeError(f"registry manifest lookup failed for {image}: {exc.reason}") from exc + if observed and observed != digest: + raise RuntimeError(f"registry manifest digest mismatch for {image}: observed {observed}") + return {"image": image, "manifestDigest": observed or digest, "configDigest": str((payload.get("config") or {}).get("digest") or ""), "layerCount": len(payload.get("layers") or [])} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", required=True) + args = parser.parse_args() + try: + docker_config = json.load(sys.stdin) + result = verify(args.image, docker_config) + except (RuntimeError, json.JSONDecodeError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/backend/kaniko_operator_base_preflight_test.py b/tests/backend/kaniko_operator_base_preflight_test.py new file mode 100644 index 00000000..141368fa --- /dev/null +++ b/tests/backend/kaniko_operator_base_preflight_test.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +def load_module(name: str, relative: str): + spec = importlib.util.spec_from_file_location(name, ROOT / relative) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +BASES = load_module("validate_kaniko_operator_bases", "scripts/validate_kaniko_operator_bases.py") +REGISTRY = load_module("verify_registry_manifest", "scripts/verify_registry_manifest.py") +GPU_DIGEST = "registry.example/dataflow/gpu-base@sha256:" + "a" * 64 + + +class FakeResponse: + def __init__(self, digest: str): + self.headers = {"Docker-Content-Digest": digest} + + def read(self): + return json.dumps({"config": {"digest": "sha256:" + "b" * 64}, "layers": [{}, {}]}).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + +class TestKanikoOperatorBasePreflight(unittest.TestCase): + def test_skipped_base_build_requires_exact_digest_only_for_selected_arg(self): + with tempfile.TemporaryDirectory() as directory: + dockerfile = Path(directory) / "Dockerfile" + dockerfile.write_text("ARG OPERATOR_GPU_BASE_IMAGE=registry.example/gpu:tag\nFROM ${OPERATOR_GPU_BASE_IMAGE}\n", encoding="utf-8") + values = {"operator_base": "", "operator_gpu_base": "registry.example/gpu:tag", "operator_holobrain_base": "", "operator_yolo_base": ""} + with self.assertRaisesRegex(RuntimeError, "must be digest-pinned"): + BASES.validate([dockerfile], values, True) + values["operator_gpu_base"] = GPU_DIGEST + result = BASES.validate([dockerfile], values, True) + self.assertEqual(result[0]["baseArgs"], {"OPERATOR_GPU_BASE_IMAGE": GPU_DIGEST}) + + def test_dockerfile_without_operator_base_does_not_require_unrelated_digest(self): + with tempfile.TemporaryDirectory() as directory: + dockerfile = Path(directory) / "Dockerfile" + dockerfile.write_text("FROM python:3.12-slim\n", encoding="utf-8") + values = {"operator_base": "", "operator_gpu_base": "", "operator_holobrain_base": "", "operator_yolo_base": ""} + self.assertEqual(BASES.validate([dockerfile], values, True), [{"dockerfile": str(dockerfile), "baseArgs": {}}]) + + def test_registry_preflight_uses_digest_and_never_echoes_auth(self): + image = "registry.example/dataflow/gpu-base@sha256:" + "c" * 64 + captured = {} + + def opener(request, timeout): + captured["url"] = request.full_url + captured["auth"] = request.get_header("Authorization") + captured["timeout"] = timeout + return FakeResponse("sha256:" + "c" * 64) + + result = REGISTRY.verify(image, {"auths": {"registry.example": {"auth": "not-a-real-secret"}}}, opener) + self.assertEqual(result["manifestDigest"], "sha256:" + "c" * 64) + self.assertEqual(result["layerCount"], 2) + self.assertIn("/v2/dataflow/gpu-base/manifests/sha256:", captured["url"]) + self.assertEqual(captured["auth"], "Basic not-a-real-secret") + self.assertEqual( + REGISTRY.bearer_challenge('Bearer realm="https://auth.example/token",service="registry.example",scope="repository:dataflow/gpu-base:pull"'), + {"realm": "https://auth.example/token", "service": "registry.example", "scope": "repository:dataflow/gpu-base:pull"}, + ) + with self.assertRaisesRegex(RuntimeError, "digest-pinned"): + REGISTRY.parse_image("registry.example/dataflow/gpu-base:tag") + + def test_build_script_preserves_operator_only_component_selection_and_reports_safe_plan(self): + script = (ROOT / "scripts" / "build-poc-images-kaniko.sh").read_text(encoding="utf-8") + self.assertIn('POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS-api,worker,drosync-coordinator,runner,headpose-service}"', script) + self.assertIn("POC_KANIKO_OPERATOR_FILTER selected zero operator images", script) + self.assertIn("KANIKO_BUILD_PLAN=contextSource=%s platformComponents=%s", script) + self.assertIn("failed at line %s (exit %s)", script) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/backend/qwen_node_local_pv_template_test.py b/tests/backend/qwen_node_local_pv_template_test.py new file mode 100644 index 00000000..2cf5d023 --- /dev/null +++ b/tests/backend/qwen_node_local_pv_template_test.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "render-qwen-node-cache-local-pv.sh" + + +class TestQwenNodeLocalPVTemplate(unittest.TestCase): + def render(self, *args: str, **extra: str) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env.update(extra) + return subprocess.run(["bash", str(SCRIPT), *args], cwd=ROOT, env=env, text=True, capture_output=True, check=False) + + def test_default_pv_and_pvc_are_local_nvme_bound_and_retained(self) -> None: + result = self.render() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("path: /home/cce/droboticflow-qwen-cache", result.stdout) + self.assertIn("storage: 80Gi", result.stdout) + self.assertIn("persistentVolumeReclaimPolicy: Retain", result.stdout) + self.assertIn("- 192.168.80.136", result.stdout) + self.assertIn("name: droboticflow-qwen-node-cache", result.stdout) + self.assertNotIn("hostPath:", result.stdout) + self.assertNotIn("kubectl apply", result.stdout) + + def test_prepare_job_is_strictly_scoped_to_single_cache_directory(self) -> None: + result = self.render("--prepare-job") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("path: /home/cce", result.stdout) + self.assertIn("install -d -m 0750 -o 10001 -g 10001 /host-cache/droboticflow-qwen-cache", result.stdout) + self.assertNotIn("chmod -R", result.stdout) + self.assertNotIn("path: /\n", result.stdout) + + def test_rejects_root_and_outside_nvme_paths(self) -> None: + for path in ("/", "/var/lib/droboticflow-qwen-cache", "/home/cce", "/home/cce/../root"): + with self.subTest(path=path): + result = self.render(QWEN_NODE_CACHE_HOST_PATH=path) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be a clean unique directory under /home/cce", result.stderr) + + +if __name__ == "__main__": + unittest.main()