🔒 Publish Qwen cache release evidence

This commit is contained in:
eust-w 2026-08-22 07:29:25 +08:00
parent b0734fec38
commit d960fab4d0
14 changed files with 581 additions and 23 deletions

View File

@ -16,8 +16,8 @@
],
"containerEntrypoint": "backend/operator-runner/main.py",
"entrypoint": "operator.py",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7",
"metadata": "operator.json",
"name": "Qwen3-VL 语义行为粗切(PTS)",
"operatorId": "operator-builtin-qwen-behavior-rough-cut",

View File

@ -53,6 +53,10 @@ manifest SHA、Qwen repository/revision、config/processor/index 和完整布局
1. 保持 feature disabled 部署并验证 Worker 健康。
2. 在 GPU node 上只创建 `/home/cce/droboticflow-qwen-cache`,权限 `10001:10001`、`0750`;
不执行递归 chmod,也不格式化或挂载任何盘。
使用 `scripts/render-qwen-node-cache-local-pv.sh --prepare-job` 只渲染受限的一次性
准备 Job;该脚本不会执行 apply。默认 PV/PVC 同样通过
`scripts/render-qwen-node-cache-local-pv.sh` 仅渲染,固定为 80Gi、Retain 与 hostname
affinity,模板不属于 `deploy-k8s.sh`。
3. 创建一个带 hostname nodeAffinity 的 `ReadWriteOnce` Local PV/PVC,再在 Worker 配置中
填入相同 PVC、mount path 与 hostname selector。
4. 先用真实固定 manifest 预热,检查 receipt、17.5GB 级别文件和 SHA 证据;再启用开关。

View File

@ -13102,8 +13102,8 @@
}
],
"id": "operator-builtin-qwen-behavior-rough-cut",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7",
"inputs": [
{
"description": "视频人脸脱敏输出:blurred-manifest 与完整脱敏视频 artifact",

View File

@ -0,0 +1,45 @@
# Render-only template. It is deliberately not included by deploy-k8s.sh.
# First run the node disk preflight and the separate, explicit directory
# preparation Job. Do not point this PV at a root-disk or kubelet path.
apiVersion: v1
kind: PersistentVolume
metadata:
name: __QWEN_NODE_CACHE_PV_NAME__
labels:
app: droboticflow
component: qwen-node-cache
spec:
capacity:
storage: __QWEN_NODE_CACHE_SIZE__
volumeMode: Filesystem
accessModes:
- ReadWriteOnce
persistentVolumeReclaimPolicy: Retain
storageClassName: droboticflow-qwen-node-local
local:
path: __QWEN_NODE_CACHE_HOST_PATH__
nodeAffinity:
required:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: In
values:
- __QWEN_NODE_CACHE_NODE_NAME__
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: __QWEN_NODE_CACHE_PVC_NAME__
namespace: __QWEN_NODE_CACHE_NAMESPACE__
labels:
app: droboticflow
component: qwen-node-cache
spec:
accessModes:
- ReadWriteOnce
storageClassName: droboticflow-qwen-node-local
volumeName: __QWEN_NODE_CACHE_PV_NAME__
resources:
requests:
storage: __QWEN_NODE_CACHE_SIZE__

View File

@ -0,0 +1,41 @@
# Render-only, one-shot directory preparation Job. It is never applied by the
# platform deployment. Its only mutation is creating/chowning the exact cache
# directory under the independently verified /home/cce NVMe mount.
apiVersion: batch/v1
kind: Job
metadata:
name: __QWEN_NODE_CACHE_PREPARE_JOB_NAME__
namespace: __QWEN_NODE_CACHE_NAMESPACE__
labels:
app: droboticflow
component: qwen-node-cache-prepare
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
nodeName: __QWEN_NODE_CACHE_NODE_NAME__
restartPolicy: Never
containers:
- name: prepare
image: __QWEN_NODE_CACHE_PREPARE_IMAGE__
command: ["/bin/sh", "-lc"]
args:
- |
set -eu
install -d -m 0750 -o 10001 -g 10001 /host-cache/__QWEN_NODE_CACHE_DIR_NAME__
stat -c 'mode=%a owner=%u:%g path=%n' /host-cache/__QWEN_NODE_CACHE_DIR_NAME__
securityContext:
runAsUser: 0
runAsGroup: 0
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
volumeMounts:
- name: host-cache-parent
mountPath: /host-cache
volumes:
- name: host-cache-parent
hostPath:
path: /home/cce
type: Directory

View File

@ -1469,8 +1469,8 @@
"orbbec-pts-csv"
],
"dockerfile": "operator-repos/builtin/qwen_behavior_rough_cut/Dockerfile",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7",
"operatorId": "operator-builtin-qwen-behavior-rough-cut",
"operatorSlug": "qwen_behavior_rough_cut",
"requiresGpu": true,

View File

@ -1,24 +1,30 @@
{
"acceptance": {},
"builtAt": "2026-08-21T09:11:39Z",
"digest": "sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc",
"acceptance": {
"artifactId": "fanyu-qwen-cache-disabled-compat-20260822-fanyu-v5-rough-cut-clips-manifest-clips-manifest.json",
"projectId": "project-271c97276a31d97dd3eff93e3f1ff3b0",
"runId": "fanyu-qwen-cache-disabled-compat-r2-20260822",
"verifiedAt": "2026-08-21T23:26:03Z",
"workflowId": "workflow-20b87364482c94d9df485d64453c9fac"
},
"builtAt": "2026-08-21T23:26:03Z",
"digest": "sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7",
"environmentReleases": {
"poc": {
"digest": "sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc",
"digest": "sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7",
"registry": "ccr-29eug8s3-pub.cnc.bj.baidubce.com",
"tag": "c859766-poc-fanyu-roughcut-20260821"
"tag": "b0734fe-poc-qwen-node-cache-r3-20260822063300"
}
},
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc",
"kanikoJob": "droboticflow-kaniko-op-qwen-behavior-rough-cut-c859766-poc-fany",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7",
"kanikoJob": "droboticflow-kaniko-op-qwen-behavior-rough-cut-b0734fe-poc-qwen",
"operatorId": "operator-builtin-qwen-behavior-rough-cut",
"registry": "ccr-29eug8s3-pub.cnc.bj.baidubce.com",
"schemaVersion": "droboticflow.operator.release/v1",
"slug": "qwen_behavior_rough_cut",
"sourceCommit": "0f67e13fc5866353ce1f00152043e4b6bb6bf27b",
"sourceTreeSha256": "b33dd4212aace5bcfabb83ff55f1f52146c90e87840c09ae9b377f89cab4c6d2",
"tag": "c859766-poc-fanyu-roughcut-20260821"
"sourceCommit": "b0734fec382eac0f5a367064a378887dca7f2326",
"sourceTreeSha256": "c76621f817fe66bc11508a33cf28e168b1d7036b7e3f69855d40c254960261b8",
"tag": "b0734fe-poc-qwen-node-cache-r3-20260822063300"
}

View File

@ -19,8 +19,8 @@
],
"containerEntrypoint": "backend/operator-runner/main.py",
"entrypoint": "src/operator.py",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:c859766-poc-fanyu-roughcut-20260821",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:ffa94d27b80e6f5a2333fcef8b7d496e0a5830c95f62d968a7364ac5325e6afc",
"image": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut:b0734fe-poc-qwen-node-cache-r3-20260822063300",
"imageId": "ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-operator-qwen-behavior-rough-cut@sha256:f1c4889b664d8bc24bf61ec9abf230a0c95c479fe54716514e370620450c25c7",
"metadata": "operator.json",
"name": "Qwen3-VL 语义行为粗切(PTS)",
"operatorId": "operator-builtin-qwen-behavior-rough-cut",
@ -90,7 +90,7 @@
"runtimeSource": "operator-repos/builtin/qwen_behavior_rough_cut/src/operator.py",
"schemaVersion": "droboticflow.operator.repository/v1",
"sourceRoot": "operator-repos/builtin/qwen_behavior_rough_cut",
"sourceTreeSha256": "b33dd4212aace5bcfabb83ff55f1f52146c90e87840c09ae9b377f89cab4c6d2",
"sourceTreeSha256": "c76621f817fe66bc11508a33cf28e168b1d7036b7e3f69855d40c254960261b8",
"strictPocAcceptanceExternals": [],
"type": "builtin"
}

View File

@ -35,7 +35,9 @@ POC_KANIKO_NODE_NAME="${POC_KANIKO_NODE_NAME:-}"
POC_KANIKO_BUILD_OPERATORS="${POC_KANIKO_BUILD_OPERATORS:-false}"
POC_KANIKO_BUILD_OPERATOR_BASES="${POC_KANIKO_BUILD_OPERATOR_BASES:-true}"
POC_KANIKO_OPERATOR_BASE_FILTER="${POC_KANIKO_OPERATOR_BASE_FILTER:-}"
POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS:-api,worker,drosync-coordinator,runner,headpose-service}"
# An explicit empty value is meaningful for an operator-only build. Use '-' so
# it is not silently expanded back to every platform component.
POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS-api,worker,drosync-coordinator,runner,headpose-service}"
POC_KANIKO_OPERATOR_MANIFEST="${POC_KANIKO_OPERATOR_MANIFEST:-operator-images/builtin/operator-images.json}"
POC_KANIKO_OPERATOR_FILTER="${POC_KANIKO_OPERATOR_FILTER:-}"
POC_KANIKO_OPERATOR_CONCURRENCY="${POC_KANIKO_OPERATOR_CONCURRENCY:-4}"
@ -45,6 +47,9 @@ POC_KANIKO_OPERATOR_GPU_BASE_IMAGE="${POC_KANIKO_OPERATOR_GPU_BASE_IMAGE:-}"
POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE="${POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE:-}"
POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}"
POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}"
POC_KANIKO_PRECHECK_ONLY="${POC_KANIKO_PRECHECK_ONLY:-false}"
trap 'status=$?; printf "ERROR: build-poc-images-kaniko.sh failed at line %s (exit %s)\n" "$LINENO" "$status" >&2; exit "$status"' ERR
if [[ -z "${IMAGE_TAG:-}" ]]; then
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)"
@ -88,6 +93,7 @@ validate_boolean POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK "$POC_KANIKO_SKIP_DEF
validate_boolean POC_KANIKO_CLEAN_CONTEXT "$POC_KANIKO_CLEAN_CONTEXT"
validate_boolean POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS"
validate_boolean POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"
validate_boolean POC_KANIKO_PRECHECK_ONLY "$POC_KANIKO_PRECHECK_ONLY"
case "$POC_KANIKO_CONTEXT_SOURCE" in
worktree|head|git|existing)
;;
@ -418,6 +424,65 @@ for component, dockerfile, image, aliases in specs:
PY
}
operator_dockerfile_uses_base_arg() {
local dockerfile="$1"
local base_arg="$2"
grep -Eq "^[[:space:]]*ARG[[:space:]]+${base_arg}([=[:space:]]|$)" "$ROOT_DIR/$dockerfile"
}
operator_base_value() {
case "$1" in
OPERATOR_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE" ;;
OPERATOR_GPU_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" ;;
OPERATOR_HOLOBRAIN_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" ;;
OPERATOR_YOLO_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" ;;
*) echo "ERROR: unsupported operator base arg $1" >&2; exit 1 ;;
esac
}
verify_registry_manifest_exists() {
local image="$1"
kubectl -n "$K8S_NAMESPACE" get secret "$K8S_IMAGE_PULL_SECRET_NAME" -o jsonpath='{.data.\.dockerconfigjson}' \
| base64 --decode \
| python3 "$ROOT_DIR/scripts/verify_registry_manifest.py" --image "$image" >&2
}
validate_prebuilt_operator_bases() {
local -a checked_images=()
local spec component dockerfile image base_arg base_image checked
for spec in "${operator_specs[@]}"; do
IFS=$'\t' read -r component dockerfile image <<<"$spec"
python3 "$ROOT_DIR/scripts/validate_kaniko_operator_bases.py" \
--dockerfile "$ROOT_DIR/$dockerfile" \
--operator-base "$POC_KANIKO_OPERATOR_BASE_IMAGE" \
--operator-gpu-base "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" \
--operator-holobrain-base "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" \
--operator-yolo-base "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" \
--require-immutable >&2
for base_arg in OPERATOR_BASE_IMAGE OPERATOR_GPU_BASE_IMAGE OPERATOR_HOLOBRAIN_BASE_IMAGE OPERATOR_YOLO_BASE_IMAGE; do
if ! operator_dockerfile_uses_base_arg "$dockerfile" "$base_arg"; then
continue
fi
base_image="$(operator_base_value "$base_arg")"
# Bash 3 treats an empty array as unset under `set -u`; the fallback
# expansion keeps the first immutable-base preflight reachable.
for checked in "${checked_images[@]:-}"; do
if [[ "$checked" == "$base_image" ]]; then
base_image=""
break
fi
done
if [[ -z "$base_image" ]]; then
continue
fi
# This intentionally happens before any Kaniko Job is created. A base
# tag that has been GC'd must fail fast rather than consume build time.
verify_registry_manifest_exists "$base_image"
checked_images+=("$base_image")
done
done
}
upload_build_context() {
case "$POC_KANIKO_CONTEXT_SOURCE" in
head)
@ -504,6 +569,20 @@ if boolean_true POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS"; then
operator_components+=("$component:$image")
operator_component_count=$((operator_component_count + 1))
done < <(operator_image_specs)
if (( operator_component_count == 0 )); then
echo "ERROR: POC_KANIKO_OPERATOR_FILTER selected zero operator images." >&2
exit 1
fi
fi
if (( operator_component_count > 0 )) && ! boolean_true POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"; then
validate_prebuilt_operator_bases
fi
if boolean_true POC_KANIKO_PRECHECK_ONLY "$POC_KANIKO_PRECHECK_ONLY"; then
printf 'KANIKO_BUILD_PLAN=contextSource=%s platformComponents=%s buildOperators=%s operatorFilter=%s buildOperatorBases=%s\n' \
"$POC_KANIKO_CONTEXT_SOURCE" "${POC_KANIKO_COMPONENTS:-<empty>}" "$POC_KANIKO_BUILD_OPERATORS" "$POC_KANIKO_OPERATOR_FILTER" "$POC_KANIKO_BUILD_OPERATOR_BASES"
printf 'KANIKO_PRECHECK_ONLY=true\n'
exit 0
fi
platform_specs=()

View File

@ -0,0 +1,64 @@
#!/usr/bin/env bash
set -euo pipefail
# This helper only renders YAML to stdout. Applying the rendered PV/PVC or the
# explicit directory-preparation Job remains a separate operator action.
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
MODE="pv"
if [[ "${1:-}" == "--prepare-job" ]]; then
MODE="prepare"
shift
fi
if [[ $# -gt 0 ]]; then
echo "usage: $0 [--prepare-job]" >&2
exit 2
fi
QWEN_NODE_CACHE_NAMESPACE="${QWEN_NODE_CACHE_NAMESPACE:-droboticflow}"
QWEN_NODE_CACHE_NODE_NAME="${QWEN_NODE_CACHE_NODE_NAME:-192.168.80.136}"
QWEN_NODE_CACHE_HOST_PATH="${QWEN_NODE_CACHE_HOST_PATH:-/home/cce/droboticflow-qwen-cache}"
QWEN_NODE_CACHE_SIZE="${QWEN_NODE_CACHE_SIZE:-80Gi}"
QWEN_NODE_CACHE_PV_NAME="${QWEN_NODE_CACHE_PV_NAME:-droboticflow-qwen-node-cache-pv}"
QWEN_NODE_CACHE_PVC_NAME="${QWEN_NODE_CACHE_PVC_NAME:-droboticflow-qwen-node-cache}"
QWEN_NODE_CACHE_PREPARE_JOB_NAME="${QWEN_NODE_CACHE_PREPARE_JOB_NAME:-droboticflow-qwen-node-cache-prepare}"
QWEN_NODE_CACHE_PREPARE_IMAGE="${QWEN_NODE_CACHE_PREPARE_IMAGE:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-api@sha256:23fa92a39016d7c31f88b63974453d67a6da9bcc02783d4b24bff68566f86828}"
export QWEN_NODE_CACHE_NAMESPACE QWEN_NODE_CACHE_NODE_NAME QWEN_NODE_CACHE_HOST_PATH QWEN_NODE_CACHE_SIZE \
QWEN_NODE_CACHE_PV_NAME QWEN_NODE_CACHE_PVC_NAME QWEN_NODE_CACHE_PREPARE_JOB_NAME QWEN_NODE_CACHE_PREPARE_IMAGE
python3 - "$ROOT_DIR" "$MODE" <<'PY'
import os
import pathlib
import re
import sys
root = pathlib.Path(sys.argv[1])
mode = sys.argv[2]
values = {key: os.environ[key] for key in (
"QWEN_NODE_CACHE_NAMESPACE", "QWEN_NODE_CACHE_NODE_NAME", "QWEN_NODE_CACHE_HOST_PATH",
"QWEN_NODE_CACHE_SIZE", "QWEN_NODE_CACHE_PV_NAME", "QWEN_NODE_CACHE_PVC_NAME",
"QWEN_NODE_CACHE_PREPARE_JOB_NAME", "QWEN_NODE_CACHE_PREPARE_IMAGE",
)}
label = re.compile(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$")
for key in ("QWEN_NODE_CACHE_NAMESPACE", "QWEN_NODE_CACHE_PV_NAME", "QWEN_NODE_CACHE_PVC_NAME", "QWEN_NODE_CACHE_PREPARE_JOB_NAME"):
if len(values[key]) > 63 or not label.fullmatch(values[key]):
raise SystemExit(f"{key} must be a lowercase DNS label")
if not re.fullmatch(r"[a-z0-9]([a-z0-9.-]*[a-z0-9])?", values["QWEN_NODE_CACHE_NODE_NAME"]):
raise SystemExit("QWEN_NODE_CACHE_NODE_NAME must be a hostname or node IP")
path = pathlib.PurePosixPath(values["QWEN_NODE_CACHE_HOST_PATH"])
if str(path) != values["QWEN_NODE_CACHE_HOST_PATH"] or not str(path).startswith("/home/cce/") or ".." in path.parts or str(path) == "/home/cce":
raise SystemExit("QWEN_NODE_CACHE_HOST_PATH must be a clean unique directory under /home/cce")
if not re.fullmatch(r"[1-9][0-9]*(Mi|Gi|Ti)", values["QWEN_NODE_CACHE_SIZE"]):
raise SystemExit("QWEN_NODE_CACHE_SIZE must be a positive Mi/Gi/Ti quantity")
if "@sha256:" not in values["QWEN_NODE_CACHE_PREPARE_IMAGE"]:
raise SystemExit("QWEN_NODE_CACHE_PREPARE_IMAGE must be digest-pinned")
template = root / "k8s" / ("qwen-node-cache.prepare-dir.template.yaml" if mode == "prepare" else "qwen-node-cache.local-pv.template.yaml")
rendered = template.read_text(encoding="utf-8")
replacements = {"__" + key + "__": value for key, value in values.items()}
replacements["__QWEN_NODE_CACHE_DIR_NAME__"] = path.name
for marker, value in replacements.items():
rendered = rendered.replace(marker, value)
remaining = sorted(set(re.findall(r"__[A-Z0-9_]+__", rendered)))
if remaining:
raise SystemExit("unresolved placeholders: " + ", ".join(remaining))
print(rendered, end="" if rendered.endswith("\n") else "\n")
PY

View File

@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Validate immutable base arguments for selected Kaniko operator builds."""
from __future__ import annotations
import argparse
import json
import pathlib
import re
import sys
BASE_ARGS = {
"OPERATOR_BASE_IMAGE": "operator_base",
"OPERATOR_GPU_BASE_IMAGE": "operator_gpu_base",
"OPERATOR_HOLOBRAIN_BASE_IMAGE": "operator_holobrain_base",
"OPERATOR_YOLO_BASE_IMAGE": "operator_yolo_base",
}
_ARG = re.compile(r"^\s*ARG\s+(OPERATOR(?:_GPU|_HOLOBRAIN|_YOLO)?_BASE_IMAGE)(?:=|\s|$)", re.MULTILINE)
_DIGEST = re.compile(r"^.+@sha256:[a-f0-9]{64}$")
def required_base_args(dockerfile: pathlib.Path) -> list[str]:
try:
text = dockerfile.read_text(encoding="utf-8")
except OSError as exc:
raise RuntimeError(f"cannot read operator Dockerfile {dockerfile}: {exc}") from exc
return sorted(set(_ARG.findall(text)))
def validate(dockerfiles: list[pathlib.Path], values: dict[str, str], require_immutable: bool) -> list[dict[str, object]]:
result: list[dict[str, object]] = []
for dockerfile in dockerfiles:
base_args = required_base_args(dockerfile)
selected: dict[str, str] = {}
for arg in base_args:
value = values[BASE_ARGS[arg]].strip()
if not value:
raise RuntimeError(f"{dockerfile}: {arg} is required")
if require_immutable and not _DIGEST.fullmatch(value):
raise RuntimeError(f"{dockerfile}: {arg} must be digest-pinned when operator base builds are disabled")
selected[arg] = value
result.append({"dockerfile": str(dockerfile), "baseArgs": selected})
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dockerfile", action="append", default=[], required=True)
parser.add_argument("--operator-base", default="")
parser.add_argument("--operator-gpu-base", default="")
parser.add_argument("--operator-holobrain-base", default="")
parser.add_argument("--operator-yolo-base", default="")
parser.add_argument("--require-immutable", action="store_true")
args = parser.parse_args()
values = {
"operator_base": args.operator_base,
"operator_gpu_base": args.operator_gpu_base,
"operator_holobrain_base": args.operator_holobrain_base,
"operator_yolo_base": args.operator_yolo_base,
}
try:
result = validate([pathlib.Path(value) for value in args.dockerfile], values, args.require_immutable)
except RuntimeError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
print(json.dumps(result, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,111 @@
#!/usr/bin/env python3
"""Fail closed when an immutable registry image manifest is unavailable.
Docker auth configuration is read from stdin so credentials never need to be
placed in command arguments, temporary files, or output.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
_IMAGE = re.compile(r"^(?P<host>[^/@\s]+)/(?P<repo>[^@\s]+)@(?P<digest>sha256:[a-f0-9]{64})$")
_ACCEPT = "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json"
def parse_image(image: str) -> tuple[str, str, str]:
match = _IMAGE.fullmatch(image.strip())
if match is None:
raise RuntimeError("registry preflight image must be a digest-pinned host/repository@sha256 reference")
return match.group("host"), match.group("repo"), match.group("digest")
def auth_value(config: dict[str, Any], host: str) -> str:
auths = config.get("auths")
if not isinstance(auths, dict):
raise RuntimeError("registry preflight docker config has no auths object")
entry = auths.get(host) or auths.get("https://" + host)
if not isinstance(entry, dict):
entry = next((value for key, value in auths.items() if host in str(key) and isinstance(value, dict)), None)
value = str((entry or {}).get("auth") or "").strip()
if not value:
raise RuntimeError("registry preflight docker auth is unavailable")
return value
def bearer_challenge(value: str) -> dict[str, str]:
match = re.match(r"^Bearer\s+(.+)$", value.strip(), re.IGNORECASE)
if match is None:
return {}
fields = {key.lower(): field for key, field in re.findall(r'([A-Za-z]+)="([^"]*)"', match.group(1))}
return fields if fields.get("realm") else {}
def read_manifest(request: Request, basic_auth: str, opener: Callable[..., Any]) -> tuple[str, dict[str, Any]]:
try:
with opener(request, timeout=30) as response:
return str(response.headers.get("Docker-Content-Digest") or ""), json.loads(response.read().decode("utf-8"))
except HTTPError as exc:
challenge = bearer_challenge(str(exc.headers.get("WWW-Authenticate") or "")) if exc.code == 401 else {}
if not challenge:
raise RuntimeError(f"registry manifest is unavailable for {request.full_url.rsplit('/manifests/', 1)[0]}: HTTP {exc.code}") from exc
token_query = {key: value for key, value in challenge.items() if key in {"service", "scope"}}
token_request = Request(
challenge["realm"] + (("?" + urlencode(token_query)) if token_query else ""),
headers={"Authorization": "Basic " + basic_auth},
)
try:
with opener(token_request, timeout=30) as token_response:
token_payload = json.loads(token_response.read().decode("utf-8"))
except (HTTPError, URLError, json.JSONDecodeError) as token_error:
raise RuntimeError("registry bearer token lookup failed") from token_error
token = str(token_payload.get("token") or token_payload.get("access_token") or "").strip()
if not token:
raise RuntimeError("registry bearer token response is missing a token")
retry = Request(request.full_url, headers={"Authorization": "Bearer " + token, "Accept": _ACCEPT})
try:
with opener(retry, timeout=30) as response:
return str(response.headers.get("Docker-Content-Digest") or ""), json.loads(response.read().decode("utf-8"))
except HTTPError as retry_error:
raise RuntimeError(f"registry manifest is unavailable after bearer auth: HTTP {retry_error.code}") from retry_error
def verify(image: str, docker_config: dict[str, Any], opener: Callable[..., Any] = urlopen) -> dict[str, Any]:
host, repository, digest = parse_image(image)
request = Request(
f"https://{host}/v2/{repository}/manifests/{digest}",
headers={"Authorization": "Basic " + auth_value(docker_config, host), "Accept": _ACCEPT},
)
basic_auth = auth_value(docker_config, host)
try:
observed, payload = read_manifest(request, basic_auth, opener)
except URLError as exc:
raise RuntimeError(f"registry manifest lookup failed for {image}: {exc.reason}") from exc
if observed and observed != digest:
raise RuntimeError(f"registry manifest digest mismatch for {image}: observed {observed}")
return {"image": image, "manifestDigest": observed or digest, "configDigest": str((payload.get("config") or {}).get("digest") or ""), "layerCount": len(payload.get("layers") or [])}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--image", required=True)
args = parser.parse_args()
try:
docker_config = json.load(sys.stdin)
result = verify(args.image, docker_config)
except (RuntimeError, json.JSONDecodeError) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
print(json.dumps(result, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@ -0,0 +1,90 @@
from __future__ import annotations
import importlib.util
import json
from pathlib import Path
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[2]
def load_module(name: str, relative: str):
spec = importlib.util.spec_from_file_location(name, ROOT / relative)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
BASES = load_module("validate_kaniko_operator_bases", "scripts/validate_kaniko_operator_bases.py")
REGISTRY = load_module("verify_registry_manifest", "scripts/verify_registry_manifest.py")
GPU_DIGEST = "registry.example/dataflow/gpu-base@sha256:" + "a" * 64
class FakeResponse:
def __init__(self, digest: str):
self.headers = {"Docker-Content-Digest": digest}
def read(self):
return json.dumps({"config": {"digest": "sha256:" + "b" * 64}, "layers": [{}, {}]}).encode("utf-8")
def __enter__(self):
return self
def __exit__(self, *_args):
return False
class TestKanikoOperatorBasePreflight(unittest.TestCase):
def test_skipped_base_build_requires_exact_digest_only_for_selected_arg(self):
with tempfile.TemporaryDirectory() as directory:
dockerfile = Path(directory) / "Dockerfile"
dockerfile.write_text("ARG OPERATOR_GPU_BASE_IMAGE=registry.example/gpu:tag\nFROM ${OPERATOR_GPU_BASE_IMAGE}\n", encoding="utf-8")
values = {"operator_base": "", "operator_gpu_base": "registry.example/gpu:tag", "operator_holobrain_base": "", "operator_yolo_base": ""}
with self.assertRaisesRegex(RuntimeError, "must be digest-pinned"):
BASES.validate([dockerfile], values, True)
values["operator_gpu_base"] = GPU_DIGEST
result = BASES.validate([dockerfile], values, True)
self.assertEqual(result[0]["baseArgs"], {"OPERATOR_GPU_BASE_IMAGE": GPU_DIGEST})
def test_dockerfile_without_operator_base_does_not_require_unrelated_digest(self):
with tempfile.TemporaryDirectory() as directory:
dockerfile = Path(directory) / "Dockerfile"
dockerfile.write_text("FROM python:3.12-slim\n", encoding="utf-8")
values = {"operator_base": "", "operator_gpu_base": "", "operator_holobrain_base": "", "operator_yolo_base": ""}
self.assertEqual(BASES.validate([dockerfile], values, True), [{"dockerfile": str(dockerfile), "baseArgs": {}}])
def test_registry_preflight_uses_digest_and_never_echoes_auth(self):
image = "registry.example/dataflow/gpu-base@sha256:" + "c" * 64
captured = {}
def opener(request, timeout):
captured["url"] = request.full_url
captured["auth"] = request.get_header("Authorization")
captured["timeout"] = timeout
return FakeResponse("sha256:" + "c" * 64)
result = REGISTRY.verify(image, {"auths": {"registry.example": {"auth": "not-a-real-secret"}}}, opener)
self.assertEqual(result["manifestDigest"], "sha256:" + "c" * 64)
self.assertEqual(result["layerCount"], 2)
self.assertIn("/v2/dataflow/gpu-base/manifests/sha256:", captured["url"])
self.assertEqual(captured["auth"], "Basic not-a-real-secret")
self.assertEqual(
REGISTRY.bearer_challenge('Bearer realm="https://auth.example/token",service="registry.example",scope="repository:dataflow/gpu-base:pull"'),
{"realm": "https://auth.example/token", "service": "registry.example", "scope": "repository:dataflow/gpu-base:pull"},
)
with self.assertRaisesRegex(RuntimeError, "digest-pinned"):
REGISTRY.parse_image("registry.example/dataflow/gpu-base:tag")
def test_build_script_preserves_operator_only_component_selection_and_reports_safe_plan(self):
script = (ROOT / "scripts" / "build-poc-images-kaniko.sh").read_text(encoding="utf-8")
self.assertIn('POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS-api,worker,drosync-coordinator,runner,headpose-service}"', script)
self.assertIn("POC_KANIKO_OPERATOR_FILTER selected zero operator images", script)
self.assertIn("KANIKO_BUILD_PLAN=contextSource=%s platformComponents=%s", script)
self.assertIn("failed at line %s (exit %s)", script)
if __name__ == "__main__":
unittest.main()

View File

@ -0,0 +1,47 @@
from __future__ import annotations
import os
from pathlib import Path
import subprocess
import unittest
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "render-qwen-node-cache-local-pv.sh"
class TestQwenNodeLocalPVTemplate(unittest.TestCase):
def render(self, *args: str, **extra: str) -> subprocess.CompletedProcess[str]:
env = os.environ.copy()
env.update(extra)
return subprocess.run(["bash", str(SCRIPT), *args], cwd=ROOT, env=env, text=True, capture_output=True, check=False)
def test_default_pv_and_pvc_are_local_nvme_bound_and_retained(self) -> None:
result = self.render()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("path: /home/cce/droboticflow-qwen-cache", result.stdout)
self.assertIn("storage: 80Gi", result.stdout)
self.assertIn("persistentVolumeReclaimPolicy: Retain", result.stdout)
self.assertIn("- 192.168.80.136", result.stdout)
self.assertIn("name: droboticflow-qwen-node-cache", result.stdout)
self.assertNotIn("hostPath:", result.stdout)
self.assertNotIn("kubectl apply", result.stdout)
def test_prepare_job_is_strictly_scoped_to_single_cache_directory(self) -> None:
result = self.render("--prepare-job")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("path: /home/cce", result.stdout)
self.assertIn("install -d -m 0750 -o 10001 -g 10001 /host-cache/droboticflow-qwen-cache", result.stdout)
self.assertNotIn("chmod -R", result.stdout)
self.assertNotIn("path: /\n", result.stdout)
def test_rejects_root_and_outside_nvme_paths(self) -> None:
for path in ("/", "/var/lib/droboticflow-qwen-cache", "/home/cce", "/home/cce/../root"):
with self.subTest(path=path):
result = self.render(QWEN_NODE_CACHE_HOST_PATH=path)
self.assertNotEqual(result.returncode, 0)
self.assertIn("must be a clean unique directory under /home/cce", result.stderr)
if __name__ == "__main__":
unittest.main()