droboticflow/scripts/build-poc-images-kaniko.sh
2026-09-01 04:02:18 +08:00

693 lines
29 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-}"
if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then
DEPLOY_ENV="$1"
shift
fi
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
# shellcheck source=scripts/lib/build-metadata.sh
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
REGISTRY_HOST="${REGISTRY_HOST:-}"
REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-}"
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
POC_KANIKO_IMAGE="${POC_KANIKO_IMAGE:-gcr.io/kaniko-project/executor:v1.23.2-debug}"
POC_KANIKO_REGISTRY_MIRRORS="${POC_KANIKO_REGISTRY_MIRRORS:-mirror.gcr.io}"
POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK="${POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK:-true}"
POC_KANIKO_CLEAN_CONTEXT="${POC_KANIKO_CLEAN_CONTEXT:-true}"
POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS="${POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS:-30}"
POC_KANIKO_WAIT_TIMEOUT="${POC_KANIKO_WAIT_TIMEOUT:-1200s}"
POC_KANIKO_GOPROXY="${POC_KANIKO_GOPROXY:-}"
POC_KANIKO_CONTEXT_SOURCE="${POC_KANIKO_CONTEXT_SOURCE:-worktree}"
POC_KANIKO_PACKED_CONTEXT="${POC_KANIKO_PACKED_CONTEXT:-true}"
POC_KANIKO_CONTEXT_PREPARE_IMAGE="${POC_KANIKO_CONTEXT_PREPARE_IMAGE:-busybox:1.36.1}"
POC_KANIKO_CONTEXT_POD="${POC_KANIKO_CONTEXT_POD:-}"
POC_KANIKO_GIT_CONTEXT="${POC_KANIKO_GIT_CONTEXT:-}"
POC_KANIKO_NODE_NAME="${POC_KANIKO_NODE_NAME:-}"
POC_KANIKO_BUILD_OPERATORS="${POC_KANIKO_BUILD_OPERATORS:-false}"
POC_KANIKO_BUILD_OPERATOR_BASES="${POC_KANIKO_BUILD_OPERATOR_BASES:-true}"
POC_KANIKO_OPERATOR_BASE_FILTER="${POC_KANIKO_OPERATOR_BASE_FILTER:-}"
# An explicit empty value is meaningful for an operator-only build. Use '-' so
# it is not silently expanded back to every platform component.
POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS-api,worker,drosync-coordinator,runner,headpose-service}"
POC_KANIKO_OPERATOR_MANIFEST="${POC_KANIKO_OPERATOR_MANIFEST:-operator-images/builtin/operator-images.json}"
POC_KANIKO_OPERATOR_FILTER="${POC_KANIKO_OPERATOR_FILTER:-}"
POC_KANIKO_OPERATOR_CONCURRENCY="${POC_KANIKO_OPERATOR_CONCURRENCY:-4}"
POC_KANIKO_OPERATOR_RETRIES="${POC_KANIKO_OPERATOR_RETRIES:-2}"
POC_KANIKO_HEADPOSE_BASE_IMAGE="${POC_KANIKO_HEADPOSE_BASE_IMAGE:-}"
POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME="${POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME:-false}"
POC_KANIKO_OPERATOR_BASE_IMAGE="${POC_KANIKO_OPERATOR_BASE_IMAGE:-}"
POC_KANIKO_OPERATOR_GPU_BASE_IMAGE="${POC_KANIKO_OPERATOR_GPU_BASE_IMAGE:-}"
POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE="${POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE:-}"
POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}"
POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}"
POC_KANIKO_PRECHECK_ONLY="${POC_KANIKO_PRECHECK_ONLY:-false}"
trap 'status=$?; printf "ERROR: build-poc-images-kaniko.sh failed at line %s (exit %s)\n" "$LINENO" "$status" >&2; exit "$status"' ERR
if [[ -z "${IMAGE_TAG:-}" ]]; then
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)"
fi
POC_KANIKO_SOURCE_COMMIT="${POC_KANIKO_SOURCE_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-}"
POC_KANIKO_BUILD_TIME="$(droboticflow_normalize_build_time "$POC_KANIKO_BUILD_TIME")"
HEADPOSE_SERVICE_IMAGE="${HEADPOSE_SERVICE_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-headpose-service:$IMAGE_TAG}"
# shellcheck source=scripts/lib/image-config.sh
source "$ROOT_DIR/scripts/lib/image-config.sh"
droboticflow_resolve_image_configuration
POC_KANIKO_OPERATOR_BASE_IMAGE="${POC_KANIKO_OPERATOR_BASE_IMAGE:-$OPERATOR_RUNNER_IMAGE}"
POC_KANIKO_OPERATOR_GPU_BASE_IMAGE="${POC_KANIKO_OPERATOR_GPU_BASE_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-operator-gpu-base:$IMAGE_TAG}"
POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE="${POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-operator-holobrain-base:$IMAGE_TAG}"
POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-operator-yolo-base:$IMAGE_TAG}"
# shellcheck source=scripts/lib/deploy-validation.sh
source "$ROOT_DIR/scripts/lib/deploy-validation.sh"
validate_dns_label K8S_NAMESPACE "$K8S_NAMESPACE"
validate_dns_subdomain K8S_IMAGE_PULL_SECRET_NAME "$K8S_IMAGE_PULL_SECRET_NAME"
validate_dns_subdomain WORKSPACE_PVC_NAME "$WORKSPACE_PVC_NAME"
validate_absolute_mount_path WORKSPACE_MOUNT_PATH "$WORKSPACE_MOUNT_PATH"
validate_absolute_mount_path POC_BUILD_CONTEXT_ROOT "$POC_BUILD_CONTEXT_ROOT"
if [[ -n "$POC_KANIKO_NODE_NAME" ]]; then
validate_dns_subdomain POC_KANIKO_NODE_NAME "$POC_KANIKO_NODE_NAME"
fi
if [[ -n "$POC_KANIKO_CONTEXT_POD" ]]; then
validate_dns_subdomain POC_KANIKO_CONTEXT_POD "$POC_KANIKO_CONTEXT_POD"
fi
validate_image_reference POC_KANIKO_IMAGE "$POC_KANIKO_IMAGE"
validate_image_reference API_IMAGE "$API_IMAGE"
validate_image_reference WORKER_IMAGE "$WORKER_IMAGE"
validate_image_reference DROSYNC_COORDINATOR_IMAGE "$DROSYNC_COORDINATOR_IMAGE"
validate_image_reference OPERATOR_RUNNER_IMAGE "$OPERATOR_RUNNER_IMAGE"
validate_image_reference HEADPOSE_SERVICE_IMAGE "$HEADPOSE_SERVICE_IMAGE"
validate_image_reference POC_KANIKO_OPERATOR_BASE_IMAGE "$POC_KANIKO_OPERATOR_BASE_IMAGE"
validate_image_reference POC_KANIKO_OPERATOR_GPU_BASE_IMAGE "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
validate_image_reference POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE"
validate_image_reference POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE"
validate_boolean POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK "$POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK"
validate_boolean POC_KANIKO_CLEAN_CONTEXT "$POC_KANIKO_CLEAN_CONTEXT"
validate_boolean POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS"
validate_boolean POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"
validate_boolean POC_KANIKO_PRECHECK_ONLY "$POC_KANIKO_PRECHECK_ONLY"
validate_boolean POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME "$POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME"
validate_boolean POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"
validate_image_reference POC_KANIKO_CONTEXT_PREPARE_IMAGE "$POC_KANIKO_CONTEXT_PREPARE_IMAGE"
if [[ -n "$POC_KANIKO_HEADPOSE_BASE_IMAGE" ]]; then
validate_image_reference POC_KANIKO_HEADPOSE_BASE_IMAGE "$POC_KANIKO_HEADPOSE_BASE_IMAGE"
fi
if boolean_true POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME "$POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME" && [[ -z "$POC_KANIKO_HEADPOSE_BASE_IMAGE" ]]; then
echo "ERROR: POC_KANIKO_HEADPOSE_BASE_IMAGE is required when POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME=true" >&2
exit 1
fi
case "$POC_KANIKO_CONTEXT_SOURCE" in
worktree|head|git|existing)
;;
*)
echo "ERROR: POC_KANIKO_CONTEXT_SOURCE must be worktree, head, git, or existing." >&2
exit 1
;;
esac
if [[ "$POC_KANIKO_CONTEXT_SOURCE" == "git" ]]; then
if [[ -z "$POC_KANIKO_GIT_CONTEXT" || "$POC_KANIKO_GIT_CONTEXT" != git://* || "$POC_KANIKO_GIT_CONTEXT" =~ [[:space:]] ]]; then
echo "ERROR: POC_KANIKO_GIT_CONTEXT must be a whitespace-free git:// context when POC_KANIKO_CONTEXT_SOURCE=git." >&2
exit 1
fi
fi
if [[ "$POC_KANIKO_GOPROXY" =~ [[:space:]] ]]; then
echo "ERROR: POC_KANIKO_GOPROXY must not contain whitespace." >&2
exit 1
fi
if ! [[ "$POC_KANIKO_OPERATOR_CONCURRENCY" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: POC_KANIKO_OPERATOR_CONCURRENCY must be a positive integer." >&2
exit 1
fi
if ! [[ "$POC_KANIKO_OPERATOR_RETRIES" =~ ^[0-9]+$ ]]; then
echo "ERROR: POC_KANIKO_OPERATOR_RETRIES must be a non-negative integer." >&2
exit 1
fi
if ! [[ "$POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
echo "ERROR: POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS must be a positive integer." >&2
exit 1
fi
if [[ "$POC_KANIKO_COMPONENTS" =~ [[:space:]] ]]; then
echo "ERROR: POC_KANIKO_COMPONENTS must be a comma-separated list without whitespace." >&2
exit 1
fi
if [[ "$POC_BUILD_CONTEXT_ROOT" != "$WORKSPACE_MOUNT_PATH" && "$POC_BUILD_CONTEXT_ROOT" != "$WORKSPACE_MOUNT_PATH"/* ]]; then
echo "ERROR: POC_BUILD_CONTEXT_ROOT must point under WORKSPACE_MOUNT_PATH." >&2
exit 1
fi
if boolean_true POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS" && [[ ! -f "$ROOT_DIR/$POC_KANIKO_OPERATOR_MANIFEST" ]]; then
echo "ERROR: POC_KANIKO_OPERATOR_MANIFEST does not exist: $POC_KANIKO_OPERATOR_MANIFEST" >&2
exit 1
fi
CONTEXT_DIR="${POC_BUILD_CONTEXT_ROOT%/}/droboticflow-${IMAGE_TAG}"
CONTEXT_ARCHIVE="${CONTEXT_DIR}.tar"
api_pod=""
context_json=""
context_uploaded=false
json_string() {
python3 - "$1" <<'PY'
import json
import sys
print(json.dumps(sys.argv[1]))
PY
}
go_duration_seconds() {
python3 - "$1" <<'PY'
import re
import sys
value = sys.argv[1]
units = {"ns": 0, "us": 0, "ms": 0, "s": 1, "m": 60, "h": 3600}
total = 0
for number, unit in re.findall(r"([1-9][0-9]*)(ns|us|ms|s|m|h)", value):
total += int(number) * units[unit]
print(max(total, 1))
PY
}
job_name() {
python3 - "$1" "$2" <<'PY'
import hashlib
import re
import sys
component = sys.argv[1]
tag = sys.argv[2].lower()
suffix = re.sub(r"[^a-z0-9-]+", "-", tag).strip("-") or "build"
component_slug = re.sub(r"[^a-z0-9-]+", "-", component.lower()).strip("-") or "component"
prefix = f"droboticflow-kaniko-{component_slug}-"
if len(prefix) >= 48:
digest = hashlib.sha1(component_slug.encode("utf-8")).hexdigest()[:8]
prefix = f"droboticflow-kaniko-{component_slug[:24].strip('-')}-{digest}-"
print(prefix + suffix[: 63 - len(prefix)].strip("-"))
PY
}
kaniko_args_yaml() {
local destination="$1"
local dockerfile="$2"
local mirror
local context_value="dir://${CONTEXT_DIR}"
local dockerfile_value="${CONTEXT_DIR}/${dockerfile}"
if [[ "$POC_KANIKO_CONTEXT_SOURCE" == "git" ]]; then
context_value="$POC_KANIKO_GIT_CONTEXT"
dockerfile_value="$dockerfile"
elif boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then
context_value="dir:///kaniko-build-context"
dockerfile_value="/kaniko-build-context/${dockerfile}"
fi
cat <<YAML
- --context=${context_value}
- --dockerfile=${dockerfile_value}
- --destination=${destination}
YAML
if [[ -n "$POC_KANIKO_GOPROXY" ]]; then
printf ' - --build-arg=GOPROXY=%s\n' "$POC_KANIKO_GOPROXY"
fi
printf ' - --build-arg=OPERATOR_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE"
printf ' - --build-arg=OPERATOR_GPU_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
printf ' - --build-arg=OPERATOR_HOLOBRAIN_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE"
printf ' - --build-arg=OPERATOR_YOLO_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE"
printf ' - --build-arg=DROBOTICFLOW_BUILD_COMMIT=%s\n' "$POC_KANIKO_SOURCE_COMMIT"
printf ' - --build-arg=DROBOTICFLOW_BUILD_TIME=%s\n' "$POC_KANIKO_BUILD_TIME"
if [[ "$dockerfile" == "Dockerfile.headpose-service" ]]; then
if [[ -n "$POC_KANIKO_HEADPOSE_BASE_IMAGE" ]]; then
printf ' - --build-arg=HEADPOSE_BASE_IMAGE=%s\n' "$POC_KANIKO_HEADPOSE_BASE_IMAGE"
fi
printf ' - --build-arg=HEADPOSE_REUSE_BASE_RUNTIME=%s\n' "$POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME"
fi
IFS=',' read -r -a mirrors <<<"$POC_KANIKO_REGISTRY_MIRRORS"
for mirror in "${mirrors[@]}"; do
mirror="$(printf '%s' "$mirror" | xargs)"
if [[ -z "$mirror" ]]; then
continue
fi
if [[ "$mirror" =~ [[:space:]] ]]; then
echo "ERROR: POC_KANIKO_REGISTRY_MIRRORS entries must not contain whitespace." >&2
exit 1
fi
printf ' - --registry-mirror=%s\n' "$mirror"
done
if boolean_true POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK "$POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK"; then
printf ' - --skip-default-registry-fallback\n'
fi
cat <<YAML
- --snapshot-mode=redo
- --use-new-run
- --verbosity=info
YAML
}
create_kaniko_job() {
local component="$1"
local dockerfile="$2"
local image="$3"
local job
job="$(job_name "$component" "$IMAGE_TAG")"
kubectl -n "$K8S_NAMESPACE" delete job "$job" --ignore-not-found=true >/dev/null
kubectl -n "$K8S_NAMESPACE" apply -f - <<YAML
apiVersion: batch/v1
kind: Job
metadata:
name: ${job}
namespace: ${K8S_NAMESPACE}
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 86400
template:
spec:
restartPolicy: Never
$(if [[ -n "$POC_KANIKO_NODE_NAME" ]]; then printf ' nodeName: %s\n' "$POC_KANIKO_NODE_NAME"; fi)
$(if [[ "$POC_KANIKO_CONTEXT_SOURCE" != "git" ]] && boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then cat <<CONTEXT_INIT
initContainers:
- name: prepare-context
image: ${POC_KANIKO_CONTEXT_PREPARE_IMAGE}
command: ["sh", "-ec"]
args:
- rm -rf /kaniko-build-context/*; tar -m -xf ${CONTEXT_ARCHIVE} -C /kaniko-build-context; test -f /kaniko-build-context/${dockerfile}
volumeMounts:
- name: workspace
mountPath: ${WORKSPACE_MOUNT_PATH}
readOnly: true
- name: build-context
mountPath: /kaniko-build-context
CONTEXT_INIT
fi)
containers:
- name: kaniko
image: ${POC_KANIKO_IMAGE}
args:
$(kaniko_args_yaml "$image" "$dockerfile")
volumeMounts:
- name: workspace
mountPath: ${WORKSPACE_MOUNT_PATH}
- name: docker-config
mountPath: /kaniko/.docker
$(if [[ "$POC_KANIKO_CONTEXT_SOURCE" != "git" ]] && boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then cat <<CONTEXT_MOUNT
- name: build-context
mountPath: /kaniko-build-context
CONTEXT_MOUNT
fi)
volumes:
- name: workspace
persistentVolumeClaim:
claimName: ${WORKSPACE_PVC_NAME}
- name: docker-config
secret:
secretName: ${K8S_IMAGE_PULL_SECRET_NAME}
items:
- key: .dockerconfigjson
path: config.json
$(if [[ "$POC_KANIKO_CONTEXT_SOURCE" != "git" ]] && boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then cat <<CONTEXT_VOLUME
- name: build-context
emptyDir: {}
CONTEXT_VOLUME
fi)
YAML
}
wait_for_job() {
local component="$1"
local job
job="$(job_name "$component" "$IMAGE_TAG")"
local timeout_seconds
local deadline
timeout_seconds="$(go_duration_seconds "$POC_KANIKO_WAIT_TIMEOUT")"
deadline=$(($(date +%s) + timeout_seconds))
while true; do
if kubectl -n "$K8S_NAMESPACE" wait --for=condition=complete "job/${job}" --timeout=5s >/dev/null 2>&1; then
printf 'job.batch/%s condition met\n' "$job"
return 0
fi
if kubectl -n "$K8S_NAMESPACE" wait --for=condition=failed "job/${job}" --timeout=1s >/dev/null 2>&1; then
echo "ERROR: Kaniko build job failed: $job" >&2
kubectl -n "$K8S_NAMESPACE" logs "job/${job}" --tail=200 >&2 || true
return 1
fi
if (( $(date +%s) >= deadline )); then
echo "ERROR: Kaniko build job timed out: $job" >&2
kubectl -n "$K8S_NAMESPACE" logs "job/${job}" --tail=200 >&2 || true
return 1
fi
done
}
run_kaniko_components() {
local -a specs=("$@")
local -a batch=()
local spec
local component
local dockerfile
local image
run_batch() {
local -a current=("${batch[@]}")
local attempt
local max_attempts=$((POC_KANIKO_OPERATOR_RETRIES + 1))
local item
for item in "${current[@]}"; do
IFS=$'\t' read -r component dockerfile image <<<"$item"
create_kaniko_job "$component" "$dockerfile" "$image"
done
for item in "${current[@]}"; do
IFS=$'\t' read -r component dockerfile image <<<"$item"
attempt=1
while true; do
if wait_for_job "$component"; then
break
fi
if (( attempt >= max_attempts )); then
return 1
fi
attempt=$((attempt + 1))
printf 'retrying Kaniko build for %s (attempt %s/%s)\n' "$component" "$attempt" "$max_attempts" >&2
create_kaniko_job "$component" "$dockerfile" "$image"
done
done
batch=()
}
for spec in "${specs[@]}"; do
batch+=("$spec")
if (( ${#batch[@]} >= POC_KANIKO_OPERATOR_CONCURRENCY )); then
run_batch
fi
done
if (( ${#batch[@]} > 0 )); then
run_batch
fi
}
component_enabled() {
local component="$1"
local item
IFS=',' read -r -a enabled_components <<<"$POC_KANIKO_COMPONENTS"
for item in "${enabled_components[@]}"; do
if [[ "$item" == "$component" ]]; then
return 0
fi
done
return 1
}
enabled_platform_specs() {
local -a specs=(
"api"$'\t'"Dockerfile.api"$'\t'"$API_IMAGE"
"worker"$'\t'"Dockerfile.worker"$'\t'"$WORKER_IMAGE"
"drosync-coordinator"$'\t'"Dockerfile.drosync-coordinator"$'\t'"$DROSYNC_COORDINATOR_IMAGE"
"runner"$'\t'"Dockerfile.operator-runner"$'\t'"$OPERATOR_RUNNER_IMAGE"
"headpose-service"$'\t'"Dockerfile.headpose-service"$'\t'"$HEADPOSE_SERVICE_IMAGE"
)
local spec
local component
local dockerfile
local image
for spec in "${specs[@]}"; do
IFS=$'\t' read -r component dockerfile image <<<"$spec"
if component_enabled "$component"; then
printf '%s\t%s\t%s\n' "$component" "$dockerfile" "$image"
fi
done
}
operator_image_specs() {
python3 - "$ROOT_DIR/$POC_KANIKO_OPERATOR_MANIFEST" "$REGISTRY_HOST" "$REGISTRY_NAMESPACE" "$IMAGE_TAG" "$POC_KANIKO_OPERATOR_FILTER" <<'PY'
import json
import pathlib
import sys
manifest_path = pathlib.Path(sys.argv[1])
registry_host = sys.argv[2].strip()
registry_namespace = sys.argv[3].strip()
tag = sys.argv[4].strip()
filters = {part.strip().replace("-", "_") for part in sys.argv[5].split(",") if part.strip()}
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
if not isinstance(payload, dict) or not isinstance(payload.get("items"), list):
raise SystemExit(f"{manifest_path} must contain items[]")
if not registry_host or not registry_namespace:
raise SystemExit("REGISTRY_HOST and REGISTRY_NAMESPACE are required for operator image builds")
for item in sorted(payload["items"], key=lambda value: value.get("operatorSlug", "")):
slug = str(item.get("operatorSlug", "")).strip()
dockerfile = str(item.get("dockerfile", "")).strip()
if not slug or not dockerfile:
raise SystemExit(f"invalid operator image manifest item: {item!r}")
if filters and slug not in filters and str(item.get("operatorId", "")).removeprefix("operator-builtin-").replace("-", "_") not in filters:
continue
image = f"{registry_host}/{registry_namespace}/droboticflow-operator-{slug.replace('_', '-')}:{tag}"
component = f"op-{slug.replace('_', '-')}"
print("\t".join([component, dockerfile, image]))
PY
}
operator_base_specs() {
python3 - "$POC_KANIKO_OPERATOR_BASE_FILTER" "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" <<'PY'
import sys
filters = {part.strip().lower().replace("_", "-") for part in sys.argv[1].split(",") if part.strip()}
specs = [
("operator-gpu-base", "operator-images/builtin/gpu-base.Dockerfile", sys.argv[2], {"gpu", "operator-gpu-base"}),
("operator-holobrain-base", "operator-images/builtin/holobrain-base.Dockerfile", sys.argv[3], {"holobrain", "operator-holobrain-base"}),
("operator-yolo-base", "operator-images/builtin/yolo-base.Dockerfile", sys.argv[4], {"yolo", "operator-yolo-base"}),
]
for component, dockerfile, image, aliases in specs:
if filters and not (filters & aliases):
continue
print("\t".join([component, dockerfile, image]))
PY
}
operator_dockerfile_uses_base_arg() {
local dockerfile="$1"
local base_arg="$2"
grep -Eq "^[[:space:]]*ARG[[:space:]]+${base_arg}([=[:space:]]|$)" "$ROOT_DIR/$dockerfile"
}
operator_base_value() {
case "$1" in
OPERATOR_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE" ;;
OPERATOR_GPU_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" ;;
OPERATOR_HOLOBRAIN_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" ;;
OPERATOR_YOLO_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" ;;
*) echo "ERROR: unsupported operator base arg $1" >&2; exit 1 ;;
esac
}
verify_registry_manifest_exists() {
local image="$1"
kubectl -n "$K8S_NAMESPACE" get secret "$K8S_IMAGE_PULL_SECRET_NAME" -o jsonpath='{.data.\.dockerconfigjson}' \
| base64 --decode \
| python3 "$ROOT_DIR/scripts/verify_registry_manifest.py" --image "$image" >&2
}
validate_prebuilt_operator_bases() {
local -a checked_images=()
local spec component dockerfile image base_arg base_image checked
for spec in "${operator_specs[@]}"; do
IFS=$'\t' read -r component dockerfile image <<<"$spec"
python3 "$ROOT_DIR/scripts/validate_kaniko_operator_bases.py" \
--dockerfile "$ROOT_DIR/$dockerfile" \
--operator-base "$POC_KANIKO_OPERATOR_BASE_IMAGE" \
--operator-gpu-base "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" \
--operator-holobrain-base "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" \
--operator-yolo-base "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" \
--require-immutable >&2
for base_arg in OPERATOR_BASE_IMAGE OPERATOR_GPU_BASE_IMAGE OPERATOR_HOLOBRAIN_BASE_IMAGE OPERATOR_YOLO_BASE_IMAGE; do
if ! operator_dockerfile_uses_base_arg "$dockerfile" "$base_arg"; then
continue
fi
base_image="$(operator_base_value "$base_arg")"
# Bash 3 treats an empty array as unset under `set -u`; the fallback
# expansion keeps the first immutable-base preflight reachable.
for checked in "${checked_images[@]:-}"; do
if [[ "$checked" == "$base_image" ]]; then
base_image=""
break
fi
done
if [[ -z "$base_image" ]]; then
continue
fi
# This intentionally happens before any Kaniko Job is created. A base
# tag that has been GC'd must fail fast rather than consume build time.
verify_registry_manifest_exists "$base_image"
checked_images+=("$base_image")
done
done
}
upload_build_context() {
case "$POC_KANIKO_CONTEXT_SOURCE" in
head)
git -C "$ROOT_DIR" archive --format=tar HEAD
;;
worktree)
git -C "$ROOT_DIR" ls-files -z | ROOT_DIR="$ROOT_DIR" python3 -c 'import os,pathlib,sys; root=pathlib.Path(os.environ["ROOT_DIR"]); [sys.stdout.buffer.write(item + b"\0") for item in sys.stdin.buffer.read().split(b"\0") if item and (root / item.decode("utf-8", errors="surrogateescape")).exists()]' | COPYFILE_DISABLE=1 tar --no-xattrs -C "$ROOT_DIR" --null -T - -cf -
;;
esac
}
cleanup_context() {
if [[ "${context_uploaded:-false}" != "true" ]]; then
return 0
fi
if ! boolean_true POC_KANIKO_CLEAN_CONTEXT "$POC_KANIKO_CLEAN_CONTEXT"; then
return 0
fi
if [[ -z "${api_pod:-}" || -z "${context_json:-}" ]]; then
return 0
fi
local cleanup_pid
local deadline
kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -lc "rm -rf $context_json" >/dev/null 2>&1 &
cleanup_pid="$!"
deadline=$(($(date +%s) + POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS))
while kill -0 "$cleanup_pid" >/dev/null 2>&1; do
if (( $(date +%s) >= deadline )); then
kill "$cleanup_pid" >/dev/null 2>&1 || true
wait "$cleanup_pid" >/dev/null 2>&1 || true
printf 'warning: timed out after %ss while cleaning Kaniko build context %s from pod %s; continuing because image build already finished.\n' "$POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS" "$CONTEXT_DIR" "$api_pod" >&2
return 0
fi
sleep 0.25
done
if ! wait "$cleanup_pid" >/dev/null 2>&1; then
printf 'warning: failed to clean Kaniko build context %s from pod %s; continuing because image build already finished.\n' "$CONTEXT_DIR" "$api_pod" >&2
fi
}
trap cleanup_context EXIT
if [[ "$POC_KANIKO_CONTEXT_SOURCE" == "existing" ]]; then
api_pod="$POC_KANIKO_CONTEXT_POD"
if [[ -z "$api_pod" ]]; then
api_pod="$(kubectl -n "$K8S_NAMESPACE" get pod -l app=droboticflow-api --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')"
fi
if boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then
context_json="$(json_string "$CONTEXT_ARCHIVE")"
existing_test="test -s $context_json"
else
context_json="$(json_string "$CONTEXT_DIR")"
existing_test="test -f $context_json/Dockerfile.api"
fi
if [[ -z "$api_pod" ]] || ! kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -lc "$existing_test"; then
echo "ERROR: existing Kaniko context is missing or unreadable: $CONTEXT_DIR" >&2
exit 1
fi
elif [[ "$POC_KANIKO_CONTEXT_SOURCE" != "git" ]]; then
api_pod="$POC_KANIKO_CONTEXT_POD"
if [[ -z "$api_pod" ]]; then
api_pod="$(kubectl -n "$K8S_NAMESPACE" get pod -l app=droboticflow-api --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')"
fi
if [[ -z "$api_pod" ]]; then
echo "ERROR: no droboticflow-api pod is available for uploading the build context." >&2
exit 1
fi
if ! kubectl -n "$K8S_NAMESPACE" get pod "$api_pod" >/dev/null 2>&1; then
echo "ERROR: POC_KANIKO_CONTEXT_POD does not reference an available pod: $api_pod" >&2
exit 1
fi
if boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then
context_json="$(json_string "$CONTEXT_ARCHIVE")"
context_parent_json="$(json_string "$(dirname "$CONTEXT_ARCHIVE")")"
kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -lc "mkdir -p $context_parent_json && rm -f $context_json"
else
context_json="$(json_string "$CONTEXT_DIR")"
kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -lc "rm -rf $context_json && mkdir -p $context_json"
fi
context_uploaded=true
# Object-backed workspace mounts can reject restoring archived mtimes while
# directory entries are still becoming visible. Build contents are immutable
# by digest, so preserve bytes and paths while assigning mount-native mtimes.
if boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then
upload_build_context | kubectl -n "$K8S_NAMESPACE" exec -i "$api_pod" -- sh -lc "cat > $context_json"
else
upload_build_context | kubectl -n "$K8S_NAMESPACE" exec -i "$api_pod" -- tar -m -xf - -C "$CONTEXT_DIR"
fi
fi
operator_components=()
operator_specs=()
operator_component_count=0
if boolean_true POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS"; then
while IFS=$'\t' read -r component dockerfile image; do
if [[ -z "$component" ]]; then
continue
fi
validate_image_reference "operator image $component" "$image"
operator_specs+=("$component"$'\t'"$dockerfile"$'\t'"$image")
operator_components+=("$component:$image")
operator_component_count=$((operator_component_count + 1))
done < <(operator_image_specs)
if (( operator_component_count == 0 )); then
echo "ERROR: POC_KANIKO_OPERATOR_FILTER selected zero operator images." >&2
exit 1
fi
fi
if (( operator_component_count > 0 )) && ! boolean_true POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"; then
validate_prebuilt_operator_bases
fi
if boolean_true POC_KANIKO_PRECHECK_ONLY "$POC_KANIKO_PRECHECK_ONLY"; then
printf 'KANIKO_BUILD_PLAN=contextSource=%s platformComponents=%s buildOperators=%s operatorFilter=%s buildOperatorBases=%s\n' \
"$POC_KANIKO_CONTEXT_SOURCE" "${POC_KANIKO_COMPONENTS:-<empty>}" "$POC_KANIKO_BUILD_OPERATORS" "$POC_KANIKO_OPERATOR_FILTER" "$POC_KANIKO_BUILD_OPERATOR_BASES"
printf 'KANIKO_PRECHECK_ONLY=true\n'
exit 0
fi
platform_specs=()
while IFS=$'\t' read -r component dockerfile image; do
if [[ -n "$component" ]]; then
platform_specs+=("$component"$'\t'"$dockerfile"$'\t'"$image")
fi
done < <(enabled_platform_specs)
if (( ${#platform_specs[@]} > 0 )); then
run_kaniko_components "${platform_specs[@]}"
fi
if (( operator_component_count > 0 )); then
if boolean_true POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"; then
operator_base_specs=()
while IFS=$'\t' read -r component dockerfile image; do
if [[ -n "$component" ]]; then
operator_base_specs+=("$component"$'\t'"$dockerfile"$'\t'"$image")
fi
done < <(operator_base_specs)
if (( ${#operator_base_specs[@]} > 0 )); then
run_kaniko_components "${operator_base_specs[@]}"
fi
fi
run_kaniko_components "${operator_specs[@]}"
fi
printf 'IMAGE_TAG=%s\n' "$IMAGE_TAG"
printf 'API_IMAGE=%s\n' "$API_IMAGE"
printf 'WORKER_IMAGE=%s\n' "$WORKER_IMAGE"
printf 'DROSYNC_COORDINATOR_IMAGE=%s\n' "$DROSYNC_COORDINATOR_IMAGE"
printf 'OPERATOR_RUNNER_IMAGE=%s\n' "$OPERATOR_RUNNER_IMAGE"
printf 'HEADPOSE_SERVICE_IMAGE=%s\n' "$HEADPOSE_SERVICE_IMAGE"
if (( operator_component_count > 0 )); then
printf 'OPERATOR_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE"
printf 'OPERATOR_GPU_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
printf 'OPERATOR_HOLOBRAIN_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE"
printf 'OPERATOR_YOLO_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE"
for spec in "${operator_components[@]}"; do
printf 'OPERATOR_IMAGE=%s\n' "${spec#*:}"
done
fi