forked from longtao.wu/droboticflow
693 lines
29 KiB
Bash
Executable File
693 lines
29 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-}"
|
|
|
|
if [[ $# -gt 0 && -z "$DEPLOY_ENV" && "${1:-}" != -* ]]; then
|
|
DEPLOY_ENV="$1"
|
|
shift
|
|
fi
|
|
|
|
export DEPLOY_ENV
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
# shellcheck source=scripts/lib/build-metadata.sh
|
|
source "$ROOT_DIR/scripts/lib/build-metadata.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
REGISTRY_HOST="${REGISTRY_HOST:-}"
|
|
REGISTRY_NAMESPACE="${REGISTRY_NAMESPACE:-}"
|
|
K8S_IMAGE_PULL_SECRET_NAME="${K8S_IMAGE_PULL_SECRET_NAME:-droboticflow-registry}"
|
|
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
|
WORKSPACE_MOUNT_PATH="${WORKSPACE_MOUNT_PATH:-/workspace}"
|
|
POC_KANIKO_IMAGE="${POC_KANIKO_IMAGE:-gcr.io/kaniko-project/executor:v1.23.2-debug}"
|
|
POC_KANIKO_REGISTRY_MIRRORS="${POC_KANIKO_REGISTRY_MIRRORS:-mirror.gcr.io}"
|
|
POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK="${POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK:-true}"
|
|
POC_KANIKO_CLEAN_CONTEXT="${POC_KANIKO_CLEAN_CONTEXT:-true}"
|
|
POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS="${POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS:-30}"
|
|
POC_KANIKO_WAIT_TIMEOUT="${POC_KANIKO_WAIT_TIMEOUT:-1200s}"
|
|
POC_KANIKO_GOPROXY="${POC_KANIKO_GOPROXY:-}"
|
|
POC_KANIKO_CONTEXT_SOURCE="${POC_KANIKO_CONTEXT_SOURCE:-worktree}"
|
|
POC_KANIKO_PACKED_CONTEXT="${POC_KANIKO_PACKED_CONTEXT:-true}"
|
|
POC_KANIKO_CONTEXT_PREPARE_IMAGE="${POC_KANIKO_CONTEXT_PREPARE_IMAGE:-busybox:1.36.1}"
|
|
POC_KANIKO_CONTEXT_POD="${POC_KANIKO_CONTEXT_POD:-}"
|
|
POC_KANIKO_GIT_CONTEXT="${POC_KANIKO_GIT_CONTEXT:-}"
|
|
POC_KANIKO_NODE_NAME="${POC_KANIKO_NODE_NAME:-}"
|
|
POC_KANIKO_BUILD_OPERATORS="${POC_KANIKO_BUILD_OPERATORS:-false}"
|
|
POC_KANIKO_BUILD_OPERATOR_BASES="${POC_KANIKO_BUILD_OPERATOR_BASES:-true}"
|
|
POC_KANIKO_OPERATOR_BASE_FILTER="${POC_KANIKO_OPERATOR_BASE_FILTER:-}"
|
|
# An explicit empty value is meaningful for an operator-only build. Use '-' so
|
|
# it is not silently expanded back to every platform component.
|
|
POC_KANIKO_COMPONENTS="${POC_KANIKO_COMPONENTS-api,worker,drosync-coordinator,runner,headpose-service}"
|
|
POC_KANIKO_OPERATOR_MANIFEST="${POC_KANIKO_OPERATOR_MANIFEST:-operator-images/builtin/operator-images.json}"
|
|
POC_KANIKO_OPERATOR_FILTER="${POC_KANIKO_OPERATOR_FILTER:-}"
|
|
POC_KANIKO_OPERATOR_CONCURRENCY="${POC_KANIKO_OPERATOR_CONCURRENCY:-4}"
|
|
POC_KANIKO_OPERATOR_RETRIES="${POC_KANIKO_OPERATOR_RETRIES:-2}"
|
|
POC_KANIKO_HEADPOSE_BASE_IMAGE="${POC_KANIKO_HEADPOSE_BASE_IMAGE:-}"
|
|
POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME="${POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME:-false}"
|
|
POC_KANIKO_OPERATOR_BASE_IMAGE="${POC_KANIKO_OPERATOR_BASE_IMAGE:-}"
|
|
POC_KANIKO_OPERATOR_GPU_BASE_IMAGE="${POC_KANIKO_OPERATOR_GPU_BASE_IMAGE:-}"
|
|
POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE="${POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE:-}"
|
|
POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-}"
|
|
POC_BUILD_CONTEXT_ROOT="${POC_BUILD_CONTEXT_ROOT:-$WORKSPACE_MOUNT_PATH/system/builds/contexts}"
|
|
POC_KANIKO_PRECHECK_ONLY="${POC_KANIKO_PRECHECK_ONLY:-false}"
|
|
|
|
trap 'status=$?; printf "ERROR: build-poc-images-kaniko.sh failed at line %s (exit %s)\n" "$LINENO" "$status" >&2; exit "$status"' ERR
|
|
|
|
if [[ -z "${IMAGE_TAG:-}" ]]; then
|
|
IMAGE_TAG="$(git -C "$ROOT_DIR" rev-parse --short HEAD)-${DEPLOY_ENV:-poc}-$(TZ=Asia/Shanghai date +%Y%m%d%H%M%S)"
|
|
fi
|
|
POC_KANIKO_SOURCE_COMMIT="${POC_KANIKO_SOURCE_COMMIT:-$(git -C "$ROOT_DIR" rev-parse HEAD)}"
|
|
POC_KANIKO_BUILD_TIME="${POC_KANIKO_BUILD_TIME:-}"
|
|
POC_KANIKO_BUILD_TIME="$(droboticflow_normalize_build_time "$POC_KANIKO_BUILD_TIME")"
|
|
HEADPOSE_SERVICE_IMAGE="${HEADPOSE_SERVICE_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-headpose-service:$IMAGE_TAG}"
|
|
|
|
# shellcheck source=scripts/lib/image-config.sh
|
|
source "$ROOT_DIR/scripts/lib/image-config.sh"
|
|
droboticflow_resolve_image_configuration
|
|
POC_KANIKO_OPERATOR_BASE_IMAGE="${POC_KANIKO_OPERATOR_BASE_IMAGE:-$OPERATOR_RUNNER_IMAGE}"
|
|
POC_KANIKO_OPERATOR_GPU_BASE_IMAGE="${POC_KANIKO_OPERATOR_GPU_BASE_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-operator-gpu-base:$IMAGE_TAG}"
|
|
POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE="${POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-operator-holobrain-base:$IMAGE_TAG}"
|
|
POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE="${POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE:-$REGISTRY_HOST/$REGISTRY_NAMESPACE/droboticflow-operator-yolo-base:$IMAGE_TAG}"
|
|
# shellcheck source=scripts/lib/deploy-validation.sh
|
|
source "$ROOT_DIR/scripts/lib/deploy-validation.sh"
|
|
validate_dns_label K8S_NAMESPACE "$K8S_NAMESPACE"
|
|
validate_dns_subdomain K8S_IMAGE_PULL_SECRET_NAME "$K8S_IMAGE_PULL_SECRET_NAME"
|
|
validate_dns_subdomain WORKSPACE_PVC_NAME "$WORKSPACE_PVC_NAME"
|
|
validate_absolute_mount_path WORKSPACE_MOUNT_PATH "$WORKSPACE_MOUNT_PATH"
|
|
validate_absolute_mount_path POC_BUILD_CONTEXT_ROOT "$POC_BUILD_CONTEXT_ROOT"
|
|
if [[ -n "$POC_KANIKO_NODE_NAME" ]]; then
|
|
validate_dns_subdomain POC_KANIKO_NODE_NAME "$POC_KANIKO_NODE_NAME"
|
|
fi
|
|
if [[ -n "$POC_KANIKO_CONTEXT_POD" ]]; then
|
|
validate_dns_subdomain POC_KANIKO_CONTEXT_POD "$POC_KANIKO_CONTEXT_POD"
|
|
fi
|
|
validate_image_reference POC_KANIKO_IMAGE "$POC_KANIKO_IMAGE"
|
|
validate_image_reference API_IMAGE "$API_IMAGE"
|
|
validate_image_reference WORKER_IMAGE "$WORKER_IMAGE"
|
|
validate_image_reference DROSYNC_COORDINATOR_IMAGE "$DROSYNC_COORDINATOR_IMAGE"
|
|
validate_image_reference OPERATOR_RUNNER_IMAGE "$OPERATOR_RUNNER_IMAGE"
|
|
validate_image_reference HEADPOSE_SERVICE_IMAGE "$HEADPOSE_SERVICE_IMAGE"
|
|
validate_image_reference POC_KANIKO_OPERATOR_BASE_IMAGE "$POC_KANIKO_OPERATOR_BASE_IMAGE"
|
|
validate_image_reference POC_KANIKO_OPERATOR_GPU_BASE_IMAGE "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
|
|
validate_image_reference POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE"
|
|
validate_image_reference POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE"
|
|
validate_boolean POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK "$POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK"
|
|
validate_boolean POC_KANIKO_CLEAN_CONTEXT "$POC_KANIKO_CLEAN_CONTEXT"
|
|
validate_boolean POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS"
|
|
validate_boolean POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"
|
|
validate_boolean POC_KANIKO_PRECHECK_ONLY "$POC_KANIKO_PRECHECK_ONLY"
|
|
validate_boolean POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME "$POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME"
|
|
validate_boolean POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"
|
|
validate_image_reference POC_KANIKO_CONTEXT_PREPARE_IMAGE "$POC_KANIKO_CONTEXT_PREPARE_IMAGE"
|
|
if [[ -n "$POC_KANIKO_HEADPOSE_BASE_IMAGE" ]]; then
|
|
validate_image_reference POC_KANIKO_HEADPOSE_BASE_IMAGE "$POC_KANIKO_HEADPOSE_BASE_IMAGE"
|
|
fi
|
|
if boolean_true POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME "$POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME" && [[ -z "$POC_KANIKO_HEADPOSE_BASE_IMAGE" ]]; then
|
|
echo "ERROR: POC_KANIKO_HEADPOSE_BASE_IMAGE is required when POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME=true" >&2
|
|
exit 1
|
|
fi
|
|
case "$POC_KANIKO_CONTEXT_SOURCE" in
|
|
worktree|head|git|existing)
|
|
;;
|
|
*)
|
|
echo "ERROR: POC_KANIKO_CONTEXT_SOURCE must be worktree, head, git, or existing." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
if [[ "$POC_KANIKO_CONTEXT_SOURCE" == "git" ]]; then
|
|
if [[ -z "$POC_KANIKO_GIT_CONTEXT" || "$POC_KANIKO_GIT_CONTEXT" != git://* || "$POC_KANIKO_GIT_CONTEXT" =~ [[:space:]] ]]; then
|
|
echo "ERROR: POC_KANIKO_GIT_CONTEXT must be a whitespace-free git:// context when POC_KANIKO_CONTEXT_SOURCE=git." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
if [[ "$POC_KANIKO_GOPROXY" =~ [[:space:]] ]]; then
|
|
echo "ERROR: POC_KANIKO_GOPROXY must not contain whitespace." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$POC_KANIKO_OPERATOR_CONCURRENCY" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: POC_KANIKO_OPERATOR_CONCURRENCY must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$POC_KANIKO_OPERATOR_RETRIES" =~ ^[0-9]+$ ]]; then
|
|
echo "ERROR: POC_KANIKO_OPERATOR_RETRIES must be a non-negative integer." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "$POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "ERROR: POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
if [[ "$POC_KANIKO_COMPONENTS" =~ [[:space:]] ]]; then
|
|
echo "ERROR: POC_KANIKO_COMPONENTS must be a comma-separated list without whitespace." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$POC_BUILD_CONTEXT_ROOT" != "$WORKSPACE_MOUNT_PATH" && "$POC_BUILD_CONTEXT_ROOT" != "$WORKSPACE_MOUNT_PATH"/* ]]; then
|
|
echo "ERROR: POC_BUILD_CONTEXT_ROOT must point under WORKSPACE_MOUNT_PATH." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if boolean_true POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS" && [[ ! -f "$ROOT_DIR/$POC_KANIKO_OPERATOR_MANIFEST" ]]; then
|
|
echo "ERROR: POC_KANIKO_OPERATOR_MANIFEST does not exist: $POC_KANIKO_OPERATOR_MANIFEST" >&2
|
|
exit 1
|
|
fi
|
|
|
|
CONTEXT_DIR="${POC_BUILD_CONTEXT_ROOT%/}/droboticflow-${IMAGE_TAG}"
|
|
CONTEXT_ARCHIVE="${CONTEXT_DIR}.tar"
|
|
api_pod=""
|
|
context_json=""
|
|
context_uploaded=false
|
|
|
|
json_string() {
|
|
python3 - "$1" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
print(json.dumps(sys.argv[1]))
|
|
PY
|
|
}
|
|
|
|
go_duration_seconds() {
|
|
python3 - "$1" <<'PY'
|
|
import re
|
|
import sys
|
|
|
|
value = sys.argv[1]
|
|
units = {"ns": 0, "us": 0, "ms": 0, "s": 1, "m": 60, "h": 3600}
|
|
total = 0
|
|
for number, unit in re.findall(r"([1-9][0-9]*)(ns|us|ms|s|m|h)", value):
|
|
total += int(number) * units[unit]
|
|
print(max(total, 1))
|
|
PY
|
|
}
|
|
|
|
job_name() {
|
|
python3 - "$1" "$2" <<'PY'
|
|
import hashlib
|
|
import re
|
|
import sys
|
|
|
|
component = sys.argv[1]
|
|
tag = sys.argv[2].lower()
|
|
suffix = re.sub(r"[^a-z0-9-]+", "-", tag).strip("-") or "build"
|
|
component_slug = re.sub(r"[^a-z0-9-]+", "-", component.lower()).strip("-") or "component"
|
|
prefix = f"droboticflow-kaniko-{component_slug}-"
|
|
if len(prefix) >= 48:
|
|
digest = hashlib.sha1(component_slug.encode("utf-8")).hexdigest()[:8]
|
|
prefix = f"droboticflow-kaniko-{component_slug[:24].strip('-')}-{digest}-"
|
|
print(prefix + suffix[: 63 - len(prefix)].strip("-"))
|
|
PY
|
|
}
|
|
|
|
kaniko_args_yaml() {
|
|
local destination="$1"
|
|
local dockerfile="$2"
|
|
local mirror
|
|
local context_value="dir://${CONTEXT_DIR}"
|
|
local dockerfile_value="${CONTEXT_DIR}/${dockerfile}"
|
|
if [[ "$POC_KANIKO_CONTEXT_SOURCE" == "git" ]]; then
|
|
context_value="$POC_KANIKO_GIT_CONTEXT"
|
|
dockerfile_value="$dockerfile"
|
|
elif boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then
|
|
context_value="dir:///kaniko-build-context"
|
|
dockerfile_value="/kaniko-build-context/${dockerfile}"
|
|
fi
|
|
cat <<YAML
|
|
- --context=${context_value}
|
|
- --dockerfile=${dockerfile_value}
|
|
- --destination=${destination}
|
|
YAML
|
|
if [[ -n "$POC_KANIKO_GOPROXY" ]]; then
|
|
printf ' - --build-arg=GOPROXY=%s\n' "$POC_KANIKO_GOPROXY"
|
|
fi
|
|
printf ' - --build-arg=OPERATOR_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE"
|
|
printf ' - --build-arg=OPERATOR_GPU_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
|
|
printf ' - --build-arg=OPERATOR_HOLOBRAIN_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE"
|
|
printf ' - --build-arg=OPERATOR_YOLO_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE"
|
|
printf ' - --build-arg=DROBOTICFLOW_BUILD_COMMIT=%s\n' "$POC_KANIKO_SOURCE_COMMIT"
|
|
printf ' - --build-arg=DROBOTICFLOW_BUILD_TIME=%s\n' "$POC_KANIKO_BUILD_TIME"
|
|
if [[ "$dockerfile" == "Dockerfile.headpose-service" ]]; then
|
|
if [[ -n "$POC_KANIKO_HEADPOSE_BASE_IMAGE" ]]; then
|
|
printf ' - --build-arg=HEADPOSE_BASE_IMAGE=%s\n' "$POC_KANIKO_HEADPOSE_BASE_IMAGE"
|
|
fi
|
|
printf ' - --build-arg=HEADPOSE_REUSE_BASE_RUNTIME=%s\n' "$POC_KANIKO_HEADPOSE_REUSE_BASE_RUNTIME"
|
|
fi
|
|
IFS=',' read -r -a mirrors <<<"$POC_KANIKO_REGISTRY_MIRRORS"
|
|
for mirror in "${mirrors[@]}"; do
|
|
mirror="$(printf '%s' "$mirror" | xargs)"
|
|
if [[ -z "$mirror" ]]; then
|
|
continue
|
|
fi
|
|
if [[ "$mirror" =~ [[:space:]] ]]; then
|
|
echo "ERROR: POC_KANIKO_REGISTRY_MIRRORS entries must not contain whitespace." >&2
|
|
exit 1
|
|
fi
|
|
printf ' - --registry-mirror=%s\n' "$mirror"
|
|
done
|
|
if boolean_true POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK "$POC_KANIKO_SKIP_DEFAULT_REGISTRY_FALLBACK"; then
|
|
printf ' - --skip-default-registry-fallback\n'
|
|
fi
|
|
cat <<YAML
|
|
- --snapshot-mode=redo
|
|
- --use-new-run
|
|
- --verbosity=info
|
|
YAML
|
|
}
|
|
|
|
create_kaniko_job() {
|
|
local component="$1"
|
|
local dockerfile="$2"
|
|
local image="$3"
|
|
local job
|
|
job="$(job_name "$component" "$IMAGE_TAG")"
|
|
kubectl -n "$K8S_NAMESPACE" delete job "$job" --ignore-not-found=true >/dev/null
|
|
kubectl -n "$K8S_NAMESPACE" apply -f - <<YAML
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: ${job}
|
|
namespace: ${K8S_NAMESPACE}
|
|
spec:
|
|
backoffLimit: 0
|
|
ttlSecondsAfterFinished: 86400
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
$(if [[ -n "$POC_KANIKO_NODE_NAME" ]]; then printf ' nodeName: %s\n' "$POC_KANIKO_NODE_NAME"; fi)
|
|
$(if [[ "$POC_KANIKO_CONTEXT_SOURCE" != "git" ]] && boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then cat <<CONTEXT_INIT
|
|
initContainers:
|
|
- name: prepare-context
|
|
image: ${POC_KANIKO_CONTEXT_PREPARE_IMAGE}
|
|
command: ["sh", "-ec"]
|
|
args:
|
|
- rm -rf /kaniko-build-context/*; tar -m -xf ${CONTEXT_ARCHIVE} -C /kaniko-build-context; test -f /kaniko-build-context/${dockerfile}
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: ${WORKSPACE_MOUNT_PATH}
|
|
readOnly: true
|
|
- name: build-context
|
|
mountPath: /kaniko-build-context
|
|
CONTEXT_INIT
|
|
fi)
|
|
containers:
|
|
- name: kaniko
|
|
image: ${POC_KANIKO_IMAGE}
|
|
args:
|
|
$(kaniko_args_yaml "$image" "$dockerfile")
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: ${WORKSPACE_MOUNT_PATH}
|
|
- name: docker-config
|
|
mountPath: /kaniko/.docker
|
|
$(if [[ "$POC_KANIKO_CONTEXT_SOURCE" != "git" ]] && boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then cat <<CONTEXT_MOUNT
|
|
- name: build-context
|
|
mountPath: /kaniko-build-context
|
|
CONTEXT_MOUNT
|
|
fi)
|
|
volumes:
|
|
- name: workspace
|
|
persistentVolumeClaim:
|
|
claimName: ${WORKSPACE_PVC_NAME}
|
|
- name: docker-config
|
|
secret:
|
|
secretName: ${K8S_IMAGE_PULL_SECRET_NAME}
|
|
items:
|
|
- key: .dockerconfigjson
|
|
path: config.json
|
|
$(if [[ "$POC_KANIKO_CONTEXT_SOURCE" != "git" ]] && boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then cat <<CONTEXT_VOLUME
|
|
- name: build-context
|
|
emptyDir: {}
|
|
CONTEXT_VOLUME
|
|
fi)
|
|
YAML
|
|
}
|
|
|
|
wait_for_job() {
|
|
local component="$1"
|
|
local job
|
|
job="$(job_name "$component" "$IMAGE_TAG")"
|
|
local timeout_seconds
|
|
local deadline
|
|
timeout_seconds="$(go_duration_seconds "$POC_KANIKO_WAIT_TIMEOUT")"
|
|
deadline=$(($(date +%s) + timeout_seconds))
|
|
while true; do
|
|
if kubectl -n "$K8S_NAMESPACE" wait --for=condition=complete "job/${job}" --timeout=5s >/dev/null 2>&1; then
|
|
printf 'job.batch/%s condition met\n' "$job"
|
|
return 0
|
|
fi
|
|
if kubectl -n "$K8S_NAMESPACE" wait --for=condition=failed "job/${job}" --timeout=1s >/dev/null 2>&1; then
|
|
echo "ERROR: Kaniko build job failed: $job" >&2
|
|
kubectl -n "$K8S_NAMESPACE" logs "job/${job}" --tail=200 >&2 || true
|
|
return 1
|
|
fi
|
|
if (( $(date +%s) >= deadline )); then
|
|
echo "ERROR: Kaniko build job timed out: $job" >&2
|
|
kubectl -n "$K8S_NAMESPACE" logs "job/${job}" --tail=200 >&2 || true
|
|
return 1
|
|
fi
|
|
done
|
|
}
|
|
|
|
run_kaniko_components() {
|
|
local -a specs=("$@")
|
|
local -a batch=()
|
|
local spec
|
|
local component
|
|
local dockerfile
|
|
local image
|
|
|
|
run_batch() {
|
|
local -a current=("${batch[@]}")
|
|
local attempt
|
|
local max_attempts=$((POC_KANIKO_OPERATOR_RETRIES + 1))
|
|
local item
|
|
for item in "${current[@]}"; do
|
|
IFS=$'\t' read -r component dockerfile image <<<"$item"
|
|
create_kaniko_job "$component" "$dockerfile" "$image"
|
|
done
|
|
for item in "${current[@]}"; do
|
|
IFS=$'\t' read -r component dockerfile image <<<"$item"
|
|
attempt=1
|
|
while true; do
|
|
if wait_for_job "$component"; then
|
|
break
|
|
fi
|
|
if (( attempt >= max_attempts )); then
|
|
return 1
|
|
fi
|
|
attempt=$((attempt + 1))
|
|
printf 'retrying Kaniko build for %s (attempt %s/%s)\n' "$component" "$attempt" "$max_attempts" >&2
|
|
create_kaniko_job "$component" "$dockerfile" "$image"
|
|
done
|
|
done
|
|
batch=()
|
|
}
|
|
|
|
for spec in "${specs[@]}"; do
|
|
batch+=("$spec")
|
|
if (( ${#batch[@]} >= POC_KANIKO_OPERATOR_CONCURRENCY )); then
|
|
run_batch
|
|
fi
|
|
done
|
|
if (( ${#batch[@]} > 0 )); then
|
|
run_batch
|
|
fi
|
|
}
|
|
|
|
component_enabled() {
|
|
local component="$1"
|
|
local item
|
|
IFS=',' read -r -a enabled_components <<<"$POC_KANIKO_COMPONENTS"
|
|
for item in "${enabled_components[@]}"; do
|
|
if [[ "$item" == "$component" ]]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
enabled_platform_specs() {
|
|
local -a specs=(
|
|
"api"$'\t'"Dockerfile.api"$'\t'"$API_IMAGE"
|
|
"worker"$'\t'"Dockerfile.worker"$'\t'"$WORKER_IMAGE"
|
|
"drosync-coordinator"$'\t'"Dockerfile.drosync-coordinator"$'\t'"$DROSYNC_COORDINATOR_IMAGE"
|
|
"runner"$'\t'"Dockerfile.operator-runner"$'\t'"$OPERATOR_RUNNER_IMAGE"
|
|
"headpose-service"$'\t'"Dockerfile.headpose-service"$'\t'"$HEADPOSE_SERVICE_IMAGE"
|
|
)
|
|
local spec
|
|
local component
|
|
local dockerfile
|
|
local image
|
|
for spec in "${specs[@]}"; do
|
|
IFS=$'\t' read -r component dockerfile image <<<"$spec"
|
|
if component_enabled "$component"; then
|
|
printf '%s\t%s\t%s\n' "$component" "$dockerfile" "$image"
|
|
fi
|
|
done
|
|
}
|
|
|
|
operator_image_specs() {
|
|
python3 - "$ROOT_DIR/$POC_KANIKO_OPERATOR_MANIFEST" "$REGISTRY_HOST" "$REGISTRY_NAMESPACE" "$IMAGE_TAG" "$POC_KANIKO_OPERATOR_FILTER" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
manifest_path = pathlib.Path(sys.argv[1])
|
|
registry_host = sys.argv[2].strip()
|
|
registry_namespace = sys.argv[3].strip()
|
|
tag = sys.argv[4].strip()
|
|
filters = {part.strip().replace("-", "_") for part in sys.argv[5].split(",") if part.strip()}
|
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict) or not isinstance(payload.get("items"), list):
|
|
raise SystemExit(f"{manifest_path} must contain items[]")
|
|
if not registry_host or not registry_namespace:
|
|
raise SystemExit("REGISTRY_HOST and REGISTRY_NAMESPACE are required for operator image builds")
|
|
for item in sorted(payload["items"], key=lambda value: value.get("operatorSlug", "")):
|
|
slug = str(item.get("operatorSlug", "")).strip()
|
|
dockerfile = str(item.get("dockerfile", "")).strip()
|
|
if not slug or not dockerfile:
|
|
raise SystemExit(f"invalid operator image manifest item: {item!r}")
|
|
if filters and slug not in filters and str(item.get("operatorId", "")).removeprefix("operator-builtin-").replace("-", "_") not in filters:
|
|
continue
|
|
image = f"{registry_host}/{registry_namespace}/droboticflow-operator-{slug.replace('_', '-')}:{tag}"
|
|
component = f"op-{slug.replace('_', '-')}"
|
|
print("\t".join([component, dockerfile, image]))
|
|
PY
|
|
}
|
|
|
|
operator_base_specs() {
|
|
python3 - "$POC_KANIKO_OPERATOR_BASE_FILTER" "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" <<'PY'
|
|
import sys
|
|
|
|
filters = {part.strip().lower().replace("_", "-") for part in sys.argv[1].split(",") if part.strip()}
|
|
specs = [
|
|
("operator-gpu-base", "operator-images/builtin/gpu-base.Dockerfile", sys.argv[2], {"gpu", "operator-gpu-base"}),
|
|
("operator-holobrain-base", "operator-images/builtin/holobrain-base.Dockerfile", sys.argv[3], {"holobrain", "operator-holobrain-base"}),
|
|
("operator-yolo-base", "operator-images/builtin/yolo-base.Dockerfile", sys.argv[4], {"yolo", "operator-yolo-base"}),
|
|
]
|
|
for component, dockerfile, image, aliases in specs:
|
|
if filters and not (filters & aliases):
|
|
continue
|
|
print("\t".join([component, dockerfile, image]))
|
|
PY
|
|
}
|
|
|
|
operator_dockerfile_uses_base_arg() {
|
|
local dockerfile="$1"
|
|
local base_arg="$2"
|
|
grep -Eq "^[[:space:]]*ARG[[:space:]]+${base_arg}([=[:space:]]|$)" "$ROOT_DIR/$dockerfile"
|
|
}
|
|
|
|
operator_base_value() {
|
|
case "$1" in
|
|
OPERATOR_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE" ;;
|
|
OPERATOR_GPU_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" ;;
|
|
OPERATOR_HOLOBRAIN_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" ;;
|
|
OPERATOR_YOLO_BASE_IMAGE) printf '%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" ;;
|
|
*) echo "ERROR: unsupported operator base arg $1" >&2; exit 1 ;;
|
|
esac
|
|
}
|
|
|
|
verify_registry_manifest_exists() {
|
|
local image="$1"
|
|
kubectl -n "$K8S_NAMESPACE" get secret "$K8S_IMAGE_PULL_SECRET_NAME" -o jsonpath='{.data.\.dockerconfigjson}' \
|
|
| base64 --decode \
|
|
| python3 "$ROOT_DIR/scripts/verify_registry_manifest.py" --image "$image" >&2
|
|
}
|
|
|
|
validate_prebuilt_operator_bases() {
|
|
local -a checked_images=()
|
|
local spec component dockerfile image base_arg base_image checked
|
|
for spec in "${operator_specs[@]}"; do
|
|
IFS=$'\t' read -r component dockerfile image <<<"$spec"
|
|
python3 "$ROOT_DIR/scripts/validate_kaniko_operator_bases.py" \
|
|
--dockerfile "$ROOT_DIR/$dockerfile" \
|
|
--operator-base "$POC_KANIKO_OPERATOR_BASE_IMAGE" \
|
|
--operator-gpu-base "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE" \
|
|
--operator-holobrain-base "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE" \
|
|
--operator-yolo-base "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE" \
|
|
--require-immutable >&2
|
|
for base_arg in OPERATOR_BASE_IMAGE OPERATOR_GPU_BASE_IMAGE OPERATOR_HOLOBRAIN_BASE_IMAGE OPERATOR_YOLO_BASE_IMAGE; do
|
|
if ! operator_dockerfile_uses_base_arg "$dockerfile" "$base_arg"; then
|
|
continue
|
|
fi
|
|
base_image="$(operator_base_value "$base_arg")"
|
|
# Bash 3 treats an empty array as unset under `set -u`; the fallback
|
|
# expansion keeps the first immutable-base preflight reachable.
|
|
for checked in "${checked_images[@]:-}"; do
|
|
if [[ "$checked" == "$base_image" ]]; then
|
|
base_image=""
|
|
break
|
|
fi
|
|
done
|
|
if [[ -z "$base_image" ]]; then
|
|
continue
|
|
fi
|
|
# This intentionally happens before any Kaniko Job is created. A base
|
|
# tag that has been GC'd must fail fast rather than consume build time.
|
|
verify_registry_manifest_exists "$base_image"
|
|
checked_images+=("$base_image")
|
|
done
|
|
done
|
|
}
|
|
|
|
upload_build_context() {
|
|
case "$POC_KANIKO_CONTEXT_SOURCE" in
|
|
head)
|
|
git -C "$ROOT_DIR" archive --format=tar HEAD
|
|
;;
|
|
worktree)
|
|
git -C "$ROOT_DIR" ls-files -z | ROOT_DIR="$ROOT_DIR" python3 -c 'import os,pathlib,sys; root=pathlib.Path(os.environ["ROOT_DIR"]); [sys.stdout.buffer.write(item + b"\0") for item in sys.stdin.buffer.read().split(b"\0") if item and (root / item.decode("utf-8", errors="surrogateescape")).exists()]' | COPYFILE_DISABLE=1 tar --no-xattrs -C "$ROOT_DIR" --null -T - -cf -
|
|
;;
|
|
esac
|
|
}
|
|
|
|
cleanup_context() {
|
|
if [[ "${context_uploaded:-false}" != "true" ]]; then
|
|
return 0
|
|
fi
|
|
if ! boolean_true POC_KANIKO_CLEAN_CONTEXT "$POC_KANIKO_CLEAN_CONTEXT"; then
|
|
return 0
|
|
fi
|
|
if [[ -z "${api_pod:-}" || -z "${context_json:-}" ]]; then
|
|
return 0
|
|
fi
|
|
local cleanup_pid
|
|
local deadline
|
|
kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -lc "rm -rf $context_json" >/dev/null 2>&1 &
|
|
cleanup_pid="$!"
|
|
deadline=$(($(date +%s) + POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS))
|
|
while kill -0 "$cleanup_pid" >/dev/null 2>&1; do
|
|
if (( $(date +%s) >= deadline )); then
|
|
kill "$cleanup_pid" >/dev/null 2>&1 || true
|
|
wait "$cleanup_pid" >/dev/null 2>&1 || true
|
|
printf 'warning: timed out after %ss while cleaning Kaniko build context %s from pod %s; continuing because image build already finished.\n' "$POC_KANIKO_CLEAN_CONTEXT_TIMEOUT_SECONDS" "$CONTEXT_DIR" "$api_pod" >&2
|
|
return 0
|
|
fi
|
|
sleep 0.25
|
|
done
|
|
if ! wait "$cleanup_pid" >/dev/null 2>&1; then
|
|
printf 'warning: failed to clean Kaniko build context %s from pod %s; continuing because image build already finished.\n' "$CONTEXT_DIR" "$api_pod" >&2
|
|
fi
|
|
}
|
|
trap cleanup_context EXIT
|
|
|
|
if [[ "$POC_KANIKO_CONTEXT_SOURCE" == "existing" ]]; then
|
|
api_pod="$POC_KANIKO_CONTEXT_POD"
|
|
if [[ -z "$api_pod" ]]; then
|
|
api_pod="$(kubectl -n "$K8S_NAMESPACE" get pod -l app=droboticflow-api --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')"
|
|
fi
|
|
if boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then
|
|
context_json="$(json_string "$CONTEXT_ARCHIVE")"
|
|
existing_test="test -s $context_json"
|
|
else
|
|
context_json="$(json_string "$CONTEXT_DIR")"
|
|
existing_test="test -f $context_json/Dockerfile.api"
|
|
fi
|
|
if [[ -z "$api_pod" ]] || ! kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -lc "$existing_test"; then
|
|
echo "ERROR: existing Kaniko context is missing or unreadable: $CONTEXT_DIR" >&2
|
|
exit 1
|
|
fi
|
|
elif [[ "$POC_KANIKO_CONTEXT_SOURCE" != "git" ]]; then
|
|
api_pod="$POC_KANIKO_CONTEXT_POD"
|
|
if [[ -z "$api_pod" ]]; then
|
|
api_pod="$(kubectl -n "$K8S_NAMESPACE" get pod -l app=droboticflow-api --field-selector=status.phase=Running -o jsonpath='{.items[0].metadata.name}')"
|
|
fi
|
|
if [[ -z "$api_pod" ]]; then
|
|
echo "ERROR: no droboticflow-api pod is available for uploading the build context." >&2
|
|
exit 1
|
|
fi
|
|
if ! kubectl -n "$K8S_NAMESPACE" get pod "$api_pod" >/dev/null 2>&1; then
|
|
echo "ERROR: POC_KANIKO_CONTEXT_POD does not reference an available pod: $api_pod" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then
|
|
context_json="$(json_string "$CONTEXT_ARCHIVE")"
|
|
context_parent_json="$(json_string "$(dirname "$CONTEXT_ARCHIVE")")"
|
|
kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -lc "mkdir -p $context_parent_json && rm -f $context_json"
|
|
else
|
|
context_json="$(json_string "$CONTEXT_DIR")"
|
|
kubectl -n "$K8S_NAMESPACE" exec "$api_pod" -- sh -lc "rm -rf $context_json && mkdir -p $context_json"
|
|
fi
|
|
context_uploaded=true
|
|
# Object-backed workspace mounts can reject restoring archived mtimes while
|
|
# directory entries are still becoming visible. Build contents are immutable
|
|
# by digest, so preserve bytes and paths while assigning mount-native mtimes.
|
|
if boolean_true POC_KANIKO_PACKED_CONTEXT "$POC_KANIKO_PACKED_CONTEXT"; then
|
|
upload_build_context | kubectl -n "$K8S_NAMESPACE" exec -i "$api_pod" -- sh -lc "cat > $context_json"
|
|
else
|
|
upload_build_context | kubectl -n "$K8S_NAMESPACE" exec -i "$api_pod" -- tar -m -xf - -C "$CONTEXT_DIR"
|
|
fi
|
|
fi
|
|
|
|
operator_components=()
|
|
operator_specs=()
|
|
operator_component_count=0
|
|
if boolean_true POC_KANIKO_BUILD_OPERATORS "$POC_KANIKO_BUILD_OPERATORS"; then
|
|
while IFS=$'\t' read -r component dockerfile image; do
|
|
if [[ -z "$component" ]]; then
|
|
continue
|
|
fi
|
|
validate_image_reference "operator image $component" "$image"
|
|
operator_specs+=("$component"$'\t'"$dockerfile"$'\t'"$image")
|
|
operator_components+=("$component:$image")
|
|
operator_component_count=$((operator_component_count + 1))
|
|
done < <(operator_image_specs)
|
|
if (( operator_component_count == 0 )); then
|
|
echo "ERROR: POC_KANIKO_OPERATOR_FILTER selected zero operator images." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if (( operator_component_count > 0 )) && ! boolean_true POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"; then
|
|
validate_prebuilt_operator_bases
|
|
fi
|
|
if boolean_true POC_KANIKO_PRECHECK_ONLY "$POC_KANIKO_PRECHECK_ONLY"; then
|
|
printf 'KANIKO_BUILD_PLAN=contextSource=%s platformComponents=%s buildOperators=%s operatorFilter=%s buildOperatorBases=%s\n' \
|
|
"$POC_KANIKO_CONTEXT_SOURCE" "${POC_KANIKO_COMPONENTS:-<empty>}" "$POC_KANIKO_BUILD_OPERATORS" "$POC_KANIKO_OPERATOR_FILTER" "$POC_KANIKO_BUILD_OPERATOR_BASES"
|
|
printf 'KANIKO_PRECHECK_ONLY=true\n'
|
|
exit 0
|
|
fi
|
|
|
|
platform_specs=()
|
|
while IFS=$'\t' read -r component dockerfile image; do
|
|
if [[ -n "$component" ]]; then
|
|
platform_specs+=("$component"$'\t'"$dockerfile"$'\t'"$image")
|
|
fi
|
|
done < <(enabled_platform_specs)
|
|
if (( ${#platform_specs[@]} > 0 )); then
|
|
run_kaniko_components "${platform_specs[@]}"
|
|
fi
|
|
if (( operator_component_count > 0 )); then
|
|
if boolean_true POC_KANIKO_BUILD_OPERATOR_BASES "$POC_KANIKO_BUILD_OPERATOR_BASES"; then
|
|
operator_base_specs=()
|
|
while IFS=$'\t' read -r component dockerfile image; do
|
|
if [[ -n "$component" ]]; then
|
|
operator_base_specs+=("$component"$'\t'"$dockerfile"$'\t'"$image")
|
|
fi
|
|
done < <(operator_base_specs)
|
|
if (( ${#operator_base_specs[@]} > 0 )); then
|
|
run_kaniko_components "${operator_base_specs[@]}"
|
|
fi
|
|
fi
|
|
run_kaniko_components "${operator_specs[@]}"
|
|
fi
|
|
|
|
printf 'IMAGE_TAG=%s\n' "$IMAGE_TAG"
|
|
printf 'API_IMAGE=%s\n' "$API_IMAGE"
|
|
printf 'WORKER_IMAGE=%s\n' "$WORKER_IMAGE"
|
|
printf 'DROSYNC_COORDINATOR_IMAGE=%s\n' "$DROSYNC_COORDINATOR_IMAGE"
|
|
printf 'OPERATOR_RUNNER_IMAGE=%s\n' "$OPERATOR_RUNNER_IMAGE"
|
|
printf 'HEADPOSE_SERVICE_IMAGE=%s\n' "$HEADPOSE_SERVICE_IMAGE"
|
|
if (( operator_component_count > 0 )); then
|
|
printf 'OPERATOR_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_BASE_IMAGE"
|
|
printf 'OPERATOR_GPU_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_GPU_BASE_IMAGE"
|
|
printf 'OPERATOR_HOLOBRAIN_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_HOLOBRAIN_BASE_IMAGE"
|
|
printf 'OPERATOR_YOLO_BASE_IMAGE=%s\n' "$POC_KANIKO_OPERATOR_YOLO_BASE_IMAGE"
|
|
for spec in "${operator_components[@]}"; do
|
|
printf 'OPERATOR_IMAGE=%s\n' "${spec#*:}"
|
|
done
|
|
fi
|