real2sim-platform/scripts/deploy_poc_cluster.sh

250 lines
11 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
KUBECONFIG_PATH="${REAL2SIM_POC_KUBECONFIG:-/Users/d-robotics/Downloads/poc.config}"
NAMESPACE="real2sim-poc"
GPU_NODE="${REAL2SIM_POC_GPU_NODE:-192.168.80.136}"
REGISTRY="${REAL2SIM_POC_REGISTRY:-ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc}"
SOURCE_COMMIT="$(git -C "$ROOT" rev-parse --short=12 HEAD)"
BUILD_ID="${REAL2SIM_POC_BUILD_ID:-${SOURCE_COMMIT}-$(date -u +%Y%m%d%H%M%S)}"
OUT="${REAL2SIM_POC_ACCEPTANCE_OUT:-$ROOT/acceptance_runs/poc/$BUILD_ID}"
BUILD_IMAGES=1
DEPLOY_APP=1
tls_dir=""
context_archive=""
rendered_manifest=""
trap 'rm -rf "${tls_dir:-}" "${context_archive:-}" "${rendered_manifest:-}"' EXIT
for arg in "$@"; do
case "$arg" in
--skip-build) BUILD_IMAGES=0 ;;
--build-only) DEPLOY_APP=0 ;;
--help)
echo "usage: $0 [--skip-build|--build-only]"
exit 0
;;
*) echo "unknown argument: $arg" >&2; exit 2 ;;
esac
done
K=(kubectl --kubeconfig "$KUBECONFIG_PATH")
mkdir -p "$OUT"
require() { command -v "$1" >/dev/null || { echo "missing required command: $1" >&2; exit 1; }; }
for command in kubectl jq openssl tar git sed; do require "$command"; done
test -f "$KUBECONFIG_PATH" || { echo "kubeconfig not found: $KUBECONFIG_PATH" >&2; exit 1; }
test "$(git -C "$ROOT" branch --show-current)" = "feat/poc-cluster-productization" || {
echo "deployment is only allowed from feat/poc-cluster-productization" >&2
exit 1
}
gpu_capacity="$("${K[@]}" get node "$GPU_NODE" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}')"
gpu_in_use="$("${K[@]}" get pods -A --field-selector "spec.nodeName=$GPU_NODE" -o json | jq '[.items[] | select(.status.phase=="Running" or .status.phase=="Pending") | ([.spec.containers[]?.resources.requests["nvidia.com/gpu"] // "0" | tonumber] | add // 0)] | add // 0')"
if (( gpu_capacity - gpu_in_use < 1 )); then
echo "A800 node has no free GPU: capacity=$gpu_capacity in_use=$gpu_in_use" >&2
exit 1
fi
printf 'gpu_capacity=%s\ngpu_in_use=%s\nbuild_id=%s\n' "$gpu_capacity" "$gpu_in_use" "$BUILD_ID" > "$OUT/deploy_preflight.txt"
"${K[@]}" apply -f "$ROOT/deploy/poc/bootstrap.yaml"
ensure_host_directory() {
local pod="real2sim-hostpath-bootstrap"
"${K[@]}" -n "$NAMESPACE" delete pod "$pod" --ignore-not-found --wait=true >/dev/null
cat <<YAML | "${K[@]}" apply -f - >/dev/null
apiVersion: v1
kind: Pod
metadata:
name: $pod
namespace: $NAMESPACE
labels:
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/component: storage-bootstrap
spec:
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: "$GPU_NODE"
containers:
- name: bootstrap
image: busybox:1.36
command: [sh, -eu, -c]
args:
- mkdir -p /host-var-lib/real2sim-poc && chmod 0770 /host-var-lib/real2sim-poc && chown 10001:10001 /host-var-lib/real2sim-poc
resources:
requests: {cpu: 20m, memory: 32Mi}
limits: {cpu: 100m, memory: 64Mi}
securityContext:
runAsUser: 0
allowPrivilegeEscalation: false
capabilities: {drop: [ALL], add: [CHOWN, FOWNER]}
volumeMounts:
- {name: host-var-lib, mountPath: /host-var-lib}
volumes:
- name: host-var-lib
hostPath: {path: /var/lib, type: Directory}
YAML
if ! "${K[@]}" -n "$NAMESPACE" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$pod" --timeout=180s; then
"${K[@]}" -n "$NAMESPACE" describe "pod/$pod" >&2 || true
exit 1
fi
"${K[@]}" -n "$NAMESPACE" delete pod "$pod" --wait=true >/dev/null
}
ensure_host_directory
clone_secret() {
local source_namespace="$1" source_name="$2" target_name="$3"
"${K[@]}" -n "$source_namespace" get secret "$source_name" -o json |
jq --arg ns "$NAMESPACE" --arg name "$target_name" '
del(.metadata.uid,.metadata.resourceVersion,.metadata.creationTimestamp,.metadata.managedFields,.metadata.ownerReferences) |
.metadata.namespace=$ns | .metadata.name=$name |
.metadata.labels={"app.kubernetes.io/part-of":"real2sim-poc"}' |
"${K[@]}" apply -f - >/dev/null
}
clone_secret droboticflow droboticflow-registry real2sim-registry
clone_secret droboticflow droboticflow-bos-workspace real2sim-bos
if ! "${K[@]}" -n "$NAMESPACE" get secret real2sim-app-secrets >/dev/null 2>&1; then
admin_password="$(openssl rand -base64 30 | tr -d '/+=' | cut -c1-28)Aa1!"
session_secret="$(openssl rand -hex 32)"
"${K[@]}" -n "$NAMESPACE" create secret generic real2sim-app-secrets \
--from-literal=bootstrap-admin-password="$admin_password" \
--from-literal=session-secret="$session_secret" \
--dry-run=client -o yaml |
"${K[@]}" label --local -f - app.kubernetes.io/part-of=real2sim-poc -o yaml |
"${K[@]}" apply -f - >/dev/null
fi
if ! "${K[@]}" -n "$NAMESPACE" get secret real2sim-tls >/dev/null 2>&1; then
tls_dir="$(mktemp -d)"
openssl req -x509 -newkey rsa:3072 -sha256 -nodes -days 365 \
-subj "/CN=real2sim-poc.local" \
-addext "subjectAltName=DNS:real2sim-poc.local,DNS:localhost,IP:127.0.0.1" \
-keyout "$tls_dir/tls.key" -out "$tls_dir/tls.crt" >/dev/null 2>&1
"${K[@]}" -n "$NAMESPACE" create secret tls real2sim-tls \
--key "$tls_dir/tls.key" --cert "$tls_dir/tls.crt" --dry-run=client -o yaml |
"${K[@]}" label --local -f - app.kubernetes.io/part-of=real2sim-poc -o yaml |
"${K[@]}" apply -f - >/dev/null
fi
"${K[@]}" -n "$NAMESPACE" get secret real2sim-app-secrets -o jsonpath='{.data.bootstrap-admin-password}' |
base64 --decode > "$OUT/bootstrap-admin-password.txt"
chmod 600 "$OUT/bootstrap-admin-password.txt"
printf 'admin\n' > "$OUT/bootstrap-admin-username.txt"
chmod 600 "$OUT/bootstrap-admin-username.txt"
prepare_context() {
context_archive="$(mktemp -t real2sim-poc-context.XXXXXX.tgz)"
tar -C "$ROOT" -czf "$context_archive" \
--exclude='.git' --exclude='.venv' --exclude='.pytest_cache' --exclude='workspace' \
--exclude='acceptance_runs' --exclude='qa' --exclude='docs' --exclude='tests' .
local loader="real2sim-context-${BUILD_ID//[^a-zA-Z0-9-]/-}"
loader="${loader:0:60}"
"${K[@]}" -n "$NAMESPACE" delete pod "$loader" --ignore-not-found --wait=true >/dev/null
"${K[@]}" -n "$NAMESPACE" run "$loader" \
--image=busybox:1.36 --restart=Never \
--labels="app.kubernetes.io/part-of=real2sim-poc,app.kubernetes.io/component=build-context" \
--overrides="$(jq -cn --arg node "$GPU_NODE" '{spec:{nodeSelector:{"kubernetes.io/hostname":$node},containers:[{name:"context-loader",image:"busybox:1.36",command:["sh","-c","sleep 3600"],resources:{requests:{cpu:"100m",memory:"64Mi"},limits:{cpu:"500m",memory:"256Mi"}},volumeMounts:[{name:"data",mountPath:"/data"}]}],volumes:[{name:"data",persistentVolumeClaim:{claimName:"real2sim-poc-data"}}]}}')" >/dev/null
"${K[@]}" -n "$NAMESPACE" wait --for=condition=Ready "pod/$loader" --timeout=180s
"${K[@]}" -n "$NAMESPACE" cp "$context_archive" "$loader:/tmp/context.tgz"
"${K[@]}" -n "$NAMESPACE" exec "$loader" -- sh -eu -c \
"mkdir -p '/data/build-context/$BUILD_ID' /data/workspace /data/cache /data/scratch; rm -rf '/data/build-context/$BUILD_ID/'*; tar -xzf /tmp/context.tgz -C '/data/build-context/$BUILD_ID'; chown -R 10001:10001 /data/workspace /data/cache /data/scratch; chmod 0770 /data/workspace /data/cache /data/scratch"
"${K[@]}" -n "$NAMESPACE" delete pod "$loader" --wait=true >/dev/null
}
build_image() {
local component="$1" dockerfile="$2" destination="$3"
local job="real2sim-build-${component}-${BUILD_ID//[^a-zA-Z0-9-]/-}"
job="${job:0:60}"
"${K[@]}" -n "$NAMESPACE" delete job "$job" --ignore-not-found --wait=true >/dev/null
cat <<YAML | "${K[@]}" apply -f - >/dev/null
apiVersion: batch/v1
kind: Job
metadata:
name: $job
namespace: $NAMESPACE
labels:
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/component: image-build
spec:
backoffLimit: 0
ttlSecondsAfterFinished: 86400
template:
metadata:
labels:
app.kubernetes.io/part-of: real2sim-poc
app.kubernetes.io/component: image-build
spec:
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: "$GPU_NODE"
containers:
- name: kaniko
image: gcr.io/kaniko-project/executor:v1.23.2-debug
args:
- --context=dir:///data/build-context/$BUILD_ID
- --dockerfile=/data/build-context/$BUILD_ID/$dockerfile
- --destination=$destination
- --build-arg=BUILD_ID=$BUILD_ID
- --build-arg=SOURCE_COMMIT=$SOURCE_COMMIT
- --snapshot-mode=redo
- --use-new-run
- --cache=true
- --cache-repo=$REGISTRY/real2sim-build-cache
- --verbosity=info
resources:
requests: {cpu: "2", memory: 4Gi}
limits: {cpu: "16", memory: 32Gi}
volumeMounts:
- {name: data, mountPath: /data}
- {name: docker-config, mountPath: /kaniko/.docker, readOnly: true}
volumes:
- name: data
persistentVolumeClaim: {claimName: real2sim-poc-data}
- name: docker-config
secret:
secretName: real2sim-registry
items:
- {key: .dockerconfigjson, path: config.json}
YAML
if ! "${K[@]}" -n "$NAMESPACE" wait --for=condition=complete "job/$job" --timeout=7200s; then
"${K[@]}" -n "$NAMESPACE" logs "job/$job" --all-containers=true > "$OUT/build-${component}.log" 2>&1 || true
"${K[@]}" -n "$NAMESPACE" describe "job/$job" > "$OUT/build-${component}-describe.txt" 2>&1 || true
return 1
fi
"${K[@]}" -n "$NAMESPACE" logs "job/$job" --all-containers=true > "$OUT/build-${component}.log"
"${K[@]}" -n "$NAMESPACE" delete job "$job" --wait=true >/dev/null
}
FRONTEND_IMAGE="$REGISTRY/real2sim-frontend:$BUILD_ID"
API_IMAGE="$REGISTRY/real2sim-api:$BUILD_ID"
WORKER_IMAGE="$REGISTRY/real2sim-worker:$BUILD_ID"
printf '%s\n' "$FRONTEND_IMAGE" "$API_IMAGE" "$WORKER_IMAGE" > "$OUT/images.txt"
if [[ "$BUILD_IMAGES" == "1" ]]; then
prepare_context
build_image frontend docker/frontend.Dockerfile "$FRONTEND_IMAGE"
build_image api docker/api.Dockerfile "$API_IMAGE"
build_image worker docker/worker.Dockerfile "$WORKER_IMAGE"
fi
if [[ "$DEPLOY_APP" == "1" ]]; then
rendered_manifest="$(mktemp -t real2sim-poc-manifest.XXXXXX.yaml)"
sed \
-e "s|__BUILD_ID__|$BUILD_ID|g" \
-e "s|__FRONTEND_IMAGE__|$FRONTEND_IMAGE|g" \
-e "s|__API_IMAGE__|$API_IMAGE|g" \
-e "s|__WORKER_IMAGE__|$WORKER_IMAGE|g" \
"$ROOT/deploy/poc/app-template.yaml" > "$rendered_manifest"
"${K[@]}" apply -f "$rendered_manifest"
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-api --timeout=600s
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-frontend --timeout=600s
"${K[@]}" -n "$NAMESPACE" rollout status deployment/real2sim-worker --timeout=3600s
"${K[@]}" -n "$NAMESPACE" get pods,svc,pvc -l app.kubernetes.io/part-of=real2sim-poc -o wide > "$OUT/resources.txt"
"${K[@]}" -n "$NAMESPACE" get svc real2sim-frontend -o json > "$OUT/frontend-service.json"
fi
printf 'build_id=%s\nnamespace=%s\ncredentials=%s\n' "$BUILD_ID" "$NAMESPACE" "$OUT/bootstrap-admin-password.txt"