droboticflow-new/scripts/configure-poc-environment.sh
2026-06-22 19:38:44 +08:00

275 lines
7.9 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DEPLOY_ENV="${DEPLOY_ENV:-poc}"
export DEPLOY_ENV
# shellcheck source=scripts/load-deploy-env.sh
source "$ROOT_DIR/scripts/load-deploy-env.sh"
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
WORKSPACE_BOS_BUCKET="${WORKSPACE_BOS_BUCKET:?WORKSPACE_BOS_BUCKET is required}"
WORKSPACE_BOS_DRIVER="${WORKSPACE_BOS_DRIVER:-csi-bosplugin}"
BOS_ACCESS_KEY_ID="${BOS_ACCESS_KEY_ID:?BOS_ACCESS_KEY_ID is required}"
BOS_SECRET_ACCESS_KEY="${BOS_SECRET_ACCESS_KEY:?BOS_SECRET_ACCESS_KEY is required}"
cluster_id="$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.cluster-id}')"
if [[ -z "$cluster_id" ]]; then
echo "ERROR: failed to infer cluster-id from node labels" >&2
exit 1
fi
kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$K8S_NAMESPACE"
kubectl -n "$K8S_NAMESPACE" create secret generic droboticflow-bos-workspace \
--from-literal=accessKeyId="$BOS_ACCESS_KEY_ID" \
--from-literal=ak="$BOS_ACCESS_KEY_ID" \
--from-literal=secretAccessKey="$BOS_SECRET_ACCESS_KEY" \
--from-literal=sk="$BOS_SECRET_ACCESS_KEY" \
--dry-run=client \
-o yaml | kubectl apply -f -
manifest="$(cat <<YAML
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-bos-external-runner
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: csi-bos-external-runner
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["get","list","watch","update","create","patch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get","list","watch","update","create","delete","patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get","list","watch","update"]
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
verbs: ["patch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["get","list","watch","update","create","delete"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get","list","watch","update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get","list","watch","update","patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments/status"]
verbs: ["patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get","list","watch"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get","list"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get","list","watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get","list","watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["*"]
verbs: ["create","get","list","watch","update","patch"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["*"]
verbs: ["create","get","list","watch"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csidrivers"]
verbs: ["create","delete","get","list","watch"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get","list","watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: csi-bos-role-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: csi-bos-external-runner
subjects:
- kind: ServiceAccount
name: csi-bos-external-runner
namespace: kube-system
---
apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
name: ${WORKSPACE_BOS_DRIVER}
spec:
attachRequired: false
fsGroupPolicy: ReadWriteOnceWithFSType
podInfoOnMount: false
requiresRepublish: false
seLinuxMount: false
storageCapacity: false
volumeLifecycleModes:
- Persistent
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: ${WORKSPACE_BOS_DRIVER}-node-server-2
namespace: kube-system
spec:
selector:
matchLabels:
app: ${WORKSPACE_BOS_DRIVER}-node-server-2
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
maxSurge: 0
template:
metadata:
labels:
app: ${WORKSPACE_BOS_DRIVER}-node-server-2
spec:
serviceAccountName: csi-bos-external-runner
hostNetwork: true
priorityClassName: system-node-critical
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: cce.baidubce.com/kubelet-dir
operator: In
values:
- 2f7661722f6c69622f6b7562656c6574
tolerations:
- operator: Exists
containers:
- name: node-driver-registrar
image: registry.baidubce.com/cce-plugin-pro/csi-node-driver-registrar:v2.1.0
imagePullPolicy: Always
args:
- --csi-address=\$(ADDRESS)
- --kubelet-registration-path=\$(ADDRESS)
- --v=2
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/${WORKSPACE_BOS_DRIVER}/csi.sock
volumeMounts:
- name: reg-dir
mountPath: /registration
- name: kubelet-dir
mountPath: /var/lib/kubelet
- name: ${WORKSPACE_BOS_DRIVER}
image: registry.baidubce.com/cce-plugin-pro/cce-csi-plugin:v1.3.4
imagePullPolicy: Always
securityContext:
privileged: true
allowPrivilegeEscalation: true
capabilities:
add: ["SYS_ADMIN"]
args:
- --override-driver-name=${WORKSPACE_BOS_DRIVER}
- --driver-type=bos
- --csi-endpoint=\$(CSI_ENDPOINT)
- --driver-mode=node
- --auth-mode=key
- --max-volumes-per-node=5
- --topology-mode=auto
- --region=bj
- --cluster-id=${cluster_id}
- --bosfs-image=registry.baidubce.com/cce-plugin-pro/bosfs:bosfs-1.0.0.13.4.3
- --logtostderr
- --v=2
env:
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/${WORKSPACE_BOS_DRIVER}/csi.sock
- name: NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: CCE_GATEWAY_ENDPOINT
value: cce-gateway.bj.baidubce.com
volumeMounts:
- name: kubelet-dir
mountPath: /var/lib/kubelet
mountPropagation: Bidirectional
- name: host-sys
mountPath: /sys
- name: host-var-run
mountPath: /var/run
- name: bosfs-cred-dir
mountPath: /etc/bosfs/credentials
volumes:
- name: kubelet-dir
hostPath:
path: /var/lib/kubelet
type: DirectoryOrCreate
- name: reg-dir
hostPath:
path: /var/lib/kubelet/plugins_registry
type: DirectoryOrCreate
- name: host-sys
hostPath:
path: /sys
- name: host-var-run
hostPath:
path: /var/run
- name: bosfs-cred-dir
hostPath:
path: /etc/bosfs/credentials
type: DirectoryOrCreate
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: ${WORKSPACE_PVC_NAME}-pv
spec:
capacity:
storage: 200Gi
accessModes:
- ReadWriteOnce
- ReadOnlyMany
persistentVolumeReclaimPolicy: Retain
storageClassName: csi-bos
csi:
driver: ${WORKSPACE_BOS_DRIVER}
nodePublishSecretRef:
name: droboticflow-bos-workspace
namespace: ${K8S_NAMESPACE}
volumeHandle: ${WORKSPACE_BOS_BUCKET}
mountOptions:
- -o meta_expires=0
- -o allow_other
- -o mount_umask=000
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ${WORKSPACE_PVC_NAME}
namespace: ${K8S_NAMESPACE}
spec:
accessModes:
- ReadWriteOnce
- ReadOnlyMany
resources:
requests:
storage: 200Gi
storageClassName: csi-bos
volumeName: ${WORKSPACE_PVC_NAME}-pv
YAML
)"
printf '%s\n' "$manifest" | kubectl apply -f -
kubectl -n kube-system rollout status "daemonset/${WORKSPACE_BOS_DRIVER}-node-server-2" --timeout=180s
kubectl -n "$K8S_NAMESPACE" wait --for=jsonpath='{.status.phase}'=Bound "pvc/${WORKSPACE_PVC_NAME}" --timeout=180s