forked from longtao.wu/droboticflow
275 lines
7.9 KiB
Bash
Executable File
275 lines
7.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
DEPLOY_ENV="${DEPLOY_ENV:-poc}"
|
|
export DEPLOY_ENV
|
|
|
|
# shellcheck source=scripts/load-deploy-env.sh
|
|
source "$ROOT_DIR/scripts/load-deploy-env.sh"
|
|
|
|
K8S_NAMESPACE="${K8S_NAMESPACE:-droboticflow}"
|
|
WORKSPACE_PVC_NAME="${WORKSPACE_PVC_NAME:-droboticflow-workspace}"
|
|
WORKSPACE_BOS_BUCKET="${WORKSPACE_BOS_BUCKET:?WORKSPACE_BOS_BUCKET is required}"
|
|
WORKSPACE_BOS_DRIVER="${WORKSPACE_BOS_DRIVER:-csi-bosplugin}"
|
|
BOS_ACCESS_KEY_ID="${BOS_ACCESS_KEY_ID:?BOS_ACCESS_KEY_ID is required}"
|
|
BOS_SECRET_ACCESS_KEY="${BOS_SECRET_ACCESS_KEY:?BOS_SECRET_ACCESS_KEY is required}"
|
|
|
|
cluster_id="$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.cluster-id}')"
|
|
if [[ -z "$cluster_id" ]]; then
|
|
echo "ERROR: failed to infer cluster-id from node labels" >&2
|
|
exit 1
|
|
fi
|
|
|
|
kubectl get namespace "$K8S_NAMESPACE" >/dev/null 2>&1 || kubectl create namespace "$K8S_NAMESPACE"
|
|
|
|
kubectl -n "$K8S_NAMESPACE" create secret generic droboticflow-bos-workspace \
|
|
--from-literal=accessKeyId="$BOS_ACCESS_KEY_ID" \
|
|
--from-literal=ak="$BOS_ACCESS_KEY_ID" \
|
|
--from-literal=secretAccessKey="$BOS_SECRET_ACCESS_KEY" \
|
|
--from-literal=sk="$BOS_SECRET_ACCESS_KEY" \
|
|
--dry-run=client \
|
|
-o yaml | kubectl apply -f -
|
|
|
|
manifest="$(cat <<YAML
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: csi-bos-external-runner
|
|
namespace: kube-system
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRole
|
|
metadata:
|
|
name: csi-bos-external-runner
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["events"]
|
|
verbs: ["get","list","watch","update","create","patch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumes"]
|
|
verbs: ["get","list","watch","update","create","delete","patch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims"]
|
|
verbs: ["get","list","watch","update"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims/status"]
|
|
verbs: ["patch"]
|
|
- apiGroups: [""]
|
|
resources: ["pods"]
|
|
verbs: ["get","list","watch","update","create","delete"]
|
|
- apiGroups: [""]
|
|
resources: ["nodes"]
|
|
verbs: ["get","list","watch","update"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["volumeattachments"]
|
|
verbs: ["get","list","watch","update","patch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["volumeattachments/status"]
|
|
verbs: ["patch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["csinodes"]
|
|
verbs: ["get","list","watch"]
|
|
- apiGroups: [""]
|
|
resources: ["namespaces"]
|
|
verbs: ["get","list"]
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
verbs: ["get","list","watch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["storageclasses"]
|
|
verbs: ["get","list","watch"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["*"]
|
|
verbs: ["create","get","list","watch","update","patch"]
|
|
- apiGroups: ["apiextensions.k8s.io"]
|
|
resources: ["*"]
|
|
verbs: ["create","get","list","watch"]
|
|
- apiGroups: ["csi.storage.k8s.io"]
|
|
resources: ["csidrivers"]
|
|
verbs: ["create","delete","get","list","watch"]
|
|
- apiGroups: ["csi.storage.k8s.io"]
|
|
resources: ["csinodeinfos"]
|
|
verbs: ["get","list","watch"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: ClusterRoleBinding
|
|
metadata:
|
|
name: csi-bos-role-binding
|
|
roleRef:
|
|
apiGroup: rbac.authorization.k8s.io
|
|
kind: ClusterRole
|
|
name: csi-bos-external-runner
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: csi-bos-external-runner
|
|
namespace: kube-system
|
|
---
|
|
apiVersion: storage.k8s.io/v1
|
|
kind: CSIDriver
|
|
metadata:
|
|
name: ${WORKSPACE_BOS_DRIVER}
|
|
spec:
|
|
attachRequired: false
|
|
fsGroupPolicy: ReadWriteOnceWithFSType
|
|
podInfoOnMount: false
|
|
requiresRepublish: false
|
|
seLinuxMount: false
|
|
storageCapacity: false
|
|
volumeLifecycleModes:
|
|
- Persistent
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: ${WORKSPACE_BOS_DRIVER}-node-server-2
|
|
namespace: kube-system
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: ${WORKSPACE_BOS_DRIVER}-node-server-2
|
|
updateStrategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxUnavailable: 1
|
|
maxSurge: 0
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: ${WORKSPACE_BOS_DRIVER}-node-server-2
|
|
spec:
|
|
serviceAccountName: csi-bos-external-runner
|
|
hostNetwork: true
|
|
priorityClassName: system-node-critical
|
|
affinity:
|
|
nodeAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
nodeSelectorTerms:
|
|
- matchExpressions:
|
|
- key: cce.baidubce.com/kubelet-dir
|
|
operator: In
|
|
values:
|
|
- 2f7661722f6c69622f6b7562656c6574
|
|
tolerations:
|
|
- operator: Exists
|
|
containers:
|
|
- name: node-driver-registrar
|
|
image: registry.baidubce.com/cce-plugin-pro/csi-node-driver-registrar:v2.1.0
|
|
imagePullPolicy: Always
|
|
args:
|
|
- --csi-address=\$(ADDRESS)
|
|
- --kubelet-registration-path=\$(ADDRESS)
|
|
- --v=2
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/kubelet/plugins/${WORKSPACE_BOS_DRIVER}/csi.sock
|
|
volumeMounts:
|
|
- name: reg-dir
|
|
mountPath: /registration
|
|
- name: kubelet-dir
|
|
mountPath: /var/lib/kubelet
|
|
- name: ${WORKSPACE_BOS_DRIVER}
|
|
image: registry.baidubce.com/cce-plugin-pro/cce-csi-plugin:v1.3.4
|
|
imagePullPolicy: Always
|
|
securityContext:
|
|
privileged: true
|
|
allowPrivilegeEscalation: true
|
|
capabilities:
|
|
add: ["SYS_ADMIN"]
|
|
args:
|
|
- --override-driver-name=${WORKSPACE_BOS_DRIVER}
|
|
- --driver-type=bos
|
|
- --csi-endpoint=\$(CSI_ENDPOINT)
|
|
- --driver-mode=node
|
|
- --auth-mode=key
|
|
- --max-volumes-per-node=5
|
|
- --topology-mode=auto
|
|
- --region=bj
|
|
- --cluster-id=${cluster_id}
|
|
- --bosfs-image=registry.baidubce.com/cce-plugin-pro/bosfs:bosfs-1.0.0.13.4.3
|
|
- --logtostderr
|
|
- --v=2
|
|
env:
|
|
- name: CSI_ENDPOINT
|
|
value: unix://var/lib/kubelet/plugins/${WORKSPACE_BOS_DRIVER}/csi.sock
|
|
- name: NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
apiVersion: v1
|
|
fieldPath: spec.nodeName
|
|
- name: CCE_GATEWAY_ENDPOINT
|
|
value: cce-gateway.bj.baidubce.com
|
|
volumeMounts:
|
|
- name: kubelet-dir
|
|
mountPath: /var/lib/kubelet
|
|
mountPropagation: Bidirectional
|
|
- name: host-sys
|
|
mountPath: /sys
|
|
- name: host-var-run
|
|
mountPath: /var/run
|
|
- name: bosfs-cred-dir
|
|
mountPath: /etc/bosfs/credentials
|
|
volumes:
|
|
- name: kubelet-dir
|
|
hostPath:
|
|
path: /var/lib/kubelet
|
|
type: DirectoryOrCreate
|
|
- name: reg-dir
|
|
hostPath:
|
|
path: /var/lib/kubelet/plugins_registry
|
|
type: DirectoryOrCreate
|
|
- name: host-sys
|
|
hostPath:
|
|
path: /sys
|
|
- name: host-var-run
|
|
hostPath:
|
|
path: /var/run
|
|
- name: bosfs-cred-dir
|
|
hostPath:
|
|
path: /etc/bosfs/credentials
|
|
type: DirectoryOrCreate
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolume
|
|
metadata:
|
|
name: ${WORKSPACE_PVC_NAME}-pv
|
|
spec:
|
|
capacity:
|
|
storage: 200Gi
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
- ReadOnlyMany
|
|
persistentVolumeReclaimPolicy: Retain
|
|
storageClassName: csi-bos
|
|
csi:
|
|
driver: ${WORKSPACE_BOS_DRIVER}
|
|
nodePublishSecretRef:
|
|
name: droboticflow-bos-workspace
|
|
namespace: ${K8S_NAMESPACE}
|
|
volumeHandle: ${WORKSPACE_BOS_BUCKET}
|
|
mountOptions:
|
|
- -o meta_expires=0
|
|
- -o allow_other
|
|
- -o mount_umask=000
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: ${WORKSPACE_PVC_NAME}
|
|
namespace: ${K8S_NAMESPACE}
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
- ReadOnlyMany
|
|
resources:
|
|
requests:
|
|
storage: 200Gi
|
|
storageClassName: csi-bos
|
|
volumeName: ${WORKSPACE_PVC_NAME}-pv
|
|
YAML
|
|
)"
|
|
|
|
printf '%s\n' "$manifest" | kubectl apply -f -
|
|
kubectl -n kube-system rollout status "daemonset/${WORKSPACE_BOS_DRIVER}-node-server-2" --timeout=180s
|
|
kubectl -n "$K8S_NAMESPACE" wait --for=jsonpath='{.status.phase}'=Bound "pvc/${WORKSPACE_PVC_NAME}" --timeout=180s
|