From f29ec4bfae97481f9f8a4f2cd1505f85854d96fa Mon Sep 17 00:00:00 2001 From: eust-w Date: Fri, 30 Oct 2026 11:48:24 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Reap=20leaked=20bosfs=20temp=20f?= =?UTF-8?q?iles=20that=20evicted=20A800=20training=20mounts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CCE csi-bosplugin writes bosfs.tmp.* under host /tmp/bosfs on the node root disk and never removes them when a mount pod ends. Leaks filled the A800 node root disk, kubelet evicted the bosfs FUSE pod mid-training and runs failed with Errno 107. Add a DaemonSet that deletes temp files of dead mounts not held open by any process, and document the failure mode. Co-Authored-By: Claude Opus 5.5 --- docs/operations.md | 7 +++ k8s/bosfs-tmp-reaper.yaml | 126 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 k8s/bosfs-tmp-reaper.yaml diff --git a/docs/operations.md b/docs/operations.md index f6d316ca..2e5e9f86 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -484,6 +484,13 @@ kubectl -n droboticflow exec deploy/droboticflow-worker -- ls -R /workspace/runs - confirm the node directory contains `input-artifacts.json`, `result.json`, `stage-manifest.json`, and `outputs/` - inspect worker logs for `DROBOTICFLOW_RESULT_PATH` parse failures +### Training fails with `[Errno 107] Transport endpoint is not connected` + +- the BOS FUSE mount died under the pod; check `kubectl get events -A | grep -i evict` for an evicted `kube-system/bosfs-` pod and `DiskPressure` on the node +- CCE csi-bosplugin runs bosfs with `-o tmpdir=/tmp` (host `/tmp/bosfs/logs/bosfs-` on the root disk) and leaves `bosfs.tmp.*` files behind when a mount pod ends; on the A800 node each full training run leaked ~100G until the root disk reached the kubelet eviction threshold and the BestEffort bosfs pod was evicted mid-run +- keep `k8s/bosfs-tmp-reaper.yaml` deployed (`kubectl apply -f k8s/bosfs-tmp-reaper.yaml`); it removes temp files of dead mounts that no process holds open, and logs `root_avail` per node every 5 minutes (`kubectl -n droboticflow logs -l app=bosfs-tmp-reaper --prefix --tail=3`) +- the reaper pins the worker image by digest; update the digest if that image is ever garbage-collected from the registry + ### Image pull failure - recreate `droboticflow-registry` secret with valid registry credentials diff --git a/k8s/bosfs-tmp-reaper.yaml b/k8s/bosfs-tmp-reaper.yaml new file mode 100644 index 00000000..d98d6dc5 --- /dev/null +++ b/k8s/bosfs-tmp-reaper.yaml @@ -0,0 +1,126 @@ +# Reclaims bosfs temp files leaked on node root disks. +# +# CCE csi-bosplugin starts one kube-system/bosfs- pod per BOS mount with +# `-o tmpdir=/tmp`, where /tmp is hostPath /tmp/bosfs/logs/bosfs- on the +# node root disk. bosfs writes bosfs.tmp.* files there while reading/writing +# objects and does not remove them when the bosfs process is killed or the +# mount pod is torn down. Each full training run leaked ~100G on 192.168.80.136 +# until the root disk hit the kubelet eviction threshold, kubelet evicted the +# (BestEffort) bosfs pod, and training failed with +# `[Errno 107] Transport endpoint is not connected`. +# +# Every INTERVAL_SECONDS this DaemonSet deletes bosfs.tmp.* files that: +# - belong to a bosfs- directory with no live bosfs process, and +# - are not held open by any process on the node, and +# - are older than MIN_FILE_AGE_MINUTES. +# Directories are kept (a live mount pod may still reference them) and only +# removed when empty, orphaned, and older than EMPTY_DIR_AGE_MINUTES. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: bosfs-tmp-reaper + namespace: droboticflow + labels: + app: bosfs-tmp-reaper +spec: + selector: + matchLabels: + app: bosfs-tmp-reaper + template: + metadata: + labels: + app: bosfs-tmp-reaper + spec: + tolerations: + - operator: Exists + terminationGracePeriodSeconds: 5 + imagePullSecrets: + - name: droboticflow-registry + containers: + - name: reaper + image: ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-worker@sha256:540dc05908745d69c4b0f1488d833b1add18953d21196a75aba31b7b969eb990 + imagePullPolicy: IfNotPresent + securityContext: + # Root + privileged: delete root-owned temp files and read + # /proc//fd of every process to skip files still open. + runAsUser: 0 + runAsGroup: 0 + privileged: true + env: + - name: DRY_RUN + value: "false" + - name: INTERVAL_SECONDS + value: "300" + - name: MIN_FILE_AGE_MINUTES + value: "10" + - name: EMPTY_DIR_AGE_MINUTES + value: "1440" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: ["/usr/bin/bash", "-c"] + args: + - | + set -u + ROOT=/host-bosfs/logs + PROC=/host-proc + log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) node=${NODE_NAME} $*"; } + reap_once() { + [ -d "$ROOT" ] || { log "no $ROOT"; return; } + # Live mounts: bosfs argv carries credentials/ matching bosfs-. + declare -A live=() open=() + for c in "$PROC"/[0-9]*/cmdline; do + args=$(tr '\0' ' ' 2>/dev/null < "$c") || continue + case "$args" in bosfs\ *|*/bosfs\ *) ;; *) continue ;; esac + h=$(grep -o 'credentials/[0-9a-f]\{32\}' <<<"$args" | head -1) + [ -n "$h" ] && live[${h#credentials/}]=1 + done + # Open temp files, by basename (container paths differ from host paths). + while IFS= read -r t; do + b=${t##*/}; open[${b%% (deleted)}]=1 + done < <(find "$PROC"/[0-9]*/fd -maxdepth 1 -lname '*bosfs.tmp.*' -printf '%l\n' 2>/dev/null) + files=0 bytes=0 skipped_live=0 + for d in "$ROOT"/bosfs-*; do + [ -d "$d" ] || continue + h=${d##*/bosfs-} + if [ -n "${live[$h]+x}" ]; then skipped_live=$((skipped_live + 1)); continue; fi + while IFS= read -r -d '' f; do + [ -n "${open[${f##*/}]+x}" ] && continue + sz=$(stat -c %s "$f" 2>/dev/null || echo 0) + if [ "$DRY_RUN" = "true" ]; then log "would remove $f ($sz bytes)" + else rm -f -- "$f" || continue; fi + files=$((files + 1)); bytes=$((bytes + sz)) + done < <(find "$d" -maxdepth 1 -type f -name 'bosfs.tmp.*' -mmin +"$MIN_FILE_AGE_MINUTES" -print0) + if [ "$DRY_RUN" != "true" ] && [ -z "$(ls -A "$d" 2>/dev/null)" ] \ + && [ -n "$(find "$d" -maxdepth 0 -mmin +"$EMPTY_DIR_AGE_MINUTES")" ]; then + rmdir -- "$d" 2>/dev/null || true + fi + done + avail=$(df -P "$ROOT" | awk 'NR==2 {printf "%.0fG", $4/1048576}') + log "dry_run=${DRY_RUN} live_mounts=${#live[@]} skipped_live_dirs=${skipped_live} reaped_files=${files} reaped_bytes=${bytes} root_avail=${avail}" + } + trap 'exit 0' TERM INT + while true; do reap_once; sleep "$INTERVAL_SECONDS" & wait $!; done + resources: + requests: + cpu: 10m + memory: 32Mi + ephemeral-storage: 16Mi + limits: + memory: 128Mi + ephemeral-storage: 64Mi + volumeMounts: + - name: bosfs-tmp + mountPath: /host-bosfs + - name: host-proc + mountPath: /host-proc + readOnly: true + volumes: + - name: bosfs-tmp + hostPath: + path: /tmp/bosfs + type: DirectoryOrCreate + - name: host-proc + hostPath: + path: /proc