🐛 Reap leaked bosfs temp files that evicted A800 training mounts

CCE csi-bosplugin writes bosfs.tmp.* under host /tmp/bosfs on the node
root disk and never removes them when a mount pod ends. Leaks filled the
A800 node root disk, kubelet evicted the bosfs FUSE pod mid-training and
runs failed with Errno 107. Add a DaemonSet that deletes temp files of
dead mounts not held open by any process, and document the failure mode.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
eust-w 2026-10-30 11:48:24 +08:00
parent a3b0bfe29a
commit f29ec4bfae
2 changed files with 133 additions and 0 deletions

View File

@ -484,6 +484,13 @@ kubectl -n droboticflow exec deploy/droboticflow-worker -- ls -R /workspace/runs
- confirm the node directory contains `input-artifacts.json`, `result.json`, `stage-manifest.json`, and `outputs/`
- inspect worker logs for `DROBOTICFLOW_RESULT_PATH` parse failures
### Training fails with `[Errno 107] Transport endpoint is not connected`
- the BOS FUSE mount died under the pod; check `kubectl get events -A | grep -i evict` for an evicted `kube-system/bosfs-<hash>` pod and `DiskPressure` on the node
- CCE csi-bosplugin runs bosfs with `-o tmpdir=/tmp` (host `/tmp/bosfs/logs/bosfs-<hash>` on the root disk) and leaves `bosfs.tmp.*` files behind when a mount pod ends; on the A800 node each full training run leaked ~100G until the root disk reached the kubelet eviction threshold and the BestEffort bosfs pod was evicted mid-run
- keep `k8s/bosfs-tmp-reaper.yaml` deployed (`kubectl apply -f k8s/bosfs-tmp-reaper.yaml`); it removes temp files of dead mounts that no process holds open, and logs `root_avail` per node every 5 minutes (`kubectl -n droboticflow logs -l app=bosfs-tmp-reaper --prefix --tail=3`)
- the reaper pins the worker image by digest; update the digest if that image is ever garbage-collected from the registry
### Image pull failure
- recreate `droboticflow-registry` secret with valid registry credentials

126
k8s/bosfs-tmp-reaper.yaml Normal file
View File

@ -0,0 +1,126 @@
# Reclaims bosfs temp files leaked on node root disks.
#
# CCE csi-bosplugin starts one kube-system/bosfs-<hash> pod per BOS mount with
# `-o tmpdir=/tmp`, where /tmp is hostPath /tmp/bosfs/logs/bosfs-<hash> on the
# node root disk. bosfs writes bosfs.tmp.* files there while reading/writing
# objects and does not remove them when the bosfs process is killed or the
# mount pod is torn down. Each full training run leaked ~100G on 192.168.80.136
# until the root disk hit the kubelet eviction threshold, kubelet evicted the
# (BestEffort) bosfs pod, and training failed with
# `[Errno 107] Transport endpoint is not connected`.
#
# Every INTERVAL_SECONDS this DaemonSet deletes bosfs.tmp.* files that:
# - belong to a bosfs-<hash> directory with no live bosfs process, and
# - are not held open by any process on the node, and
# - are older than MIN_FILE_AGE_MINUTES.
# Directories are kept (a live mount pod may still reference them) and only
# removed when empty, orphaned, and older than EMPTY_DIR_AGE_MINUTES.
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: bosfs-tmp-reaper
namespace: droboticflow
labels:
app: bosfs-tmp-reaper
spec:
selector:
matchLabels:
app: bosfs-tmp-reaper
template:
metadata:
labels:
app: bosfs-tmp-reaper
spec:
tolerations:
- operator: Exists
terminationGracePeriodSeconds: 5
imagePullSecrets:
- name: droboticflow-registry
containers:
- name: reaper
image: ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-worker@sha256:540dc05908745d69c4b0f1488d833b1add18953d21196a75aba31b7b969eb990
imagePullPolicy: IfNotPresent
securityContext:
# Root + privileged: delete root-owned temp files and read
# /proc/<pid>/fd of every process to skip files still open.
runAsUser: 0
runAsGroup: 0
privileged: true
env:
- name: DRY_RUN
value: "false"
- name: INTERVAL_SECONDS
value: "300"
- name: MIN_FILE_AGE_MINUTES
value: "10"
- name: EMPTY_DIR_AGE_MINUTES
value: "1440"
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
command: ["/usr/bin/bash", "-c"]
args:
- |
set -u
ROOT=/host-bosfs/logs
PROC=/host-proc
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) node=${NODE_NAME} $*"; }
reap_once() {
[ -d "$ROOT" ] || { log "no $ROOT"; return; }
# Live mounts: bosfs argv carries credentials/<hash> matching bosfs-<hash>.
declare -A live=() open=()
for c in "$PROC"/[0-9]*/cmdline; do
args=$(tr '\0' ' ' 2>/dev/null < "$c") || continue
case "$args" in bosfs\ *|*/bosfs\ *) ;; *) continue ;; esac
h=$(grep -o 'credentials/[0-9a-f]\{32\}' <<<"$args" | head -1)
[ -n "$h" ] && live[${h#credentials/}]=1
done
# Open temp files, by basename (container paths differ from host paths).
while IFS= read -r t; do
b=${t##*/}; open[${b%% (deleted)}]=1
done < <(find "$PROC"/[0-9]*/fd -maxdepth 1 -lname '*bosfs.tmp.*' -printf '%l\n' 2>/dev/null)
files=0 bytes=0 skipped_live=0
for d in "$ROOT"/bosfs-*; do
[ -d "$d" ] || continue
h=${d##*/bosfs-}
if [ -n "${live[$h]+x}" ]; then skipped_live=$((skipped_live + 1)); continue; fi
while IFS= read -r -d '' f; do
[ -n "${open[${f##*/}]+x}" ] && continue
sz=$(stat -c %s "$f" 2>/dev/null || echo 0)
if [ "$DRY_RUN" = "true" ]; then log "would remove $f ($sz bytes)"
else rm -f -- "$f" || continue; fi
files=$((files + 1)); bytes=$((bytes + sz))
done < <(find "$d" -maxdepth 1 -type f -name 'bosfs.tmp.*' -mmin +"$MIN_FILE_AGE_MINUTES" -print0)
if [ "$DRY_RUN" != "true" ] && [ -z "$(ls -A "$d" 2>/dev/null)" ] \
&& [ -n "$(find "$d" -maxdepth 0 -mmin +"$EMPTY_DIR_AGE_MINUTES")" ]; then
rmdir -- "$d" 2>/dev/null || true
fi
done
avail=$(df -P "$ROOT" | awk 'NR==2 {printf "%.0fG", $4/1048576}')
log "dry_run=${DRY_RUN} live_mounts=${#live[@]} skipped_live_dirs=${skipped_live} reaped_files=${files} reaped_bytes=${bytes} root_avail=${avail}"
}
trap 'exit 0' TERM INT
while true; do reap_once; sleep "$INTERVAL_SECONDS" & wait $!; done
resources:
requests:
cpu: 10m
memory: 32Mi
ephemeral-storage: 16Mi
limits:
memory: 128Mi
ephemeral-storage: 64Mi
volumeMounts:
- name: bosfs-tmp
mountPath: /host-bosfs
- name: host-proc
mountPath: /host-proc
readOnly: true
volumes:
- name: bosfs-tmp
hostPath:
path: /tmp/bosfs
type: DirectoryOrCreate
- name: host-proc
hostPath:
path: /proc