forked from longtao.wu/droboticflow
🐛 Reap leaked bosfs temp files that evicted A800 training mounts
CCE csi-bosplugin writes bosfs.tmp.* under host /tmp/bosfs on the node root disk and never removes them when a mount pod ends. Leaks filled the A800 node root disk, kubelet evicted the bosfs FUSE pod mid-training and runs failed with Errno 107. Add a DaemonSet that deletes temp files of dead mounts not held open by any process, and document the failure mode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a3b0bfe29a
commit
f29ec4bfae
@ -484,6 +484,13 @@ kubectl -n droboticflow exec deploy/droboticflow-worker -- ls -R /workspace/runs
|
||||
- confirm the node directory contains `input-artifacts.json`, `result.json`, `stage-manifest.json`, and `outputs/`
|
||||
- inspect worker logs for `DROBOTICFLOW_RESULT_PATH` parse failures
|
||||
|
||||
### Training fails with `[Errno 107] Transport endpoint is not connected`
|
||||
|
||||
- the BOS FUSE mount died under the pod; check `kubectl get events -A | grep -i evict` for an evicted `kube-system/bosfs-<hash>` pod and `DiskPressure` on the node
|
||||
- CCE csi-bosplugin runs bosfs with `-o tmpdir=/tmp` (host `/tmp/bosfs/logs/bosfs-<hash>` on the root disk) and leaves `bosfs.tmp.*` files behind when a mount pod ends; on the A800 node each full training run leaked ~100G until the root disk reached the kubelet eviction threshold and the BestEffort bosfs pod was evicted mid-run
|
||||
- keep `k8s/bosfs-tmp-reaper.yaml` deployed (`kubectl apply -f k8s/bosfs-tmp-reaper.yaml`); it removes temp files of dead mounts that no process holds open, and logs `root_avail` per node every 5 minutes (`kubectl -n droboticflow logs -l app=bosfs-tmp-reaper --prefix --tail=3`)
|
||||
- the reaper pins the worker image by digest; update the digest if that image is ever garbage-collected from the registry
|
||||
|
||||
### Image pull failure
|
||||
|
||||
- recreate `droboticflow-registry` secret with valid registry credentials
|
||||
|
||||
126
k8s/bosfs-tmp-reaper.yaml
Normal file
126
k8s/bosfs-tmp-reaper.yaml
Normal file
@ -0,0 +1,126 @@
|
||||
# Reclaims bosfs temp files leaked on node root disks.
|
||||
#
|
||||
# CCE csi-bosplugin starts one kube-system/bosfs-<hash> pod per BOS mount with
|
||||
# `-o tmpdir=/tmp`, where /tmp is hostPath /tmp/bosfs/logs/bosfs-<hash> on the
|
||||
# node root disk. bosfs writes bosfs.tmp.* files there while reading/writing
|
||||
# objects and does not remove them when the bosfs process is killed or the
|
||||
# mount pod is torn down. Each full training run leaked ~100G on 192.168.80.136
|
||||
# until the root disk hit the kubelet eviction threshold, kubelet evicted the
|
||||
# (BestEffort) bosfs pod, and training failed with
|
||||
# `[Errno 107] Transport endpoint is not connected`.
|
||||
#
|
||||
# Every INTERVAL_SECONDS this DaemonSet deletes bosfs.tmp.* files that:
|
||||
# - belong to a bosfs-<hash> directory with no live bosfs process, and
|
||||
# - are not held open by any process on the node, and
|
||||
# - are older than MIN_FILE_AGE_MINUTES.
|
||||
# Directories are kept (a live mount pod may still reference them) and only
|
||||
# removed when empty, orphaned, and older than EMPTY_DIR_AGE_MINUTES.
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: bosfs-tmp-reaper
|
||||
namespace: droboticflow
|
||||
labels:
|
||||
app: bosfs-tmp-reaper
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: bosfs-tmp-reaper
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: bosfs-tmp-reaper
|
||||
spec:
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
terminationGracePeriodSeconds: 5
|
||||
imagePullSecrets:
|
||||
- name: droboticflow-registry
|
||||
containers:
|
||||
- name: reaper
|
||||
image: ccr-29eug8s3-pub.cnc.bj.baidubce.com/dataflow-poc/droboticflow-worker@sha256:540dc05908745d69c4b0f1488d833b1add18953d21196a75aba31b7b969eb990
|
||||
imagePullPolicy: IfNotPresent
|
||||
securityContext:
|
||||
# Root + privileged: delete root-owned temp files and read
|
||||
# /proc/<pid>/fd of every process to skip files still open.
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
privileged: true
|
||||
env:
|
||||
- name: DRY_RUN
|
||||
value: "false"
|
||||
- name: INTERVAL_SECONDS
|
||||
value: "300"
|
||||
- name: MIN_FILE_AGE_MINUTES
|
||||
value: "10"
|
||||
- name: EMPTY_DIR_AGE_MINUTES
|
||||
value: "1440"
|
||||
- name: NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
command: ["/usr/bin/bash", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -u
|
||||
ROOT=/host-bosfs/logs
|
||||
PROC=/host-proc
|
||||
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) node=${NODE_NAME} $*"; }
|
||||
reap_once() {
|
||||
[ -d "$ROOT" ] || { log "no $ROOT"; return; }
|
||||
# Live mounts: bosfs argv carries credentials/<hash> matching bosfs-<hash>.
|
||||
declare -A live=() open=()
|
||||
for c in "$PROC"/[0-9]*/cmdline; do
|
||||
args=$(tr '\0' ' ' 2>/dev/null < "$c") || continue
|
||||
case "$args" in bosfs\ *|*/bosfs\ *) ;; *) continue ;; esac
|
||||
h=$(grep -o 'credentials/[0-9a-f]\{32\}' <<<"$args" | head -1)
|
||||
[ -n "$h" ] && live[${h#credentials/}]=1
|
||||
done
|
||||
# Open temp files, by basename (container paths differ from host paths).
|
||||
while IFS= read -r t; do
|
||||
b=${t##*/}; open[${b%% (deleted)}]=1
|
||||
done < <(find "$PROC"/[0-9]*/fd -maxdepth 1 -lname '*bosfs.tmp.*' -printf '%l\n' 2>/dev/null)
|
||||
files=0 bytes=0 skipped_live=0
|
||||
for d in "$ROOT"/bosfs-*; do
|
||||
[ -d "$d" ] || continue
|
||||
h=${d##*/bosfs-}
|
||||
if [ -n "${live[$h]+x}" ]; then skipped_live=$((skipped_live + 1)); continue; fi
|
||||
while IFS= read -r -d '' f; do
|
||||
[ -n "${open[${f##*/}]+x}" ] && continue
|
||||
sz=$(stat -c %s "$f" 2>/dev/null || echo 0)
|
||||
if [ "$DRY_RUN" = "true" ]; then log "would remove $f ($sz bytes)"
|
||||
else rm -f -- "$f" || continue; fi
|
||||
files=$((files + 1)); bytes=$((bytes + sz))
|
||||
done < <(find "$d" -maxdepth 1 -type f -name 'bosfs.tmp.*' -mmin +"$MIN_FILE_AGE_MINUTES" -print0)
|
||||
if [ "$DRY_RUN" != "true" ] && [ -z "$(ls -A "$d" 2>/dev/null)" ] \
|
||||
&& [ -n "$(find "$d" -maxdepth 0 -mmin +"$EMPTY_DIR_AGE_MINUTES")" ]; then
|
||||
rmdir -- "$d" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
avail=$(df -P "$ROOT" | awk 'NR==2 {printf "%.0fG", $4/1048576}')
|
||||
log "dry_run=${DRY_RUN} live_mounts=${#live[@]} skipped_live_dirs=${skipped_live} reaped_files=${files} reaped_bytes=${bytes} root_avail=${avail}"
|
||||
}
|
||||
trap 'exit 0' TERM INT
|
||||
while true; do reap_once; sleep "$INTERVAL_SECONDS" & wait $!; done
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
ephemeral-storage: 16Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
ephemeral-storage: 64Mi
|
||||
volumeMounts:
|
||||
- name: bosfs-tmp
|
||||
mountPath: /host-bosfs
|
||||
- name: host-proc
|
||||
mountPath: /host-proc
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: bosfs-tmp
|
||||
hostPath:
|
||||
path: /tmp/bosfs
|
||||
type: DirectoryOrCreate
|
||||
- name: host-proc
|
||||
hostPath:
|
||||
path: /proc
|
||||
Loading…
x
Reference in New Issue
Block a user