Proxmox Storage Health Check (Read-Only)

#!/usr/bin/env bash
# proxmox-storage-health-check.sh — read-only storage audit for one PVE node.
# Catches the quiet failures before they bite. Changes NOTHING.
#
# Usage: run as root on a Proxmox VE node:
#   ./proxmox-storage-health-check.sh
set -u

# ==== MAKE THESE VALUES YOUR OWN ==========================================
WARN_PCT=80                     # warn when a filesystem or pool passes this %
BACKUP_MAX_AGE_DAYS=3           # warn when the newest backup is older than this
BACKUP_DIRS="/mnt/backup/dump /var/lib/vz/dump"   # where vzdump files land
VG="pve"                        # your volume group (pve = installer default)
# ==========================================================================

WARNS=0
warn() { echo "WARN  $1"; WARNS=$((WARNS+1)); }
ok()   { echo "OK    $1"; }

echo "--- 1) root filesystem ---"
ROOT_PCT=$(df -P / | awk 'NR==2 {gsub(/%/,""); print $5}')
if [ "$ROOT_PCT" -ge "$WARN_PCT" ]; then
  warn "root filesystem at ${ROOT_PCT}% (threshold ${WARN_PCT}%)"
else
  ok "root filesystem at ${ROOT_PCT}%"
fi

echo "--- 2) thin pool data + metadata ---"
if lvs "$VG/data" >/dev/null 2>&1; then
  # LC_ALL=C matters: on comma-decimal locales lvs prints "21,39" and the
  # integer comparison below would silently pass a nearly-full pool as OK.
  DATA_PCT=$(LC_ALL=C lvs --noheadings -o data_percent "$VG/data" | tr -d ' ' | cut -d. -f1)
  META_PCT=$(LC_ALL=C lvs --noheadings -o metadata_percent "$VG/data" | tr -d ' ' | cut -d. -f1)
  for pair in "data:$DATA_PCT" "metadata:$META_PCT"; do
    name=${pair%%:*}; pct=${pair##*:}
    if [ -n "$pct" ] && [ "$pct" -ge "$WARN_PCT" ]; then
      warn "thin pool $name at ${pct}%"
    else
      ok "thin pool $name at ${pct:-?}%"
    fi
  done
else
  ok "no thin pool $VG/data on this node (check skipped)"
fi

echo "--- 3) backups on the root filesystem? ---"
DUMP_DEV=$(df -P /var/lib/vz/dump 2>/dev/null | awk 'NR==2 {print $1}')
ROOT_DEV=$(df -P / | awk 'NR==2 {print $1}')
if [ -d /var/lib/vz/dump ] && [ "$DUMP_DEV" = "$ROOT_DEV" ] && \
   [ -n "$(ls -A /var/lib/vz/dump 2>/dev/null)" ]; then
  SIZE=$(du -xsh /var/lib/vz/dump 2>/dev/null | cut -f1)
  warn "backups live on the ROOT filesystem: /var/lib/vz/dump holds $SIZE (move them — they compete with the OS for space)"
else
  ok "no backup files competing with the root filesystem"
fi

echo "--- 4) newest backup age ---"
FOUND_DIR=0
FRESH=0
HAS_FILES=0
for d in $BACKUP_DIRS; do
  [ -d "$d" ] || continue
  FOUND_DIR=1
  if [ -n "$(find "$d" -maxdepth 1 -name 'vzdump-*' -mtime -"$BACKUP_MAX_AGE_DAYS" -print -quit 2>/dev/null)" ]; then
    ok "$d has a backup newer than ${BACKUP_MAX_AGE_DAYS} day(s)"
    FRESH=1
    HAS_FILES=1
  elif [ -n "$(find "$d" -maxdepth 1 -name 'vzdump-*' -print -quit 2>/dev/null)" ]; then
    warn "$d newest backup is OLDER than ${BACKUP_MAX_AGE_DAYS} day(s) — is the job silently failing?"
    HAS_FILES=1
  else
    echo "INFO  $d holds no vzdump files (not a backup target on this node?)"
  fi
done
if [ "$FOUND_DIR" -eq 0 ]; then
  warn "none of BACKUP_DIRS exist on this node — where do backups go?"
elif [ "$HAS_FILES" -eq 0 ]; then
  warn "no vzdump files anywhere in BACKUP_DIRS — wrong paths, or the job has never run?"
fi

echo "--- 5) stale vzdump snapshots + locked guests ---"
STALE=$(grep -l '^\[vzdump\]' /etc/pve/lxc/*.conf /etc/pve/qemu-server/*.conf 2>/dev/null)
if [ -n "$STALE" ]; then
  warn "leftover [vzdump] snapshot section in: $(echo "$STALE" | tr '\n' ' ')"
else
  ok "no leftover vzdump snapshot sections in guest configs"
fi
LOCKED=$(grep -H '^lock:' /etc/pve/lxc/*.conf /etc/pve/qemu-server/*.conf 2>/dev/null)
if [ -n "$LOCKED" ]; then
  warn "locked guest(s): $(echo "$LOCKED" | tr '\n' ' ')"
else
  ok "no locked guests"
fi
SNAP_LVS=$(lvs --noheadings -o lv_name "$VG" 2>/dev/null | grep -i vzdump)
if [ -n "$SNAP_LVS" ]; then
  warn "leftover vzdump snapshot LV(s): $(echo "$SNAP_LVS" | tr -d ' ' | tr '\n' ' ')"
else
  ok "no leftover vzdump snapshot LVs"
fi

echo
if [ "$WARNS" -gt 0 ]; then
  echo "Result: $WARNS warning(s) — read them top to bottom; each maps to a fix in the storage-issues post."
  exit 1
fi
echo "Result: all clear."