Measure Your Proxmox Restore RTO (Honestly)

#!/usr/bin/env bash
# measure-restore-rto.sh — honestly measure Proxmox/PBS restore time (RTO).
#
#   Creates a small + large test VM, backs them up to a PBS datastore, restores
#   each while timing it, reports GB restored and effective MB/s, then DESTROYS
#   everything it created.
#
#   *** THIS CREATES AND DESTROYS VMs AND BACKUPS. ***  Dry-run ("plan") is the
#   default; nothing changes until you pass an action. It is NOT a product — no
#   scheduling, no daemon, no support, no warranty. A measurement aid you read
#   before you run.
#
# Usage (run as root on a Proxmox VE node):
#   ./measure-restore-rto.sh plan        # default: print the plan, change nothing
#   ./measure-restore-rto.sh setup       # create the test VMs and back them up
#   ./measure-restore-rto.sh restore [N] # N timed restores of each (default 1)
#   ./measure-restore-rto.sh teardown    # destroy test VMs + try to prune backups
#   ./measure-restore-rto.sh run [N]     # setup -> restore N -> teardown
#
# Config via env (see `plan`) — CHANGE THESE to match your cluster:
STORAGE="${STORAGE:-local-lvm}"          # target storage for disks/restores
PBS_STORAGE="${PBS_STORAGE:-pbs-local}"  # PBS datastore to back up to / restore from
BRIDGE="${BRIDGE:-vmbr0}"
SMALL_ID="${SMALL_ID:-9001}"; LARGE_ID="${LARGE_ID:-9002}"
RESTORE_BASE="${RESTORE_BASE:-9100}"     # restores use RESTORE_BASE+1 / +2
LARGE_DATA_GIB="${LARGE_DATA_GIB:-12}"   # incompressible data disk on the large VM
IMAGE_URL="${IMAGE_URL:-https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2}"
IMAGE="${IMAGE:-/var/lib/vz/rto-debian12.qcow2}"
RESULTS="${RESULTS:-/tmp/rto-results.csv}"
set -uo pipefail
# keep the PBS API-token id out of any shared logs — redact both the --repository
# arg AND the "connecting to repository '<user>@<realm>!<token>@host'" line pbs-restore prints
RED='s/[A-Za-z0-9_.-]+@[A-Za-z0-9_.-]+![A-Za-z0-9_.-]+/<REDACTED-TOKEN>/g; s/(--repository )[^ ]+/\1<REDACTED>/g'

die(){ echo "ERROR: $*" >&2; exit 1; }
have(){ command -v "$1" >/dev/null || die "missing $1 (run on a PVE node)"; }

plan(){
  cat <<EOF
DRY RUN — nothing will change. This script would:
  1. download a Debian cloud image to $IMAGE (if absent)
  2. create small VM $SMALL_ID (cloud image only, ~1 GiB real)
  3. create large VM $LARGE_ID (cloud image + a ${LARGE_DATA_GIB} GiB disk of
     INCOMPRESSIBLE random data — the worst case for a dedup/compress store)
  4. back both up to PBS datastore '$PBS_STORAGE'
  5. restore each to '$STORAGE' as $((RESTORE_BASE+1))/$((RESTORE_BASE+2)),
     timing wall-clock and computing effective MB/s, then destroy the restore
  6. teardown: destroy $SMALL_ID/$LARGE_ID and prune the test backups
WARNING: steps 2-6 CREATE and DESTROY guests. Review IDs above first.
Storage=$STORAGE  PBS=$PBS_STORAGE  bridge=$BRIDGE  results->$RESULTS
Run:  $0 setup   then   $0 restore 3   then   $0 teardown
EOF
}

fetch_image(){
  [ -f "$IMAGE" ] && { echo "image present: $IMAGE"; return; }
  echo "downloading cloud image..."; wget -q -O "$IMAGE.part" "$IMAGE_URL" || die "download failed"
  mv "$IMAGE.part" "$IMAGE"
}

mkvm(){ # $1=id $2=name $3=mem $4=cores
  qm create "$1" --name "$2" --memory "$3" --cores "$4" --net0 "virtio,bridge=$BRIDGE" \
     --scsihw virtio-scsi-single --ostype l26 --agent 1 || die "create $1"
  qm set "$1" --scsi0 "$STORAGE:0,import-from=$IMAGE,discard=on,ssd=1" >/dev/null || die "import $1"
  qm set "$1" --boot order=scsi0 --ide2 "$STORAGE:cloudinit" --ipconfig0 ip=dhcp >/dev/null
}

setup(){
  have qm; have vzdump; fetch_image
  echo "== create small VM $SMALL_ID =="; mkvm "$SMALL_ID" rto-small 1024 1
  echo "== create large VM $LARGE_ID (+${LARGE_DATA_GIB} GiB incompressible) =="
  mkvm "$LARGE_ID" rto-large 2048 2
  qm set "$LARGE_ID" --scsi1 "$STORAGE:${LARGE_DATA_GIB},discard=on,ssd=1" >/dev/null
  local dev; dev="$(pvesm path "$STORAGE:vm-${LARGE_ID}-disk-1" 2>/dev/null || echo /dev/pve/vm-${LARGE_ID}-disk-1)"
  [ -b "$dev" ] || die "data disk $dev not found"
  echo "filling $dev with random data..."; dd if=/dev/urandom of="$dev" bs=1M count=$((LARGE_DATA_GIB*1000)) conv=fsync status=none
  echo "== back up both to $PBS_STORAGE =="
  vzdump "$SMALL_ID" "$LARGE_ID" --storage "$PBS_STORAGE" --mode stop 2>&1 | sed -E "$RED" | grep -E 'Backup of|transferred|Finished' || true
  echo "setup complete."
}

latest(){ pvesm list "$PBS_STORAGE" | awk -v id="$1" '$2=="pbs-vm" && $NF==id{print $1}' | sort | tail -1; }

time_one(){ # $1=snapshot $2=target-id $3=label
  qm destroy "$2" --purge --destroy-unreferenced-disks 1 >/dev/null 2>&1 || true
  local t0 t1 secs; t0="$(date +%s)"
  qmrestore "$1" "$2" --storage "$STORAGE" 2>&1 | sed -E "$RED" | grep -E 'restore image complete|bytes=' || true
  t1="$(date +%s)"; secs=$((t1-t0))
  echo "$3,$2,$1,$secs" | tee -a "$RESULTS"
  qm destroy "$2" --purge --destroy-unreferenced-disks 1 >/dev/null 2>&1 || true
}

restore(){ # $1 = number of runs
  have qmrestore
  local runs="${1:-1}" s l i
  s="$(latest "$SMALL_ID")"; l="$(latest "$LARGE_ID")"
  [ -n "$s" ] && [ -n "$l" ] || die "no backups found for $SMALL_ID/$LARGE_ID on $PBS_STORAGE (run setup)"
  echo "label,target,snapshot,seconds" | tee "$RESULTS"
  i=1; while [ "$i" -le "$runs" ]; do
    echo "== restore run $i/$runs =="
    time_one "$s" "$((RESTORE_BASE+1))" "small-run$i"
    time_one "$l" "$((RESTORE_BASE+2))" "large-run$i"
    i=$((i+1))
  done
  echo "== results ($RESULTS) =="; cat "$RESULTS"
}

teardown(){
  for id in "$SMALL_ID" "$LARGE_ID" "$((RESTORE_BASE+1))" "$((RESTORE_BASE+2))"; do
    qm destroy "$id" --purge --destroy-unreferenced-disks 1 >/dev/null 2>&1 && echo "destroyed $id" || true
  done
  echo "pruning test backups (needs Datastore.Prune on the token)..."
  for grp in "$SMALL_ID" "$LARGE_ID"; do
    for v in $(pvesm list "$PBS_STORAGE" | awk -v id="$grp" '$2=="pbs-vm" && $NF==id{print $1}'); do
      pvesm free "$v" 2>&1 | sed -E "$RED" | tail -1 || \
        echo "  could not prune $v — forget it in the PBS UI (backup tokens often lack Prune, by design)"
    done
  done
  rm -f "$IMAGE"; echo "teardown done."
}

case "${1:-plan}" in
  plan) plan ;; setup) setup ;; restore) restore "${2:-1}" ;;
  teardown) teardown ;; run) setup && restore "${2:-1}" && teardown ;;
  *) die "unknown action '$1' (try: plan|setup|restore|teardown|run)" ;;
esac