From 87bfbab8dadf2e1f845eb9c5a6c1d048b6063728 Mon Sep 17 00:00:00 2001
From: Ian Kelling <iank@fsf.org>
Date: Thu, 20 Mar 2025 01:31:07 -0400
Subject: [PATCH] fix lots of bugs.

---
 .../files/simple/usr/local/bin/attach-vm-disk | 105 -----
 .../files/simple/usr/local/bin/create-vm      |  64 ++-
 .../files/simple/usr/local/bin/mount-vm       |   8 +-
 .../files/simple/usr/local/bin/savannah-virsh | 118 +++--
 .../files/simple/usr/local/bin/umount-vm      |   7 +-
 .../simple/usr/local/bin/unsafe-remove-vm     | 257 +----------
 .../files/simple/usr/local/bin/vm-disk        | 416 ++++++++++++++++++
 .../simple/usr/local/lib/fsf-vm-disk-wipe     | 147 +++++++
 8 files changed, 709 insertions(+), 413 deletions(-)
 delete mode 100755 roles/kvmhost/files/simple/usr/local/bin/attach-vm-disk
 create mode 100755 roles/kvmhost/files/simple/usr/local/bin/vm-disk
 create mode 100644 roles/kvmhost/files/simple/usr/local/lib/fsf-vm-disk-wipe

diff --git a/roles/kvmhost/files/simple/usr/local/bin/attach-vm-disk b/roles/kvmhost/files/simple/usr/local/bin/attach-vm-disk
deleted file mode 100755
index 2288ad1..0000000
--- a/roles/kvmhost/files/simple/usr/local/bin/attach-vm-disk
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/bin/bash
-
-
-# in addition, we need virsh detach, and preferably rm.
-
-set -e; . /usr/local/lib/bash-bear; set +e
-shopt -s nullglob
-
-# note: you have to run shellcheck from this directory for this to work. kinda dumb.
-# shellcheck source=../lib/fsf-virsh-bash-lib
-source /usr/local/lib/fsf-virsh-bash-lib
-
-[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
-
-usage() {
-  cat <<EOF
-Usage: ${0##*/} SOURCE_VM_NAME SOURCE_DEV_PATH TARGET_VM_NAME
-One line description
-
--h|--help  Print help and exit.
-
-
-For ceph disks, SOURCE_DEV_PATH is the name attribute in its xml:
-<source protocol='rbd' name='rbd/frontend2.savannah.gnu.org'>
-
-For non-ceph disks, the path is the dev attribute in its vm xml:
- <source dev='/dev/mapper/crypt-nfs2.savannah.gnu.org' index='4'/>
-
-The above xml must already exist for SOURCE_VM_NAME. This is so we
-can use access control to VMs.
-
-
-Note: Uses util-linux getopt option parsing: spaces between args and
-options, short options can be combined, options before args.
-EOF
-  exit $1
-}
-
-get-attached-devs() {
-  local tmps vm="$1"
-  tmps=$(virsh dumpxml $vm)
-  tmps=$(xmllint --xpath "$xpath" - <<<"$tmps" | sed 's/^[^"]*"// ; s/"[^"]*$//')
-  mapfile -t attached_devs <<<"$tmps"
-}
-
-check-source-dev() {
-  local found_source attached_dev
-  local -a attached_devs
-
-  get-attached-devs $source_vm
-
-  found_source=false
-  for attached_dev in "${attached_devs[@]}"; do
-    if [[ $source_dev_path == "$attached_dev" ]]; then
-      found_source=true
-      break
-    fi
-  done
-
-  if ! $found_source; then
-    echo "$0: error: failed to find SOURCE_DEV_PATH in SOURCE_VM_NAME's xml"
-    exit 1
-  fi
-}
-
-
-##### begin command line parsing ########
-
-# ensure we can handle args with spaces or empty.
-ret=0; getopt -T || ret=$?
-[[ $ret == 4 ]] || { echo "Install util-linux for enhanced getopt" >&2; exit 1; }
-
-temp=$(getopt -l help h "$@") || usage 1
-eval set -- "$temp"
-while true; do
-  case $1 in
-    -h|--help) usage ;;
-    --) shift; break ;;
-    *) echo "$0: unexpected args: $*" >&2 ; usage 1 ;;
-  esac
-  shift
-done
-read -r source_vm source_dev_path host <<<"$@"
-
-##### end command line parsing ########
-
-ceph=false
-xpath=/domain/devices/disk/source/@dev
-if [[ $HOSTNAME == kvmhost[234] ]]; then
-  ceph=true
-  xpath="/domain/devices/disk/source[@protocol='rbd']/@name"
-fi
-
-check-source-dev
-
-get-attached-devs $host
-
-disk_letters=( {a..z} )
-letter=${disk_letters[${#attached_devs[@]}]}
-
-if $ceph; then
-  attach-ceph-disk $source_dev_path
-else
-  attach-local-disk $source_dev_path
-fi
diff --git a/roles/kvmhost/files/simple/usr/local/bin/create-vm b/roles/kvmhost/files/simple/usr/local/bin/create-vm
index 63ea8af..2c3fa8d 100755
--- a/roles/kvmhost/files/simple/usr/local/bin/create-vm
+++ b/roles/kvmhost/files/simple/usr/local/bin/create-vm
@@ -114,10 +114,11 @@ to undo what it has done. Be sure to read the output!
 
 -i --import : For importing/migrating existing vm: creates the vm with a
                            mostly empty disk. Then you can mount the
-                           disk with the mount-vm script, rsync an OS,
-                           and umount with umount-vm.
+                           disk with the mount-vm or attach it to
+                           another vm with savannah-virsh vm-disk
+                           attach. Then rsync an OS.
 
--l : List free disk, memory & cpu stats.
+-l : List free disk & memory.
 
 -g --vol-group VOL_GROUP : Use the the local 3 disk array with
                            VOL_GROUP as one of the 3 volume groups in
@@ -406,14 +407,13 @@ mkfs-and-mount-ceph-disk() {
 }
 
 maybe-get-ceph-live-add-disk-steps() {
-  if ! $add_disk && ! vm-on; then
+  if ! $add_disk || ! vm-on; then
     return 0
   fi
 
   inside_vm_cmds=true
   cat >/root/generated-to-run-in-vm <<EOF
-# todo: accept cli argument which alters these commands for swap partition, or multiple device btrfs, etc.
-set -x
+set -xe
 mknod /dev/sd$letter b 8 $disk_num
 tmpf=\$(mktemp)
 /boot/keyscript.sh >\$tmpf
@@ -545,22 +545,59 @@ mk-common-grub-images() {
 
 set-disk-letter() {
   letter=a
+
+  local -A letter_to_number
+  local -i i=1
+  for l in {a..z}; do
+    letter_to_number[$l]=$i
+    i+=1
+  done
+
   if $add_disk; then
-    xpath=/domain/devices/disk/source/@dev
-    if $doceph; then
-      xpath="/domain/devices/disk/source[@protocol='rbd']/@name"
-    fi
+    disk_letters=( {a..z} )
+
     tmpf=$(mktemp)
     virsh dumpxml $host >$tmpf
-    disk_num=$( xmllint --xpath "count($xpath)" $tmpf)
-    disk_letters=( {a..z} )
-    letter=${disk_letters[disk_num]}
+
+    ## This method uses the count of disks, but if they are already
+    ## designated to letters which skip over one, perhaps because we
+    ## removed a disk, then we will get a conflict. Leaving commented
+    ## in case it comes in handy later.
+    #
+    # xpath=/domain/devices/disk/source/@dev
+    # if $doceph; then
+    #   xpath="/domain/devices/disk/source[@protocol='rbd']/@name"
+    # fi
+    # disk_num=$( xmllint --xpath "count($xpath)" $tmpf)
+    # letter=${disk_letters[disk_num]}
+
+
+    tmps=$(xmllint --xpath /domain/devices/disk/target/@dev $tmpf | sed -r 's/^ *//;s/ /\n/g' | sed 's/^[^"]*"// ; s/"[^"]*$//')
+    mapfile -t xml_targets <<<"$tmps"
+    last=z
+    for letter in {y..a}; do
+      found=false
+      for xml_target in ${xml_targets[@]}; do
+        if [[ $xml_target == *$letter ]]; then
+          found=true
+          break
+        fi
+      done
+      if $found; then
+        letter=$last
+        break
+      fi
+      last=$letter
+    done
+    disk_num=${letter_to_number[$letter]}
+
   fi
 }
 
 attach-disk() {
   if $doceph; then
     attach-ceph-disk $pool/$dname
+    maybe-get-ceph-live-add-disk-steps
     return 0
   fi
 
@@ -1420,7 +1457,6 @@ else
   create-vm
 fi
 
-maybe-get-ceph-live-add-disk-steps
 
 if $inside_vm_cmds; then
   cat <<EOF
diff --git a/roles/kvmhost/files/simple/usr/local/bin/mount-vm b/roles/kvmhost/files/simple/usr/local/bin/mount-vm
index f4f5bf0..3b77b24 100755
--- a/roles/kvmhost/files/simple/usr/local/bin/mount-vm
+++ b/roles/kvmhost/files/simple/usr/local/bin/mount-vm
@@ -19,12 +19,8 @@
 #    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 #
 
-
-if [[ ! -s /usr/local/lib/err ]]; then
-  echo "$0: error, missing /usr/local/lib/err" >&2
-  exit 1
-fi
-source /usr/local/lib/err
+set -eE -o pipefail
+trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?"' ERR
 
 usage() {
   cat <<EOF
diff --git a/roles/kvmhost/files/simple/usr/local/bin/savannah-virsh b/roles/kvmhost/files/simple/usr/local/bin/savannah-virsh
index dab2b51..3e643db 100755
--- a/roles/kvmhost/files/simple/usr/local/bin/savannah-virsh
+++ b/roles/kvmhost/files/simple/usr/local/bin/savannah-virsh
@@ -20,40 +20,51 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# todo: document the command that requires ssh -t, and detect
-# it with if tty -t; then terminal is allocated; fi
-
 ## Managed by Ansible, changes will be overwritten ##
 
+set -eE -o pipefail
+trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?"' ERR
+
+
 usage() {
   cat <<EOF
 ### There are many subcommands:
 
-## Usage: list
+## Usage: VIRSH_COMMAND VM
 
-runs: virsh list --name
+VIRSH_COMMAND: console|reboot|reset|start|destroy|dumpxml|list [OPTIONS]
 
-## Usage: list-verbose
+runs: virsh VIRSH_COMMAND VM
 
-runs: virsh list --all
+The most useful list options are: --name, --all, --autostart.
 
-## Usage: VIRSH_COMMAND VM
+## Usage: ceph-mv-vm OTHER_KVM_HOST LOCAL_VM
 
-VIRSH_COMMAND:  console|reboot|reset|start|destroy|dumpxml
+Move an offline, nonautostarting ceph vm to a different ceph kvm host.
 
-runs: virsh VIRSH_COMMAND VM
+## Usage: autostart [--disable] VM
+
+runs: virsh autostart [--disable] VM
+
+Enable to disable automatically starting VM on boot.
 
 ## Usage: detach-disk VM VIRTUAL_DEVICE_NAME
 
 runs: virsh detach-disk VM VIRTUAL_DEVICE_NAME --persistent
 
+VIRTUAL_DEVICE_NAME example: sdb, it is the device file name seen inside
+the vm and specified in the virsh xml as for example: <target
+dev='sda'>. WARNING: Do not detach a disk that is only attached to 1 vm,
+FSF Sysadmins will have to run some commands to deal with the orphaned
+disk.
+
 ## Usage: nmon
 
-runs nmon. You may need to use ssh -t to request a pseudo-terminal.
+runs nmon. Requires using ssh -t to request a pseudo-terminal.
 
 ### Complex FSF scripts with separate docs:
 
-attach-vm-disk --help
+vm-disk --help
 
 unsafe-remove-vm --help
 
@@ -89,7 +100,7 @@ check-host-arg() {
   done
 
   if ! $valid_host; then
-    echo "error: bad argument"; exit 1
+    echo "error: bad host argument"; exit 1
   fi
 }
 
@@ -100,9 +111,8 @@ arg-die() {
 pre="sv-virsh: +"
 m() { printf "$pre %s\n"  "$*"; "$@"; }
 
-set -x
-set -eE -o pipefail
-trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?"' ERR
+# for debugging.
+#set -x
 
 initial_input_regex='^[[:alnum:]/. _-]+$'
 general_arg_regex='^[[:alnum:]._-]+$'
@@ -161,13 +171,33 @@ case "$arg0" in
       fi
     done
     m "${arg_array[@]}"
+    exit
     ;;
-  attach-vm-disk)
-    if (( argc >= 6 )); then
-      echo "error: bad argument argc 6"; exit 1
+  vm-disk)
+    if (( argc >= 20 )); then
+      echo "error: bad argument argc 20"; exit 1
     fi
-    if [[ ! $arg =~ $path_arg_regex ]]; then arg-die path_arg_regex in attach-vm-disk; fi
+    case ${arg_array[1]} in
+      attach) check-host-arg ;;&
+      -*) : ;;
+      *)
+        host_arg="${arg_array[2]}"
+        check-host-arg
+        ;;
+    esac
+
     m "${arg_array[@]}"
+    exit
+    ;;
+
+  detach-disk)
+    if (( argc >= 4 )); then
+      echo "error: bad argument count: $argc, wanted 3"; exit 1
+    fi
+    host_arg="${arg_array[argc - 2]}"
+    check-host-arg
+    m virsh "${arg_array[@]}" --persistent
+    exit
     ;;
   *)
     for arg in "${arg_array[@]}"; do
@@ -175,26 +205,44 @@ case "$arg0" in
     done
 
     ;;&
+  ceph-mv-vm)
+    target_kvm="${arg_array[1]}"
+    src_vm="${arg_array[2]}"
+    if [[ $target_kvm != kvmhost[234].fsf.org ]]; then
+      echo "bad target kvm host arg:$target_kvm" >&2
+      exit 1
+    fi
+    if ! virsh list --state-shutoff --no-autostart --name | grep -xFq $src_vm; then
+      echo "error: vm not off with autoresume disabled"
+      exit 1
+    fi
 
-  detach-disk)
-    if (( argc > 3 )); then
-      echo "error: bad argument count: $argc, wanted 3"; exit 1
+    tmpf=$(mktemp)
+    virsh dumpxml $src_vm | tee $tmpf
+    m sleep 3
+    grub_kernel=$(xmllint --xpath '/domain/os/kernel/text()' $tmpf)
+    if [[ ! -s $grub_kernel || $grub_kernel != /var/lib/libvirt/* ]]; then
+      echo "error: failed to get grub image, got:$grub_kernel"
+      exit 1
     fi
-    host_arg="${arg_array[argc - 2]}"
-    check-host-arg
-    m virsh "${arg_array[@]}"
+    m rsync -av --backup "$grub_kernel" "$target_kvm:$grub_kernel"
+
+    m rsync -a $tmpf $target_kvm:/root/ceph-mv-$src_vm.xml
+
+    m ssh $target_kvm virsh define /root/ceph-mv-$src_vm.xml
+    m virsh undefine $src_vm
+    m ssh $target_kvm virsh start $src_vm
     ;;
   list)
-    m virsh list --name
+    m virsh "${arg_array[@]}"
+    exit
     ;;
   nmon)
     if ! type -p nmon &>/dev/null; then
       m apt-get -y install nmon
     fi
     nmon
-    ;;
-  list-verbose)
-    m virsh list --all
+    exit
     ;;
   unsafe-remove-vm)
     if (( argc != 2 )); then
@@ -202,6 +250,15 @@ case "$arg0" in
     fi
     check-host-arg
     m "${arg_array[@]}"
+    exit
+    ;;
+  autostart)
+    if (( argc >= 4 )); then
+      echo "error: bad argument count: $argc, wanted <=3"; exit 1
+    fi
+    check-host-arg
+    m virsh "${arg_array[@]}"
+    exit
     ;;
   console|reboot|reset|start|destroy|dumpxml)
     if (( argc != 2 )); then
@@ -209,6 +266,7 @@ case "$arg0" in
     fi
     check-host-arg
     m virsh "${arg_array[@]}"
+    exit
     ;;
   *)
     arg-die
diff --git a/roles/kvmhost/files/simple/usr/local/bin/umount-vm b/roles/kvmhost/files/simple/usr/local/bin/umount-vm
index 40c5054..0672e69 100755
--- a/roles/kvmhost/files/simple/usr/local/bin/umount-vm
+++ b/roles/kvmhost/files/simple/usr/local/bin/umount-vm
@@ -19,11 +19,8 @@
 
 # This script undoes mounts made by mount-vm
 
-if [[ ! -s /usr/local/lib/err ]]; then
-  echo "$0: error, missing /usr/local/lib/err" >&2
-  exit 1
-fi
-source /usr/local/lib/err
+set -eE -o pipefail
+trap 'echo "$0:$LINENO:error: \"$BASH_COMMAND\" returned $?"' ERR
 
 ### begin command line parsing ####
 if (( $# != 1 )); then
diff --git a/roles/kvmhost/files/simple/usr/local/bin/unsafe-remove-vm b/roles/kvmhost/files/simple/usr/local/bin/unsafe-remove-vm
index d3cbedc..fb7525d 100755
--- a/roles/kvmhost/files/simple/usr/local/bin/unsafe-remove-vm
+++ b/roles/kvmhost/files/simple/usr/local/bin/unsafe-remove-vm
@@ -7,6 +7,8 @@
 
 
 set -e; . /usr/local/lib/bash-bear; set +e
+# shellcheck source=../lib/fsf-vm-disk-wipe
+source /usr/local/lib/fsf-vm-disk-wipe
 
 ##### begin command line parsing ########
 
@@ -34,35 +36,7 @@ host=$1
 ##### end command line parsing ########
 
 script_gen=/root/unsafe-remove-vm-generated
-cat >$script_gen <<EOF
-#!/bin/bash
-set -xe
-# generated by unsafe-remove-vm on $(date -R)
-EOF
-chmod +x $script_gen
-
-dosleep=false
-m() {
-  # with no sleeps, i got this example output:
-  #+ integritysetup close /dev/dm-68
-  # device-mapper: remove ioctl on integrity-vgata-Samsung_SSD_850_EVO_1TB_S3PJNF0J909379K-nextcloud.fsf.org  failed: Device or resource busy
-  # however, the command didnt fail and the overall script
-  # still succeeded. anyways, just adding sleeps out of extra caution.
-  # Not like we run this very often, it also allows you to press ctrl-c
-  # if you see some output you dont like.
-  if $dosleep; then
-    echo "sleep 3" >>$script_gen
-  fi
-  dosleep=true
-  if [[ $BASH_VERSION == 5* ]]; then
-    # This is more robust, but got introduced in bash 5.1 or 5.0.
-    # too old for t8.
-    echo "${@@Q}" >>$script_gen
-  else
-    printf "%s\n" "$*" >>$script_gen
-  fi
-}
-
+initialize-script-gen
 
 ##### begin probing/sanity checking ########
 
@@ -103,8 +77,6 @@ else
   fi
 fi
 
-
-
 if virsh list --name | grep -Fx $host &>/dev/null; then
   m virsh destroy $host
 fi
@@ -134,105 +106,9 @@ if virsh list --name --all | grep -Fx $host &>/dev/null; then
   m virsh undefine $host
 fi
 
-lvrm() {
-  local lv
-  lv="$1"
-  lvs "$lv" &>/dev/null || return $?
-  echo "$0: found logical volume. output of lvs $lv :"
-  lvs "$lv"
-  # wipefs makes it so when a new lv is created later, it doesn't give a
-  # prompt warning that an existing filesystem signature is found.
-  m wipefs -a "$lv"
-  m lvremove -f "$lv"
-}
-
-declare -A keyfiles
-for devpath in ${devpaths[@]}; do
-  if lvrm "$devpath" &>/dev/null; then
-    ## we are a simple unencrypted vg-disk. lvrm took care of things
-    :
-  elif under_crypt_dev=$(cryptsetup status "$devpath" | awk '$1 == "device:" {print $2}') && [[ $under_crypt_dev ]]; then
-    ## This whole block is for community0p
-    m cryptsetup luksClose "$devpath"
-    # we assume all the libvirt xml crypt paths are /dev/mapper/
-    crypt_name=${devpath##*/}
-    keyfile=$(awk '$1 == "'$crypt_name'" {print $3}' /etc/crypttab)
-    if [[ $keyfile && -f $keyfile && ! ${keyfiles[$keyfile]} ]]; then
-      keyfiles[$keyfile]=t
-      m rm -f $keyfile
-    fi
-    m sed -i "/^$crypt_name/d" /etc/crypttab
-    if lvrm "$under_crypt_dev"; then
-      # we are in a btrfs setup on community
-      :
-    elif [[ $under_crypt_dev == /dev/md* ]]; then
-      # we are in a mdraid setup on community
-      stat=$(awk '$1 == "'${under_crypt_dev##*/}'"' /proc/mdstat)
-      m mdadm -v --stop "$under_crypt_dev"
-      # see comment at end of script for background on this.
-      m test ! -e "/sys/devices/virtual/block/${under_crypt_dev##*/}"
-      for w in $stat; do
-        dm=${w%%\[*}
-        if [[ $dm && $dm != "$w" ]]; then
-          for f in /dev/mapper/*; do
-            if [[ $(readlink -f "$f") == "/dev/$dm" ]]; then
-              under_md_dev="$f"
-              break
-            fi
-          done
-          m mdadm -v --zero-superblock "$under_md_dev"
-          if under_integrity_dev=$(integritysetup status "$under_md_dev" | awk '$1 == "device:" {print $2}') && [[ $under_integrity_dev ]]; then
-            m integritysetup close "$under_md_dev"
-            lvrm "$under_integrity_dev" ||:
-          fi
-        fi
-      done
-    fi
-  fi
-done
-
-for rbdname in ${rbdnames[@]}; do
-  snapshot_count=$(rbd info $rbdname | awk '$1 == "snapshot_count:" {print $2}')
-  if [[ $snapshot_count != 0 ]]; then
-    cat <<EOF
-$0: error found snapshot_count = $snapshot_count, != 0 for rbd image $rbdname."
-see https://docs.ceph.com/en/quincy/rbd/rbd-snapshot/
-flatten or remove snapshots first and consider vms using them.
-eg:
-
-rbd snap protect jitsi2p.fsf.org@working2p
-rbd clone jitsi2p.fsf.org@working2p jitsi2p-working
-rbd flatten rbd/jitsi2p-working
-rbd snap unprotect jitsi2p.fsf.org@working2p
-rbd snap rm jitsi2p.fsf.org@working2p
-
-
-output of rbd snap ls $rbdname:
-
-EOF
-    rbd snap ls $rbdname
-    echo
-    echo "$0: exiting 1"
-    exit 1
-  fi
-done
-
-
-for rbdname in ${rbdnames[@]}; do
-  parent=$(rbd info $rbdname | awk '$1 == "parent:" {print $2}')
-  m rbd rm $rbdname
-  if [[ $parent ]]; then
-    m rbd snap unprotect $parent
-    m rbd snap rm $parent
-  fi
-done
-
+vm-disk-wipe
 ##### end probing/sanity checking ########
 
-echo "planned commands, cat $script_gen:"
-cat $script_gen
-echo
-echo "end of cat $script_gen"
 
 echo "Do you want to completely remove $host from this kvmhost and delete its disks based on the planned commands? (y/N): "
 read -r yn
@@ -249,128 +125,3 @@ case $yn in
 esac
 
 
-# background on test ! -e "/sys/devices/virtual/block/${under_crypt_dev##*/}"
-
-# I ran unsafe-remove-vm, and it succeeded. However, a later create-vm
-# failed on md creation with a cryptic error. So I looked in dmesg saw
-# the error below. The timestamps are from when I ran unsafe-remove-vm,
-# and the 1st 2 lines are normal output after running mdadm
-# --stop. Google searches came up with very little for this error. So,
-# in order to hopefully help avoid this error, I've increased the sleep
-# between commands from 1 to 3 and added a check that the sysfs device
-# is gone so we can catch the problem.
-
-# Note: When searching for mdadm array removal, some old instructions
-# say you need to do mdadm --remove /dev/mdXXX after --stop if it
-# exists, but I've yet to see that needed.
-
-# Nov 13 02:29:27 community0p kernel: md121: detected capacity change from 914061312 to 0
-# Nov 13 02:29:27 community0p kernel: md: md121 stopped.
-# Nov 13 02:29:27 community0p kernel: sysfs: cannot create duplicate filename '/devices/virtual/block/md121'
-# Nov 13 02:29:27 community0p kernel: CPU: 10 PID: 4905 Comm: mdadm Tainted: G        W         5.15.0-47-generic #51+11.0trisquel11
-# Nov 13 02:29:27 community0p kernel: Hardware name: ASUS KGPE-D16/KGPE-D16, BIOS c57e03c 09/07/2016
-# Nov 13 02:29:27 community0p kernel: Call Trace:
-# Nov 13 02:29:27 community0p kernel:  <TASK>
-# Nov 13 02:29:27 community0p kernel:  show_stack+0x52/0x5c
-# Nov 13 02:29:27 community0p kernel:  dump_stack_lvl+0x4a/0x63
-# Nov 13 02:29:27 community0p kernel:  dump_stack+0x10/0x16
-# Nov 13 02:29:27 community0p kernel:  sysfs_warn_dup.cold+0x17/0x2b
-# Nov 13 02:29:27 community0p kernel:  sysfs_create_dir_ns+0xbc/0xd0
-# Nov 13 02:29:27 community0p kernel:  create_dir+0x28/0x170
-# Nov 13 02:29:27 community0p kernel:  kobject_add_internal+0x9e/0x1d0
-# Nov 13 02:29:27 community0p kernel:  kobject_add+0x7e/0xb0
-# Nov 13 02:29:27 community0p kernel:  ? mutex_lock+0x13/0x50
-# Nov 13 02:29:27 community0p kernel:  device_add+0x11d/0x7b0
-# Nov 13 02:29:27 community0p kernel:  ? dev_set_name+0x53/0x70
-# Nov 13 02:29:27 community0p kernel:  device_add_disk+0xfb/0x3c0
-# Nov 13 02:29:27 community0p kernel:  md_alloc+0x15e/0x400
-# Nov 13 02:29:27 community0p kernel:  md_probe+0x52/0x60
-# Nov 13 02:29:27 community0p kernel:  blk_request_module+0x7f/0x110
-# Nov 13 02:29:27 community0p kernel:  blkdev_get_no_open+0xa9/0xe0
-# Nov 13 02:29:27 community0p kernel:  blkdev_get_by_dev.part.0+0x21/0x320
-# Nov 13 02:29:27 community0p kernel:  blkdev_get_by_dev+0x55/0x70
-# Nov 13 02:29:27 community0p kernel:  ? blkdev_close+0x40/0x40
-# Nov 13 02:29:27 community0p kernel:  blkdev_open+0x50/0x90
-# Nov 13 02:29:27 community0p kernel:  do_dentry_open+0x159/0x390
-# Nov 13 02:29:27 community0p kernel:  vfs_open+0x2d/0x40
-# Nov 13 02:29:27 community0p kernel:  do_open+0x20d/0x3d0
-# Nov 13 02:29:27 community0p kernel:  path_openat+0x112/0x2b0
-# Nov 13 02:29:27 community0p kernel:  ? kmem_cache_free+0x249/0x290
-# Nov 13 02:29:27 community0p kernel:  ? rseq_get_rseq_cs.isra.0+0x1b/0x230
-# Nov 13 02:29:27 community0p kernel:  do_filp_open+0xb2/0x160
-# Nov 13 02:29:27 community0p kernel:  ? __check_object_size+0x1d/0x30
-# Nov 13 02:29:27 community0p kernel:  ? alloc_fd+0x53/0x180
-# Nov 13 02:29:27 community0p kernel:  do_sys_openat2+0x9f/0x160
-# Nov 13 02:29:27 community0p kernel:  __x64_sys_openat+0x55/0x90
-# Nov 13 02:29:27 community0p kernel:  do_syscall_64+0x5c/0xc0
-# Nov 13 02:29:27 community0p kernel:  ? irqentry_exit_to_user_mode+0x9/0x20
-# Nov 13 02:29:27 community0p kernel:  ? irqentry_exit+0x1d/0x30
-# Nov 13 02:29:27 community0p kernel:  ? sysvec_call_function_single+0x4e/0x90
-# Nov 13 02:29:27 community0p kernel:  entry_SYSCALL_64_after_hwframe+0x61/0xcb
-# Nov 13 02:29:27 community0p kernel: RIP: 0033:0x7f8a3cfa76eb
-# Nov 13 02:29:27 community0p kernel: Code: 25 00 00 41 00 3d 00 00 41 00 74 4b 64 8b 04 25 18 00 00 00 85 c0 75 67 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0
-# f 87 91 00 00 00 48 8b 54 24 28 64 48 2b 14 25
-# Nov 13 02:29:27 community0p kernel: RSP: 002b:00007fff5ca451c0 EFLAGS: 00000246 ORIG_RAX: 0000000000000101
-# Nov 13 02:29:27 community0p kernel: RAX: ffffffffffffffda RBX: 0000000000000014 RCX: 00007f8a3cfa76eb
-# Nov 13 02:29:27 community0p kernel: RDX: 0000000000000000 RSI: 0000562d70b95300 RDI: 00000000ffffff9c
-# Nov 13 02:29:27 community0p kernel: RBP: 0000562d70b95300 R08: 0000000000000014 R09: 0000000000000000
-# Nov 13 02:29:27 community0p kernel: R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
-# Nov 13 02:29:27 community0p kernel: R13: 0000000000000000 R14: 0000562d70b95270 R15: 0000562d70b95270
-# Nov 13 02:29:27 community0p kernel:  </TASK>
-# Nov 13 02:29:27 community0p kernel: kobject_add_internal failed for md121 with -EEXIST, don't try to register things with the same name in the same directory.
-# Nov 13 02:29:27 community0p kernel: ------------[ cut here ]------------
-# Nov 13 02:29:27 community0p kernel: WARNING: CPU: 10 PID: 4905 at block/genhd.c:544 device_add_disk+0x121/0x3c0
-# Nov 13 02:29:27 community0p kernel: Modules linked in: vhost_net vhost vhost_iotlb dm_integrity dm_bufio dm_crypt xt_CHECKSUM xt_MASQUERADE xt_conntrack ipt_REJECT nf_reject_ipv4 xt_tcpudp nft_compat nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nft_counter nf_tables nfnetlink macvtap macvlan tap sunrpc nls_iso8859_1 amd64_edac edac_mce_amd snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi snd_hda_codec kvm_amd snd_hda_core ccp snd_hwdep kvm snd_pcm input_leds snd_timer serio_raw snd fam15h_power k10temp soundcore mac_hid sch_fq_codel w83627ehf hwmon_vid w83795 bonding tls br_netfilter bridge stp llc usbhid ramoops hid pstore_blk reed_solomon mtd pstore_zone efi_pstore ip_tables x_tables autofs4 btrfs blake2b_generic zstd_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq libcrc32c raid1 raid0 multipath linear ast i2c_algo_bit drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops cec crct10dif_pclmul
-# Nov 13 02:29:27 community0p kernel:  pata_acpi crc32_pclmul rc_core firewire_ohci uas ghash_clmulni_intel aesni_intel ahci firewire_core crypto_simd cryptd psmouse drm usb_storage crc_itu_t e1000e i2c_piix4 pata_atiixp libahci
-# Nov 13 02:29:27 community0p kernel: CPU: 10 PID: 4905 Comm: mdadm Tainted: G        W         5.15.0-47-generic #51+11.0trisquel11
-# Nov 13 02:29:27 community0p kernel: Hardware name: ASUS KGPE-D16/KGPE-D16, BIOS c57e03c 09/07/2016
-# Nov 13 02:29:27 community0p kernel: RIP: 0010:device_add_disk+0x121/0x3c0
-# Nov 13 02:29:27 community0p kernel: Code: 15 28 26 00 85 c0 75 14 4c 89 e7 e8 99 b8 00 00 85 c0 74 34 4c 89 ff e8 ad fc 25 00 41 81 3c 24 03 01 00 00 0f 84 83 00 00 00 <0f> 0b 41 b8 01 00 00 00 48 83 c4 08 44 89 c0 5b 41 5c 41 5d 41 5e
-# Nov 13 02:29:27 community0p kernel: RSP: 0018:ffffac30016ffb00 EFLAGS: 00010287
-# Nov 13 02:29:27 community0p kernel: RAX: 00000000ffffffef RBX: ffff9df9547babc0 RCX: 00000000001c6dfd
-# Nov 13 02:29:27 community0p kernel: RDX: 00000000001c6dfc RSI: a61588aaaedf7f56 RDI: ffff9df9547bac38
-# Nov 13 02:29:27 community0p kernel: RBP: ffffac30016ffb30 R08: 0000000000000003 R09: 0000000000000001
-# Nov 13 02:29:27 community0p kernel: R10: ffff9dfc62d93d00 R11: ffffffffc040f0e0 R12: ffff9dfd28427000
-# Nov 13 02:29:27 community0p kernel: R13: 0000000000000000 R14: 0000000000000000 R15: ffff9df9547bac00
-# Nov 13 02:29:27 community0p kernel: FS:  00007f8a3d0f8740(0000) GS:ffff9e17f7a80000(0000) knlGS:0000000000000000
-# Nov 13 02:29:27 community0p kernel: CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
-# Nov 13 02:29:27 community0p kernel: CR2: 00007f4d63a67000 CR3: 000000010a158000 CR4: 00000000000406e0
-# Nov 13 02:29:27 community0p kernel: Call Trace:
-# Nov 13 02:29:27 community0p kernel:  <TASK>
-# Nov 13 02:29:27 community0p kernel:  md_alloc+0x15e/0x400
-# Nov 13 02:29:27 community0p kernel:  md_probe+0x52/0x60
-# Nov 13 02:29:27 community0p kernel:  blk_request_module+0x7f/0x110
-# Nov 13 02:29:27 community0p kernel:  blkdev_get_no_open+0xa9/0xe0
-# Nov 13 02:29:27 community0p kernel:  blkdev_get_by_dev.part.0+0x21/0x320
-# Nov 13 02:29:27 community0p kernel:  blkdev_get_by_dev+0x55/0x70
-# Nov 13 02:29:27 community0p kernel:  ? blkdev_close+0x40/0x40
-# Nov 13 02:29:27 community0p kernel:  blkdev_open+0x50/0x90
-# Nov 13 02:29:27 community0p kernel:  do_dentry_open+0x159/0x390
-# Nov 13 02:29:27 community0p kernel:  vfs_open+0x2d/0x40
-# Nov 13 02:29:27 community0p kernel:  do_open+0x20d/0x3d0
-# Nov 13 02:29:27 community0p kernel:  path_openat+0x112/0x2b0
-# Nov 13 02:29:27 community0p kernel:  ? kmem_cache_free+0x249/0x290
-# Nov 13 02:29:27 community0p kernel:  ? rseq_get_rseq_cs.isra.0+0x1b/0x230
-# Nov 13 02:29:27 community0p kernel:  do_filp_open+0xb2/0x160
-# Nov 13 02:29:27 community0p kernel:  ? __check_object_size+0x1d/0x30
-# Nov 13 02:29:27 community0p kernel:  ? alloc_fd+0x53/0x180
-# Nov 13 02:29:27 community0p kernel:  do_sys_openat2+0x9f/0x160
-# Nov 13 02:29:27 community0p kernel:  __x64_sys_openat+0x55/0x90
-# Nov 13 02:29:27 community0p kernel:  do_syscall_64+0x5c/0xc0
-# Nov 13 02:29:27 community0p kernel:  ? irqentry_exit_to_user_mode+0x9/0x20
-# Nov 13 02:29:27 community0p kernel:  ? irqentry_exit+0x1d/0x30
-# Nov 13 02:29:27 community0p kernel:  ? sysvec_call_function_single+0x4e/0x90
-# Nov 13 02:29:27 community0p kernel:  entry_SYSCALL_64_after_hwframe+0x61/0xcb
-# Nov 13 02:29:27 community0p kernel: RIP: 0033:0x7f8a3cfa76eb
-# Nov 13 02:29:27 community0p kernel: Code: 25 00 00 41 00 3d 00 00 41 00 74 4b 64 8b 04 25 18 00 00 00 85 c0 75 67 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0f 87 91 00 00 00 48 8b 54 24 28 64 48 2b 14 25
-# Nov 13 02:29:27 community0p kernel: RSP: 002b:00007fff5ca451c0 EFLAGS: 00000246 ORIG_RAX: 0000000000000101
-# Nov 13 02:29:27 community0p kernel: RAX: ffffffffffffffda RBX: 0000000000000014 RCX: 00007f8a3cfa76eb
-# Nov 13 02:29:27 community0p kernel: RDX: 0000000000000000 RSI: 0000562d70b95300 RDI: 00000000ffffff9c
-# Nov 13 02:29:27 community0p kernel: RBP: 0000562d70b95300 R08: 0000000000000014 R09: 0000000000000000
-# Nov 13 02:29:27 community0p kernel: R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
-# Nov 13 02:29:27 community0p kernel: R13: 0000000000000000 R14: 0000562d70b95270 R15: 0000562d70b95270
-# Nov 13 02:29:27 community0p kernel:  </TASK>
-# Nov 13 02:29:27 community0p kernel: ---[ end trace 4567bea8adb921ba ]---
-# Nov 13 02:29:27 community0p kernel: kobject_add_internal failed for md (error: -2 parent: md121)
-# Nov 13 02:29:27 community0p kernel: clocksource: timekeeping watchdog on CPU17: hpet wd-wd read-back delay of 262044ns
-# Nov 13 02:29:27 community0p kernel: clocksource: wd-tsc-wd read-back delay of 264000ns, clock-skew test skipped!
diff --git a/roles/kvmhost/files/simple/usr/local/bin/vm-disk b/roles/kvmhost/files/simple/usr/local/bin/vm-disk
new file mode 100755
index 0000000..4c86da4
--- /dev/null
+++ b/roles/kvmhost/files/simple/usr/local/bin/vm-disk
@@ -0,0 +1,416 @@
+#!/bin/bash
+
+
+# in addition, we need virsh detach, and preferably rm.
+
+set -e; . /usr/local/lib/bash-bear; set +e
+shopt -s nullglob
+
+# note: you have to run shellcheck from this directory for this to
+# work. kinda dumb. I ignore the warning.
+#
+# shellcheck disable=SC1091
+# shellcheck source=../lib/fsf-vm-disk-wipe
+source /usr/local/lib/fsf-vm-disk-wipe
+# shellcheck disable=SC1091
+# shellcheck source=../lib/fsf-virsh-bash-lib
+source /usr/local/lib/fsf-virsh-bash-lib
+
+[[ $EUID == 0 ]] || exec sudo -E "${BASH_SOURCE[0]}" "$@"
+
+usage() {
+  cat <<EOF
+Usage:
+${0##*/} attach VM VM_DEV_PATH TARGET_VM
+${0##*/} delete VM VM_DEV_PATH...
+${0##*/} list VM
+
+In the 1st form, make VM_DEV_PATH be ALSO attached to
+TARGET_VM. WARNING: Only 1 VM should be online during any disk
+writes. detach-disk or unsafe-remove-vm should be run within the same
+day as this command. If both VMs are set to autostart, VM will have
+autostart disabled and a large warning printed.
+
+In the 2nd form, delete VM_DEV_PATH... and detach them from the
+VM. The source dev paths should include all members of any raid arrays.
+
+In the 3rd form, list the device paths currently attached to
+VM. If run over ssh, this requires running with -t to allocate a pty so
+that all user prompts will happen properly.
+
+-h|--help  Print help and exit.
+
+For ceph disks, VM_DEV_PATH is the name attribute in its xml:
+<source protocol='rbd' name='rbd/frontend2.savannah.gnu.org'>
+
+For non-ceph disks, the path is the dev attribute in its vm xml:
+ <source dev='/dev/mapper/crypt-nfs2.savannah.gnu.org' index='4'/>
+
+The above xml must already exist for VM.
+
+TODO: add some mechanism to share vm xml across ceph kvm hosts.
+
+Note: Uses util-linux getopt option parsing: spaces between args and
+options, short options can be combined, options before args.
+EOF
+  exit $1
+}
+
+get-attached-devs() {
+  local tmps vm="$1"
+  dumped_xml=$(virsh dumpxml $vm)
+  tmps=$(xmllint --xpath "$xpath" - <<<"$dumped_xml" | sed -r 's/^ *//;s/ /\n/g' | sed 's/^[^"]*"// ; s/"[^"]*$//')
+  mapfile -t attached_devs <<<"$tmps"
+
+  # note: when you xmldump to a file, there are more newlines.
+  # we just add them here.
+  tmps=$(xmllint --xpath /domain/devices/disk/target/@dev - <<<"$dumped_xml" | sed -r 's/^ *//;s/ /\n/g' | sed 's/^[^"]*"// ; s/"[^"]*$//')
+  mapfile -t xml_targets <<<"$tmps"
+}
+
+check-source-dev() {
+  local found_source attached_dev
+  local -a attached_devs
+
+  get-attached-devs $source_vm
+
+  found_source=false
+  for attached_dev in "${attached_devs[@]}"; do
+    if [[ $disk == "$attached_dev" ]]; then
+      found_source=true
+      break
+    fi
+  done
+
+  if ! $found_source; then
+    echo "$0: error: failed to find VM_DEV_PATH:$disk in VM's xml"
+    exit 1
+  fi
+}
+
+
+##### begin command line parsing ########
+
+# ensure we can handle args with spaces or empty.
+ret=0; getopt -T || ret=$?
+[[ $ret == 4 ]] || { echo "Install util-linux for enhanced getopt" >&2; exit 1; }
+
+temp=$(getopt -l help h "$@") || usage 1
+eval set -- "$temp"
+while true; do
+  case $1 in
+    -h|--help) usage ;;
+    --) shift; break ;;
+    *) echo "$0: unexpected args: $*" >&2 ; usage 1 ;;
+  esac
+  shift
+done
+
+if (( $# < 2 )); then
+  echo "$0: error: expected more args"
+  exit 1
+fi
+
+subcmd="$1"
+source_vm="$2"
+shift 2
+
+do_attach=false
+do_list=false
+case $subcmd in
+  attach)
+    read -r disk host <<<"$@"
+    do_attach=true
+    if (( $# != 2 )); then
+      echo "$0: error: wrong remaining arg count:$# for attach"
+      exit 1
+    fi
+    ;;
+  list)
+    read -r disk <<<"$@"
+    do_list=true
+    if (( $# != 0 )); then
+      echo "$0: error: wrong remaining arg count:$# for list"
+      exit 1
+    fi
+    ;;
+  delete)
+    declare -a devpaths rbdnames all_disks
+    for disk; do
+      all_disks+=("$disk")
+      if [[ $disk == /* ]]; then
+        devpaths+=("$disk")
+      else
+        rbdnames+=("$disk")
+      fi
+    done
+    ;;
+  *)
+    echo "$0: error: expected arg of attach, delete or list. Got: $subcmd"
+    exit 1
+    ;;
+esac
+
+##### end command line parsing ########
+
+
+ceph=false
+xpath=/domain/devices/disk/source/@dev
+if [[ $HOSTNAME == kvmhost[234] ]]; then
+  ceph=true
+  xpath="/domain/devices/disk/source[@protocol='rbd']/@name"
+fi
+
+if $do_list; then
+  get-attached-devs "$source_vm"
+  for dev in "${attached_devs[@]}"; do
+    printf "%s\n" "$dev"
+  done
+  exit 0
+fi
+
+
+
+if $do_attach; then
+  check-source-dev
+
+  get-attached-devs $host
+
+  # # If testing the last letter didn't work, we could use this method of
+  # # using a count of devices. But this doesn't work if you create a
+  # # hole in the disks, like detaching sdb and leaving sdc attached.
+  # disk_letters=( {a..z} )
+  # letter=${disk_letters[${#attached_devs[@]}]}
+
+
+  # Find the first letter not used by an existing target dev.
+  for letter in {a..z}; do
+    found=false
+    for xml_target in ${xml_targets[@]}; do
+      if [[ $xml_target == *$letter ]]; then
+        found=true
+        break
+      fi
+    done
+    if ! $found; then
+      break
+    fi
+  done
+
+
+  if $ceph; then
+    attach-ceph-disk $disk
+  else
+    attach-local-disk $disk
+  fi
+  found_vms=0
+  for vm in $(virsh list --name --autostart); do
+    if [[ $vm == "$source_vm" || $vm == "$host" ]]; then
+      found_vms=$(( found_vms + 1 ))
+    fi
+  done
+  if (( found_vms == 2 )); then
+    virsh autostart --disable "$source_vm"
+    echo "
+WARNING: disabled autostart for $source_vm !!!!!!!!!!!!!"
+  fi
+
+  exit 0
+fi
+
+# Otherwise, we are deleting.
+
+if ! tty -s; then
+  echo "$0: error, no tty. Use ssh -t." >&2
+  exit 1
+fi
+
+
+for disk in "${all_disks[@]}"; do check-source-dev; done
+
+
+### fancy filename for our generated script, eg: 2025-03-19-001
+mkdir -p /root/vm-disk-generated
+date_stamp=$(date +%F)
+for (( i=0; i < 999; i++ )); do
+  script_gen="/root/vm-disk-generated/$date_stamp-$(printf %03d $i)"
+  if [[ ! -s $script_gen ]]; then
+    break
+  fi
+done
+
+initialize-script-gen
+
+target_dev_regex='^[a-z]+$'
+for disk in "${all_disks[@]}"; do
+
+  if $ceph; then
+    xpath='string(/domain/devices/disk/source[@name="'$disk'"]/'
+  else
+    xpath='string(/domain/devices/disk/source[@dev="'$disk'"]/'
+  fi
+
+  tmps=$(virsh dumpxml $source_vm)
+  # todo: make some regexps to detect invalid target_dev.
+  target_dev=$(xmllint --xpath "${xpath}following-sibling::target/@dev)" - <<<"$tmps")
+  if [[ ! $target_dev ]]; then
+    # examples I looked at were following, but I suspect they could be preceding
+    target_dev=$(xmllint --xpath "${xpath}preceding-sibling::target/@dev)" - <<<"$tmps")
+  fi
+  if [[ ! $target_dev =~ $target_dev_regex ]]; then
+    echo "$0: error: got bad target_dev:$target_dev"
+    exit 1
+  fi
+
+  m virsh detach-disk "$source_vm" "$target_dev" --persistent
+done
+
+vm-disk-wipe
+
+
+echo "Are you REALLY sure you want to run these commands?
+Sanity check: for mdraid arrays, it should include sets of 'integritysetup close', and lvremove, and mdadm --zero-superblock.
+For btrfs array, it shold mainly include lvremove, cryptsetup luksClose, virsh detach-disk.
+
+(y/N): "
+read -r yn
+
+case $yn in
+  Y|y|YES|Yes|yes)
+    $script_gen
+    exit 0
+    ;;
+  * )
+    echo "Non-yes answer, exiting"
+    exit 0
+    ;;
+esac
+
+
+
+
+# background on test ! -e "/sys/devices/virtual/block/${under_crypt_dev##*/}"
+
+# I ran unsafe-remove-vm, and it succeeded. However, a later create-vm
+# failed on md creation with a cryptic error. So I looked in dmesg saw
+# the error below. The timestamps are from when I ran unsafe-remove-vm,
+# and the 1st 2 lines are normal output after running mdadm
+# --stop. Google searches came up with very little for this error. So,
+# in order to hopefully help avoid this error, I've increased the sleep
+# between commands from 1 to 3 and added a check that the sysfs device
+# is gone so we can catch the problem.
+
+# Note: When searching for mdadm array removal, some old instructions
+# say you need to do mdadm --remove /dev/mdXXX after --stop if it
+# exists, but I've yet to see that needed.
+
+# Nov 13 02:29:27 community0p kernel: md121: detected capacity change from 914061312 to 0
+# Nov 13 02:29:27 community0p kernel: md: md121 stopped.
+# Nov 13 02:29:27 community0p kernel: sysfs: cannot create duplicate filename '/devices/virtual/block/md121'
+# Nov 13 02:29:27 community0p kernel: CPU: 10 PID: 4905 Comm: mdadm Tainted: G        W         5.15.0-47-generic #51+11.0trisquel11
+# Nov 13 02:29:27 community0p kernel: Hardware name: ASUS KGPE-D16/KGPE-D16, BIOS c57e03c 09/07/2016
+# Nov 13 02:29:27 community0p kernel: Call Trace:
+# Nov 13 02:29:27 community0p kernel:  <TASK>
+# Nov 13 02:29:27 community0p kernel:  show_stack+0x52/0x5c
+# Nov 13 02:29:27 community0p kernel:  dump_stack_lvl+0x4a/0x63
+# Nov 13 02:29:27 community0p kernel:  dump_stack+0x10/0x16
+# Nov 13 02:29:27 community0p kernel:  sysfs_warn_dup.cold+0x17/0x2b
+# Nov 13 02:29:27 community0p kernel:  sysfs_create_dir_ns+0xbc/0xd0
+# Nov 13 02:29:27 community0p kernel:  create_dir+0x28/0x170
+# Nov 13 02:29:27 community0p kernel:  kobject_add_internal+0x9e/0x1d0
+# Nov 13 02:29:27 community0p kernel:  kobject_add+0x7e/0xb0
+# Nov 13 02:29:27 community0p kernel:  ? mutex_lock+0x13/0x50
+# Nov 13 02:29:27 community0p kernel:  device_add+0x11d/0x7b0
+# Nov 13 02:29:27 community0p kernel:  ? dev_set_name+0x53/0x70
+# Nov 13 02:29:27 community0p kernel:  device_add_disk+0xfb/0x3c0
+# Nov 13 02:29:27 community0p kernel:  md_alloc+0x15e/0x400
+# Nov 13 02:29:27 community0p kernel:  md_probe+0x52/0x60
+# Nov 13 02:29:27 community0p kernel:  blk_request_module+0x7f/0x110
+# Nov 13 02:29:27 community0p kernel:  blkdev_get_no_open+0xa9/0xe0
+# Nov 13 02:29:27 community0p kernel:  blkdev_get_by_dev.part.0+0x21/0x320
+# Nov 13 02:29:27 community0p kernel:  blkdev_get_by_dev+0x55/0x70
+# Nov 13 02:29:27 community0p kernel:  ? blkdev_close+0x40/0x40
+# Nov 13 02:29:27 community0p kernel:  blkdev_open+0x50/0x90
+# Nov 13 02:29:27 community0p kernel:  do_dentry_open+0x159/0x390
+# Nov 13 02:29:27 community0p kernel:  vfs_open+0x2d/0x40
+# Nov 13 02:29:27 community0p kernel:  do_open+0x20d/0x3d0
+# Nov 13 02:29:27 community0p kernel:  path_openat+0x112/0x2b0
+# Nov 13 02:29:27 community0p kernel:  ? kmem_cache_free+0x249/0x290
+# Nov 13 02:29:27 community0p kernel:  ? rseq_get_rseq_cs.isra.0+0x1b/0x230
+# Nov 13 02:29:27 community0p kernel:  do_filp_open+0xb2/0x160
+# Nov 13 02:29:27 community0p kernel:  ? __check_object_size+0x1d/0x30
+# Nov 13 02:29:27 community0p kernel:  ? alloc_fd+0x53/0x180
+# Nov 13 02:29:27 community0p kernel:  do_sys_openat2+0x9f/0x160
+# Nov 13 02:29:27 community0p kernel:  __x64_sys_openat+0x55/0x90
+# Nov 13 02:29:27 community0p kernel:  do_syscall_64+0x5c/0xc0
+# Nov 13 02:29:27 community0p kernel:  ? irqentry_exit_to_user_mode+0x9/0x20
+# Nov 13 02:29:27 community0p kernel:  ? irqentry_exit+0x1d/0x30
+# Nov 13 02:29:27 community0p kernel:  ? sysvec_call_function_single+0x4e/0x90
+# Nov 13 02:29:27 community0p kernel:  entry_SYSCALL_64_after_hwframe+0x61/0xcb
+# Nov 13 02:29:27 community0p kernel: RIP: 0033:0x7f8a3cfa76eb
+# Nov 13 02:29:27 community0p kernel: Code: 25 00 00 41 00 3d 00 00 41 00 74 4b 64 8b 04 25 18 00 00 00 85 c0 75 67 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0
+# f 87 91 00 00 00 48 8b 54 24 28 64 48 2b 14 25
+# Nov 13 02:29:27 community0p kernel: RSP: 002b:00007fff5ca451c0 EFLAGS: 00000246 ORIG_RAX: 0000000000000101
+# Nov 13 02:29:27 community0p kernel: RAX: ffffffffffffffda RBX: 0000000000000014 RCX: 00007f8a3cfa76eb
+# Nov 13 02:29:27 community0p kernel: RDX: 0000000000000000 RSI: 0000562d70b95300 RDI: 00000000ffffff9c
+# Nov 13 02:29:27 community0p kernel: RBP: 0000562d70b95300 R08: 0000000000000014 R09: 0000000000000000
+# Nov 13 02:29:27 community0p kernel: R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
+# Nov 13 02:29:27 community0p kernel: R13: 0000000000000000 R14: 0000562d70b95270 R15: 0000562d70b95270
+# Nov 13 02:29:27 community0p kernel:  </TASK>
+# Nov 13 02:29:27 community0p kernel: kobject_add_internal failed for md121 with -EEXIST, don't try to register things with the same name in the same directory.
+# Nov 13 02:29:27 community0p kernel: ------------[ cut here ]------------
+# Nov 13 02:29:27 community0p kernel: WARNING: CPU: 10 PID: 4905 at block/genhd.c:544 device_add_disk+0x121/0x3c0
+# Nov 13 02:29:27 community0p kernel: Modules linked in: vhost_net vhost vhost_iotlb dm_integrity dm_bufio dm_crypt xt_CHECKSUM xt_MASQUERADE xt_conntrack ipt_REJECT nf_reject_ipv4 xt_tcpudp nft_compat nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nft_counter nf_tables nfnetlink macvtap macvlan tap sunrpc nls_iso8859_1 amd64_edac edac_mce_amd snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi snd_hda_codec kvm_amd snd_hda_core ccp snd_hwdep kvm snd_pcm input_leds snd_timer serio_raw snd fam15h_power k10temp soundcore mac_hid sch_fq_codel w83627ehf hwmon_vid w83795 bonding tls br_netfilter bridge stp llc usbhid ramoops hid pstore_blk reed_solomon mtd pstore_zone efi_pstore ip_tables x_tables autofs4 btrfs blake2b_generic zstd_compress raid10 raid456 async_raid6_recov async_memcpy async_pq async_xor async_tx xor raid6_pq libcrc32c raid1 raid0 multipath linear ast i2c_algo_bit drm_vram_helper drm_ttm_helper ttm drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops cec crct10dif_pclmul
+# Nov 13 02:29:27 community0p kernel:  pata_acpi crc32_pclmul rc_core firewire_ohci uas ghash_clmulni_intel aesni_intel ahci firewire_core crypto_simd cryptd psmouse drm usb_storage crc_itu_t e1000e i2c_piix4 pata_atiixp libahci
+# Nov 13 02:29:27 community0p kernel: CPU: 10 PID: 4905 Comm: mdadm Tainted: G        W         5.15.0-47-generic #51+11.0trisquel11
+# Nov 13 02:29:27 community0p kernel: Hardware name: ASUS KGPE-D16/KGPE-D16, BIOS c57e03c 09/07/2016
+# Nov 13 02:29:27 community0p kernel: RIP: 0010:device_add_disk+0x121/0x3c0
+# Nov 13 02:29:27 community0p kernel: Code: 15 28 26 00 85 c0 75 14 4c 89 e7 e8 99 b8 00 00 85 c0 74 34 4c 89 ff e8 ad fc 25 00 41 81 3c 24 03 01 00 00 0f 84 83 00 00 00 <0f> 0b 41 b8 01 00 00 00 48 83 c4 08 44 89 c0 5b 41 5c 41 5d 41 5e
+# Nov 13 02:29:27 community0p kernel: RSP: 0018:ffffac30016ffb00 EFLAGS: 00010287
+# Nov 13 02:29:27 community0p kernel: RAX: 00000000ffffffef RBX: ffff9df9547babc0 RCX: 00000000001c6dfd
+# Nov 13 02:29:27 community0p kernel: RDX: 00000000001c6dfc RSI: a61588aaaedf7f56 RDI: ffff9df9547bac38
+# Nov 13 02:29:27 community0p kernel: RBP: ffffac30016ffb30 R08: 0000000000000003 R09: 0000000000000001
+# Nov 13 02:29:27 community0p kernel: R10: ffff9dfc62d93d00 R11: ffffffffc040f0e0 R12: ffff9dfd28427000
+# Nov 13 02:29:27 community0p kernel: R13: 0000000000000000 R14: 0000000000000000 R15: ffff9df9547bac00
+# Nov 13 02:29:27 community0p kernel: FS:  00007f8a3d0f8740(0000) GS:ffff9e17f7a80000(0000) knlGS:0000000000000000
+# Nov 13 02:29:27 community0p kernel: CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
+# Nov 13 02:29:27 community0p kernel: CR2: 00007f4d63a67000 CR3: 000000010a158000 CR4: 00000000000406e0
+# Nov 13 02:29:27 community0p kernel: Call Trace:
+# Nov 13 02:29:27 community0p kernel:  <TASK>
+# Nov 13 02:29:27 community0p kernel:  md_alloc+0x15e/0x400
+# Nov 13 02:29:27 community0p kernel:  md_probe+0x52/0x60
+# Nov 13 02:29:27 community0p kernel:  blk_request_module+0x7f/0x110
+# Nov 13 02:29:27 community0p kernel:  blkdev_get_no_open+0xa9/0xe0
+# Nov 13 02:29:27 community0p kernel:  blkdev_get_by_dev.part.0+0x21/0x320
+# Nov 13 02:29:27 community0p kernel:  blkdev_get_by_dev+0x55/0x70
+# Nov 13 02:29:27 community0p kernel:  ? blkdev_close+0x40/0x40
+# Nov 13 02:29:27 community0p kernel:  blkdev_open+0x50/0x90
+# Nov 13 02:29:27 community0p kernel:  do_dentry_open+0x159/0x390
+# Nov 13 02:29:27 community0p kernel:  vfs_open+0x2d/0x40
+# Nov 13 02:29:27 community0p kernel:  do_open+0x20d/0x3d0
+# Nov 13 02:29:27 community0p kernel:  path_openat+0x112/0x2b0
+# Nov 13 02:29:27 community0p kernel:  ? kmem_cache_free+0x249/0x290
+# Nov 13 02:29:27 community0p kernel:  ? rseq_get_rseq_cs.isra.0+0x1b/0x230
+# Nov 13 02:29:27 community0p kernel:  do_filp_open+0xb2/0x160
+# Nov 13 02:29:27 community0p kernel:  ? __check_object_size+0x1d/0x30
+# Nov 13 02:29:27 community0p kernel:  ? alloc_fd+0x53/0x180
+# Nov 13 02:29:27 community0p kernel:  do_sys_openat2+0x9f/0x160
+# Nov 13 02:29:27 community0p kernel:  __x64_sys_openat+0x55/0x90
+# Nov 13 02:29:27 community0p kernel:  do_syscall_64+0x5c/0xc0
+# Nov 13 02:29:27 community0p kernel:  ? irqentry_exit_to_user_mode+0x9/0x20
+# Nov 13 02:29:27 community0p kernel:  ? irqentry_exit+0x1d/0x30
+# Nov 13 02:29:27 community0p kernel:  ? sysvec_call_function_single+0x4e/0x90
+# Nov 13 02:29:27 community0p kernel:  entry_SYSCALL_64_after_hwframe+0x61/0xcb
+# Nov 13 02:29:27 community0p kernel: RIP: 0033:0x7f8a3cfa76eb
+# Nov 13 02:29:27 community0p kernel: Code: 25 00 00 41 00 3d 00 00 41 00 74 4b 64 8b 04 25 18 00 00 00 85 c0 75 67 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0f 87 91 00 00 00 48 8b 54 24 28 64 48 2b 14 25
+# Nov 13 02:29:27 community0p kernel: RSP: 002b:00007fff5ca451c0 EFLAGS: 00000246 ORIG_RAX: 0000000000000101
+# Nov 13 02:29:27 community0p kernel: RAX: ffffffffffffffda RBX: 0000000000000014 RCX: 00007f8a3cfa76eb
+# Nov 13 02:29:27 community0p kernel: RDX: 0000000000000000 RSI: 0000562d70b95300 RDI: 00000000ffffff9c
+# Nov 13 02:29:27 community0p kernel: RBP: 0000562d70b95300 R08: 0000000000000014 R09: 0000000000000000
+# Nov 13 02:29:27 community0p kernel: R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
+# Nov 13 02:29:27 community0p kernel: R13: 0000000000000000 R14: 0000562d70b95270 R15: 0000562d70b95270
+# Nov 13 02:29:27 community0p kernel:  </TASK>
+# Nov 13 02:29:27 community0p kernel: ---[ end trace 4567bea8adb921ba ]---
+# Nov 13 02:29:27 community0p kernel: kobject_add_internal failed for md (error: -2 parent: md121)
+# Nov 13 02:29:27 community0p kernel: clocksource: timekeeping watchdog on CPU17: hpet wd-wd read-back delay of 262044ns
+# Nov 13 02:29:27 community0p kernel: clocksource: wd-tsc-wd read-back delay of 264000ns, clock-skew test skipped!
diff --git a/roles/kvmhost/files/simple/usr/local/lib/fsf-vm-disk-wipe b/roles/kvmhost/files/simple/usr/local/lib/fsf-vm-disk-wipe
new file mode 100644
index 0000000..fe649e7
--- /dev/null
+++ b/roles/kvmhost/files/simple/usr/local/lib/fsf-vm-disk-wipe
@@ -0,0 +1,147 @@
+#!/bin/bash
+
+initialize-script-gen() {
+  cat >$script_gen <<EOF
+#!/bin/bash
+set -xe
+# generated by $0 on $(date -R)
+EOF
+  chmod +x $script_gen
+}
+
+dosleep=false
+m() {
+  # with no sleeps, i got this example output:
+  #+ integritysetup close /dev/dm-68
+  # device-mapper: remove ioctl on integrity-vgata-Samsung_SSD_850_EVO_1TB_S3PJNF0J909379K-nextcloud.fsf.org  failed: Device or resource busy
+  # however, the command didnt fail and the overall script
+  # still succeeded. anyways, just adding sleeps out of extra caution.
+  # Not like we run this very often, it also allows you to press ctrl-c
+  # if you see some output you dont like.
+  if $dosleep; then
+    echo "sleep 3" >>$script_gen
+  fi
+  dosleep=true
+  if [[ $BASH_VERSION == 5* ]]; then
+    # This is more robust, but got introduced in bash 5.1 or 5.0.
+    # too old for t8.
+    echo "${@@Q}" >>$script_gen
+  else
+    printf "%s\n" "$*" >>$script_gen
+  fi
+}
+
+
+lvrm() {
+  local lv
+  lv="$1"
+  lvs "$lv" &>/dev/null || return $?
+  echo "$0: found logical volume. output of lvs $lv :"
+  lvs "$lv"
+  # wipefs makes it so when a new lv is created later, it doesn't give a
+  # prompt warning that an existing filesystem signature is found.
+  m wipefs -a "$lv"
+  m lvremove -f "$lv"
+}
+
+vm-disk-wipe() {
+  # unused
+  #local -A keyfiles
+  for devpath in ${devpaths[@]}; do
+    if lvrm "$devpath" &>/dev/null; then
+      ## we are a simple unencrypted vg-disk. lvrm took care of things
+      :
+    elif under_crypt_dev=$(cryptsetup status "$devpath" | awk '$1 == "device:" {print $2}') && [[ $under_crypt_dev ]]; then
+      ## This whole block is for community0p
+      m cryptsetup luksClose "$devpath"
+      # we assume all the libvirt xml crypt paths are /dev/mapper/
+      crypt_name=${devpath##*/}
+
+      ## Since bob asked for this, a secondary disk image can easily live beyond the original
+      ## vm, so we can't remove a vm's key unless we did some checking around to make
+      ## sure that no other disk exists with that key. Meh, not a big deal to keep around
+      ## some old unused keys.
+      ##
+      # keyfile=$(awk '$1 == "'$crypt_name'" {print $3}' /etc/crypttab)
+      # if [[ $keyfile && -f $keyfile && ! ${keyfiles[$keyfile]} ]]; then
+      #   keyfiles[$keyfile]=t
+      #   m rm -f $keyfile
+      # fi
+
+      m sed -i "/^$crypt_name/d" /etc/crypttab
+      if lvrm "$under_crypt_dev"; then
+        # we are in a btrfs setup on community
+        :
+      elif [[ $under_crypt_dev == /dev/md* ]]; then
+        # we are in a mdraid setup on community
+        stat=$(awk '$1 == "'${under_crypt_dev##*/}'"' /proc/mdstat)
+        m mdadm -v --stop "$under_crypt_dev"
+        # see comment at end of script for background on this.
+        m test ! -e "/sys/devices/virtual/block/${under_crypt_dev##*/}"
+        IFS=" " read -r -a mdstat_array <<<"$stat"
+        for word in "${mdstat_array[@]}"; do
+          dev_dm_name="${word%%\[*}"
+          if [[ $dev_dm_name && $dev_dm_name != "$word" ]]; then
+            for f in /dev/mapper/*; do
+              if [[ $(readlink -f "$f") == "/dev/$dev_dm_name" ]]; then
+                under_md_dev="$f"
+                break
+              fi
+            done
+            m mdadm -v --zero-superblock "$under_md_dev"
+            if under_integrity_dev=$(integritysetup status "$under_md_dev" | awk '$1 == "device:" {print $2}') && [[ $under_integrity_dev ]]; then
+              m integritysetup close "$under_md_dev"
+              lvrm "$under_integrity_dev" ||:
+            fi
+          fi
+        done
+      fi
+    fi
+  done
+
+  for rbdname in ${rbdnames[@]}; do
+    snapshot_count=$(rbd info $rbdname | awk '$1 == "snapshot_count:" {print $2}')
+    if [[ $snapshot_count != 0 ]]; then
+      cat <<EOF
+$0: error found snapshot_count = $snapshot_count, != 0 for rbd image $rbdname."
+see https://docs.ceph.com/en/quincy/rbd/rbd-snapshot/
+flatten or remove snapshots first and consider vms using them.
+eg:
+
+rbd snap protect jitsi2p.fsf.org@working2p
+rbd clone jitsi2p.fsf.org@working2p jitsi2p-working
+rbd flatten rbd/jitsi2p-working
+rbd snap unprotect jitsi2p.fsf.org@working2p
+rbd snap rm jitsi2p.fsf.org@working2p
+
+
+output of rbd snap ls $rbdname:
+
+EOF
+      rbd snap ls $rbdname
+      echo
+      echo "$0: exiting 1"
+      exit 1
+    fi
+  done
+
+
+  for rbdname in ${rbdnames[@]}; do
+    parent=$(rbd info $rbdname | awk '$1 == "parent:" {print $2}')
+    m rbd rm $rbdname
+    if [[ $parent ]]; then
+      m rbd snap unprotect $parent
+      m rbd snap rm $parent
+    fi
+  done
+
+  echo "
+############  planned commands, cat $script_gen: ###################
+"
+  cat $script_gen
+  echo
+  echo "
+############## end of cat $script_gen #################
+"
+
+}
-- 
2.25.1