Not able to start virtual machine/domain in KVM: Failed to get "write" lock
After host's restart, I'm not able to start virtual machine:
user@server-1:~$ virsh start docker-1
error: Failed to start domain docker-1
error: internal error: process exited while connecting to monitor: 2021-03-23T01:21:58.149079Z qemu-system-x86_64: -blockdev {"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null}: Failed to get "write" lock
Is another process using the image [/apphd/prod/kvm/storage/docker-1-volume-hd.qcow2]?
File is not in use:
user@server-1:~$ sudo fuser -u /apphd/prod/kvm/storage/docker-1-volume-hd.qcow2
user@server-1:~$ sudo lsof | grep qcow
user@server-1:~$ virsh list
Id Name State
--------------------
user@server-1:~$
I have tried on Ubuntu 18.04/qemu 2.11 and upgraded to Ubuntu 20.04/qemu 4.2.1
This upgrade didn't help to solve an issue.
This VM is very big so can't easily create new one from it, there is no available space.
Any help to recover from this situation and start this domain?
Thank you
UPDATE
attaching output of locks:
user@server-1:~$ sudo lslocks -u
COMMAND PID TYPE SIZE MODE M START END PATH
blkmapd 583 POSIX 4B WRITE 0 0 0 /run/blkmapd.pid
rpcbind 1181 FLOCK WRITE 0 0 0 /run/rpcbind.lock
lxcfs 1312 POSIX 5B WRITE 0 0 0 /run/lxcfs.pid
atd 1456 POSIX 5B WRITE 0 0 0 /run/atd.pid
whoopsie 1454 FLOCK WRITE 0 0 0 /run/lock/whoopsie/lock
virtlogd 6143 POSIX 4B WRITE 0 0 0 /run/virtlogd.pid
multipathd 1106 POSIX 4B WRITE 0 0 0 /run/multipathd.pid
containerd 1401 FLOCK 128K WRITE 0 0 0 /var/lib/containerd/io.containerd.metadata.v1.bolt/meta.db
tracker-miner-f 1561 POSIX 3.6M READ 0 1073741826 1073742335 /var/lib/gdm3/.cache/tracker/meta.db
tracker-miner-f 1561 POSIX 32K READ 0 128 128 /var/lib/gdm3/.cache/tracker/meta.db-shm
libvirtd 6057 POSIX 4B WRITE 0 0 0 /run/libvirt/network/driver.pid
libvirtd 6057 POSIX 4B WRITE 0 0 0 /run/libvirt/interface/driver.pid
libvirtd 6057 POSIX 4B WRITE 0 0 0 /run/libvirt/secrets/driver.pid
libvirtd 6057 POSIX 4B WRITE 0 0 0 /run/libvirt/storage/driver.pid
libvirtd 6057 POSIX 4B WRITE 0 0 0 /run/libvirt/nodedev/driver.pid
libvirtd 6057 POSIX 4B WRITE 0 0 0 /run/libvirt/nwfilter/driver.pid
libvirtd 6057 POSIX 4B WRITE 0 0 0 /run/libvirt/qemu/driver.pid
tracker-miner-f 8956 POSIX 3.6M READ 0 1073741826 1073742335 /home/user/.cache/tracker/meta.db
tracker-miner-f 8956 POSIX 32K READ 0 128 128 /home/user/.cache/tracker/meta.db-shm
dmeventd 581 POSIX 4B WRITE 0 0 0 /run/dmeventd.pid
cron 1445 FLOCK 5B WRITE 0 0 0 /run/crond.pid
gnome-shell 1713 FLOCK WRITE 0 0 0 /run/user/126/wayland-0.lock
libvirtd 6057 POSIX 4B WRITE 0 0 0 /run/libvirtd.pid
and attaching process table:
user@server-1:~$ ps -ef
UID PID PPID C STIME TTY TIME CMD
root 1 0 0 01:11 ? 00:00:03 /sbin/init
root 2 0 0 01:11 ? 00:00:00 [kthreadd]
root 3 2 0 01:11 ? 00:00:00 [rcu_gp]
root 4 2 0 01:11 ? 00:00:00 [rcu_par_gp]
root 6 2 0 01:11 ? 00:00:00 [kworker/0:0H-kblockd]
root 9 2 0 01:11 ? 00:00:00 [mm_percpu_wq]
root 10 2 0 01:11 ? 00:00:00 [ksoftirqd/0]
root 11 2 0 01:11 ? 00:00:01 [rcu_sched]
root 12 2 0 01:11 ? 00:00:00 [migration/0]
root 13 2 0 01:11 ? 00:00:00 [idle_inject/0]
root 14 2 0 01:11 ? 00:00:00 [cpuhp/0]
root 15 2 0 01:11 ? 00:00:00 [cpuhp/1]
root 16 2 0 01:11 ? 00:00:00 [idle_inject/1]
root 17 2 0 01:11 ? 00:00:00 [migration/1]
root 18 2 0 01:11 ? 00:00:00 [ksoftirqd/1]
root 20 2 0 01:11 ? 00:00:00 [kworker/1:0H-kblockd]
root 21 2 0 01:11 ? 00:00:00 [cpuhp/2]
root 22 2 0 01:11 ? 00:00:00 [idle_inject/2]
root 23 2 0 01:11 ? 00:00:00 [migration/2]
root 24 2 0 01:11 ? 00:00:00 [ksoftirqd/2]
root 26 2 0 01:11 ? 00:00:00 [kworker/2:0H-kblockd]
root 27 2 0 01:11 ? 00:00:00 [cpuhp/3]
root 28 2 0 01:11 ? 00:00:00 [idle_inject/3]
root 29 2 0 01:11 ? 00:00:00 [migration/3]
root 30 2 0 01:11 ? 00:00:00 [ksoftirqd/3]
root 32 2 0 01:11 ? 00:00:00 [kworker/3:0H-events_highpri]
root 33 2 0 01:11 ? 00:00:00 [kdevtmpfs]
root 34 2 0 01:11 ? 00:00:00 [netns]
root 35 2 0 01:11 ? 00:00:00 [rcu_tasks_kthre]
root 36 2 0 01:11 ? 00:00:00 [kauditd]
root 38 2 0 01:11 ? 00:00:00 [khungtaskd]
root 39 2 0 01:11 ? 00:00:00 [oom_reaper]
root 40 2 0 01:11 ? 00:00:00 [writeback]
root 41 2 0 01:11 ? 00:00:00 [kcompactd0]
root 42 2 0 01:11 ? 00:00:00 [ksmd]
root 43 2 0 01:11 ? 00:00:00 [khugepaged]
root 89 2 0 01:11 ? 00:00:00 [kintegrityd]
root 90 2 0 01:11 ? 00:00:00 [kblockd]
root 91 2 0 01:11 ? 00:00:00 [blkcg_punt_bio]
root 93 2 0 01:11 ? 00:00:00 [tpm_dev_wq]
root 94 2 0 01:11 ? 00:00:00 [ata_sff]
root 95 2 0 01:11 ? 00:00:00 [md]
root 96 2 0 01:11 ? 00:00:00 [edac-poller]
root 97 2 0 01:11 ? 00:00:00 [devfreq_wq]
root 98 2 0 01:11 ? 00:00:00 [watchdogd]
root 101 2 0 01:11 ? 00:00:00 [kswapd0]
root 102 2 0 01:11 ? 00:00:00 [ecryptfs-kthrea]
root 104 2 0 01:11 ? 00:00:00 [kthrotld]
root 105 2 0 01:11 ? 00:00:00 [irq/122-aerdrv]
root 106 2 0 01:11 ? 00:00:00 [acpi_thermal_pm]
root 107 2 0 01:11 ? 00:00:00 [vfio-irqfd-clea]
root 111 2 0 01:11 ? 00:00:00 [ipv6_addrconf]
root 120 2 0 01:11 ? 00:00:00 [kstrp]
root 123 2 0 01:11 ? 00:00:00 [kworker/u9:0-xprtiod]
root 138 2 0 01:11 ? 00:00:00 [charger_manager]
root 197 2 0 01:11 ? 00:00:00 [cryptd]
root 224 2 0 01:11 ? 00:00:00 [scsi_eh_0]
root 225 2 0 01:11 ? 00:00:00 [scsi_tmf_0]
root 226 2 0 01:11 ? 00:00:00 [scsi_eh_1]
root 227 2 0 01:11 ? 00:00:00 [scsi_tmf_1]
root 228 2 0 01:11 ? 00:00:00 [scsi_eh_2]
root 229 2 0 01:11 ? 00:00:00 [scsi_tmf_2]
root 230 2 0 01:11 ? 00:00:00 [scsi_eh_3]
root 231 2 0 01:11 ? 00:00:00 [scsi_tmf_3]
root 232 2 0 01:11 ? 00:00:00 [scsi_eh_4]
root 233 2 0 01:11 ? 00:00:00 [scsi_tmf_4]
root 234 2 0 01:11 ? 00:00:00 [scsi_eh_5]
root 235 2 0 01:11 ? 00:00:00 [scsi_tmf_5]
root 241 2 0 01:11 ? 00:00:00 [kworker/0:1H]
root 245 2 0 01:11 ? 00:00:00 [scsi_eh_6]
root 246 2 0 01:11 ? 00:00:00 [scsi_tmf_6]
root 247 2 0 01:11 ? 00:00:02 [usb-storage]
root 248 2 0 01:11 ? 00:00:00 [scsi_eh_7]
root 249 2 0 01:11 ? 00:00:00 [scsi_tmf_7]
root 250 2 0 01:11 ? 00:00:00 [usb-storage]
root 251 2 0 01:11 ? 00:00:00 [kworker/3:1H-kblockd]
root 252 2 0 01:11 ? 00:00:00 [uas]
root 253 2 0 01:11 ? 00:00:00 [kworker/2:1H-kblockd]
root 254 2 0 01:11 ? 00:00:00 [kworker/1:1H-kblockd]
root 286 2 0 01:11 ? 00:00:00 [raid5wq]
root 287 2 0 01:11 ? 00:00:00 [kdmflush]
root 288 2 0 01:11 ? 00:00:00 [kdmflush]
root 290 2 0 01:11 ? 00:00:00 [kdmflush]
root 292 2 0 01:11 ? 00:00:00 [kdmflush]
root 297 2 0 01:11 ? 00:00:00 [kdmflush]
root 319 2 0 01:11 ? 00:00:00 [mdX_raid1]
root 326 2 0 01:11 ? 00:00:00 [kdmflush]
root 327 2 0 01:11 ? 00:00:00 [kdmflush]
root 328 2 0 01:11 ? 00:00:00 [kdmflush]
root 330 2 0 01:11 ? 00:00:00 [kdmflush]
root 331 2 0 01:11 ? 00:00:00 [kdmflush]
root 363 2 0 01:11 ? 00:00:00 [mdX_raid1]
root 476 2 0 01:11 ? 00:00:00 [jbd2/sda2-8]
root 477 2 0 01:11 ? 00:00:00 [ext4-rsv-conver]
root 552 2 0 01:11 ? 00:00:00 [rpciod]
root 553 2 0 01:11 ? 00:00:00 [xprtiod]
root 554 1 0 01:11 ? 00:00:00 /lib/systemd/systemd-journald
root 581 1 0 01:11 ? 00:00:01 /sbin/dmeventd -f
root 583 1 0 01:11 ? 00:00:00 /usr/sbin/blkmapd
root 597 1 0 01:11 ? 00:00:01 /lib/systemd/systemd-udevd
root 635 2 0 01:11 ? 00:00:00 [irq/133-mei_me]
root 697 2 0 01:11 ? 00:00:00 [led_workqueue]
root 1102 2 0 01:11 ? 00:00:00 [kaluad]
root 1103 2 0 01:11 ? 00:00:00 [kmpath_rdacd]
root 1104 2 0 01:11 ? 00:00:00 [kmpathd]
root 1105 2 0 01:11 ? 00:00:00 [kmpath_handlerd]
root 1106 1 0 01:11 ? 00:00:04 /sbin/multipathd -d -s
root 1115 2 0 01:11 ? 00:00:00 [jbd2/dm-4-8]
root 1117 2 0 01:11 ? 00:00:00 [ext4-rsv-conver]
root 1120 2 0 01:11 ? 00:00:00 [loop0]
root 1126 2 0 01:11 ? 00:00:00 [loop1]
root 1129 2 0 01:11 ? 00:00:00 [loop2]
root 1131 2 0 01:11 ? 00:00:00 [jbd2/dm-9-8]
root 1132 2 0 01:11 ? 00:00:00 [ext4-rsv-conver]
root 1135 2 0 01:11 ? 00:00:00 [loop3]
root 1137 2 0 01:11 ? 00:00:00 [loop4]
root 1138 2 0 01:11 ? 00:00:00 [loop5]
root 1145 2 0 01:11 ? 00:00:00 [jbd2/sde1-8]
root 1146 2 0 01:11 ? 00:00:00 [ext4-rsv-conver]
systemd+ 1176 1 0 01:11 ? 00:00:00 /lib/systemd/systemd-networkd
root 1177 1 0 01:11 ? 00:00:00 /usr/sbin/rpc.idmapd
_rpc 1181 1 0 01:11 ? 00:00:00 /sbin/rpcbind -f -w
systemd+ 1182 1 0 01:11 ? 00:00:00 /lib/systemd/systemd-timesyncd
systemd+ 1187 1 0 01:11 ? 00:00:00 /lib/systemd/systemd-resolved
root 1296 1 0 01:11 ? 00:00:00 /usr/lib/accountsservice/accounts-daemon
root 1297 1 0 01:11 ? 00:00:00 /usr/sbin/acpid
avahi 1301 1 0 01:11 ? 00:00:00 avahi-daemon: running [server-1.local]
root 1302 1 0 01:11 ? 00:00:00 /usr/sbin/cupsd -l
message+ 1303 1 0 01:11 ? 00:00:01 /usr/bin/dbus-daemon --system --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
root 1304 1 0 01:11 ? 00:00:01 /usr/sbin/NetworkManager --no-daemon
root 1310 1 0 01:11 ? 00:00:02 /usr/sbin/irqbalance --foreground
root 1312 1 0 01:11 ? 00:00:00 /usr/bin/lxcfs /var/lib/lxcfs
root 1314 1 0 01:11 ? 00:00:00 /usr/bin/python3 /usr/bin/networkd-dispatcher --run-startup-triggers
root 1322 1 0 01:11 ? 00:00:02 /usr/lib/policykit-1/polkitd --no-debug
syslog 1329 1 0 01:11 ? 00:00:00 /usr/sbin/rsyslogd -n -iNONE
root 1335 1 0 01:11 ? 00:00:00 /usr/sbin/smartd -n
root 1340 1 0 01:11 ? 00:00:00 /usr/libexec/switcheroo-control
root 1341 1 0 01:11 ? 00:00:00 /lib/systemd/systemd-logind
root 1342 1 0 01:11 ? 00:00:00 /lib/systemd/systemd-machined
root 1343 1 0 01:11 ? 00:00:09 /usr/lib/udisks2/udisksd
root 1344 1 0 01:11 ? 00:00:00 /sbin/wpa_supplicant -u -s -O /run/wpa_supplicant
avahi 1353 1301 0 01:11 ? 00:00:00 avahi-daemon: chroot helper
root 1383 1 0 01:11 ? 00:00:00 /usr/sbin/cups-browsed
root 1386 1 0 01:11 ? 00:00:00 /usr/sbin/ModemManager --filter-policy=strict
root 1401 1 0 01:11 ? 00:02:22 /usr/bin/containerd
root 1416 1 0 01:11 ? 00:00:00 /usr/sbin/rpc.mountd --manage-gids
root 1445 1 0 01:11 ? 00:00:00 /usr/sbin/cron -f
whoopsie 1454 1 0 01:11 ? 00:00:00 /usr/bin/whoopsie -f
daemon 1456 1 0 01:11 ? 00:00:00 /usr/sbin/atd -f
root 1457 2 0 01:11 ? 00:00:00 [kworker/u9:1-xprtiod]
root 1458 1 0 01:11 ? 00:00:00 sshd: /usr/sbin/sshd -D [listener] 0 of 10-100 startups
root 1460 2 0 01:11 ? 00:00:00 [lockd]
kernoops 1463 1 0 01:11 ? 00:00:01 /usr/sbin/kerneloops --test
kernoops 1474 1 0 01:11 ? 00:00:01 /usr/sbin/kerneloops
root 1477 1 0 01:11 ? 00:00:00 /usr/bin/python3 /usr/share/unattended-upgrades/unattended-upgrade-shutdown --wait-for-signal
root 1486 1 0 01:11 ? 00:00:00 /usr/sbin/gdm3
root 1496 1486 0 01:11 ? 00:00:00 gdm-session-worker [pam/gdm-launch-environment]
gdm 1527 1 0 01:11 ? 00:00:00 /lib/systemd/systemd --user
gdm 1528 1527 0 01:11 ? 00:00:00 (sd-pam)
root 1552 2 0 01:11 ? 00:00:00 bpfilter_umh
gdm 1559 1527 0 01:11 ? 00:00:00 /usr/bin/pulseaudio --daemonize=no --log-target=journal
gdm 1561 1527 0 01:11 ? 00:00:00 /usr/libexec/tracker-miner-fs
gdm 1568 1496 0 01:11 tty1 00:00:00 /usr/lib/gdm3/gdm-wayland-session dbus-run-session -- gnome-session --autostart /usr/share/gdm/greeter/autostart
gdm 1577 1527 0 01:11 ? 00:00:00 /usr/bin/dbus-daemon --session --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
gdm 1584 1568 0 01:11 tty1 00:00:00 dbus-run-session -- gnome-session --autostart /usr/share/gdm/greeter/autostart
gdm 1585 1584 0 01:11 tty1 00:00:00 dbus-daemon --nofork --print-address 4 --session
rtkit 1586 1 0 01:11 ? 00:00:00 /usr/libexec/rtkit-daemon
gdm 1589 1584 0 01:11 tty1 00:00:00 /usr/libexec/gnome-session-binary --systemd --autostart /usr/share/gdm/greeter/autostart
gdm 1590 1527 0 01:11 ? 00:00:00 /usr/libexec/gvfsd
gdm 1600 1527 0 01:11 ? 00:00:00 /usr/libexec/gvfsd-fuse /run/user/126/gvfs -f -o big_writes
gdm 1608 1527 0 01:11 ? 00:00:01 /usr/libexec/gvfs-udisks2-volume-monitor
gdm 1640 1527 0 01:11 ? 00:00:00 /usr/libexec/gvfs-mtp-volume-monitor
gdm 1648 1527 0 01:11 ? 00:00:00 /usr/libexec/gvfs-goa-volume-monitor
gdm 1653 1527 0 01:11 ? 00:00:00 /usr/libexec/goa-daemon
gdm 1686 1 0 01:11 tty1 00:00:00 /usr/libexec/dconf-service
gdm 1702 1527 0 01:11 ? 00:00:00 /usr/libexec/goa-identity-service
gdm 1711 1527 0 01:11 ? 00:00:01 /usr/libexec/gvfs-afc-volume-monitor
gdm 1713 1589 0 01:11 tty1 00:00:13 /usr/bin/gnome-shell
gdm 1723 1527 0 01:11 ? 00:00:00 /usr/libexec/gvfs-gphoto2-volume-monitor
root 1729 1 0 01:11 ? 00:00:00 /usr/lib/upower/upowerd
root 1800 2 0 01:11 ? 00:00:00 [nfsd]
root 1801 2 0 01:11 ? 00:00:00 [nfsd]
root 1802 2 0 01:11 ? 00:00:00 [nfsd]
root 1803 2 0 01:11 ? 00:00:00 [nfsd]
root 1804 2 0 01:11 ? 00:00:00 [nfsd]
root 1805 2 0 01:11 ? 00:00:00 [nfsd]
root 1806 2 0 01:11 ? 00:00:00 [nfsd]
root 1807 2 0 01:11 ? 00:00:00 [nfsd]
gdm 1868 1 0 01:11 tty1 00:00:00 /usr/libexec/at-spi-bus-launcher
gdm 1874 1868 0 01:11 tty1 00:00:00 /usr/bin/dbus-daemon --config-file=/usr/share/defaults/at-spi2/accessibility.conf --nofork --print-address 3
gdm 1880 1713 0 01:11 tty1 00:00:00 /usr/bin/Xwayland :1024 -rootless -noreset -accessx -core -auth /run/user/126/.mutter-Xwaylandauth.XH3U00 -listen 4 -listen 5 -displayfd 6 -listen 7
libvirt+ 1916 1 0 01:11 ? 00:00:00 /usr/sbin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/default.conf --leasefile-ro --dhcp-script=/usr/lib/libvirt/libvirt_leaseshelper
root 1917 1916 0 01:11 ? 00:00:00 /usr/sbin/dnsmasq --conf-file=/var/lib/libvirt/dnsmasq/default.conf --leasefile-ro --dhcp-script=/usr/lib/libvirt/libvirt_leaseshelper
gdm 2003 1 0 01:11 tty1 00:00:00 /usr/libexec/xdg-permission-store
gdm 2052 1 0 01:11 tty1 00:00:00 /usr/bin/gjs /usr/share/gnome-shell/org.gnome.Shell.Notifications
gdm 2054 1 0 01:11 tty1 00:00:00 /usr/libexec/at-spi2-registryd --use-gnome-session
gdm 2066 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-sharing
gdm 2069 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-wacom
gdm 2070 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-color
gdm 2075 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-keyboard
gdm 2078 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-print-notifications
gdm 2079 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-rfkill
gdm 2084 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-smartcard
gdm 2090 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-datetime
gdm 2103 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-media-keys
gdm 2110 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-screensaver-proxy
gdm 2111 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-sound
gdm 2112 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-a11y-settings
gdm 2114 1589 0 01:11 tty1 00:00:03 /usr/libexec/gsd-housekeeping
gdm 2116 1589 0 01:11 tty1 00:00:00 /usr/libexec/gsd-power
gdm 2179 1713 0 01:11 tty1 00:00:00 ibus-daemon --panel disable -r --xim
gdm 2183 1 0 01:11 tty1 00:00:00 /usr/libexec/gsd-printer
gdm 2185 2179 0 01:11 tty1 00:00:00 /usr/libexec/ibus-dconf
gdm 2192 1 0 01:11 tty1 00:00:00 /usr/libexec/ibus-x11 --kill-daemon
gdm 2199 2179 0 01:11 tty1 00:00:00 /usr/libexec/ibus-engine-simple
gdm 2202 1 0 01:11 tty1 00:00:00 /usr/libexec/ibus-portal
colord 2212 1 0 01:11 ? 00:00:00 /usr/libexec/colord
gdm 2268 1527 0 01:11 ? 00:00:00 /usr/libexec/gvfsd-metadata
root 6057 1 0 01:18 ? 00:00:01 /usr/sbin/libvirtd
root 6143 1 0 01:19 ? 00:00:00 /usr/sbin/virtlogd
root 6562 2 0 01:34 ? 00:00:01 [kworker/2:3-events]
root 7924 2 0 06:06 ? 00:00:00 [loop6]
root 7981 1 0 06:06 ? 00:00:03 /usr/lib/snapd/snapd
root 8320 2 0 08:34 ? 00:00:00 [kworker/0:0-rcu_gp]
root 8891 2 0 09:30 ? 00:00:00 [kworker/1:0-events]
root 8919 1458 0 10:02 ? 00:00:00 sshd: user [priv]
user 8938 1 0 10:02 ? 00:00:00 /lib/systemd/systemd --user
user 8939 8938 0 10:02 ? 00:00:00 (sd-pam)
root 8951 2 0 10:02 ? 00:00:00 [kworker/0:2-events]
user 8954 8938 0 10:02 ? 00:00:00 /usr/bin/pulseaudio --daemonize=no --log-target=journal
user 8956 8938 0 10:02 ? 00:00:00 /usr/libexec/tracker-miner-fs
user 8958 8938 0 10:02 ? 00:00:00 /usr/bin/dbus-daemon --session --address=systemd: --nofork --nopidfile --systemd-activation --syslog-only
user 8975 8938 0 10:02 ? 00:00:00 /usr/libexec/gvfsd
user 8983 8938 0 10:02 ? 00:00:00 /usr/libexec/gvfsd-fuse /run/user/1000/gvfs -f -o big_writes
user 8995 8938 0 10:02 ? 00:00:00 /usr/libexec/gvfs-udisks2-volume-monitor
user 9007 8938 0 10:02 ? 00:00:00 /usr/libexec/gvfs-mtp-volume-monitor
user 9011 8938 0 10:02 ? 00:00:00 /usr/libexec/gvfs-goa-volume-monitor
user 9015 8938 0 10:02 ? 00:00:00 /usr/libexec/goa-daemon
user 9022 8938 0 10:02 ? 00:00:00 /usr/libexec/goa-identity-service
user 9029 8938 0 10:02 ? 00:00:00 /usr/libexec/gvfs-afc-volume-monitor
user 9035 8938 0 10:02 ? 00:00:00 /usr/libexec/gvfs-gphoto2-volume-monitor
user 9185 8919 0 10:02 ? 00:00:00 sshd: user@pts/0
user 9186 9185 0 10:02 pts/0 00:00:00 -bash
root 9258 2 0 10:13 ? 00:00:00 [kworker/3:3-events]
root 9259 2 0 10:13 ? 00:00:00 [kworker/3:4-cgroup_destroy]
root 9294 2 0 10:31 ? 00:00:00 [kworker/1:1]
root 9330 2 0 11:31 ? 00:00:00 [kworker/2:0-events]
root 9334 2 0 11:41 ? 00:00:00 [kworker/u8:2-events_freezable_power_]
root 9348 2 0 11:53 ? 00:00:00 [kworker/u8:0-events_power_efficient]
root 9352 2 0 12:07 ? 00:00:00 [kworker/u8:3-events_unbound]
root 9400 2 0 12:09 ? 00:00:00 [kworker/3:0-events]
root 9403 2 0 12:09 ? 00:00:00 [kworker/0:1-rcu_gp]
root 9413 2 0 12:09 ? 00:00:00 [kworker/3:1-cgroup_destroy]
root 9414 2 0 12:09 ? 00:00:00 [kworker/3:2-events]
root 9415 2 0 12:09 ? 00:00:00 [kworker/3:5-events]
root 9418 2 0 12:09 ? 00:00:00 [kworker/2:1]
root 9419 2 0 12:09 ? 00:00:00 [kworker/3:6]
root 9459 2 0 12:13 ? 00:00:00 [kworker/u8:1-events_unbound]
user 9463 9186 0 12:14 pts/0 00:00:00 ps -ef
user@server-1:~$
attaching XML dump of this VM:
user@server-1:~$ virsh dumpxml docker-1
<domain type='kvm'>
<name>docker-1</name>
<uuid>dfb49ea5-f6e7-45d1-9422-e3ce97cf6320</uuid>
<memory unit='KiB'>10485760</memory>
<currentMemory unit='KiB'>10485760</currentMemory>
<vcpu placement='static'>4</vcpu>
<os>
<type arch='x86_64' machine='pc-i440fx-bionic'>hvm</type>
<boot dev='hd'/>
<boot dev='network'/>
</os>
<features>
<acpi/>
<apic/>
<pae/>
</features>
<cpu mode='custom' match='exact' check='none'>
<model fallback='forbid'>qemu64</model>
</cpu>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/bin/kvm-spice</emulator>
<disk type='volume' device='disk'>
<driver name='qemu' type='qcow2'/>
<source pool='default' volume='docker-1-volume-resized.qcow2'/>
<target dev='vda' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='raw'/>
<source file='/apphd/prod/kvm/storage/docker-1-volume-hd.qcow2'/>
<target dev='vdb' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/apphd/prod/kvm/storage/docker-1-volume-hd.qcow2'/>
<target dev='vdc' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/apphd/prod/kvm/storage/docker-1-volume-hd.qcow2'/>
<target dev='vdx' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/>
</disk>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<source file='/app/prod/kvm/storage/common-init-docker-1.iso'/>
<target dev='hdd' bus='ide'/>
<readonly/>
<address type='drive' controller='0' bus='1' target='0' unit='1'/>
</disk>
<controller type='usb' index='0' model='piix3-uhci'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'/>
<controller type='ide' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</controller>
<interface type='bridge'>
<mac address='00:01:00:00:00:01'/>
<source bridge='br0'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<console type='pty'>
<target type='virtio' port='1'/>
</console>
<channel type='pty'>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='spice' autoport='yes' listen='127.0.0.1'>
<listen type='address' address='127.0.0.1'/>
</graphics>
<video>
<model type='vga' vram='16384' heads='1' primary='yes'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</video>
<memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
</memballoon>
<rng model='virtio'>
<backend model='random'>/dev/urandom</backend>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
</rng>
</devices>
</domain>
user@server-1:~$
In this case, it was momentary lock conflict or we shouldn't understand this error directly in some circumstances.
So issue was not visible in lslocks output.
Solution was to remove 2 disk duplicates via virsh edit docker-1:
<disk type='file' device='disk'>
<driver name='qemu' type='raw'/>
<source file='/apphd/prod/kvm/storage/docker-1-volume-hd.qcow2'/>
<target dev='vdb' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/apphd/prod/kvm/storage/docker-1-volume-hd.qcow2'/>
<target dev='vdc' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/apphd/prod/kvm/storage/docker-1-volume-hd.qcow2'/>
<target dev='vdx' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/>
</disk>
It can't call same file more than once!
Hope that helps someone
I think, I've left this quite long ago when experimented with virsh attach-disk docker-1 --target vdx --source /apphd/prod/kvm/storage/docker-1-volume-hd.qcow2 --persistent --driver qemu --subdriver qcow2 --targetbus virtio to attach the storage