Optimization and Troubleshooting

A complete guide to optimizing MicroVM performance and solving the problems you will commonly encounter.

Boot Time Optimization

Disable Legacy Devices

# Fastest boot time (requires KVM + TSC_DEADLINE)
qemu-system-x86_64 \
  -M microvm,x-option-roms=off,pit=off,pic=off,rtc=off,isa-serial=off \
  -enable-kvm -cpu host -m 256m -smp 2 \
  -kernel vmlinuz \
  -append "console=hvc0 root=/dev/vda rw quiet" \
  -nographic \
  -chardev stdio,id=virtiocon0 \
  -device virtio-serial-device \
  -device virtconsole,chardev=virtiocon0 \
  -drive id=rootfs,file=rootfs.img,format=raw,if=none,cache=none \
  -device virtio-blk-device,drive=rootfs \
  -netdev tap,id=net0,script=no,downscript=no \
  -device virtio-net-device,netdev=net0

Kernel Optimization

# Add to the kernel cmdline
quiet loglevel=0 systemd.show_status=0

# Disable initramfs (if all modules are built-in)
# Build the kernel with built-in modules, not loadable modules

Filesystem Optimization

# Mount options for the rootfs
mount -o noatime,nodiratime,discard /dev/vda1 /

# Disable the journal (for a read-only rootfs)
tune2fs -O ^has_journal /dev/vda1

# or use ext2 (no journal)
mkfs.ext2 rootfs.img

Memory Optimization

QEMU Memory Tuning

# Use hugepages (Linux host)
echo 512 > /proc/sys/vm/nr_hugepages

qemu-system-x86_64 \
  -M microvm -enable-kvm -cpu host \
  -m 256m \
  -mem-path /dev/hugepages \
  ...

# Disable swap in the guest
swapoff -a

Guest Memory Optimization

# Disable unused services
systemctl disable --now bluetooth
systemctl disable --now cups
systemctl disable --now avahi-daemon

# Clear page cache
echo 3 > /proc/sys/vm/drop_caches

# Optimize kernel memory
echo "vm.swappiness=10" >> /etc/sysctl.conf
echo "vm.dirty_ratio=60" >> /etc/sysctl.conf
echo "vm.dirty_background_ratio=2" >> /etc/sysctl.conf
sysctl -p

Disk I/O Optimization

QEMU Drive Options

# Optimal I/O settings
-drive id=rootfs,file=rootfs.img,format=raw,if=none,\
  cache=none,aio=linux,discard=unmap,\
  format=raw,serial=ROOTFS

# Virtio-blk with multi-queue
-device virtio-blk-device,drive=rootfs,num-queues=4

Guest I/O Optimization

# Use the right I/O scheduler
echo "none" > /sys/block/vda/queue/scheduler

# Optimize readahead
blockdev --setra 4096 /dev/vda

# Disable barriers (if the host has battery backup)
mount -o remount,nobarrier /

Network Optimization

QEMU Network Tuning

# Virtio-net with multi-queue
-device virtio-net-device,netdev=net0,mrg_rxbuf=on,indirect_desc=on

# TAP with multi-queue
-netdev tap,id=net0,ifname=tap0,script=no,downscript=no,queues=4

Guest Network Optimization

# TCP tuning
echo "net.core.rmem_max=16777216" >> /etc/sysctl.conf
echo "net.core.wmem_max=16777216" >> /etc/sysctl.conf
echo "net.ipv4.tcp_rmem=4096 87380 16777216" >> /etc/sysctl.conf
echo "net.ipv4.tcp_wmem=4096 65536 16777216" >> /etc/sysctl.conf
echo "net.core.netdev_max_backlog=5000" >> /etc/sysctl.conf
sysctl -p

Monitoring

Resource Monitoring

# CPU usage
top -bn1 | head -5

# Memory
free -h

# Disk I/O
iostat -x 1 5

# Network
iftop -i eth0

# or use dstat
dstat -cdnm 1

QEMU Monitoring

# QEMU monitor (Ctrl+A, C)
info status
info block
info network
info mem

# Stats
info status
info snapshots

Monitoring Script

#!/bin/bash
# Script: monitor-microvm.sh

echo "=== MicroVM Monitor ==="
echo "Time: $(date)"
echo ""
echo "--- CPU ---"
uptime
echo ""
echo "--- Memory ---"
free -h
echo ""
echo "--- Disk ---"
df -h /
echo ""
echo "--- Network ---"
ip -s link show eth0
echo ""
echo "--- Processes ---"
ps aux --sort=-%mem | head -10
echo ""

Troubleshooting

The VM Does Not Boot

# Check the kernel and rootfs
file vmlinuz
file rootfs.img

# Check the kernel cmdline
# Make sure root=/dev/vda is correct

# Check with the serial console
# Add earlyprintk=serial earlyprintk=ttyS0

No Network Connectivity

# Check the interface
ip link show
ip addr show

# Check the routing
ip route show

# Check DNS
cat /etc/resolv.conf
ping 8.8.8.8

# Check the QEMU network backend
# Make sure the netdev configuration is correct

High CPU Usage

# Check the processes using CPU
top -bn1 | head -20

# Check the QEMU process on the host
ps aux | grep qemu

# Disable unnecessary services
systemctl list-unit-files --type=service --state=enabled
systemctl disable <service>

Disk Full

# Check disk usage
df -h

# Find large files
du -sh /* | sort -rh | head -10

# Clean the package cache
apt clean  # Debian
apk cache clean  # Alpine

# Remove old logs
journalctl --vacuum-time=7d
find /var/log -name "*.gz" -delete

OOM (Out of Memory)

# Check the OOM logs
dmesg | grep -i "out of memory"

# Check memory usage
ps aux --sort=-%mem | head -10

# Reduce memory usage
# - Reduce application buffers
# - Disable swap (or add swap)
# - Use a lighter version of the application

Crash / Panic

# Enable kernel panic handler
echo 1 > /proc/sys/kernel/panic

# Or in the kernel cmdline
panic=1

# QEMU will exit on kernel panic with:
-no-reboot

Backup Strategy

Full Image Backup

# Backup full image
dd if=/dev/vda bs=4M | gzip > backup-$(date +%F).img.gz

# Restore
gunzip -c backup-2024-01-01.img.gz | dd of=/dev/vda bs=4M

Incremental Backup

# Using a QCOW2 backing file
qemu-img create -f qcow2 -b base.qcow2 -F qcow2 overlay.qcow2

# Back up only the changes
qemu-img commit overlay.qcow2

Snapshot Management

# Create a snapshot
qemu-img snapshot -c snap1 rootfs.img

# List snapshots
qemu-img snapshot -l rootfs.img

# Restore snapshot
qemu-img snapshot -a snap1 rootfs.img

# Delete snapshot
qemu-img snapshot -d snap1 rootfs.img

Security Hardening

QEMU Security

# Run as non-root
# Requires capabilities for KVM
setcap cap_net_admin+ep /usr/bin/qemu-system-x86_64

# Disable QEMU monitor
# Add -monitor /dev/null

# Disable USB
# Add -device usb-none

Guest Security

# Disable root login via SSH
PermitRootLogin no

# Enable firewall
apt install ufw
ufw default deny incoming
ufw default allow outgoing
ufw allow ssh
ufw enable

# Disable unused modules
blacklist nouveau
blacklist snd
blacklist pcspkr

Performance Benchmark

Boot Time Test

#!/bin/bash
# Script: benchmark-boot.sh

START=$(date +%s%N)

qemu-system-x86_64 \
  -M microvm,pit=off,pic=off,rtc=off \
  -enable-kvm -cpu host -m 256m -smp 2 \
  -kernel vmlinuz \
  -append "console=ttyS0 root=/dev/vda rw quiet reboot=t" \
  -drive id=rootfs,file=rootfs.img,format=raw,if=none,cache=none \
  -device virtio-blk-device,drive=rootfs \
  -nographic -serial stdio \
  -no-reboot 2>&1 | tail -1

END=$(date +%s%N)
DURATION=$(( (END - START) / 1000000 ))

echo "Boot time: ${DURATION}ms"

I/O Benchmark

# Install fio
apt install fio

# Sequential read
fio --name=seq-read --rw=read --bs=1M --size=100M --numjobs=1

# Random read/write
fio --name=rand-rw --rw=randrw --bs=4k --size=100M --numjobs=4

# IOPS test
fio --name=iops --rw=randread --bs=4k --size=100M --numjobs=4

Network Benchmark

# Bandwidth test
iperf3 -s  # on the server
iperf3 -c 10.0.0.1  # on the client

# Latency test
ping -c 100 10.0.0.1

# HTTP benchmark
ab -n 10000 -c 100 http://localhost/

Best Practices

Production Checklist

• [ ] Use KVM for performance
• [ ] Disable legacy devices (PIT, PIC, RTC)
• [ ] Use virtio-mmio for all devices
• [ ] Set the memory exactly (do not over-allocate)
• [ ] Use cache=none for the disk
• [ ] Enable aio=linux for async I/O
• [ ] Setup monitoring (CPU, memory, disk, network)
• [ ] A backup strategy using snapshots
• [ ] Security hardening (firewall, SSH keys)
• [ ] Log rotation for disk space

Development Checklist

• [ ] Use user-mode networking for convenience
• [ ] Port forwarding for access from the host
• [ ] QCOW2 for development snapshots
• [ ] A shared folder with 9p (or virtio-fs)
• [ ] Debug with the serial console