Caddy on a MicroVM

Caddy is a modern web server with automatic HTTPS, zero-config, and a simpler configuration than Nginx. It is very well suited to MicroVMs thanks to its static binary and ease of deployment.

Why Caddy?

Preparing the Image

sudo mount -o loop rootfs.img /mnt/microvm

# Download Caddy binary
wget -q https://caddyserver.com/api/download?os=linux&arch=amd64 -O /tmp/caddy
chmod +x /tmp/caddy
cp /tmp/caddy /mnt/microvm/usr/local/bin/caddy

# Or install via the package manager
# Debian/Ubuntu
sudo chroot /mnt/microvm bash -c '
  apt update && apt install -y debian-keyring debian-archive-keyring apt-transport-https
  curl -1sLf "https://dl.cloudsmith.io/public/caddy/stable/gpg.key" | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
  curl -1sLf "https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt" | tee /etc/apt/sources.list.d/caddy-stable.list
  apt update
  apt install caddy
'

# Alpine
sudo chroot /mnt/microvm bash -c '
  apk add caddy
'

sudo umount /mnt/microvm

Configuring the Caddyfile

cat > /etc/caddy/Caddyfile << 'CADDY'
# Global options
{
    email [email protected]
    # default_sni example.com
}

# Static site
example.com {
    root * /var/www/html
    file_server
    encode gzip zstd

    header {
        X-Frame-Options "SAMEORIGIN"
        X-Content-Type-Options "nosniff"
        Referrer-Policy "strict-origin-when-cross-origin"
    }

    log {
        output file /var/log/caddy/access.log
        format json
    }
}

# Reverse proxy
api.example.com {
    reverse_proxy 127.0.0.1:3000 {
        header_up X-Real-IP {remote_host}
        header_up X-Forwarded-For {remote_host}
        header_up X-Forwarded-Proto {scheme}
    }
}

# API with rate limiting
api.example.com/api/* {
    reverse_proxy 127.0.0.1:3000

    rate_limit {
        zone api {
            key {remote_host}
            events 100
            window 1m
        }
    }
}
CADDY

Running the MicroVM

qemu-system-x86_64 \
  -M microvm -enable-kvm -cpu host -m 256m -smp 2 \
  -kernel vmlinuz \
  -append "console=ttyS0 root=/dev/vda rw" \
  -drive id=rootfs,file=rootfs.img,format=raw,if=none,cache=none \
  -device virtio-blk-device,drive=rootfs \
  -netdev user,id=net0,hostfwd=tcp::80-:80,hostfwd=tcp::443-:443 \
  -device virtio-net-device,dev=net0 \
  -nographic -serial stdio

Setting Up the Guest

# Start Caddy
caddy start --config /etc/caddy/Caddyfile

# or with systemd
systemctl start caddy

# Test
curl -I http://localhost

Auto Start

sudo tee /mnt/microvm/etc/init.d/caddy << 'CADDYINIT'
#!/sbin/openrc-run

CADDY=/usr/local/bin/caddy
CONFIG=/etc/caddy/Caddyfile
PIDFILE=/run/caddy.pid

depend() {
    need net
    keyword timeout 60
}

start() {
    ebegin "Starting Caddy"
    mkdir -p /var/log/caddy /run
    start-stop-daemon --start --background \
        --make-pidfile --pidfile $PIDFILE \
        --exec $CADDY -- start --config $CONFIG
    eend $?
}

stop() {
    ebegin "Stopping Caddy"
    start-stop-daemon --stop --pidfile $PIDFILE
    eend $?
}
CADDYINIT

sudo chmod +x /mnt/microvm/etc/init.d/caddy
sudo chroot /mnt/microvm rc-update add caddy default

Automatic HTTPS

Caddy automatically obtains and renews SSL certificates:

# Caddyfile - automatic HTTPS
example.com {
    root * /var/www/html
    file_server
}

# Wildcard domain (requires a DNS challenge)
*.example.com {
    tls {
        dns cloudflare {env.CLOUDFLARE_API_TOKEN}
    }
    reverse_proxy /* 127.0.0.1:3000
}

Self-Signed Certificate (No Domain)

# For testing without a domain
localhost {
    tls internal
    reverse_proxy 127.0.0.1:3000
}

Reverse Proxy

# Single backend
example.com {
    reverse_proxy 127.0.0.1:3000
}

# Multiple backends (load balancing)
example.com {
    reverse_proxy 127.0.0.1:3000 127.0.0.1:3001 127.0.0.1:3002 {
        lb_policy round_robin
        health_uri /health
        health_interval 10s
    }
}

# With headers
example.com {
    reverse_proxy 127.0.0.1:3000 {
        header_up Host {host}
        header_up X-Real-IP {remote_host}
        header_up X-Forwarded-For {remote_host}
        header_up X-Forwarded-Proto {scheme}

        header_down -Server
    }
}

File Server

# Serve static files
files.example.com {
    root * /var/www/files
    file_server {
        browse
    }

    # Hide sensitive files
    @hidden path */.*
    handle @hidden {
        respond "Forbidden" 403
    }
}

Log Configuration

# JSON format (for ELK/Loki)
log {
    output file /var/log/caddy/access.log
    format json
    level INFO
}

# Custom format
log {
    output file /var/log/caddy/access.log
    format console
    level INFO
}

# Per-site log
example.com {
    log {
        output file /var/log/caddy/example.log
    }
}

Caddy API

Caddy can be configured via the API without a restart:

# Fetch the current configuration
curl http://localhost:2019/config/

# Update config via API
curl -X POST http://localhost:2019/config/ \
  -H "Content-Type: application/json" \
  -d '{
    "apps": {
      "http": {
        "servers": {
          "srv0": {
            "listen": [":80"],
            "routes": [{
              "match": [{"host": ["example.com"]}],
              "handle": [{"handler": "file_server", "root": "/var/www/html"}]
            }]
          }
        }
      }
    }
  }'

Monitoring

# Check the status
caddy version

# Check the configuration validity
caddy validate --config /etc/caddy/Caddyfile

# Reload config
caddy reload --config /etc/caddy/Caddyfile

# Logs
tail -f /var/log/caddy/access.log

Troubleshooting

Ports 80/443 are already in use

# Check what is using the ports
ss -tlnp | grep :80
ss -tlnp | grep :443

# Stop the other service or change Caddy's port
:8080 {
    reverse_proxy 127.0.0.1:3000
}

Certificate fails

# Check DNS resolution
dig example.com

# For self-signed testing
localhost {
    tls internal
}

# Check the Caddy logs
journalctl -u caddy -f

Permission error

# Caddy needs permission to bind ports below 1024
# Solution: run as root or use capabilities
setcap 'cap_net_bind_service=+ep' /usr/local/bin/caddy