Nginx on a MicroVM

Nginx is a lightweight, high-performance web server. It is ideal for MicroVMs thanks to its minimal memory consumption and its ability to handle thousands of simultaneous connections.

Preparing the Image

sudo mount -o loop rootfs.img /mnt/microvm

# Debian/Ubuntu
sudo chroot /mnt/microvm bash -c '
  apt update && apt install -y nginx
'

# Alpine
sudo chroot /mnt/microvm bash -c '
  apk add nginx
  mkdir -p /run/nginx
'

sudo umount /mnt/microvm

Optimal Configuration

sudo mount -o loop rootfs.img /mnt/microvm

sudo tee /mnt/microvm/etc/nginx/nginx.conf << 'NGINXCONF'
user www-data;
worker_processes auto;
pid /run/nginx.pid;
error_log /var/log/nginx/error.log warn;

events {
    worker_connections 512;
    multi_accept on;
    use epoll;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    # Logging
    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                    '$status $body_bytes_sent "$http_referer" '
                    '"$http_user_agent"';
    access_log /var/log/nginx/access.log main;

    # Performance
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    server_tokens off;

    # Gzip
    gzip on;
    gzip_vary on;
    gzip_proxied any;
    gzip_comp_level 6;
    gzip_types text/plain text/css application/json application/javascript
               text/xml application/xml application/xml+rss text/javascript;

    # Buffer
    client_body_buffer_size 16K;
    client_header_buffer_size 1k;
    large_client_header_buffers 4 8k;

    # Temp paths
    client_temp_path /var/cache/nginx/client_temp;
    proxy_temp_path /var/cache/nginx/proxy_temp;
    fastcgi_temp_path /var/cache/nginx/fastcgi_temp;
    uwsgi_temp_path /var/cache/nginx/uwsgi_temp;
    scgi_temp_path /var/cache/nginx/scgi_temp;

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}
NGINXCONF

# Make sure the cache directory exists
mkdir -p /mnt/microvm/var/cache/nginx

sudo umount /mnt/microvm

Virtual Host

sudo tee /mnt/microvm/etc/nginx/sites-available/default << 'NGINXVHOST'
server {
    listen 80 default_server;
    listen [::]:80 default_server;
    server_name _;

    root /var/www/html;
    index index.html index.htm;

    location / {
        try_files $uri $uri/ =404;
    }

    # Static files caching
    location ~* \.(jpg|jpeg|png|gif|ico|css|js|woff2)$ {
        expires 30d;
        add_header Cache-Control "public, immutable";
    }

    # Security headers
    add_header X-Frame-Options "SAMEORIGIN" always;
    add_header X-Content-Type-Options "nosniff" always;
    add_header X-XSS-Protection "1; mode=block" always;

    # Error pages
    error_page 404 /404.html;
    error_page 500 502 503 504 /50x.html;
}
NGINXVHOST

Running the MicroVM

qemu-system-x86_64 \
  -M microvm -enable-kvm -cpu host -m 256m -smp 2 \
  -kernel vmlinuz \
  -append "console=ttyS0 root=/dev/vda rw" \
  -drive id=rootfs,file=rootfs.img,format=raw,if=none,cache=none \
  -device virtio-blk-device,drive=rootfs \
  -netdev user,id=net0,hostfwd=tcp::8080-:80 \
  -device virtio-net-device,netdev=net0 \
  -nographic -serial stdio

Setting Up the Guest

# Create an index page
cat > /var/www/html/index.html << 'HTML'
<!DOCTYPE html>
<html>
<head><title>Nginx on MicroVM</title></head>
<body>
  <h1>Nginx on QEMU MicroVM</h1>
  <p>Running successfully!</p>
</body>
</html>
HTML

# Test config
nginx -t

# Start nginx
nginx

# or via the service
service nginx start

Auto Start

sudo tee /mnt/microvm/etc/init.d/nginx << 'NGINXINIT'
#!/sbin/openrc-run

NGINX=/usr/sbin/nginx
PIDFILE=/run/nginx.pid

depend() {
    need net
    keyword timeout 60
}

start() {
    ebegin "Starting Nginx"
    mkdir -p /run/nginx /var/cache/nginx
    $NGINX -g "pid $PIDFILE;"
    eend $?
}

stop() {
    ebegin "Stopping Nginx"
    start-stop-daemon --stop --pidfile $PIDFILE
    eend $?
}
NGINXINIT

sudo chmod +x /mnt/microvm/etc/init.d/nginx
sudo chroot /mnt/microvm rc-update add nginx default

SSL/TLS

# Generate self-signed certificate
openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
  -keyout /etc/ssl/private/nginx-selfsigned.key \
  -out /etc/ssl/certs/nginx-selfsigned.crt

# SSL server block
cat > /etc/nginx/sites-available/ssl << 'SSLCONF'
server {
    listen 443 ssl http2;
    server_name example.com;

    ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt;
    ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key;

    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers HIGH:!aNULL:!MD5;
    ssl_prefer_server_ciphers on;

    root /var/www/html;
    index index.html;
}
SSLCONF

ln -s /etc/nginx/sites-available/ssl /etc/nginx/sites-enabled/
nginx -t && service nginx restart

Performance Tuning

# Check the worker processes
nginx -T | grep worker_processes

# Reload the config without downtime
nginx -s reload

# Test performance
ab -n 1000 -c 10 http://localhost/

# Check active connections
ss -s

Monitoring

# Check the nginx status
service nginx status

# Check the logs
tail -f /var/log/nginx/access.log
tail -f /var/log/nginx/error.log

# Check the stats (requires the stub_status module)
cat > /etc/nginx/conf.d/status.conf << 'STATUS'
server {
    listen 8080;
    location /nginx_status {
        stub_status;
        allow 127.0.0.1;
        deny all;
    }
}
STATUS

Troubleshooting

Address already in use

# Check port 80
ss -tlnp | grep :80

# Stop the other service or change Nginx's port
listen 8080;

Permission denied

# Fix the permissions
chown -R www-data:www-data /var/www/html
chmod -R 755 /var/www/html
chown -R www-data:www-data /var/cache/nginx

Worker connections not enough

# Increase worker_connections
events {
    worker_connections 1024;
}

# Check ulimit
ulimit -n