NGINX Ingress with Nomad
NGINX Ingress with Nomad
Running several web applications on a single server usually requires a web proxy to handle to have a single ingress to your server that handles TLS encryption. I already prepared a NGINX Ingress template that can be used to spin up an Ingress inside a Docker container:
Let's migrate this to Nomad.
Preparation
Certbot
First I need to create an TLS certificate with certbot:
apt install certbot
Since NGINX will be running inside a docker container we cannot use the --nginx flag to auto-configure NGINX to use the certificate. Instead we have to use:
certbot certonly --standalone
Before running this command make sure that you have a domain pointing towards your server, e.g. my.server.com and you will have to have opened port 80, e.g. ufw allow 80/tcp to have certbot verify your domain!
The command is going to create your certificates in /etc/letsencrypt/live/my.server.com which we will have to provide as a volume to the Nomad process - check out the Hashicorp Nomad Client Configuration for how to configure the client on your server that should run the NGNIX ingress.
Nomad Job File
There are two ways on how to provide the NGINX configuration to your Docker container. The first one is to source your configuration in a Git repository and add it to your Nomad job using the Artifact stanza inside the Task directive:
artifact {
source = "git::git@my.gitlab.com:nginx_ingress.git"
destination = "local/nginx"
options {
sshkey = "${base64encode(file(pathexpand("/etc/nomad.d/.ssh/id_rsa")))}"
depth = 1
}
}
Create an SSH key /etc/nomad.d/.ssh/id_rsa and add it to your Git repository to allow Nomad to clone it. Now you can mount the local/nginx directory as volumes:
config {
network_mode = "host"
image = "nginx:alpine"
ports = ["http","https"]
volumes = [
"local/nginx/configuration/conf.d:/etc/nginx/conf.d",
"local/nginx/configuration/ssl:/etc/nginx/ssl",
"local/nginx/configuration/nginx.conf:/etc/nginx/nginx.conf"
]
}
But I don't want to create another repo and just use the Template stanza to generate the entire configuration from my job file:
nginx_ingress.nomad
job "nginx_ingress" {
datacenters = ["nginxNTS"]
group "nginx" {
count = 1
network {
mode = "host"
port "http" {
static = "80"
}
port "https" {
static = "443"
}
}
service {
name = "nginx-ingress"
port = "http"
check {
name = "HTTP Health"
path = "/"
type = "http"
protocol = "http"
interval = "10s"
timeout = "2s"
}
}
volume "letsencrypt" {
type = "host"
read_only = true
source = "letsencrypt"
}
task "ingress_container" {
driver = "docker"
volume_mount {
volume = "letsencrypt"
destination = "/opt/letsencrypt" #in the container
read_only = false
}
config {
network_mode = "host"
image = "nginx:alpine"
ports = ["http","https"]
volumes = [
"local/nginx/nginx.conf:/etc/nginx/nginx.conf",
"local/nginx/dhparam.pem:/etc/nginx/ssl/dhparam.pem",
"local/nginx/ssl-params.conf:/etc/nginx/ssl/ssl-params.conf",
"local/nginx/default.conf:/etc/nginx/conf.d/default.conf",
"local/nginx/buffers.conf:/etc/nginx/conf.d/buffers.conf",
"local/nginx/timeouts.conf:/etc/nginx/conf.d/timeouts.conf",
"local/nginx/header.conf:/etc/nginx/conf.d/header.conf",
"local/nginx/cache.conf:/etc/nginx/conf.d/cache.conf",
"local/nginx/gzip.conf:/etc/nginx/conf.d/gzip.conf",
"local/nginx/index.html:/usr/share/nginx/html/index.html"
]
}
template {
data = <<EOH
user nginx;
worker_processes auto;
worker_rlimit_nofile 15000;
pid /var/run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 2048;
multi_accept on;
use epoll;
}
http {
default_type application/octet-stream;
# access_log /var/log/nginx/access.log;
# activate the server access log only when needed
access_log off;
error_log /var/log/nginx/error.log;
# don't display server version on error pages
server_tokens off;
server_names_hash_bucket_size 64;
include /etc/nginx/mime.types;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
charset utf-8;
source_charset utf-8;
charset_types text/xml text/plain text/vnd.wap.wml application/javascript application/rss+xml;
include /etc/nginx/conf.d/default.conf;
include /etc/nginx/conf.d/buffers.conf;
include /etc/nginx/conf.d/timeouts.conf;
include /etc/nginx/conf.d/cache.conf;
include /etc/nginx/conf.d/gzip.conf;
}
EOH
destination = "local/nginx/nginx.conf"
}
template {
data = <<EOH
server {
listen 80;
listen [::]:80;
server_name my.server.com;
return 301 https://$server_name$request_uri;
}
server {
listen 443 ssl http2 default_server;
listen [::]:443 ssl;
ssl_certificate /opt/letsencrypt/live/my.server.com/fullchain.pem;
ssl_certificate_key /opt/letsencrypt/live/my.server.com/privkey.pem;
include ssl/ssl-params.conf;
include /etc/nginx/conf.d/header.conf;
server_name my.server.com;
#access_log /var/log/nginx/host.access.log main;
location / {
root /usr/share/nginx/html;
index index.html index.htm;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /usr/share/nginx/html;
}
}
EOH
destination = "local/nginx/default.conf"
}
template {
data = <<EOH
<!DOCTYPE html>
<html>
<head>
<title>Hello World!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Hello World!</h1>
</body>
</html>
EOH
destination = "local/nginx/index.html"
}
template {
data = <<EOH
-----BEGIN DH PARAMETERS-----
MIICCAKCAgEAt59...
...dfszADFDGV346dfzxg9AC=
-----END DH PARAMETERS-----
EOH
destination = "local/nginx/dhparam.pem"
}
template {
data = <<EOH
ssl_protocols TLSv1.3;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/nginx/ssl/dhparam.pem;
ssl_ciphers ECDH+AESGCM:ECDH+CHACHA20:ECDH+AES256:ECDH+AES128:!aNULL:!SHA1:!AESCCM;
ssl_conf_command Options PrioritizeChaCha;
ssl_conf_command Ciphersuites TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256;
ssl_ecdh_curve secp384r1; # Requires nginx >= 1.1.0
ssl_session_timeout 10m;
ssl_session_cache shared:SSL:10m;
ssl_session_tickets off; # Requires nginx >= 1.5.9
ssl_stapling on; # Requires nginx >= 1.3.7
ssl_stapling_verify on; # Requires nginx => 1.3.7
resolver 8.8.8.8 8.8.4.4 valid=300s;
resolver_timeout 5s;
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
add_header X-Frame-Options "";
add_header X-Content-Type-Options nosniff;
add_header X-XSS-Protection "1; mode=block";
EOH
destination = "local/nginx/ssl-params.conf"
}
template {
data = <<EOH
client_body_buffer_size 10k;
client_header_buffer_size 1k;
client_max_body_size 8m;
large_client_header_buffers 2 1k;
# Directive needs to be increased for certain site types to prevent ERROR 400
# large_client_header_buffers 4 32k;
EOH
destination = "local/nginx/buffers.conf"
}
template {
data = <<EOH
add_header Cache-Control "public, must-revalidate, proxy-revalidate, max-age=0";
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-NginX-Proxy true;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-Proto http;
proxy_hide_header X-Frame-Options;
proxy_set_header Accept-Encoding "";
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
proxy_max_temp_file_size 0;
proxy_redirect off;
proxy_read_timeout 240s;
EOH
destination = "local/nginx/header.conf"
}
template {
data = <<EOH
open_file_cache max=1500 inactive=20s;
open_file_cache_valid 30s;
open_file_cache_min_uses 5;
open_file_cache_errors off;
EOH
destination = "local/nginx/cache.conf"
}
template {
data = <<EOH
client_header_timeout 3m;
client_body_timeout 3m;
keepalive_timeout 100;
keepalive_requests 1000;
send_timeout 3m;
EOH
destination = "local/nginx/timeouts.conf"
}
template {
data = <<EOH
gzip on;
gzip_disable "msie6";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 5;
gzip_min_length 256;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types text/plain text/css application/json application/javascript
text/xml application/xml application/xml+rss text/javascript
image/svg+xml application/xhtml+xml application/atom+xml;
EOH
destination = "local/nginx/gzip.conf"
}
}
}
}