Nginx redirect loop when using proxy_pass over SSL
We are using Nginx as a load balancer for our Rails app. Since we're are moving to a multicloud hosting solution, we want our load balancer to start using SSL for every connection when it's forwarding requests to frontends, since some of them might go over the internet.
The problem I'm facing is that the non https page created a redirect loop. This seems to be caused by the X-Forwarded-Proto header not being set properly. So when rails gets a request on http it thinks it's an https request even though it's not, so it redirects it to http, and it thinks it an https request and so on.
No amount of Googling seems to help me fix that problem. So I would like to know:
- Is this supported by NGINX (I think it is)
- Is there something wrong with my config
- Is there something conceptually wrong with my approach
Thanks!
user www-data;
worker_processes 2;
events {
worker_connections 1024;
use epoll;
}
http {
passenger_root /opt/ruby-enterprise/lib/ruby/gems/1.8/gems/passenger-3.0.2;
passenger_ruby /opt/ruby-enterprise/bin/ruby;
passenger_pool_idle_time 0;
passenger_max_pool_size 20; # Over all apps
passenger_min_instances 5; # Over each app
passenger_use_global_queue on;
rails_env production;
include mime.types;
default_type application/octet-stream;
log_format main '"$remote_addr", "$remote_user", "$time_local", "$request", '
'"$uid_got", "$uid_set", "$status", "$body_bytes_sent", "$http_referer", '
'"$http_user_agent", "$http_x_forwarded_for"';
## Compression
gzip on;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_proxied any;
gzip_min_length 1100;
gzip_buffers 16 8k;
gzip_types text/plain text/css application/x-javascript text/xml application/xml application/xml+rss text/javascript;
# Some version of IE 6 don't handle compression well on some mime-types, so just disable for them
gzip_disable "MSIE [1-6].(?!.*SV1)";
# Set a vary header so downstream proxies don't send cached gzipped content to IE6
gzip_vary on;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
client_max_body_size 20M;
server {
listen 80 default;
return 404;
}
server {
listen 80;
server_name MYDOMAIN.com;
rewrite ^(.*) http://www.MYDOMAIN.com$1 permanent;
}
server {
listen 80;
server_name www.MYDOMAIN.com;
root /u/apps/MYDOMAIN_marketing/current/public;
passenger_enabled on;
userid on;
userid_domain MYDOMAIN.com;
userid_expires max;
access_log /u/apps/MYDOMAIN_marketing/shared/log/nginx/access.log main;
error_log /u/apps/MYDOMAIN_marketing/shared/log/nginx/error.log info;
}
#####################
upstream app_backend {
server www01:8000;
server www02:8000;
server www03:8000;
server www04:8000;
server www05:8000;
server www06:8000;
}
server {
listen 80;
server_name *.MYDOMAIN.com;
root /u/apps/MYDOMAIN/current/public;
userid on;
userid_domain MYDOMAIN.com;
userid_expires max;
access_log /u/apps/MYDOMAIN/shared/log/nginx/lb_access.log main;
error_log /u/apps/MYDOMAIN/shared/log/nginx/lb_error.log info;
if (-f $document_root/system/maintenance.html) {
# I don't know how to get NGINX to both show a page and give a return code.
# So just return 503 with a generic error page.
return 503;
}
location / {
## General Rails error page stuff
error_page 404 /404.html;
error_page 422 /422.html;
error_page 500 502 503 504 /500.html;
error_page 403 /403.html;
# If the file exists then stop here. Saves 4 more stats and some
# rewrites.
if (-f $request_filename) {
break;
}
# Rails page caching
if (-f $request_filename/index.html) {
rewrite (.*) $1/index.html break;
}
if (-f $request_filename.html) {
rewrite (.*) $1.html break;
}
# If it hasn't been handled above, and isn't a static file
# then send to passenger.
proxy_pass https://app_backend;
proxy_connect_timeout 1;
##proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto http;
}
}
server {
ssl on;
listen 8000;
server_name *.MYDOMAIN.com;
root /u/apps/MYDOMAIN/current/public;
passenger_enabled on;
access_log /u/apps/MYDOMAIN/shared/log/nginx/access.log main;
error_log /u/apps/MYDOMAIN/shared/log/nginx/error.log info;
}
########################
# SSL configuration from:
# http://tumblelog.jauderho.com/post/121851623/nginx-and-stronger-ssl
# http://articles.slicehost.com/2007/12/19/ubuntu-gutsy-nginx-ssl-and-vhosts
ssl_certificate MYDOMAIN.com.combined.crt;
ssl_certificate_key MYDOMAIN.com.key;
ssl_prefer_server_ciphers on;
ssl_protocols SSLv3 TLSv1;
ssl_session_cache shared:SSL:2m;
ssl_ciphers ALL:!ADH:RC4+RSA:+HIGH:+MEDIUM:-LOW:-SSLv2:-EXP;
# This is necessary to catch random crap thrown at us during the
# SecurityMetrics scans before it hits Passenger. Without this,
# Passenger becomes confused and stops serving requests.
server {
listen 443 default;
ssl on;
return 404;
}
server {
# If we're willing to log secure and non-secure together,
# we can probably just merge this with the config above.
listen 443;
server_name *.MYDOMAIN.com;
ssl on;
root /u/apps/MYDOMAIN/current/public;
userid on;
userid_domain MYDOMAIN.com;
userid_expires max;
access_log /u/apps/MYDOMAIN/shared/log/nginx/lb_secure_access.log main;
error_log /u/apps/MYDOMAIN/shared/log/nginx/lb_secure_error.log info;
if (-f $document_root/system/maintenance.html) {
# I don't know how to get NGINX to both show a page and give a return code.
# So just return 503 with a generic error page.
return 503;
}
location / {
## General Rails error page stuff
error_page 404 /404.html;
error_page 422 /422.html;
error_page 500 502 503 504 /500.html;
error_page 403 /403.html;
# If the file exists then stop here. Saves 4 more stats and some
# rewrites.
if (-f $request_filename) {
break;
}
# Rails page caching
if (-f $request_filename/index.html) {
rewrite (.*) $1/index.html break;
}
if (-f $request_filename.html) {
rewrite (.*) $1.html break;
}
# If it hasn't been handled above, and isn't a static file
# then send to passenger.
proxy_pass https://app_backend;
proxy_connect_timeout 1;
##proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto https;
}
}
}
Solution 1:
As it turns out, Passenger and Nginx don't work like most other. You have to manually force the HTTPS header in your nginx.conf file.
server {
listen 8000;
ssl on;
server_name *.pagerduty.com *.supportduty.com;
root /u/apps/pagerduty/current/public;
set $my_https "off";
if ($http_x_forwarded_proto = "https") {
set $my_https "on";
}
passenger_enabled on;
passenger_set_cgi_param HTTPS $my_https;
access_log /u/apps/pagerduty/shared/log/nginx/access.log main;
error_log /u/apps/pagerduty/shared/log/nginx/error.log info;
}
Solution 2:
The if statements may not set the proxy headers properly.
See : http://wiki.nginx.org/IfIsEvil -> "Directive if has problems when used in location context, in some cases it doesn't do what you expect but something completely different instead."
The if statements could be replaced by try_files statements which is more efficient.
Your conf file should look like this :
server {
listen 80;
server_name *.MYDOMAIN.com;
root /u/apps/MYDOMAIN/current/public;
userid on;
userid_domain MYDOMAIN.com;
userid_expires max;
access_log /u/apps/MYDOMAIN/shared/log/nginx/lb_access.log main;
error_log /u/apps/MYDOMAIN/shared/log/nginx/lb_error.log info;
location $document_root/system/maintenance.html {
return 503;
}
# Nginx will try thoses locations and will stop after the first success
try_files $uri $uri.html $uri/index.html @passenger
location / {
## General Rails error page stuff
error_page 404 /404.html;
error_page 422 /422.html;
error_page 500 502 503 504 /500.html;
error_page 403 /403.html;
}
location @passenger {
# Here you're sure that if the request goes to the backends, the header will be set. If this doesn't work, then you should consider charging the ruby app.
proxy_pass https://app_backend;
proxy_connect_timeout 1;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto http;
}
}