Kong DNS resolution failed without default gateway
I am attempting to send a request to endpoint /v1/resource, and I am getting the following error:
2021/06/02 23:17:17 [crit] 30#0: *2963 connect() failed (101: Network unreachable), context: ngx.timer, client: 172.13.0.21, server: 0.0.0.0:8443
2021/06/02 23:17:18 [crit] 30#0: *2969 connect() failed (101: Network unreachable), context: ngx.timer, client: 172.13.0.21, server: 0.0.0.0:8443
2021/06/02 23:17:19 [crit] 30#0: *2988 connect() failed (101: Network unreachable), context: ngx.timer, client: 172.13.0.21, server: 0.0.0.0:8443
2021/06/02 23:17:20 [crit] 30#0: *3007 connect() failed (101: Network unreachable), context: ngx.timer, client: 172.13.0.21, server: 0.0.0.0:8443
2021/06/02 23:17:21 [crit] 30#0: *3039 connect() failed (101: Network unreachable), context: ngx.timer, client: 172.13.0.21, server: 0.0.0.0:8443
2021/06/02 23:17:21 [crit] 30#0: *3053 connect() failed (101: Network unreachable), context: ngx.timer, client: 172.13.0.21, server: 0.0.0.0:8443
2021/06/02 22:16:27 [error] 30#0: *29609 [lua] balancer.lua:1064: execute(): DNS resolution failed: dns lookup pool exceeded retries (1): failed to create a resolver: failed to set peer name: network unreachable. Tried: ["(short)my.hostname.com:(na) - cache-miss","my.hostname.com:33 - cache-miss/scheduled/try 1 error: failed to create a resolver: failed to set peer name: network unreachable/scheduled/try 2 error: failed to create a resolver: failed to set peer name: network unreachable/dns lookup pool exceeded retries (1): failed to create a resolver: failed to set peer name: network unreachable","my.hostname.com:1 - cache-miss/scheduled/try 1 error: failed to create a resolver: failed to set peer name: network unreachable/scheduled/try 2 error: failed to create a resolver: failed to set peer name: network unreachable/dns lookup pool exceeded retries (1): failed to create a resolver: failed to set peer name: network unreachable","my.hostname.com:5 - cache-miss/scheduled/try 1 error: failed to create a resolver: failed to set peer name: network unreachable/scheduled/try 2 error: failed to create a resolver: failed to set peer name: network unreachable/dns lookup pool exceeded retries (1): failed to create a resolver: failed to set peer name: network unreachable"], client: 172.13.0.21, server: kong, request: "GET /v1/resource HTTP/1.1", host: "my.hostname.com"
172.18.0.11 - - [02/Jun/2021:23:17:22 +0000] "GET /v1/resource HTTP/1.1" 500 42 "-" "curl/7.29.0"
It complains that i cannot resolve my.hostname.com, however, when I exec into the Kong docker, I am able to successfully get a response, and able to ping my.hostname.com successfully.
bash-5.0# nslookup my.hostname.com
Server: 172.13.0.1
Address: 172.13.0.1:5
Non-authoritative answer:
Name: my.hostname.com
Address: 172.13.0.21
Non-authoritative answer:
And ping:
bash-5.0# ping my.hostname.com
PING my.hostname.com (172.13.0.21): 56 data bytes
64 bytes from 172.13.0.21: seq=0 ttl=64 time=0.065 ms
64 bytes from 172.13.0.21: seq=1 ttl=64 time=0.055 ms
64 bytes from 172.13.0.21: seq=2 ttl=64 time=0.063 ms
The issue can be fixed if a default route is added, however, on our setup this is not possible and this should work because all services are localhost.
bash-5.0$ kong version
2.4.1
nginx.conf:
charset UTF-8;
server_tokens off;
error_log /dev/stderr debug;
lua_package_path './?.lua;./?/init.lua;;;;';
lua_package_cpath ';;;';
lua_socket_pool_size 30;
lua_socket_log_errors off;
lua_max_running_timers 4096;
lua_max_pending_timers 16384;
lua_ssl_verify_depth 1;
lua_shared_dict kong 5m;
lua_shared_dict kong_locks 8m;
lua_shared_dict kong_healthchecks 5m;
lua_shared_dict kong_process_events 5m;
lua_shared_dict kong_cluster_events 5m;
lua_shared_dict kong_rate_limiting_counters 12m;
lua_shared_dict kong_core_db_cache 128m;
lua_shared_dict kong_core_db_cache_miss 12m;
lua_shared_dict kong_db_cache 128m;
lua_shared_dict kong_db_cache_miss 12m;
underscores_in_headers on;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
# injected nginx_http_* directives
client_body_buffer_size 8k;
client_max_body_size 0;
lua_shared_dict prometheus_metrics 5m;
lua_ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3;
ssl_dhparam /usr/local/kong/ssl/ffdhe2048.pem;
ssl_prefer_server_ciphers off;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_session_tickets on;
ssl_session_timeout 1d;
init_by_lua_block {
Kong = require 'kong'
Kong.init()
}
init_worker_by_lua_block {
Kong.init_worker()
}
upstream kong_upstream {
server 0.0.0.1;
# injected nginx_upstream_* directives
balancer_by_lua_block {
Kong.balancer()
}
}
server {
server_name kong;
listen 0.0.0.0:8000 reuseport backlog=16384;
listen 0.0.0.0:8443 ssl http2 reuseport backlog=16384;
error_page 400 404 405 408 411 412 413 414 417 494 /kong_error_handler;
error_page 500 502 503 504 /kong_error_handler;
access_log /dev/stdout;
error_log /dev/stderr debug;
ssl_certificate /usr/local/kong/ssl/kong-default.crt;
ssl_certificate_key /usr/local/kong/ssl/kong-default.key;
ssl_certificate /usr/local/kong/ssl/kong-default-ecdsa.crt;
ssl_certificate_key /usr/local/kong/ssl/kong-default-ecdsa.key;
ssl_session_cache shared:SSL:10m;
ssl_certificate_by_lua_block {
Kong.ssl_certificate()
}
# injected nginx_proxy_* directives
real_ip_header X-Real-IP;
real_ip_recursive off;
rewrite_by_lua_block {
Kong.rewrite()
}
access_by_lua_block {
Kong.access()
}
header_filter_by_lua_block {
Kong.header_filter()
}
body_filter_by_lua_block {
Kong.body_filter()
}
log_by_lua_block {
Kong.log()
}
location / {
default_type '';
set $ctx_ref '';
set $upstream_te '';
set $upstream_host '';
set $upstream_upgrade '';
set $upstream_connection '';
set $upstream_scheme '';
set $upstream_uri '';
set $upstream_x_forwarded_for '';
set $upstream_x_forwarded_proto '';
set $upstream_x_forwarded_host '';
set $upstream_x_forwarded_port '';
set $upstream_x_forwarded_path '';
set $upstream_x_forwarded_prefix '';
set $kong_proxy_mode 'http';
proxy_http_version 1.1;
proxy_buffering on;
proxy_request_buffering on;
proxy_set_header TE $upstream_te;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Forwarded-For $upstream_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $upstream_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $upstream_x_forwarded_host;
proxy_set_header X-Forwarded-Port $upstream_x_forwarded_port;
proxy_set_header X-Forwarded-Path $upstream_x_forwarded_path;
proxy_set_header X-Forwarded-Prefix $upstream_x_forwarded_prefix;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Server;
proxy_pass_header Date;
proxy_ssl_name $upstream_host;
proxy_ssl_server_name on;
proxy_pass $upstream_scheme://kong_upstream$upstream_uri;
}
location @unbuffered {
internal;
default_type '';
set $kong_proxy_mode 'unbuffered';
proxy_http_version 1.1;
proxy_buffering off;
proxy_request_buffering off;
proxy_set_header TE $upstream_te;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Forwarded-For $upstream_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $upstream_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $upstream_x_forwarded_host;
proxy_set_header X-Forwarded-Port $upstream_x_forwarded_port;
proxy_set_header X-Forwarded-Path $upstream_x_forwarded_path;
proxy_set_header X-Forwarded-Prefix $upstream_x_forwarded_prefix;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Server;
proxy_pass_header Date;
proxy_ssl_name $upstream_host;
proxy_ssl_server_name on;
proxy_pass $upstream_scheme://kong_upstream$upstream_uri;
}
location @unbuffered_request {
internal;
default_type '';
set $kong_proxy_mode 'unbuffered';
proxy_http_version 1.1;
proxy_buffering on;
proxy_request_buffering off;
proxy_set_header TE $upstream_te;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Forwarded-For $upstream_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $upstream_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $upstream_x_forwarded_host;
proxy_set_header X-Forwarded-Port $upstream_x_forwarded_port;
proxy_set_header X-Forwarded-Path $upstream_x_forwarded_path;
proxy_set_header X-Forwarded-Prefix $upstream_x_forwarded_prefix;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Server;
proxy_pass_header Date;
proxy_ssl_name $upstream_host;
proxy_ssl_server_name on;
proxy_pass $upstream_scheme://kong_upstream$upstream_uri;
}
location @unbuffered_response {
internal;
default_type '';
set $kong_proxy_mode 'unbuffered';
proxy_http_version 1.1;
proxy_buffering off;
proxy_request_buffering on;
proxy_set_header TE $upstream_te;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Forwarded-For $upstream_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $upstream_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $upstream_x_forwarded_host;
proxy_set_header X-Forwarded-Port $upstream_x_forwarded_port;
proxy_set_header X-Forwarded-Path $upstream_x_forwarded_path;
proxy_set_header X-Forwarded-Prefix $upstream_x_forwarded_prefix;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Server;
proxy_pass_header Date;
proxy_ssl_name $upstream_host;
proxy_ssl_server_name on;
proxy_pass $upstream_scheme://kong_upstream$upstream_uri;
}
location @grpc {
internal;
default_type '';
set $kong_proxy_mode 'grpc';
grpc_set_header TE $upstream_te;
grpc_set_header X-Forwarded-For $upstream_x_forwarded_for;
grpc_set_header X-Forwarded-Proto $upstream_x_forwarded_proto;
grpc_set_header X-Forwarded-Host $upstream_x_forwarded_host;
grpc_set_header X-Forwarded-Port $upstream_x_forwarded_port;
grpc_set_header X-Forwarded-Path $upstream_x_forwarded_path;
grpc_set_header X-Forwarded-Prefix $upstream_x_forwarded_prefix;
grpc_set_header X-Real-IP $remote_addr;
grpc_pass_header Server;
grpc_pass_header Date;
grpc_ssl_name $upstream_host;
grpc_ssl_server_name on;
grpc_pass $upstream_scheme://kong_upstream;
}
location = /kong_buffered_http {
internal;
default_type '';
set $kong_proxy_mode 'http';
rewrite_by_lua_block {;}
access_by_lua_block {;}
header_filter_by_lua_block {;}
body_filter_by_lua_block {;}
log_by_lua_block {;}
proxy_http_version 1.1;
proxy_set_header TE $upstream_te;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Forwarded-For $upstream_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $upstream_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $upstream_x_forwarded_host;
proxy_set_header X-Forwarded-Port $upstream_x_forwarded_port;
proxy_set_header X-Forwarded-Path $upstream_x_forwarded_path;
proxy_set_header X-Forwarded-Prefix $upstream_x_forwarded_prefix;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Server;
proxy_pass_header Date;
proxy_ssl_name $upstream_host;
proxy_ssl_server_name on;
proxy_pass $upstream_scheme://kong_upstream$upstream_uri;
}
location = /kong_error_handler {
internal;
default_type '';
uninitialized_variable_warn off;
rewrite_by_lua_block {;}
access_by_lua_block {;}
content_by_lua_block {
Kong.handle_error()
}
}
}
server {
server_name kong_admin;
listen 0.0.0.0:8001;
access_log /dev/stdout;
error_log /dev/stderr debug;
# injected nginx_admin_* directives
client_body_buffer_size 10m;
client_max_body_size 10m;
location / {
default_type application/json;
content_by_lua_block {
Kong.admin_content()
}
header_filter_by_lua_block {
Kong.admin_header_filter()
}
}
location /nginx_status {
internal;
access_log off;
stub_status;
}
location /robots.txt {
return 200 'User-agent: *\nDisallow: /';
}
}
Does anyone know what may cause this issue, can kong operate without network? or if there is a setting that is needed to get around this issue?
Solution 1:
DNS resolving is done by using UDP request/response actions or TCP connections. Both require that the system can send packets to the IP address of the DNS resolver.
In order to do that, one needs to have a route defined at least to the name server.
In your case, you would need to have a route to 172.13.0.21.