Memory leak on Apache

apache-2.4

I have a light memory leak apparently due to semaphores which are not released by Apache (configured as a reverse proxy). When the platform gets more loaded with additional traffic this leak gets more important over time. Memory can be released when we issue the command below as well as restarting Apache with a hard restart. Graceful restart doesn’t release memory.

/usr/bin/ipcrm sem $(/usr/bin/ipcs -s | grep www-data | awk '{print$2}')

If we don’t force the release for the semaphores we get into an issue similar to the one described in the post below. What are semaphores and how are they caused?

Apache then refuses to restart and we get no error message in the logs. We already made several tests with no success in fixing the problem. We changed the sysctl.conf file with the kernel.sem values

Current values are :

$ cat /proc/sys/kernel/sem
500     64000   64      256

I deactivated the modules below one by one (and changed the Apache configuration files accordingly):

LoadModule security2_module /usr/lib/apache2/modules/mod_security2.so
LoadModule geoip_module /usr/lib/apache2/modules/mod_geoip.so
LoadModule pagespeed_module /usr/lib/apache2/modules/mod_pagespeed_ap24.so
LoadModule evasive20_module   /usr/lib/apache2/modules/mod_evasive20.so
LoadModule proxy_protocol_module   /usr/lib/apache2/modules/mod_proxy_protocol.so

Then I tried to move from mpm_event to mpm_worker, without much success as well.

The current versions of Apache and the modules are:

$ apache2 -V
Server version: Apache/2.4.7 (Ubuntu)
Server built:   Mar 10 2015 13:05:59
Server's Module Magic Number: 20120211:27
Server loaded:  APR 1.5.1-dev, APR-UTIL 1.5.3
Compiled using: APR 1.5.1-dev, APR-UTIL 1.5.3
Architecture:   64-bit
Server MPM:     worker
threaded:     yes (fixed thread count)
forked:     yes (variable process count)

Server compiled with:

-D APR_HAS_SENDFILE
-D APR_HAS_MMAP
-D APR_HAVE_IPV6 (IPv4-mapped addresses enabled)
-D APR_USE_SYSVSEM_SERIALIZE
-D APR_USE_PTHREAD_SERIALIZE
-D SINGLE_LISTEN_UNSERIALIZED_ACCEPT
-D APR_HAS_OTHER_CHILD
-D AP_HAVE_RELIABLE_PIPED_LOGS
-D DYNAMIC_MODULE_LIMIT=256
-D HTTPD_ROOT="/etc/apache2"
-D SUEXEC_BIN="/usr/lib/apache2/suexec"
-D DEFAULT_PIDLOG="/var/run/apache2.pid"
-D DEFAULT_SCOREBOARD="logs/apache_runtime_status"
-D DEFAULT_ERRORLOG="logs/error_log"
-D AP_TYPES_CONFIG_FILE="mime.types"
-D SERVER_CONFIG_FILE="apache2.conf"

Ubuntu packages:

apache2 : 2.4.7-1ubuntu4.4
apache2-bin : 2.4.7-1ubuntu4.4
apache2-data : 2.4.7-1ubuntu4.4
apache2-dev :  2.4.7-1ubuntu4.4
libapache2-mod-evasive: 1.10.1-2
libapache2-mod-security2: 2.7.7-2
libapache2-modsecurity: 2.7.7-2
libgeoip-dev : 1.6.0-1

Other compiled packages

geoip-api-mod_geoip2 : 1.2.9
mod_pagespeed : latest stable version
proxy-protocol (https://github.com/roadrunner2/mod-proxy-protocol): latest version

Here is my apache2.conf

# GENERIC DECLARATIONS
User www-data
Group www-data
Listen 80
ServerAdmin admin@xxx
ServerName server:80
ServerRoot "/etc/apache2"

# MODULES LOADING
# Generic modulesE
#LoadModule mpm_event_module /usr/lib/apache2/modules/mod_mpm_event.so
LoadModule mpm_worker_module /usr/lib/apache2/modules/mod_mpm_worker.so
LoadModule ratelimit_module /usr/lib/apache2/modules/mod_ratelimit.so
LoadModule reqtimeout_module /usr/lib/apache2/modules/mod_reqtimeout.so
LoadModule mime_module /usr/lib/apache2/modules/mod_mime.so
LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so
LoadModule setenvif_module /usr/lib/apache2/modules/mod_setenvif.so
LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so
LoadModule proxy_connect_module /usr/lib/apache2/modules/mod_proxy_connect.so
LoadModule proxy_http_module /usr/lib/apache2/modules/mod_proxy_http.so
LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so
LoadModule status_module /usr/lib/apache2/modules/mod_status.so
LoadModule autoindex_module /usr/lib/apache2/modules/mod_autoindex.so
LoadModule dir_module /usr/lib/apache2/modules/mod_dir.so
LoadModule alias_module /usr/lib/apache2/modules/mod_alias.so
LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so
LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
LoadModule authz_host_module /usr/lib/apache2/modules/mod_authz_host.so
LoadModule unique_id_module /usr/lib/apache2/modules/mod_unique_id.so
LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so
LoadModule substitute_module /usr/lib/apache2/modules/mod_substitute.so
LoadModule filter_module /usr/lib/apache2/modules/mod_filter.so
LoadModule deflate_module /usr/lib/apache2/modules/mod_deflate.so

# Specific modules
LoadModule security2_module /usr/lib/apache2/modules/mod_security2.so
LoadModule geoip_module /usr/lib/apache2/modules/mod_geoip.so
LoadModule pagespeed_module /usr/lib/apache2/modules/mod_pagespeed_ap24.so
#LoadModule evasive20_module   /usr/lib/apache2/modules/mod_evasive20.so
LoadModule proxy_protocol_module       /usr/lib/apache2/modules/mod_proxy_protocol.so

#### PROXY SETTINGS ####
# LOG FORMATS
LogFormat "%t \"%r\" %>s %I %O [%{GEOIP_COUNTRY_CODE}e] %{UNIQUE_ID}e" o
CustomLog "/var/log/access_log" o

# TIMEOUT SETTINGS
Timeout 60
ProxyTimeout 60

# ERROR LOGS
ErrorLog "/var/log/error_log"
LogLevel error

#### WEB PAGES CONFIGURATION ####
DocumentRoot "/var/www"
<Directory "/var/www">
    Options Indexes FollowSymLinks
    AllowOverride None
    Require all granted
</Directory>

DirectoryIndex index.html

<Files ".ht*">
    Require all denied
</Files>

TypesConfig /etc/apache2/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz



#### MODULES CONFIGURATION ####
# Server-pool management (MPM specific)
Include /etc/apache2/httpd-mpm.conf
# Secure (SSL/TLS) connections
Include /etc/apache2/httpd-ssl.conf

#### GEOIP CONF ####
GeoIPEnable On
GeoIPScanProxyHeaders On
GeoIPDBFile /usr/share/GeoIP/GeoIP.dat IndexCache

#### MODSECURITY CONF ####
SecPcreMatchLimit 15000
SecPcreMatchLimitRecursion 15000
SecTmpDir /tmp/
SecDataDir /tmp/

#### MODPAGESPEED CONF ####
ModPagespeed on
ModPagespeedInheritVHostConfig on
AddOutputFilterByType MOD_PAGESPEED_OUTPUT_FILTER text/html
AddOutputFilterByType MOD_PAGESPEED_OUTPUT_FILTER text/html
ModPagespeedXHeaderValue "Powered"
ModPagespeedRewriteLevel OptimizeForBandwidth
ModPagespeedFileCachePath "/var/cache/mod_pagespeed/"
ModPagespeedFileCacheInodeLimit 500000
ModPagespeedStatisticsLogging off
ModPagespeedMessageBufferSize 100000
Include /etc/apache2/pagespeed_libraries.conf

#### DDOS PROTECTION ####
#DOSHashTableSize 3097
#DOSPageCount 2
#DOSPageInterval 1
#DOSSiteCount 150
#DOSSiteInterval 1
#DOSBlockingPeriod 10
#DOSLogDir "/var/lock/mod_evasive"

#### VHOSTS #####
Include /etc/apache2/sites/*.conf

httpd-mpm.conf : PidFile "/var/log/apache2/httpd.pid"

<IfModule mod_mpm_event.c>
StartServers                  3
MinSpareThreads              75
MaxSpareThreads             250
ThreadsPerChild              25
MaxRequestWorkers           400
MaxConnectionsPerChild    10000
</IfModule>

<IfModule mod_mpm_worker.c>
# Default values
ServerLimit                  16
MaxMemFree                 2048
StartServers                  2
MinSpareThreads              75
MaxSpareThreads             250

# Custom values
MaxConnectionsPerChild    10000
ThreadsPerChild            1000
MaxRequestWorkers         16000
</IfModule>

Vhosts configuration example :

<VirtualHost *:80>
    #### SSL FALLBACK ####
    ServerName server
    ProxyProtocol on
    RewriteEngine On
    RewriteRule ^(.*)$ https://%{HTTP_HOST}$1 [R=301,L]
    ######################
</VirtualHost>

<VirtualHost *:443>

#### GENERIC CONFIGURATION ####
ServerName server
ProxyProtocol on
ProxyPreserveHost On
RewriteEngine On
RewriteRule "^/errors/(.*)" "https://server/v1/errors/$1" [R]
ProxyPass / https://server/ keepalive=On
ProxyPassReverse / https://server/
###############################

#### COMPRESSION ####
ModPagespeedFetchHttps enable
DeflateFilterNote Input instream
DeflateFilterNote Output outstream
DeflateFilterNote Ratio ratio

#####################

#### LOG FORMATS ####
LogFormat "[3] [%{Host}i] %t \"%r\" %>s [response_time %D] [request_size %I] [response_size_before_compression %O] [response_size_after_compression %{outstream}n] [country \"%{GEOIP_COUNTRY_CODE}e\"] [uuid \"%{uuid}C\"] [user-agent \"%{User-agent}i\"] [unique_id \"%{UNIQUE_ID}e\"]" O
LogFormat "[3] [%{Host}i] %t [pid %{pid}P:tid %{tid}P] [tag \"GEOBLOCKING\"] [tag \"%{GEOIP_COUNTRY_CODE}e\"] [client %a] [uri \"%U\"] [unique_id \"%{UNIQUE_ID}e\"]" GeoIPblocked
ErrorLog "/var/log/error_log"
CustomLog "/var/log/access_log" O
##########################

#### SSL CONFIGURATION ####
SSLEngine on
SSLCertificateFile "/etc/apache2/sites/3/3.crt"
SSLCertificateKeyFile "/etc/apache2/sites/3/3.key"
###########################

#### DDOS PROTECTION ####
#DOSHashTableSize    3097
#DOSPageCount        10
#DOSSiteCount        150
#DOSPageInterval     1
#DOSSiteInterval     1
#DOSBlockingPeriod   10
#############################


#### MODSECURITY ####
SecRuleEngine on
SecDefaultAction "setenv:unique_id=%{UNIQUE_ID},phase:2,redirect:/errors/block?unique_id=%{unique_id}&site_id=3"
Include /etc/apache2/modsecurity.conf
Include /etc/apache2/sites/3/3-exceptions.conf
Include /etc/apache2/sites/3/3-cms-exceptions.conf
Include /etc/apache2/sites/3/3-vp.conf
##########################

#### SCRIPT INJECTION ####
SetInputFilter DEFLATE
AddOutputFilterByType SUBSTITUTE text/html
Substitute "s\</body>\<script src=\"script_source"></script><script type=\"text/javascript\"> 1kB script </script></body>\in"
#########################################

Modsecurity.conf SecRequestBodyAccess On

SecRule REQUEST_HEADERS:Content-Type "text/xml" "id:'200000',phase:1,nolog,allow"
SecRule REQUEST_HEADERS:Content-Type "application/json" "id:'200001',phase:1,nolog,allow"

SecRequestBodyLimit 13107200
SecRequestBodyNoFilesLimit 13107200
SecRequestBodyInMemoryLimit 13107200
SecRequestBodyLimitAction Reject
SecRule REQBODY_ERROR "!@eq 0" \
"id:'200002', phase:2,t:none,log,deny,status:400,msg:'Failed to parse request body.',logdata:'%{reqbody_error_msg}',severity:2"

SecRule MULTIPART_STRICT_ERROR "!@eq 0" \
"id:'200003',phase:2,t:none,log,deny,status:400, \
msg:'Multipart request body failed strict validation: \
PE %{REQBODY_PROCESSOR_ERROR}, \
BQ %{MULTIPART_BOUNDARY_QUOTED}, \
BW %{MULTIPART_BOUNDARY_WHITESPACE}, \
DB %{MULTIPART_DATA_BEFORE}, \
DA %{MULTIPART_DATA_AFTER}, \
HF %{MULTIPART_HEADER_FOLDING}, \
LF %{MULTIPART_LF_LINE}, \
SM %{MULTIPART_MISSING_SEMICOLON}, \
IQ %{MULTIPART_INVALID_QUOTING}, \
IP %{MULTIPART_INVALID_PART}, \
IH %{MULTIPART_INVALID_HEADER_FOLDING}, \
FL %{MULTIPART_FILE_LIMIT_EXCEEDED}'"

SecRule TX:/^MSC_/ "!@streq 0" \
    "id:'200005',phase:2,t:none,deny,msg:'ModSecurity internal error flagged:     %{MATCHED_VAR_NAME}'"

SecResponseBodyAccess Off

SecArgumentSeparator &
SecCookieFormat 0
SecUnicodeMapFile unicode.mapping 20127

Include /etc/apache2/rules/*.conf

Apache rules are the standard crs rules with about 50 exceptions set via the SecRuleRemoveById directive.

Solution 1:

We use Apache a lot in a reverse proxy configuration and used to see memory leaks which were noticeable as the server only had 512MB RAM.

I don't know if we had a semaphore-related leak or something else, but we cured it by reducing the MaxConnectionsPerChild directive to one quarter of the default i.e. 2500. At our traffic levels this meant child processes were recycled about twice an hour, which cured the problem. There was no noticeable performance impact.

Memory leak on Apache

Solution 1:

Related

Recent Posts