Varnish CPU Usage
MAGNIEN, Thierry
thierry.magnien at sfr.com
Tue Aug 2 16:43:08 CEST 2016
Hi,
There are known CPU issues due to WordPress HearBeat API, for example, or misbehaving plugins.
See http://www.inmotionhosting.com/support/website/wordpress/heartbeat-ajax-php-usage for example.
Maybe one of your plugins has some difficulties going through varnish.
If you enable debug on your browser, can you see specific calls to /wp-admin/admin-ajax.php that take a very long time with varnish and not when varnish is disabled ?
Thierry
De : varnish-misc-bounces+thierry.magnien=sfr.com at varnish-cache.org [mailto:varnish-misc-bounces+thierry.magnien=sfr.com at varnish-cache.org] De la part de Ayberk Kimsesiz
Envoyé : mardi 2 août 2016 15:18
À : Stig Bakken
Cc : varnish-misc
Objet : Re: Varnish CPU Usage
I used Default.VCL in two different ways. The first with default settings and the second with settings customized for Wordpress (either case, the CPU usage increases). That is as follows:
Also i couldn't find a Centos 6 installation guide for Prometheus.
/* SET THE HOST AND PORT OF WORDPRESS
* *********************************************************/
vcl 4.0;
import std;
backend default {
.host = "SERVER IP";
.port = "8080";
.first_byte_timeout = 60s;
.connect_timeout = 300s;
}
# SET THE ALLOWED IP OF PURGE REQUESTS
# ##########################################################
acl purge {
"localhost";
"127.0.0.1";
}
#THE RECV FUNCTION
# ##########################################################
sub vcl_recv {
# set realIP by trimming CloudFlare IP which will be used for various checks
set req.http.X-Actual-IP = regsub(req.http.X-Forwarded-For, "[, ].*$", "");
# FORWARD THE IP OF THE REQUEST
if (req.restarts == 0) {
if (req.http.x-forwarded-for) {
set req.http.X-Forwarded-For =
req.http.X-Forwarded-For + ", " + client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
# Purge request check sections for hash_always_miss, purge and ban
# BLOCK IF NOT IP is not in purge acl
# ##########################################################
# Enable smart refreshing using hash_always_miss
if (req.http.Cache-Control ~ "no-cache") {
if (client.ip ~ purge || !std.ip(req.http.X-Actual-IP, "1.2.3.4") ~ purge) {
set req.hash_always_miss = true;
}
}
if (req.method == "PURGE") {
if (!client.ip ~ purge || !std.ip(req.http.X-Actual-IP, "1.2.3.4") ~ purge) {
return(synth(405,"Not allowed."));
}
return (purge);
}
if (req.method == "BAN") {
# Same ACL check as above:
if (!client.ip ~ purge || !std.ip(req.http.X-Actual-IP, "1.2.3.4") ~ purge) {
return(synth(403, "Not allowed."));
}
ban("req.http.host == " + req.http.host +
" && req.url == " + req.url);
# Throw a synthetic page so the
# request won't go to the backend.
return(synth(200, "Ban added"));
}
# Unset cloudflare cookies
# Remove has_js and CloudFlare/Google Analytics __* cookies.
set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", "");
# Remove a ";" prefix, if present.
set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
# For Testing: If you want to test with Varnish passing (not caching) uncomment
# return( pass );
# FORWARD THE IP OF THE REQUEST
if (req.restarts == 0) {
if (req.http.x-forwarded-for) {
set req.http.X-Forwarded-For =
req.http.X-Forwarded-For + ", " + client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
# DO NOT CACHE RSS FEED
if (req.url ~ "/feed(/)?") {
return ( pass );
}
## Do not cache search results, comment these 3 lines if you do want to cache them
if (req.url ~ "/\?s\=") {
return ( pass );
}
# CLEAN UP THE ENCODING HEADER.
# SET TO GZIP, DEFLATE, OR REMOVE ENTIRELY. WITH VARY ACCEPT-ENCODING
# VARNISH WILL CREATE SEPARATE CACHES FOR EACH
# DO NOT ACCEPT-ENCODING IMAGES, ZIPPED FILES, AUDIO, ETC.
# ##########################################################
if (req.http.Accept-Encoding) {
if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
# No point in compressing these
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
# unknown algorithm
unset req.http.Accept-Encoding;
}
}
# PIPE ALL NON-STANDARD REQUESTS
# ##########################################################
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "DELETE") {
return (pipe);
}
# ONLY CACHE GET AND HEAD REQUESTS
# ##########################################################
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# OPTIONAL: DO NOT CACHE LOGGED IN USERS (THIS OCCURS IN FETCH TOO, EITHER
# COMMENT OR UNCOMMENT BOTH
# ##########################################################
if ( req.http.cookie ~ "wordpress_logged_in" ) {
return( pass );
}
# IF THE REQUEST IS NOT FOR A PREVIEW, WP-ADMIN OR WP-LOGIN
# THEN UNSET THE COOKIES
# ##########################################################
if (!(req.url ~ "wp-(login|admin)")
&& !(req.url ~ "&preview=true" )
){
unset req.http.cookie;
}
# IF BASIC AUTH IS ON THEN DO NOT CACHE
# ##########################################################
if (req.http.Authorization || req.http.Cookie) {
return (pass);
}
# IF YOU GET HERE THEN THIS REQUEST SHOULD BE CACHED
# ##########################################################
return (hash);
# This is for phpmyadmin
if (req.http.Host == "ki1.org<http://ki1.org>") {
return (pass);
}
if (req.http.Host == "mysql.ki1.org<http://mysql.ki1.org>") {
return (pass);
}
}
# HIT FUNCTION
# ##########################################################
sub vcl_hit {
# IF THIS IS A PURGE REQUEST THEN DO THE PURGE
# ##########################################################
if (req.method == "PURGE") {
#
# This is now handled in vcl_recv.
#
# purge;
return (synth(200, "Purged."));
}
return (deliver);
}
# MISS FUNCTION
# ##########################################################
sub vcl_miss {
if (req.method == "PURGE") {
#
# This is now handled in vcl_recv.
#
# purge;
return (synth(200, "Purged."));
}
return (fetch);
}
# FETCH FUNCTION
# ##########################################################
sub vcl_backend_response {
# I SET THE VARY TO ACCEPT-ENCODING, THIS OVERRIDES W3TC
# TENDANCY TO SET VARY USER-AGENT. YOU MAY OR MAY NOT WANT
# TO DO THIS
# ##########################################################
set beresp.http.Vary = "Accept-Encoding";
# IF NOT WP-ADMIN THEN UNSET COOKIES AND SET THE AMOUNT OF
# TIME THIS PAGE WILL STAY CACHED (TTL)
# ##########################################################
if (!(bereq.url ~ "wp-(login|admin)") && !bereq.http.cookie ~ "wordpress_logged_in" ) {
unset beresp.http.set-cookie;
set beresp.ttl = 52w;
# set beresp.grace =1w;
}
if (beresp.ttl <= 0s ||
beresp.http.Set-Cookie ||
beresp.http.Vary == "*") {
set beresp.ttl = 120 s;
# set beresp.ttl = 120s;
set beresp.uncacheable = true;
return (deliver);
}
return (deliver);
}
# DELIVER FUNCTION
# ##########################################################
sub vcl_deliver {
# IF THIS PAGE IS ALREADY CACHED THEN RETURN A 'HIT' TEXT
# IN THE HEADER (GREAT FOR DEBUGGING)
# ##########################################################
if (obj.hits > 0) {
set resp.http.X-Cache = "HIT";
# IF THIS IS A MISS RETURN THAT IN THE HEADER
# ##########################################################
} else {
set resp.http.X-Cache = "MISS";
}
}
2016-08-02 15:07 GMT+03:00 Stig Bakken <stig at zedge.net<mailto:stig at zedge.net>>:
Could it be that Varnish is not caching those POST requests, and that they are what makes Apache consume a lot of CPU?
On MPMs: with PHP prefork is the safest choice, and it will work just fine as long as you have a reasonable config, especially since you have Varnish in front offloading the job of shuffling bytes back to the users.
If I were you I'd try to get more data on which requests take a long time, some variant of "varnishncsa -F %D"..
If you're into using Prometheus for metrics, you can try out this exporter which will give you a lot of insight into which requests take a long time to process: https://github.com/stigsb/varnish_request_exporter (Prometheus is awesome!)
- Stig
On Thu, Jul 28, 2016 at 6:57 PM, Ayberk Kimsesiz <ayberk.kimsesiz at gmail.com<mailto:ayberk.kimsesiz at gmail.com>> wrote:
Hi,
CPU Monitor:
http://i.imgur.com/5KT1xRu.jpg
Apache status:
Srv
PID
Acc
M
CPU
SS
Req
Conn
Child
Slot
Client
Protocol
VHost
Request
0-0
-
0/0/9766
.
134.59
37
0
0.0
0.00
64.40
::1
http/1.1
ns1.***com:8080
OPTIONS * HTTP/1.0
1-0
14612
0/16/9058
_
17.83
13
1498
0.0
0.02
53.29
176.***.10
http/1.1
www.***.com:8080<http://www.***.com:8080>
POST /wp-admin/admin-ajax.php HTTP/1.1
2-0
10863
0/179/9795
_
185.14
6
1424
0.0
0.58
60.32
176.***.10
http/1.1
www.***.com:8080<http://www.***.com:8080>
POST /wp-admin/admin-ajax.php HTTP/1.1
3-0
13127
0/127/9435
_
119.80
4
1419
0.0
0.42
56.51
176.***.10
http/1.1
www.***.com:8080<http://www.***.com:8080>
POST /wp-admin/admin-ajax.php HTTP/1.1
4-0
-
0/0/9187
.
0.00
50
0
0.0
0.00
56.60
::1
http/1.1
ns1.***.com:8080
OPTIONS * HTTP/1.0
5-0
14851
0/9/8761
_
8.95
13
1559
0.0
0.01
57.90
176.***.10
http/1.1
www.***.com:8080<http://www.***.com:8080>
POST /wp-admin/admin-ajax.php HTTP/1.1
6-0
14852
0/6/8130
_
6.67
4
1482
0.0
0.01
51.88
176.***.10
http/1.1
www.***.com:8080<http://www.***.com:8080>
POST /wp-admin/admin-ajax.php HTTP/1.1
7-0
14192
11/57/8355
K
72.73
0
1363
106.6
0.44
52.79
176.***.10
http/1.1
www.***.com:8080<http://www.***.com:8080>
POST /wp-admin/admin-ajax.php HTTP/1.1
8-0
13067
0/125/7795
_
121.19
13
1759
0.0
0.65
68.27
176.***.10
http/1.1
www.***.com:8080<http://www.***.com:8080>
POST /wp-admin/admin-ajax.php HTTP/1.1
Apache error logs don't show anything about CPU.
2016-07-28 19:31 GMT+03:00 Guillaume Quintard <guillaume at varnish-software.com<mailto:guillaume at varnish-software.com>>:
You seem to have a good hit ratio, are you seeing anything on the apache logs that would explain the cpu usage?
--
Guillaume Quintard
_______________________________________________
varnish-misc mailing list
varnish-misc at varnish-cache.org<mailto:varnish-misc at varnish-cache.org>
https://www.varnish-cache.org/lists/mailman/listinfo/varnish-misc
--
Stig Bakken
CTO, Zedge.net - free your phone!
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20160802/10e4eb07/attachment-0001.html>
More information about the varnish-misc
mailing list