benchmarking varnish

[Miguel González]

On 12/03/2013 12:35, Jonathan Matthews wrote:
> On 12 March 2013 11:26, Miguel Gonzalez <miguel_3_gonzalez at yahoo.es> wrote:
>> Dear all,
>>
>>    I have a Varnish 3 installation pointing to an Apache Server. Any manual
>> of how to tune Varnish, OS, Apache and any manual of how to test Varnish
>> considering it's a cache tool?
> Have you worked through
> https://www.varnish-cache.org/docs/3.0/tutorial/increasing_your_hitrate.html
> yet?
>
> Jonathan
Sorry I haven't replied for so long. I was trying to make sense of the docs.

 From my munin monitoring and varnishstat it seems I'm getting around 
80% of hitrate in my Varnish cache. This Varnish cache has a remote 
Apache backup (Varnish is in a server in Europe while Apache is in a 
server in the US.

How can I pinpoint why is not working fast?

My default.vcl and /etc/sysconfig/varnish

backend default {
   .host = "XXX.XXX.XXX.XXX";
   .port = "8000";
   .connect_timeout = 600s;
   .first_byte_timeout = 600s;
   .between_bytes_timeout = 600s;
}

acl purge {
   "localhost";
}

acl ban {
   "localhost";
}

sub vcl_recv {
   // Strip cookies for static files:
   if (req.url ~ 
"\.(jpg|jpeg|gif|png|ico|css|zip|tgz|gz|rar|bz2|pdf|txt|tar|wav|bmp|rtf|js|flv|swf|html|htm|xml)$") 
{
     unset req.http.Cookie;
   }

   // Remove specific cookies:
   set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", 
"");
   set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", 
"");
   set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
   if (req.http.cookie ~ "^ *$") { unset req.http.cookie; }
   if (req.http.Cookie ~ "^\s*$") { unset req.http.Cookie; }
   if (req.http.Cookie == "") { remove req.http.Cookie; }

   // Normalize Accept-Encoding header (straight from the manual: 
https://www.varnish-cache.org/docs/3.0/tutorial/vary.html)
   if (req.http.Accept-Encoding) {
     if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
       // No point in compressing these
       remove req.http.Accept-Encoding;
     } elsif (req.http.Accept-Encoding ~ "gzip") {
       set req.http.Accept-Encoding = "gzip";
     } elsif (req.http.Accept-Encoding ~ "deflate") {
       set req.http.Accept-Encoding = "deflate";
     } else {
       // unkown algorithm
       remove req.http.Accept-Encoding;
     }
   }

   if (req.restarts == 0) {
     if (req.http.x-forwarded-for) {
       set req.http.X-Forwarded-For = req.http.X-Forwarded-For;
     } else {
       set req.http.X-Forwarded-For = client.ip;
     }
   }

   if (req.request != "GET" &&
     req.request != "HEAD" &&
     req.request != "PUT" &&
     req.request != "POST" &&
     req.request != "TRACE" &&
     req.request != "OPTIONS" &&
     req.request != "DELETE") {
       /* Non-RFC2616 or CONNECT which is weird. */
       return (pipe);
   }

   if (req.request != "GET" && req.request != "HEAD") {
     /* We only deal with GET and HEAD by default */
     return (pass);
   }

   if (req.http.Authorization || req.http.Cookie) {
     /* Not cacheable by default */
     return (pass);
   }

   if (req.request == "PURGE") {
     if (!client.ip ~ purge) {
       error 405 "Not allowed.";
     }
     return (lookup);
   }

   if (req.request == "BAN") {
     if (!client.ip ~ ban) {
       error 405 "Not allowed.";
     }
     ban("req.http.host == " + req.http.host + "&& req.url == " + req.url);
     error 200 "Ban added";
   }

   return (lookup);
}

sub vcl_pipe {
   set bereq.http.connection = "close";
   return (pipe);
}

sub vcl_pass {
   return (pass);
}

sub vcl_hash {
   hash_data(req.url);
   if (req.http.host) {
     hash_data(req.http.host);
   } else {
     hash_data(server.ip);
   }

   // If the client supports compression, keep that in a different cache
   if (req.http.Accept-Encoding) {
     hash_data(req.http.Accept-Encoding);
   }

   return (hash);
}

sub vcl_hit {
   if (req.request == "PURGE") {
     purge;
     error 200 "Purged.";
   }

   //Listen to browser force refresh
   if (req.http.Cache-Control ~ "no-cache") {
     if (! (req.http.Via || req.http.User-Agent ~ "bot|MSIE")) {
       set obj.ttl = 0s;
       return (restart);
     }
   }

   return (deliver);
}

sub vcl_miss {
   if (req.request == "PURGE") {
     purge;
     error 200 "Purged.";
   }

   return (fetch);
}

sub vcl_fetch {
   if (beresp.ttl <= 0s ||
     beresp.http.Set-Cookie ||
     beresp.http.Vary == "*") {
       set beresp.ttl = 120 s;
       return (hit_for_pass);
   }
   return (deliver);
}

sub vcl_deliver {
   // Debugging
   if (obj.hits > 0) {
     set resp.http.X-Cache = "HIT";
   } else {
     set resp.http.X-Cache = "MISS";
   }

   // Remove some headers: PHP version
   unset resp.http.X-Powered-By;

   // Remove some headers: Apache version & OS
   unset resp.http.Server;

   return (deliver);
}

sub vcl_error {
   set obj.http.Content-Type = "text/html; charset=utf-8";
   set obj.http.Retry-After = "5";
   synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
   <head>
     <title>"} + obj.status + " " + obj.response + {"</title>
   </head>
   <body>
     <h1>Error "} + obj.status + " " + obj.response + {"</h1>
     <p>"} + obj.response + {"</p>
     <h3>Guru Meditation:</h3>
     <p>XID: "} + req.xid + {"</p>
     <hr>
     <p>Varnish cache server</p>
   </body>
</html>
"};
   return (deliver);
}

sub vcl_init {
   return (ok);
}

sub vcl_fini {
   return (ok);
}

--------------------------------------

# Configuration file for varnish
#
# /etc/init.d/varnish expects the variable $DAEMON_OPTS to be set from this
# shell script fragment.
#

# Maximum number of open files (for ulimit -n)
NFILES=131072

# Locked shared memory (for ulimit -l)
# Default log size is 82MB + header
MEMLOCK=82000

# Maximum size of corefile (for ulimit -c). Default in Fedora is 0
# DAEMON_COREFILE_LIMIT="unlimited"

# Set this to 1 to make init script reload try to switch vcl without 
restart.
# To make this work, you need to set the following variables
# explicit: VARNISH_VCL_CONF, VARNISH_ADMIN_LISTEN_ADDRESS,
# VARNISH_ADMIN_LISTEN_PORT, VARNISH_SECRET_FILE, or in short,
# use Alternative 3, Advanced configuration, below
RELOAD_VCL=1

# This file contains 4 alternatives, please use only one.

## Alternative 1, Minimal configuration, no VCL
#
# Listen on port 6081, administration on localhost:6082, and forward to
# content server on localhost:8080.  Use a fixed-size cache file.
#
#DAEMON_OPTS="-a :6081 \
#             -T localhost:6082 \
#             -b localhost:8080 \
#             -u varnish -g varnish \
#             -s file,/var/lib/varnish/varnish_storage.bin,1G"


## Alternative 2, Configuration with VCL
#
# Listen on port 6081, administration on localhost:6082, and forward to
# one content server selected by the vcl file, based on the request.  Use a
# fixed-size cache file.
#
#DAEMON_OPTS="-a :6081 \
#             -T localhost:6082 \
#             -f /etc/varnish/default.vcl \
#             -u varnish -g varnish \
#             -S /etc/varnish/secret \
#             -s file,/var/lib/varnish/varnish_storage.bin,1G"


## Alternative 3, Advanced configuration
#
# See varnishd(1) for more information.
#
# # Main configuration file. You probably want to change it :)
VARNISH_VCL_CONF=/etc/varnish/default.vcl
#
# # Default address and port to bind to
# # Blank address means all IPv4 and IPv6 interfaces, otherwise specify
# # a host name, an IPv4 dotted quad, or an IPv6 address in brackets.
# VARNISH_LISTEN_ADDRESS=
VARNISH_LISTEN_PORT=80
#
# # Telnet admin interface listen address and port
VARNISH_ADMIN_LISTEN_ADDRESS=127.0.0.1
VARNISH_ADMIN_LISTEN_PORT=6082
#
# # Shared secret file for admin interface
VARNISH_SECRET_FILE=/etc/varnish/secret
#
# # The minimum number of worker threads to start
VARNISH_MIN_THREADS=800
#
# # The Maximum number of worker threads to start
VARNISH_MAX_THREADS=1000
#
# # Idle timeout for worker threads
VARNISH_THREAD_TIMEOUT=120

VARNISH_CACHE_SIZE=18G
VARNISH_CACHE="malloc,${VARNISH_CACHE_SIZE}"

#
# # Cache file location
#VARNISH_STORAGE_FILE=/var/lib/varnish/varnish_storage.bin
#
# # Cache file size: in bytes, optionally using k / M / G / T suffix,
# # or in percentage of available disk space using the % suffix.
#VARNISH_STORAGE_SIZE=1G
#
# # Backend storage specification
#VARNISH_STORAGE="file,${VARNISH_STORAGE_FILE},${VARNISH_STORAGE_SIZE}"
#
# # Default TTL used when the backend does not specify one
VARNISH_TTL=120
#
# # DAEMON_OPTS is used by the init script.  If you add or remove 
options, make
# # sure you update this section, too.
DAEMON_OPTS="-a ${VARNISH_LISTEN_ADDRESS}:${VARNISH_LISTEN_PORT} \
              -f ${VARNISH_VCL_CONF} \
              -T 
${VARNISH_ADMIN_LISTEN_ADDRESS}:${VARNISH_ADMIN_LISTEN_PORT} \
              -t ${VARNISH_TTL} \
              -w 
${VARNISH_MIN_THREADS},${VARNISH_MAX_THREADS},${VARNISH_THREAD_TIMEOUT} \
              -u varnish -g varnish \
              -S ${VARNISH_SECRET_FILE} \
              -s ${VARNISH_CACHE}"
#


## Alternative 4, Do It Yourself. See varnishd(1) for more information.
#
# DAEMON_OPTS=""






This message and any attachments are intended for the use of the addressee or addressees only. The unauthorised disclosure, use, dissemination or copying (either in whole or in part) of its content is not permitted. If you received this message in error, please notify the sender and delete it from your system. Emails can be altered and their integrity cannot be guaranteed by the sender.

Please consider the environment before printing this email.