Is anyone using ESI with a lot of traffic?
Artur Bergman
sky at crucially.net
Mon Mar 2 22:40:16 CET 2009
On Feb 27, 2009, at 2:24 PM, John Adams wrote:
> cc'ing the varnish dev list for comments...
>
> On Feb 27, 2009, at 1:33 PM, Cloude Porteus wrote:
>
>> John,
>> Goodto hear from you. You must be slammed at Twitter. I'm happy to
>> hear that ESI is holding up for you. It's been in my backlog since
>> you
>> mentioned it to me pre-Twitter.
>>
>> Any performance info would be great.
>>
>
> Any comments on our setup are welcome. You may also choose to call
> us crazypants. Many, many thanks to Artur Bergman of Wikia for
> helping us get this configuration straightened out.
>
Thanks John :)
I'll describe the settings we use. (We don't use ESI because of gzip)
The first important step is that we put the shmlog on tmpfs
tmpfs /usr/var/varnish/ tmpfs noatime,defaults,size=150M 0 0
/dev/md0 /var/lib/varnish ext2
noatime,nodiratime,norelatime 0 0
Notice also ext2 we don't care about journaling. (Ignore the broken
paths)
This is because linux will asynchronously write the log to disk, this
puts a large io pressure on the system (interfering with your normal
reads if you use the same disks) It also scales the IO load with
traffic and not working set.
# Maximum number of open files (for ulimit -n)
NFILES=131072
# Locked shared memory (for ulimit -l)
# Default log size is 82MB + header
MEMLOCK=90000
DAEMON_COREFILE_LIMIT="unlimited"
DAEMON_OPTS="-a :80 \
-T localhost:6082 \
-f /etc/varnish/wikia.vcl \
-p obj_workspace=4096 \
# We have lots of objects
-p sess_workspace=32768 \
# Need lots of sessoin space
-p listen_depth=8192 \
-p ping_interval=1 \
-s file,/var/lib/varnish/mmap,120G \
# lots of mmap
-p log_hashstring=off \
-h classic,250007 \
# 2.5 mmilion objects
-p thread_pool_max=4000 \
-p lru_interval=60 \
-p esi_syntax=0x00000003 \
-p sess_timeout=10 \
-p thread_pools=4 \
-p thread_pool_min=500 \
# we force 4000 threads pre-created
# otherwise we run into overflows
-p shm_workspace=32768 \
# avoid shm_mtx
-p srcaddr_ttl=0"
# avoid hash lookup
# we link geoip into the vcl
CC_COMMAND='cc_command=exec cc -fpic -shared -Wl,-x -L/usr/local/lib/ -
lGeoIP -o %o %s'
#### VCL
# declare the function signature
# so we can use them
C{
#include <string.h>
double TIM_real(void);
void TIM_format(double t, char *p);
}C
# init GeoIP code
C{
#include <dlfcn.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <GeoIPCity.h>
#include <pthread.h>
pthread_mutex_t geoip_mutex = PTHREAD_MUTEX_INITIALIZER;
GeoIP* gi;
void geo_init () {
if(!gi) {
gi = GeoIP_open_type(GEOIP_CITY_EDITION_REV1,GEOIP_MEMORY_CACHE);
}
}
}C
vcl_recv {
set req.url = regsub(req.url, "http://[^/]*","");
#will normalize proxied requests, specificl curl -x foo:80
# get out error handler for geoiplookup
if(req.http.host == "geoiplookup.wikia.com") {
error 200 "Ok";
}
# lvs check
if (req.url == "/lvscheck.html") {
error 200 "Ok";
}
# normalize Accept-Encoding to reduce vary
if (req.http.Accept-Encoding) {
if (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
unset req.http.Accept-Encoding;
}
}
# Yahoo uses this to check for 404
if (req.url ~ "^/SlurpConfirm404") {
error 404 "Not found";
}
set req.grace = 360000s; #if the backend is down, just serve
# check for specific cookies, otherwise nuke them
# save them so we can re-inject them later in pipe or miss
set req.http.X-Orig-Cookie = req.http.Cookie;
if(req.http.Cookie ~ "(session|UserID|UserName|Token|LoggedOut)") {
# dont do anything, the user is logged in
} else {
# dont care about any other cookies
unset req.http.Cookie;
}
}
# varnish XFF is broken, it doesn't chain them
# if you have chained varnishes, or trust AOL, you need to append them
sub vcl_pipe {
# do the right XFF processing
set bereq.http.X-Forwarded-For = req.http.X-Forwarded-For;
set bereq.http.X-Forwarded-For = regsub(bereq.http.X-Forwarded-For,
"$", ", ");
set bereq.http.X-Forwarded-For = regsub(bereq.http.X-Forwarded-For,
"$", client.ip);
set bereq.http.Cookie = req.http.X-Orig-Cookie;
}
# this implements purging (we purge all 3 versions of the accept-
encoding, none,gzip,deflate)
sub vcl_hit {
if (req.request == "PURGE") {
set obj.ttl = 0s;
error 200 "Purged.";
}
}
sub vcl_miss {
if (req.request == "PURGE") {
error 404 "Not purged";
}
set bereq.http.X-Forwarded-For = req.http.X-Forwarded-For;
set bereq.http.X-Forwarded-For = regsub(bereq.http.X-Forwarded-For,
"$", ", ");
set bereq.http.X-Forwarded-For = regsub(bereq.http.X-Forwarded-For,
"$", client.ip);
}
# this marks if something is cacheable or not, if it isn't
# say why
vcl_fetch {
# so we have access to this in deliver
set obj.http.X-Orighost = req.http.host;
set obj.http.X-Origurl = req.url;
if (!obj.cacheable) {
set obj.http.X-Cacheable = "NO:Not-Cacheable";
pass;
}
if (obj.http.Cache-Control ~ "private") {
if(req.http.Cookie ~"(UserID|_session)") {
set obj.http.X-Cacheable = "NO:Got Session";
} else {
set obj.http.X-Cacheable = "NO:Cache-
Control=private";
}
pass;
}
if (obj.http.Set-Cookie ~ "(UserID|_session)") {
set obj.http.X-Cacheable = "NO:Set-Cookie";
pass;
}
set obj.http.X-Cacheable = "YES";
set obj.grace = 360000s;
}
#Following sets X-Served-By, if it is already set it appends it
# it also says if it is a HIT, and how many hits
sub vcl_deliver {
#add or append Served By
if(!resp.http.X-Served-By) {
set resp.http.X-Served-By = "varnish8";
if (obj.hits > 0) {
set resp.http.X-Cache = "HIT";
} else {
set resp.http.X-Cache = "MISS";
}
set resp.http.X-Cache-Hits = obj.hits;
} else {
# append current data
set resp.http.X-Served-By = regsub(resp.http.X-Served-By, "$", ",
varnish8");
if (obj.hits > 0) {
set resp.http.X-Cache = regsub(resp.http.X-Cache, "$", ", HIT");
} else {
set resp.http.X-Cache = regsub(resp.http.X-Cache, "$" , ",
MISS");
}
set resp.http.X-Cache-Hits = regsub(resp.http.X-Cache-Hits, "$",
", ");
set resp.http.X-Cache-Hits = regsub(resp.http.X-Cache-Hits, "$",
obj.hits);
}
#
# if the client is another DC, just remove stuff and deliver
if ( client.ip ~ LON
|| client.ip ~ SJC
|| client.ip ~ IOWA
) {
unset resp.http.X-CPU-Time;
unset resp.http.X-Real-Time;
unset resp.http.X-Served-By-Backend;
unset resp.http.X-User-Id;
unset resp.http.X-Namespace-Number;
unset resp.http.X-Orighost;
unset resp.http.X-Origurl;
deliver;
}
# else do cache-control
# nuke the headers since they were generally meant for varnish
# these rules are mostly based on mediawiki rules
if ( resp.http.X-Pass-Cache-Control ) {
set resp.http.Cache-Control = resp.http.X-Pass-Cache-Control;
} elsif ( resp.status == 304 ) {
# no headers on if-modified since
} elsif ( resp.http.X-Origurl ~ ".*/index\.php.*(css|js)"
|| resp.http.X-Origurl ~ "raw") {
# dont touch it let mediawiki decide
} elsif (resp.http.X-Orighost ~ "images.wikia.com") {
# lighttpd knows what it is doing
} elsif (resp.http.X-Orighost ~ "geoiplookup") {
} else {
#follow squid content here
set resp.http.Cache-Control = "private, s-maxage=0, max-age=0,
must-revalidate";
}
# this will calculate an Expire headers which is based on now+max-age
# if you cache the Expire header, then it won't match max-age since it
is static
if (!resp.status == 304) {
C{
char *cache = VRT_GetHdr(sp, HDR_REQ, "\016cache-control:");
char date[40];
int max_age;
int want_equals = 0;
if(cache) {
while(*cache != '\0') {
if (want_equals && *cache == '=') {
cache++;
max_age = strtoul(cache, 0, 0);
break;
}
if (*cache == 'm' && !memcmp(cache, "max-age", 7)) {
cache += 7;
want_equals = 1;
continue;
}
cache++;
}
if (max_age) {
TIM_format(TIM_real() + max_age, date);
VRT_SetHdr(sp, HDR_RESP, "\010Expires:", date,
vrt_magic_string_end);
}
}
}C
#;
}
}
vcl_error {
# this implements geoip lookups inside varnish
# so clients can get the data without hitting the backend
if(req.http.host == "geoiplookup.wikia.com" || req.url == "/
__varnish/geoip") {
set obj.http.Content-Type = "text/plain";
set obj.http.cache-control = "private, s-maxage=0, max-age=360";
set obj.http.X-Orighost = req.http.host;
C{
char *ip = VRT_IP_string(sp, VRT_r_client_ip(sp));
char date[40];
char json[255];
pthread_mutex_lock(&geoip_mutex);
if(!gi) { geo_init(); }
GeoIPRecord *record = GeoIP_record_by_addr(gi, ip);
if(record) {
snprintf(json, 255, "Geo = {\"city\":\"%s\",\"country\":\"%s
\",\"lat\":\"%f\",\"lon\":\"%f\",\"classC\":\"%s\",\"netmask\":\"%d\"}",
record->city,
record->country_code,
record->latitude,
record->longitude,
ip,
GeoIP_last_netmask(gi)
);
pthread_mutex_unlock(&geoip_mutex);
VRT_synth_page(sp, 0, json, vrt_magic_string_end);
} else {
pthread_mutex_unlock(&geoip_mutex);
VRT_synth_page(sp, 0, "Geo = {}", vrt_magic_string_end);
}
TIM_format(TIM_real(), date);
VRT_SetHdr(sp, HDR_OBJ, "\016Last-Modified:", date,
vrt_magic_string_end);
}C
# check if site is working
if(req.url ~ "lvscheck.html") {
synthetic {"varnish is okay"};
deliver;
}
deliver;
}
#############
sysctl
net.ipv4.ip_local_port_range = 1024 65536
net.core.rmem_max=16777216
net.core.wmem_max=16777216
net.ipv4.tcp_rmem=4096 87380 16777216
net.ipv4.tcp_wmem=4096 65536 16777216
net.ipv4.tcp_fin_timeout = 3
net.ipv4.tcp_tw_recycle = 1
net.core.netdev_max_backlog = 30000
net.ipv4.tcp_no_metrics_save=1
net.core.somaxconn = 262144
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_max_orphans = 262144
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2
These are mostly cargo culted from previous emails here.
Cheers
Artur
More information about the varnish-dev
mailing list