Backend fail & 503 Service Unavailable

M L ml at tinwong.com
Tue Sep 22 19:22:06 CEST 2009


Hi list


I love varnish and really want to use it :D  Any clue to fix my problem , it
come out alot backend fail ( i guess timeout problem )

my setup

1.Centos 5.3 64bit varnish / webserver
2.nginx backend server (it run over 200+days 2Mil pv/day without any problem
& healthy hardware)
3.varnish connect to nginx in same internal switchs  (3com 5500 Giga layer4)
4.Tested different version nginx was same happen ( nginx-0.6.36 &
nginx-0.7.61)
5.Tested 2 different hardware for varnish same happen
6.Changed nginx different timeout same happen , if changed to keepalive 0
will more backend fail
7.When i changed vcl to director rr x50 times , it didnt show 503 Service
Unavailable on client side but like 2-8 sec. lag  when then Backend fail
number increase


#start

varnishd -p lru_interval=3600 -a 0.0.0.0:80 -T localhost:3500 -p
client_http11=on -f vconf2 -s file,/usr/local/varnish/cache.bin,80G -h
classic,500009 -p listen_depth=4096 -p obj_workspace=32768 -p
sess_workspace=32768 -p send_timeout=327 -p first_byte_timeout=300 -p
connect_timeout=5 -p vcl_trace=on



#varnishlog

  140 ReqStart     c 121.203.78.124 4755 1383283991
  140 RxRequest    c GET
  140 RxURL        c /thread-1131553-1-1.html
  140 RxProtocol   c HTTP/1.1
  140 RxHeader     c Host: www.zoobar.com
  140 RxHeader     c User-Agent: Mozilla/5.0 (Windows; U; Windows NT 5.1;
zh-TW; rv:1.9.0.14) Gecko/2009082707 Firefox/3.0.14
  140 RxHeader     c Accept: image/png,image/*;q=0.8,*/*;q=0.5
  140 RxHeader     c Accept-Language: zh-tw,en-us;q=0.7,en;q=0.3
  140 RxHeader     c Accept-Encoding: gzip,deflate
  140 RxHeader     c Accept-Charset: Big5,utf-8;q=0.7,*;q=0.7
  140 RxHeader     c Keep-Alive: 300
  140 RxHeader     c Connection: keep-alive
  140 RxHeader     c Referer: http://www.zoobar.com/thread-1131553-1-1.html
  140 RxHeader     c Cookie: cdb_sid=Nf6FM3;
cdb_oldtopics=D1131553D1131773D1129353D1129581D1130425D1131274D1121699D1122534D1122581D1124932D1125704D1126044D1126641D1126650D1127247D1128227D1128288D1128329D1128640D1129165D1129403D1130965D1131013D1131057D1131460D1131464D1131491D
  140 VCL_call     c recv
  140 VCL_trace    c 1 106.14
  140 VCL_trace    c 2 110.5
  140 VCL_trace    c 3 110.41
  140 VCL_trace    c 9 143.5
  140 VCL_trace    c 14 157.1
  140 VCL_trace    c 15 157.5
  140 VCL_trace    c 17 162.1
  140 VCL_trace    c 18 162.5
  140 VCL_trace    c 26 172.9
  140 VCL_trace    c 27 172.13
  140 VCL_trace    c 30 180.1
  140 VCL_trace    c 31 180.5
  140 VCL_trace    c 33 184.1
  140 VCL_trace    c 34 184.5
  140 VCL_trace    c 36 192.1
  140 VCL_trace    c 37 192.5
  140 VCL_trace    c 39 198.1
  140 VCL_trace    c 40 198.5
  140 VCL_trace    c 41 198.31
  140 VCL_trace    c 42 199.9
  140 VCL_trace    c 44 202.18
  140 VCL_trace    c 45 202.53
  140 VCL_trace    c 49 213.1
  140 VCL_trace    c 84 42.14
  140 VCL_trace    c 85 43.9
  140 VCL_trace    c 93 53.5
  140 VCL_trace    c 94 53.9
  140 VCL_trace    c 97 57.5
  140 VCL_trace    c 98 57.9
  140 VCL_trace    c 99 57.35
  140 VCL_trace    c 100 57.52
  140 VCL_return   c pass
  140 VCL_call     c pass
  140 VCL_trace    c 103 74.14
  140 VCL_return   c pass
  140 VCL_call     c error
  140 VCL_trace    c 124 129.15
  140 VCL_return   c deliver
  140 Length       c 466
  140 VCL_call     c deliver
  140 VCL_trace    c 69 327.17
  140 VCL_trace    c 70 328.21
  140 VCL_trace    c 75 344.1
  140 VCL_trace    c 120 110.17
  140 VCL_return   c deliver
  140 TxProtocol   c HTTP/1.1
  140 TxStatus     c 503
  140 TxResponse   c Service Unavailable
  140 TxHeader     c Server: Varnish
  140 TxHeader     c Retry-After: 0
  140 TxHeader     c Content-Type: text/html; charset=utf-8
  140 TxHeader     c Content-Length: 466
  140 TxHeader     c Date: Tue, 22 Sep 2009 16:15:52 GMT
  140 TxHeader     c X-Varnish: 1383283991
  140 TxHeader     c Age: 0
  140 TxHeader     c Via: 1.1 varnish
  140 TxHeader     c Connection: close
  140 ReqEnd       c 1383283991 1253636152.715658903 1253636152.715944052
0.016985893 0.000265121 0.000020027
  140 SessionClose c error
  140 StatSess     c 121.203.78.124 4755 0 1 1 0 1 0 235 466




  #varnishstat -1

uptime                    266          .   Child uptime
client_conn             13993        52.61 Client connections accepted
client_req              43378       163.08 Client requests received
cache_hit               31219       117.36 Cache hits
cache_hitpass              86         0.32 Cache hits for pass
cache_miss               3523        13.24 Cache misses
backend_conn            12054        45.32 Backend connections success
backend_unhealthy            0         0.00 Backend connections not
attempted
backend_busy                0         0.00 Backend connections too many
backend_fail             5900        22.18 Backend connections failures
backend_reuse            3503        13.17 Backend connections reuses
backend_recycle         11552        43.43 Backend connections recycles
backend_unused              0         0.00 Backend connections unused
n_srcaddr                1246          .   N struct srcaddr
n_srcaddr_act              64          .   N active struct srcaddr
n_sess_mem                974          .   N struct sess_mem
n_sess                     84          .   N struct sess
n_object                 3040          .   N struct object
n_objecthead             1972          .   N struct objecthead
n_smf                    6460          .   N struct smf
n_smf_frag                573          .   N small free smf
n_smf_large                 7          .   N large free smf
n_vbe_conn                119          .   N struct vbe_conn
n_bereq                   240          .   N struct bereq
n_wrk                     261          .   N worker threads
n_wrk_create              261         0.98 N worker threads created
n_wrk_failed                0         0.00 N worker threads not created
n_wrk_max              336496      1265.02 N worker threads limited
n_wrk_queue                 0         0.00 N queued work requests
n_wrk_overflow           4696        17.65 N overflowed work requests
n_wrk_drop                374         1.41 N dropped work requests
n_backend                  60          .   N backends
n_expired                 675          .   N expired objects
n_lru_nuked                 0          .   N LRU nuked objects
n_lru_saved                 0          .   N LRU saved objects
n_lru_moved                 0          .   N LRU moved objects
n_deathrow                  0          .   N objects on deathrow
losthdr                     0         0.00 HTTP header overflows
n_objsendfile               0         0.00 Objects sent with sendfile
n_objwrite              41590       156.35 Objects sent with write
n_objoverflow               0         0.00 Objects overflowing workspace
s_sess                  10425        39.19 Total Sessions
s_req                   43325       162.88 Total Requests
s_pipe                      0         0.00 Total pipe
s_pass                   8542        32.11 Total pass
s_fetch                 11996        45.10 Total fetch
s_hdrbytes           16332373     61399.90 Total header bytes
s_bodybytes         266640005   1002406.03 Total body bytes
sess_closed              2116         7.95 Session Closed
sess_pipeline              54         0.20 Session Pipeline
sess_readahead             34         0.13 Session Read Ahead
sess_linger                 0         0.00 Session Linger
sess_herd               41284       155.20 Session herd
shm_records           3806411     14309.82 SHM records
shm_writes             134677       506.30 SHM writes
shm_flushes                40         0.15 SHM flushes due to overflow
shm_cont                  884         3.32 SHM MTX contention
shm_cycles                  1         0.00 SHM cycles through buffer
sm_nreq                 24335        91.48 allocator requests
sm_nobj                  5880          .   outstanding allocations
sm_balloc           129392640          .   bytes allocated
sm_bfree          85769953280          .   bytes free
sma_nreq                    0         0.00 SMA allocator requests
sma_nobj                    0          .   SMA outstanding allocations
sma_nbytes                  0          .   SMA outstanding bytes
sma_balloc                  0          .   SMA bytes allocated
sma_bfree                   0          .   SMA bytes free
sms_nreq                  111         0.42 SMS allocator requests
sms_nobj                    0          .   SMS outstanding allocations
sms_nbytes                  0          .   SMS outstanding bytes
sms_balloc              50376          .   SMS bytes allocated
sms_bfree               50376          .   SMS bytes freed
backend_req             12054        45.32 Backend requests made
n_vcl                       1         0.00 N vcl total
n_vcl_avail                 1         0.00 N vcl available
n_vcl_discard               0         0.00 N vcl discarded
n_purge                     1          .   N total active purges
n_purge_add                 1         0.00 N new purges added
n_purge_retire              0         0.00 N old purges deleted
n_purge_obj_test            0         0.00 N objects tested
n_purge_re_test             0         0.00 N regexps tested against
n_purge_dups                0         0.00 N duplicate purges removed
hcb_nolock                  0         0.00 HCB Lookups without lock
hcb_lock                    0         0.00 HCB Lookups with lock
hcb_insert                  0         0.00 HCB Inserts
esi_parse                   0         0.00 Objects ESI parsed (unlock)
esi_errors                  0         0.00 ESI parse errors (unlock)


# my vcl

director srv1 round-robin {
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    { .backend = { .connect_timeout = 2s; .host = "10.0.0.5"; .port = "80";
} }
    }


acl purge {

    "localhost"; "127.0.0.1";
}

#recv
sub vcl_recv {

#set req.grace = 30s;

if (req.http.host ~ "www.zoobar.com") {
    set req.http.host = "www.zoobar.com";
    set req.backend = srv1;


}elseif ( req.http.host ~ "www.voobar.com") {
    set req.http.host = "www.voobar.com";
    set req.backend = srv1;


}elseif ( req.http.host ~ "www.hoobar.com") {
    set req.http.host = "www.hoobar.com";
    set req.backend = srv1;

}else{
    error 401 "Bad Domain";
}

# Add a unique header containing the client address
remove req.http.X-Forwarded-For;
set    req.http.X-Forwarded-For = client.ip;
# [...]


if (req.request == "PURGE") {
    if(!client.ip ~ purge) {
        error 405 "Not Allowed";
    } lookup;}

if (req.http.Expect) {
        pipe;
    }


if (req.request != "GET" &&
                req.request != "HEAD" &&
                req.request != "PUT" &&
                req.request != "POST" &&
                req.request != "TRACE" &&
                req.request != "OPTIONS" &&
                req.request != "DELETE") {
                        /* Non-RFC2616 or CONNECT which is weird. */
                        pipe;
        }
        if (req.request != "GET" && req.request != "HEAD") {
                /* We only deal with GET and HEAD by default */
                pass;
        }




if (req.http.Cache-Control ~ "no-cache") {
        pass;
    }

if (req.http.Authenticate) {
    pass;
  }

#if (req.http.Cookie) {
#    pass;
#  }

if (req.url ~
"\.(zip|ico|dat|torrent|png|gif|jpg|swf|css|js|bmp|bz2|tbz|mp3|ogg)$") {
    unset req.http.cookie;
    lookup;
#    unset req.http.authenticate;
    }

if (req.http.Accept-Encoding) {
    if (req.url ~
"\.(zip|ico|dat|torrent|png|gif|jpg|swf|css|js|bmp|bz2|tbz|mp3|ogg)$") {
            # No point in compressing these
            remove req.http.Accept-Encoding;
        } elsif (req.http.Accept-Encoding ~ "gzip") {
            set req.http.Accept-Encoding = "gzip";
        } elsif (req.http.Accept-Encoding ~ "deflate") {
            set req.http.Accept-Encoding = "deflate";
        } else {
            # unkown algorithm
            remove req.http.Accept-Encoding;
        }
    }


} #end recv


sub vcl_hash {
set req.hash += req.url;
set req.hash += req.http.host;
#set req.hash += req.http.cookie;
#set req.hash += server.ip;
hash;
}  #end hash


# strip the cookie before the image is inserted into cache.
sub vcl_fetch {

#set obj.grace = 30s;


if(obj.http.Set-Cookie){
                pass;
                }


  if(obj.http.Pragma ~ "no-cache" ||
       obj.http.Cache-Control ~ "no-cache" ||
        obj.http.Cache-Control ~ "private"){
        pass;
     }


if (req.url ~
"\.(zip|ico|dat|torrent|png|gif|jpg|swf|css|js|bmp|bz2|tbz|mp3|ogg)$") {
        unset obj.http.set-cookie;
        set obj.ttl = 1w;
               }




if (req.url ~
"\.(zip|ico|dat|torrent|png|gif|jpg|swf|css|js|bmp|bz2|tbz|mp3|ogg)$") {
unset obj.http.expires;
set obj.http.cache-control = "max-age=315360000, public";
set obj.ttl = 1w;
set obj.http.magicmarker = "1";
}


    if (obj.status == 503) {

        restart;
    }


# if (obj.cacheable) {
#                        /* Remove Expires from backend, it's not long
enough */
#                        unset obj.http.expires;

#                        /* Set the clients TTL on this object */
#                        set obj.http.cache-control = "max-age=315360000,
public";

#                        /* Set how long Varnish will keep it */
#                        set obj.ttl = 1w;

#                        /* marker for vcl_deliver to reset Age: */
#                        set obj.http.magicmarker = "1";
#                }


    } #fetch end




sub vcl_deliver {
                if (resp.http.magicmarker) {
                        /* Remove the magic marker */
                        unset resp.http.magicmarker;

                        /* By definition we have a fresh object */
                        set resp.http.age = "0";
if (obj.hits > 0) {
                 set resp.http.X-Cache = "HIT";
        } else {
                 set resp.http.X-Cache = "MISS";
       }

                }


} #deliver end


sub vcl_pipe {
    # http://varnish.projects.linpro.no/ticket/451
    # This forces every pipe request to be the first one.
    set bereq.http.connection = "close";
} #pipe end


sub vcl_hit {
         if (req.request == "PURGE") {
                 set obj.ttl = 0s;
                 error 200 "Purged.";
         }

         if (!obj.cacheable) {
                 pass;
         }
         deliver;
 }




Thank you

TW
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20090923/ab1497b1/attachment-0001.html>


More information about the varnish-misc mailing list