[Varnish] #262: Varnish crashes on Assert error in htc_header_complete(), cache_httpconn.c line 60

[Varnish]

#262: Varnish crashes on Assert error in htc_header_complete(), cache_httpconn.c
line 60
----------------------+-----------------------------------------------------
 Reporter:  anders    |       Owner:  phk  
     Type:  defect    |      Status:  new  
 Priority:  normal    |   Milestone:       
Component:  varnishd  |     Version:  trunk
 Severity:  major     |    Keywords:       
----------------------+-----------------------------------------------------
 Running Varnish trunk/2665 for around a week or so, I got the following
 assert:

 {{{
 Child said (2, 83498): <<Assert error in htc_header_complete(),
 cache_httpconn.c line 60:>>
 Child said (2, 83498): <<  Condition(*t->e == '\0') not true.>>
 Cache child died pid=83498 status=0x86
 }}}

 This is in FreeBSD 7.0/amd64 running on Intel hardware, with SCHED_ULE and
 a patch to set 2K page size in storage_file.c. At the time of the crash I
 had 4.33 million objects and 75 GB data in the cache.

 Startup parameters:

 varnishd_files="-s malloc,100G"
 varnishd_flags="-d -d -u root -g wheel -p obj_workspace=4096 -p
 lru_interval=3600 -h classic,500009 -p ping_interval=0 -p cli_timeout=30
 -p auto_restart=off -p thread_pools=4 -p thread_pool_max=1000 -p
 session_linger=40 -p listen_depth=4096 -p srcaddr_hash=20480 -p
 default_ttl=604800 -T localhost:8080 -f /usr/local/etc/varnish.vcl
 $varnishd_files -P /var/run/varnishd.pid"

 My VCL:

 {{{
 backend default {
              .host = "192.168.110.1";
              .port = "80";
         }
 acl purge {
         "192.168.100.1"/32;
 }

 sub vcl_recv {
         set req.grace = 5m;
         if (req.http.host ~ "^(bars.*.foo.no|bazcache.foo.no)$") {
                 if (req.request == "GET" || req.request == "HEAD") {
                         lookup;
                 } elsif (req.request == "PURGE") {
                         if (client.ip ~ purge) {
                                 lookup;
                         } else {
                                 error 405 "Not allowed.";
                         }
                 } else {
                         pipe;
                 }
         } else {
                 error 403 "Access denied. Contact cacheadmin at foo.no if you
 have problems.";
         }
 }


 sub vcl_miss {
         if (req.request ~ "^(PURGE)$") {
                 error 404 "Not in cache.";
         } else {
                 fetch;
         }
 }
 sub vcl_hit {
         if (req.request == "PURGE") {
                 set obj.ttl = 0s;
                 error 200 "Purged.";
         } else {
                 if (!obj.cacheable) {
                         pass;
                 } else {
                         deliver;
                 }
         }
 }
 sub vcl_fetch {
         set obj.grace = 5m;
         if (obj.status == 404 || obj.status == 401 || obj.status == 500) {
                 pass;
         }

         if (!obj.valid) {
                 error obj.status;
         }
         if (!obj.cacheable) {
                 pass;
         }

         if (obj.http.Cookie) {
                 remove obj.http.Cookie;
         }
         if (obj.http.Set-Cookie) {
                 remove obj.http.Set-Cookie;
         }
         insert;
 }
 }}}

 Backtrace (90 GB core dump, used 55 GB on a compressed filesystem):

 {{{
 (gdb) bt
 #0  0x0000000800d0efec in thr_kill () from /lib/libc.so.7
 #1  0x0000000800d98c5b in abort () from /lib/libc.so.7
 #2  0x000000080066fcef in lbv_assert (func=Variable "func" is not
 available.
 ) at assert.c:65
 #3  0x0000000000416477 in htc_header_complete (t=0x80c0010d0)
     at cache_httpconn.c:60
 #4  0x00000000004164c4 in HTC_Complete (htc=0x80c0010c0)
     at cache_httpconn.c:136
 #5  0x000000000040c108 in cnt_again (sp=0x80c001008) at cache_center.c:97
 #6  0x000000000040e209 in CNT_Session (sp=0x80c001008) at steps.h:32
 #7  0x000000000041871d in wrk_do_cnt_sess (w=0x7fff90484ac0, priv=Variable
 "priv" is not available.
 )
     at cache_pool.c:330
 #8  0x00000000004176d2 in wrk_thread (priv=0x806303250) at
 cache_pool.c:244
 #9  0x0000000800a96a88 in pthread_getprio () from /lib/libthr.so.3
 #10 0x00007fff90285000 in ?? ()
 Cannot access memory at address 0x7fff90485000
 }}}

-- 
Ticket URL: <http://varnish.projects.linpro.no/ticket/262>
Varnish <http://varnish.projects.linpro.no/>
The Varnish HTTP Accelerator