[Varnish] #414: Another Varnish crash with -h critbit

[Varnish]

#414: Another Varnish crash with -h critbit
----------------------+-----------------------------------------------------
 Reporter:  anders    |        Owner:  phk  
     Type:  defect    |       Status:  new  
 Priority:  high      |    Milestone:       
Component:  varnishd  |      Version:  trunk
 Severity:  normal    |   Resolution:       
 Keywords:            |  
----------------------+-----------------------------------------------------
Comment (by anders):

 Got this one again:

 {{{
 Child (15012) died signal=6
 Child (15012) Panic message: Assert error in HSH_Deref(), cache_hash.c
 line 440:
   Condition((oh)->magic == 0x1b96615d) not true.  thread = (cache-timeout)

 Child cleanup complete
 }}}

 Finally I got a core-dump also. Backtrace:

 {{{
 (gdb) bt
 #0  0x0000000000414807 in hsh_rush (oh=0xfcb4e48f0) at cache_hash.c:375
 #1  0x0000000000414913 in HSH_Deref (o=0x1259d97000) at cache_hash.c:448
 #2  0x000000000040f75d in cnt_deliver (sp=0x17ea972008) at
 cache_center.c:180
 #3  0x0000000000410311 in CNT_Session (sp=0x17ea972008) at steps.h:42
 #4  0x000000000041c42c in wrk_do_cnt_sess (w=0x7fffb1b8fa40, priv=Variable
 "priv" is not available.
 )
     at cache_pool.c:398
 #5  0x000000000041ba62 in wrk_thread (priv=0x806452ba0) at
 cache_pool.c:310
 #6  0x0000000800aa7a88 in pthread_getprio () from /lib/libthr.so.3
 #7  0x00007fffb1990000 in ?? ()
 Cannot access memory at address 0x7fffb1b90000
 (gdb) frame 1
 #1  0x0000000000414913 in HSH_Deref (o=0x1259d97000) at cache_hash.c:448
 448                     hsh_rush(oh);
 (gdb) print *oh
 $2 = {magic = 462840157, mtx = {priv = 0xfc8ef1d00}, refcnt = 0, objects =
 {
     vtqh_first = 0x1259d97000, vtqh_last = 0x1259d97308},
   hash = 0xfc8ef1d40
 "/mmo/6/159/361/46_903995808_hoved.jpg#cache.finn.no#",
   hashlen = 53, digest = "«u©L°ý§Aý\200'øU,ý\226Ç{:¹Öç\235!)S\nÙ[9Ç\215",
   __u = {__u_waitinglist = {vtqh_first = 0x12589e1230,
       vtqh_last = 0x12571c3fc8}, __u_coollist = {vtqe_next = 0x12589e1230,
       vtqe_prev = 0x12571c3fc8}}, u = {filler = {0x1020017, 0x0, 0x0}, n =
 {
       u_n_hoh_list = {vtqe_next = 0x1020017, vtqe_prev = 0x0},
       u_n_hoh_head = 0x0, u_n_hoh_digest = 0}}}
 (gdb) frame 0
 #0  0x0000000000414807 in hsh_rush (oh=0xfcb4e48f0) at cache_hash.c:375
 375                     VTAILQ_REMOVE(&oh->waitinglist, sp, list);
 (gdb) print *sp
 $3 = {magic = 462840157, fd = 0, id = 1532310336, xid = 18, restarts = 0,
   esis = 0, wrk = 0x0, sockaddrlen = 1486754376, mysockaddrlen = 18,
   sockaddr = 0x125b553380, mysockaddr = 0x320a1bbf00000036,
   mylsock = 0xb386f615cc218603,
   addr = 0xed04548a830894f6 <Address 0xed04548a830894f6 out of bounds>,
   port = 0x4e41ed962570257d <Address 0x4e41ed962570257d out of bounds>,
   srcaddr = 0x13f99ec4, doclose = 0x0, http = 0xfcb4e4948, http0 =
 0x4020015,
   ws = {{magic = 0, id = 0x0, s = 0x0,
       f = 0xa5a5a5a5a5a5a5a5 <Address 0xa5a5a5a5a5a5a5a5 out of bounds>,
       r = 0x1b96615d <Address 0x1b96615d out of bounds>,
       e = 0x125b5533c0 "¥È\206{", overflow = 1}},
   ws_ses = 0x125b54c800 "B\035\2052\001", ws_req = 0x125b54cb08 "", htc =
 {{
       magic = 1532310528, fd = 18, ws = 0xda72b70d00000034, rxbuf = {
         b = 0x83d8aedd3bab7182 <Address 0x83d8aedd3bab7182 out of bounds>,
         e = 0x501ff09115825eec <Address 0x501ff09115825eec out of
 bounds>},
       pipeline = {
         b = 0x61ba5b30509ff99f <Address 0x61ba5b30509ff99f out of bounds>,
         e = 0xc8712787 <Address 0xc8712787 out of bounds>}}}, t_open = 0,
   t_req = 3.8930478601126296e-313, t_resp = 1.6642861158692102e-316,
   t_end = 0, connect_timeout = 0, first_byte_timeout = 0,
   between_bytes_timeout = -2.4983353906949635e-127,
   grace = 2.2867342108946895e-315, step = 1532310592, cur_method = 18,
   handling = 1, sendbody = 0 '\0', wantbody = 0 '\0', err_code =
 899649536,
   err_reason = 0x12359f9308 "", list = {vtqe_next = 0x1255519800,
     vtqe_prev = 0x6da9ac530000002e}, director = 0x82464dd154b4f770,
   vbe = 0x5b0f3d78c6e5d1c7, bereq = 0x503e5a2b3bbcc4ca, obj = 0x5aaa6ee,
   objhead = 0x0, vcl = 0x12589e13a8, mem = 0x8020014, workreq = {list = {
       vtqe_next = 0x0, vtqe_prev = 0x0}, func = 0, priv =
 0xa5a5a5a5a5a5a5a5},
   acct = {first = 2.2867342108946895e-315, sess = 78841721984, req = 1,
     pipe = 78841696256, pass = 78841697032, fetch = 78841722048,
     hdrbytes = 5162986676217184305, bodybytes = 11177425214313201848},
   nhashptr = 3786983412, ihashptr = 51184323, lhashptr = 2412129124,
   hashptr = 0xd585f48f}
 }}}

 In order to continue testing -h critbit, I really need some feedback on
 this.

 Ticket #408 seems to be a dupe of this one, so I closed it.

-- 
Ticket URL: <http://varnish.projects.linpro.no/ticket/414#comment:1>
Varnish <http://varnish.projects.linpro.no/>
The Varnish HTTP Accelerator