[Varnish] #534: Threads stuck in trunk

[Varnish]

#534: Threads stuck in trunk
---------------------------+------------------------------------------------
 Reporter:  anders         |        Owner:  phk  
     Type:  defect         |       Status:  new  
 Priority:  high           |    Milestone:       
Component:  varnishd       |      Version:  trunk
 Severity:  critical       |   Resolution:       
 Keywords:  threads stuck  |  
---------------------------+------------------------------------------------
Comment (by anders):

 PHK asked for a backtrace from one of the stuck threads. I got one now,
 from trunk/4144.

 First of all, info threads shows most threads are in __error:

 {{{
   3298 Thread 0x806b418f0 (LWP 101198)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
   3297 Thread 0x80bb703d0 (LWP 101199)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
   3296 Thread 0x806b3fb40 (LWP 101200)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
   3295 Thread 0x82cf45500 (LWP 101204)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
   3294 Thread 0x8085e80b0 (LWP 101206)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
   3293 Thread 0x806305500 (LWP 100523)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
   3292 Thread 0xb659d4050 (LWP 100648)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
   3291 Thread 0x806305050 (LWP 100662)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
   3290 Thread 0x80cc83d40 (LWP 100731)  0x0000000800ac929c in __error ()
    from /lib/libthr.so.3
 }}}

 Some backtraces for these threads in __eroror:

 1)

 {{{
 (gdb) thread 3291
 [Switching to thread 3291 (Thread 0x806305050 (LWP 100662))]#0
 0x0000000800ac929c in __error () from /lib/libthr.so.3
 (gdb) bt
 #0  0x0000000800ac929c in __error () from /lib/libthr.so.3
 #1  0x0000000800ac8f8c in __error () from /lib/libthr.so.3
 #2  0x0000000800ac41eb in pthread_mutex_getyieldloops_np ()
    from /lib/libthr.so.3
 #3  0x000000000041f7e2 in Lck__Lock (lck=Variable "lck" is not available.
 ) at cache_lck.c:78
 #4  0x000000000042c00b in hcb_lookup (sp=0x809cc0008, noh=0x17e0ebf600)
     at hash_critbit.c:452
 #5  0x000000000041ae3a in HSH_Lookup (sp=0x809cc0008, poh=0x7fffd4aa01e0)
     at cache_hash.c:443
 #6  0x0000000000410a42 in cnt_lookup (sp=0x809cc0008) at
 cache_center.c:734
 #7  0x0000000000412ad3 in CNT_Session (sp=0x809cc0008) at steps.h:38
 #8  0x0000000000422121 in wrk_do_cnt_sess (w=0x7fffd4aa62c0, priv=Variable
 "priv" is not available.
 )
     at cache_pool.c:281
 #9  0x0000000000421417 in wrk_thread_real (qp=0x80100f740,
 shm_workspace=Variable "shm_workspace" is not available.
 )
     at cache_pool.c:176
 #10 0x0000000800abf4d1 in pthread_getprio () from /lib/libthr.so.3
 #11 0x00007fffd48a7000 in ?? ()
 Error accessing memory address 0x7fffd4aa7000: Bad address.
 }}}

 2)

 {{{
 (gdb) thread 3317
 [Switching to thread 3317 (Thread 0x806b4bc70 (LWP 101131))]#0
 0x0000000800ac929c in __error () from /lib/libthr.so.3
 (gdb) bt
 #0  0x0000000800ac929c in __error () from /lib/libthr.so.3
 #1  0x0000000800ac8f8c in __error () from /lib/libthr.so.3
 #2  0x0000000800ac41eb in pthread_mutex_getyieldloops_np ()
    from /lib/libthr.so.3
 #3  0x000000000041f7e2 in Lck__Lock (lck=Variable "lck" is not available.
 ) at cache_lck.c:78
 #4  0x000000000042c00b in hcb_lookup (sp=0x827034008, noh=0x17e0dfd200)
     at hash_critbit.c:452
 #5  0x000000000041ae3a in HSH_Lookup (sp=0x827034008, poh=0x7fffad9681e0)
     at cache_hash.c:443
 #6  0x0000000000410a42 in cnt_lookup (sp=0x827034008) at
 cache_center.c:734
 #7  0x0000000000412ad3 in CNT_Session (sp=0x827034008) at steps.h:38
 #8  0x0000000000422121 in wrk_do_cnt_sess (w=0x7fffad96e2c0, priv=Variable
 "priv" is not available.
 )
     at cache_pool.c:281
 #9  0x0000000000421417 in wrk_thread_real (qp=0x80100f6a0,
 shm_workspace=Variable "shm_workspace" is not available.
 )
     at cache_pool.c:176
 #10 0x0000000800abf4d1 in pthread_getprio () from /lib/libthr.so.3
 #11 0x00007fffad76f000 in ?? ()
 Error accessing memory address 0x7fffad96f000: Bad address.
 }}}

 3)

 {{{
 (gdb) frame 3489
 Error accessing memory address 0x7fffad96f000: Bad address.
 (gdb) bt
 #0  0x0000000800ac929c in __error () from /lib/libthr.so.3
 #1  0x0000000800ac8f8c in __error () from /lib/libthr.so.3
 #2  0x0000000800ac41eb in pthread_mutex_getyieldloops_np ()
    from /lib/libthr.so.3
 #3  0x000000000041f7e2 in Lck__Lock (lck=Variable "lck" is not available.
 ) at cache_lck.c:78
 #4  0x000000000042c00b in hcb_lookup (sp=0x827034008, noh=0x17e0dfd200)
     at hash_critbit.c:452
 #5  0x000000000041ae3a in HSH_Lookup (sp=0x827034008, poh=0x7fffad9681e0)
     at cache_hash.c:443
 #6  0x0000000000410a42 in cnt_lookup (sp=0x827034008) at
 cache_center.c:734
 #7  0x0000000000412ad3 in CNT_Session (sp=0x827034008) at steps.h:38
 #8  0x0000000000422121 in wrk_do_cnt_sess (w=0x7fffad96e2c0, priv=Variable
 "priv" is not available.
 )
     at cache_pool.c:281
 #9  0x0000000000421417 in wrk_thread_real (qp=0x80100f6a0,
 shm_workspace=Variable "shm_workspace" is not available.
 )
     at cache_pool.c:176
 #10 0x0000000800abf4d1 in pthread_getprio () from /lib/libthr.so.3
 #11 0x00007fffad76f000 in ?? ()
 Error accessing memory address 0x7fffad96f000: Bad address.
 }}}


 4)

 {{{
 (gdb) thread 2583
 [Switching to thread 2583 (Thread 0x17e3202ba0 (LWP 102654))]#0
 0x0000000800ac929c in __error () from /lib/libthr.so.3
 (gdb) bt
 #0  0x0000000800ac929c in __error () from /lib/libthr.so.3
 #1  0x0000000800ac8f8c in __error () from /lib/libthr.so.3
 #2  0x0000000800ac41eb in pthread_mutex_getyieldloops_np ()
    from /lib/libthr.so.3
 #3  0x000000000041f7e2 in Lck__Lock (lck=Variable "lck" is not available.
 ) at cache_lck.c:78
 #4  0x000000000042c00b in hcb_lookup (sp=0x828d84008, noh=0x17e0f5c380)
     at hash_critbit.c:452
 #5  0x000000000041ae3a in HSH_Lookup (sp=0x828d84008, poh=0x7fff4e6701e0)
     at cache_hash.c:443
 #6  0x0000000000410a42 in cnt_lookup (sp=0x828d84008) at
 cache_center.c:734
 #7  0x0000000000412ad3 in CNT_Session (sp=0x828d84008) at steps.h:38
 #8  0x0000000000422121 in wrk_do_cnt_sess (w=0x7fff4e6762c0, priv=Variable
 "priv" is not available.
 )
     at cache_pool.c:281
 #9  0x0000000000421417 in wrk_thread_real (qp=0x80100f420,
 shm_workspace=Variable "shm_workspace" is not available.
 )
     at cache_pool.c:176
 #10 0x0000000800abf4d1 in pthread_getprio () from /lib/libthr.so.3
 #11 0x0000000000000000 in ?? ()
 Error accessing memory address 0x7fff4e677000: Bad address.
 }}}

-- 
Ticket URL: <http://varnish.projects.linpro.no/ticket/534#comment:3>
Varnish <http://varnish.projects.linpro.no/>
The Varnish HTTP Accelerator