[Varnish] #275: Varnish child stops responding
Varnish
varnish-bugs at projects.linpro.no
Wed Jul 16 12:18:05 CEST 2008
#275: Varnish child stops responding
----------------------+-----------------------------------------------------
Reporter: chenxy | Owner: phk
Type: defect | Status: new
Priority: high | Milestone: Varnish 2.0 code complete
Component: varnishd | Version: trunk
Severity: normal | Keywords: Varnish child stuck
----------------------+-----------------------------------------------------
I am running Varnish trunk up to date to commit 2945, on FreeBSD/amd64
7.0-RELEASE.
many times,my Varnish server stopped responding (just getting timeouts).
I changed some codes for core dump
changed file:
mgt_child.c line 171 :
"(void)kill(child_pid, SIGKILL); " changed to "(void)kill(child_pid,
SIGABRT);"
run environment:
{{{
varnishd -F -a 0.0.0.0:80 -f /usr/local/etc/vcl.conf \
-u nobody -g nobody \
-T 10.71.5.25:8080 \
-s file,/data0/varnish/v,16G \
-s file,/data1/varnish/v,16G \
-s file,/data2/varnish/v,16G \
-s file,/data3/varnish/v,16G \
-w 300,3000,600 \
-h classic,500009 \
-p thread_pools=2 \
-p thread_pool_max=3000 \
-p sess_workspace=16384 -p listen_depth=4096 \
-p lru_interval=3600 \
-p ping_interval=10 >> $logdir/varnish.out 2>&1
}}}
{{{
ulimit -a
cpu time (seconds, -t) unlimited
file size (512-blocks, -f) unlimited
data seg size (kbytes, -d) 33554432
stack size (kbytes, -s) 524288
core file size (512-blocks, -c) unlimited
max memory size (kbytes, -m) unlimited
locked memory (kbytes, -l) unlimited
max user processes (-u) 5547
open files (-n) 98304
virtual mem size (kbytes, -v) unlimited
sbsize (bytes, -b) unlimited
}}}
log information:
{{{
Jul 15 16:58:54 236-25 varnishd[19424]: child (19429) Started
Jul 15 16:58:54 236-25 varnishd[19424]: Child (19429) said Closed fds: 7 8
11 12 14 15
Jul 15 16:58:54 236-25 varnishd[19424]: Child (19429) said Child starts
Jul 15 16:58:54 236-25 varnishd[19424]: Child (19429) said managed to mmap
17179869184 bytes of 17179869184
Jul 15 16:58:54 236-25 varnishd[19424]: Child (19429) said Ready
Jul 16 15:30:09 236-25 varnishd[19424]: Child (19429) not responding to
ping, killing it.
Jul 16 15:30:56 236-25 varnishd[19424]: Child (19429) not responding to
ping, killing it.
Jul 16 15:30:59 236-25 kernel: pid 19429 (varnishd), uid 65534: exited on
signal 6 (core dumped)
Jul 16 15:30:59 236-25 varnishd[19424]: child (34746) Started
Jul 16 15:31:00 236-25 varnishd[19424]: Child (34746) said Closed fds: 7 8
11 12 14 15
Jul 16 15:31:00 236-25 varnishd[19424]: Child (34746) said Child starts
Jul 16 15:31:00 236-25 varnishd[19424]: Child (34746) said managed to mmap
17179869184 bytes of 17179869184
Jul 16 15:31:00 236-25 varnishd[19424]: Child (34746) said Ready
}}}
My VCL:
{{{
backend image2 {
.host = "10.71.5.248";
.port = "80";
}
backend ad4 {
.host = "10.71.5.247";
.port = "80";
}
sub vcl_hash {
set req.hash += req.url;
set req.hash += req.http.host;
hash;
}
sub vcl_hit {
deliver;
}
sub vcl_miss {
fetch;
}
sub vcl_recv {
if ( req.request != "GET" && req.request != "HEAD" ) {
error 403 "Invalid request method.";
} elseif ( req.http.host == "image2.myserver.com.cn" ||
req.http.host ~ ".myserverimg.cn$" ) {
set req.backend = image2;
} elseif ( req.http.host ~ "^d[0-9].myserver.com.cn$" ||
req.http.host == "ad4.myserver.com.cn" || req.http.host ~
"^a[0-9].myserver.com.cn$" ) {
set req.backend = ad4;
} else {
error 403 "Request denied.";
}
if (req.request == "GET" && req.http.cookie) {
lookup;
}
lookup;
}
sub vcl_fetch {
if ( req.http.host == "image2.myserver.com.cn" || req.http.host ~
".myserverimg.cn$" ) {
set obj.ttl = 86400s;
} elseif ( req.http.host ~ "^d[0-9].myserver.com.cn$" ||
req.http.host == "ad4.myserver.com.cn" || req.http.host ~
"^a[0-9].myserver.com.cn$" ) {
set obj.ttl = 300s;
}
if (obj.http.Set-Cookie) {
insert;
}
insert;
}
sub vcl_deliver {
deliver;
}
sub vcl_timeout {
discard;
}
sub vcl_discard {
discard;
}
}}}
Backtrace:
{{{
......
[New Thread 0x806105ec0 (LWP 100128)]
[New Thread 0x806103f20 (LWP 100105)]
[New Thread 0x806103960 (LWP 100101)]
[New Thread 0x806102de0 (LWP 100091)]
[New Thread 0x806102990 (LWP 100087)]
[New Thread 0x8061020f0 (LWP 100076)]
[New Thread 0x806101f80 (LWP 100074)]
[New Thread 0x806101e10 (LWP 100073)]
[New Thread 0x8061016e0 (LWP 100060)]
[New Thread 0x800f01850 (LWP 100056)]
[New Thread 0x800f016e0 (LWP 100055)]
[New Thread 0x800f01570 (LWP 100049)]
[New Thread 0x800f01400 (LWP 100048)]
[New Thread 0x800f01290 (LWP 100046)]
[New Thread 0x800f01120 (LWP 100448)]
(gdb) bt
#0 0x0000000800dae871 in fwrite () from /lib/libc.so.7
#1 0x0000000800da428b in strchr () from /lib/libc.so.7
#2 0x0000000800da5f13 in strchr () from /lib/libc.so.7
#3 0x0000000800d2702c in vsnprintf () from /lib/libc.so.7
#4 0x00000000004329b3 in VSL (tag=SLT_WorkThread, id=0, fmt=0x43c630 "%p
start") at shmlog.c:159
#5 0x000000000041c0de in wrk_thread (priv=0x800f620b0) at
cache_pool.c:221
#6 0x0000000800aaba38 in pthread_getprio () from /lib/libthr.so.3
#7 0x00007fff4dc73000 in ?? ()
Cannot access memory at address 0x7fff4de73000
(gdb) frame 5
#5 0x000000000041c0de in wrk_thread (priv=0x800f620b0) at
cache_pool.c:221
221 VSL(SLT_WorkThread, 0, "%p start", w);
(gdb) print *w
$1 = {magic = 1670491599, nobjhead = 0x0, nobj = 0x0, used =
1216193378.3797276, cond = 0x1825a1b340, list = {
vtqe_next = 0x0, vtqe_prev = 0x0}, wrq = 0x0, wfd = 0x0, werr = 0, iov
= {{iov_base = 0x0,
iov_len = 0} <repeats 64 times>}, niov = 0, liov = 0, vcl = 0x0,
srcaddr = 0x0, acct = {first = 0,
sess = 0, req = 0, pipe = 0, pass = 0, fetch = 0, hdrbytes = 0,
bodybytes = 0},
wlb = 0x7fff4de70ad0 "\004", wlp = 0x7fff4de70ad0 "\004", wle =
0x7fff4de72ad0 "?-\221c", wlr = 0}
(gdb)
}}}
--
Ticket URL: <http://varnish.projects.linpro.no/ticket/275>
Varnish <http://varnish.projects.linpro.no/>
The Varnish HTTP Accelerator
More information about the varnish-bugs
mailing list