Child panics on OpenSolaris

Paul Wright wrighty+varnishmisc at gmail.com
Wed Feb 10 12:05:30 CET 2010


Hello list,

Using the Letsgetdugg[1] article I've installed Varnish on an
OpenSolaris zone.  During testing it works as expected but when it
receives production traffic I'm seeing children die with three
different types of panics[2][3][4] that look like this:

Panic message: Assert error in TCP_nonblocking(), tcp.c line 172:
Panic message: Assert error in TCP_blocking(), tcp.c line 163:
Assert error in VCA_Prep(), cache_acceptor.c line 163:

I've tried both enabling and disabling KeepAlive on the backend server
but doesn't seem to have any effect.  I've also tried a 2GB and 1GB
malloc cache just in case it was a 32bit issue (it's not and I've
since confirmed it's running as a 64bit process).

The VCL I'm using is pretty simple[5], it normalises the host header
and unsets the cookie header if the request is for a static asset.
This is how I'm starting up Varnish at the moment:

newtask -p highfile /opt/sbin/varnishd -f /opt/etc/varnish/firebox.vcl -F \
-p cc_command='/opt/SunStudioExpress/bin/cc -Kpic -G -m64 -o %o %s' \
-T 127.0.0.1:9001 \
-s malloc,1G \
-p sess_timeout=5s \
-p max_restarts=12 \
-p waiter=poll \
-p connect_timeout=0s \
-p sess_workspace=65536

Is there anything that jumps out as incorrect?  Is there some
additional configuration required for Solaris or are these panics to
be expected?

Cheers,

Paul.

[1] - http://letsgetdugg.com/2009/12/04/varnish-on-solaris/

[2] First panic type:
Child (18997) died signal=6
Child (18997) Panic message: Assert error in TCP_nonblocking(), tcp.c line 172:
  Condition((ioctl(sock, ((int)((uint32_t)(0x80000000|(((sizeof
(int))&0xff)<<16)| ('f'<<8)|126))), &i)) == 0) not true.
errno = 9 (Bad file number)
thread = (cache-worker)
ident = -smalloc,-hcritbit,poll
Backtrace:
  44548b: /opt/sbin/varnishd'pan_backtrace+0x1b [0x44548b]
  445795: /opt/sbin/varnishd'pan_ic+0x1c5 [0x445795]
  fffffd7ff3e5dfec: /opt/lib/libvarnish.so.1.0.0'TCP_nonblocking+0x7c
[0xfffffd7ff3e5dfec]
  419091: /opt/sbin/varnishd'vca_return_session+0x1b1 [0x419091]
  42675d: /opt/sbin/varnishd'cnt_wait+0x2bd [0x42675d]
  42b94a: /opt/sbin/varnishd'CNT_Session+0x4ba [0x42b94a]
  44801b: /opt/sbin/varnishd'wrk_do_cnt_sess+0x19b [0x44801b]
  447614: /opt/sbin/varnishd'wrk_thread_real+0x854 [0x447614]
  447b73: /opt/sbin/varnishd'wrk_thread+0x123 [0x447b73]
  fffffd7ff653acf5: /lib/amd64/libc.so.1'_thrp_setup+0x8d [0xfffffd7ff653acf5]
sp = 866548 {
  fd = 25, id = 25, xid = 0,
  client = 92.41.40.169:2589,
  step = STP_WAIT,
  handling = deliver,
  restarts = 0, esis = 0
  ws = 8665b8 {
    id = "sess",
    {s,f,r,e} = {8672c0,+18,+32786,+65536},
  },
  http[req] = {
    ws = 8665b8[sess]
      "",
      "/i/template/2009/search_icon_1.gif",
      "HTTP/1.1",
      "Accept: */*",
      "Referer:
http://www.firebox.com/product/2579/Yurakoro-Lucky-Cats?aff=1781",
      "Accept-Language: en-gb",
      "User-Agent: Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1;
Trident/4.0; GTB6.4; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR
3.0.4506.2152; .NET CLR 3.5.30729; OfficeLiveConnector.1.3;
OfficeLivePatch.0.0)",
      "Accept-Encoding: gzip, deflate",
      "Connection: Keep-Alive",
      "host: media.firebox.com",
      "X-Forwarded-For: 92.41.40.169",
  },
},


[3] Second panic type:
Child (12024) said Child starts
Child (12024) died signal=6
Child (12024) Panic message: Assert error in TCP_blocking(), tcp.c line 163:
  Condition((ioctl(sock, ((int)((uint32_t)(0x80000000|(((sizeof
(int))&0xff)<<16)| ('f'<<8)|126))), &i)) == 0) not true.
errno = 9 (Bad file number)
thread = (cache-worker)
ident = -smalloc,-hcritbit,poll
Backtrace:
  44548b: /opt/sbin/varnishd'pan_backtrace+0x1b [0x44548b]
  445795: /opt/sbin/varnishd'pan_ic+0x1c5 [0x445795]
  fffffd7ff3e5df5c: /opt/lib/libvarnish.so.1.0.0'TCP_blocking+0x7c
[0xfffffd7ff3e5df5c]
  42b686: /opt/sbin/varnishd'CNT_Session+0x1f6 [0x42b686]
  44801b: /opt/sbin/varnishd'wrk_do_cnt_sess+0x19b [0x44801b]
  447614: /opt/sbin/varnishd'wrk_thread_real+0x854 [0x447614]
  447b73: /opt/sbin/varnishd'wrk_thread+0x123 [0x447b73]
  fffffd7ff653acf5: /lib/amd64/libc.so.1'_thrp_setup+0x8d [0xfffffd7ff653acf5]
  fffffd7ff653afb0: /lib/amd64/libc.so.1'_lwp_start+0x0 [0xfffffd7ff653afb0]
sp = 3491f88 {
  fd = 156, id = 156, xid = 0,
  client = ?.?.?.?:?,
  step = STP_FIRST,
  handling = deliver,
  restarts = 0, esis = 0
  ws = 3491ff8 {
    id = "sess",
    {s,f,r,e} = {3492d00,3492d00,0,+65536},
  },
  http[req] = {
    ws = 3491ff8[sess]
      "",
      "/pic/p2387_search.jpg",
      "HTTP/1.1",
      "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB;
rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20 (.NET CLR 3.5.30729)",
      "Accept: image/png,*/*;q=0.5",
      "Accept-Language: en-gb,en;q=0.5",
      "Accept-Encoding: gzip,deflate",
      "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7",
      "Keep-Alive: 300",
      "Connection: keep-alive",
      "Referer: http://www.firebox.com/admin/allproducts",
      "host: media.firebox.com",
      "X-Forwarded-For: 94.196.164.41",
  },
  worker = fffffd7ff8e08d30 {
    ws = fffffd7ff8e08e78 {
      id = "wrk",
      {s,f,r,e} = {fffffd7ff8df6c40,fffffd7ff8df6c40,0,+65536},
    },
    },
},

[4] Third panic type:
Child (14402) died signal=6
Child (14402) Panic message: Assert error in VCA_Prep(),
cache_acceptor.c line 163:
  Condition((setsockopt(sp->fd, 0xffff, 0x0080, &linger, sizeof
linger)) == 0) not true.
errno = 9 (Bad file number)
thread = (cache-worker)
ident = -smalloc,-hcritbit,poll
Backtrace:
  44548b: /opt/sbin/varnishd'pan_backtrace+0x1b [0x44548b]
  445795: /opt/sbin/varnishd'pan_ic+0x1c5 [0x445795]
  418495: /opt/sbin/varnishd'VCA_Prep+0x255 [0x418495]
  429284: /opt/sbin/varnishd'cnt_first+0xa4 [0x429284]
  42b98e: /opt/sbin/varnishd'CNT_Session+0x4fe [0x42b98e]
  44801b: /opt/sbin/varnishd'wrk_do_cnt_sess+0x19b [0x44801b]
  447614: /opt/sbin/varnishd'wrk_thread_real+0x854 [0x447614]
  447b73: /opt/sbin/varnishd'wrk_thread+0x123 [0x447b73]
  fffffd7ff653acf5: /lib/amd64/libc.so.1'_thrp_setup+0x8d [0xfffffd7ff653acf5]
  fffffd7ff653afb0: /lib/amd64/libc.so.1'_lwp_start+0x0 [0xfffffd7ff653afb0]
sp = 67ef48 {
  fd = 131, id = 131, xid = 0,
  client = 90.196.3.202:52874,
  step = STP_FIRST,
  handling = deliver,
  restarts = 0, esis = 0
  ws = 67efb8 {
    id = "sess",
    {s,f,r,e} = {67fcc0,+19,0,+65536},
  },
  http[req] = {
    ws = 67efb8[sess]
      "",
      "/i/video_graphics/blankpix.gif",
      "HTTP/1.1",
      "Accept: */*",
      "Referer: http://www.firebox.com/product/2277/Naughty-Knot",
      "Accept-Language: en-us",
      "Accept-Encoding: gzip, deflate",
      "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;
SV1; GTB0.0; .NET CLR 2.0.50727)",
      "Connection: Keep-Alive",
      "host: media.firebox.com",
      "X-Forwarded-For: 86.156.9.150",
  },
  worker = fffffd7ff87ebd30 {
    ws = fffffd7ff87ebe78 {
      id = "wrk",
      {s,f,r,e} = {fffffd7ff87d9c40,fffffd7ff87d9c40,0,+65536},
    },
    },
},

[5] firebox.vcl
backend martin {
	.host = "10.0.0.21";
	.port = "80";
}

director dynamic_director round-robin {
	{
		.backend = martin;
	}
}

director static_director round-robin {
	{
		.backend = martin;
	}
}

sub vcl_recv {
	// normalise static requests
	if ( req.http.host ~ "media([0-9]+).firebox.com" ) {
		set req.http.host = "media.firebox.com";
	}
	
	//catch any relative image URLs that haven't been repointed to media
	if ( req.http.host != "media.firebox.com" &&
	     ( req.url ~ "^/pic/.+" || req.url ~ "^/i/.+" ) ) {
			set req.http.host = "media.firebox.com";
	}
	
	// split traffic based on host name
	if ( req.http.host == "media.firebox.com" ) {
		remove req.http.cookie;
		set req.backend = static_director;
	} else {
		// dynamic content that should be cached (ie no cookies required)
		// these patterns should match up with settings.inc on the PHP side
		if ( req.url ~ "^/styles/(.+).css$" ||
		     req.url ~ "^/js/(.+).js$" ) {
			remove req.http.cookie;
		}
		//default all requests to the dynamic backend
		set req.backend = dynamic_director;
	}
}



More information about the varnish-misc mailing list