Varnish 5.2.0 child panic
Hugues Alary
hugues at betabrand.com
Tue Nov 28 23:35:49 UTC 2017
Just realized this might be better as a bug report, I'll submit one if
needed.
Also, I just had another panic:
Panic at: Tue, 28 Nov 2017 22:18:49 GMT
Assert error in HSH_Lookup(), cache/cache_hash.c line 432:
Condition((vary) != 0) not true.
version = varnish-5.2.0 revision 4c4875cbf, vrt api = 6.1
ident = Linux,4.4.64+,x86_64,-junix,-smalloc,-smalloc,-hcritbit,epoll
now = 1786235.707578 (mono), 1511907529.436702 (real)
Backtrace:
0x556f4d169e36: varnishd(+0x4ae36) [0x556f4d169e36]
0x556f4d1b4b80: varnishd(VAS_Fail+0x40) [0x556f4d1b4b80]
0x556f4d15f1b2: varnishd(HSH_Lookup+0xcb2) [0x556f4d15f1b2]
0x556f4d16e14f: varnishd(CNT_Request+0xedf) [0x556f4d16e14f]
0x556f4d18dda2: varnishd(+0x6eda2) [0x556f4d18dda2]
0x556f4d18525c: varnishd(+0x6625c) [0x556f4d18525c]
0x556f4d185780: varnishd(+0x66780) [0x556f4d185780]
0x7f13aa27c494: /lib/x86_64-linux-gnu/libpthread.so.0(+0x7494)
[0x7f13aa27c494]
0x7f13a9fbeaff: /lib/x86_64-linux-gnu/libc.so.6(clone+0x3f)
[0x7f13a9fbeaff]
thread = (cache-worker)
thr.req = 0x7f1245e0a020 {
vxid = 2869347, transport = HTTP/1 {
state = HTTP1::Proc
}
step = R_STP_LOOKUP,
req_body = R_BODY_NONE,
restarts = 0, esi_level = 0,
sp = 0x7f137a00ea20 {
fd = 65, vxid = 2869346,
t_open = 1511907453.132658,
t_idle = 1511907453.132658,
transport = HTTP/1 {
state = HTTP1::Proc
}
client = 10.44.43.4 45520,
privs = 0x7f137a00ea88 {
},
},
worker = 0x7f1388213dd0 {
stack = {0x7f1388214000 -> 0x7f1388181000},
ws = 0x7f1388213e78 {
id = \"wrk\",
{s, f, r, e} = {0x7f1388213190, +0, (nil), +2040},
},
VCL::method = DELIVER,
VCL::return = deliver,
VCL::methods = {},
},
ws = 0x7f1245e0a208 {
id = \"req\",
{s, f, r, e} = {0x7f1245e0c008, +4144, +516080, +516080},
},
http_conn = 0x7f1245e0a130 {
fd = 65 (@0x7f137a00ea38),
doclose = NULL,
ws = 0x7f1245e0a208 {
[Already dumped, see above]
},
{rxbuf_b, rxbuf_e} = {0x7f1245e0c008, 0x7f1245e0cf01},
{pipeline_b, pipeline_e} = {(nil), (nil)},
content_length = -1,
body_status = none,
first_byte_timeout = 0.000000,
between_bytes_timeout = 0.000000,
},
http[req] = 0x7f1245e0a2a0 {
ws = 0x7f1245e0a208 {
[Already dumped, see above]
},
hdrs {
\"GET\",
\"/api/rest/reviews/product/6430\",
\"HTTP/1.1\",
\"Host: www.betabrand.com\",
\"Accept-Encoding: gzip\",
\"CF-IPCountry: US\",
\"CF-RAY: 3c50b2adfddd5a56-BOS\",
\"CF-Visitor: {\"scheme\":\"https\"}\",
\"user-agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_1_2 like Mac OS
X) AppleWebKit/604.3.5 (KHTML, like Gecko) Mobile/15B202
[FBAN/FBIOS;FBAV/150.0.0.32.132;FBBV/80278251;FBDV/iPhone9,1;FBMD/iPhone;FBSN/iOS;FBSV/11.1.2;FBSS/2;FBCR/Verizon;FBID/phone;FBLC/en_US;FBOP/5;FBRV/0]\",
\"accept-language: en-us\",
\"referer:
https://www.betabrand.com/womens/pants/dress-pant-yoga-pants-collection/womens-black-boot-flare-dress-pant-yoga-pants\
",
\"cookie: NaN_hash=d7023912CNFF1437826184182031;
__utma=207700493.1027173649.1511907207.1511907209.1511907209.1;
__utmb=207700493.16.10.1511907209; __utmc=207700493;
__utmz=207700493.1511907209.1.1.utmcsr=Facebook|utmccn=BlackSheepWrap_Booyah|utmcmd=Acquisition|utmctr=ACQ_WMN;
_ga=GA1.2.1027173649.1511907207; _gat_UA-17580748-8=1;
_gid=GA1.2.363913430.1511907207;
_sp_id.ac93=16411174-95e0-4636-a008-9e3cba433d59.1511907211.1.1511907405.1511907211.756129b3-5717-460b-98c9-c7597c30c819;
_sp_ses.ac93=*;
optimizelyBuckets=%7B%223353291595%22%3A%223339030766%22%2C%223397350584%22%3A%223376220538%22%2C%223528387311%22%3A%223528387312%22%2C%228443224466%22%3A%228449793114%22%2C%228247617066%22%3A%228240095367%22%2C%228411440184%22%3A%228414131467%22%2C%228106662898%22%3A%228109860945%22%7D;
sailthru_content=bfba62eaf5e186fe0219ddc33264f66df9f47e5805766810fea9aeef193727f0d9d08cd819b052bb46ae11d6f4f2382032527e3cf48859c6a3a1764af88774438fc9ab569691ee6c97a8146c72e75c31c8d1082e407b7c337fe9b9af43106beb9eb5c66e1957b87d5165561970fe74c12fd3caaaf239d3b03d10e1dc4b5fe514;
_hp2_id.777713614=%7B%22userId%22%3A%220743002369627522%22%2C%22pageviewId%22%3A%225740794725429989%22%2C%22sessionId%22%3A%221322112537315924%22%2C%22identity%22%3Anull%2C%22trackerVersion%22%3A%223.0%22%7D;
_hp2_ses_props.777713614=%7B%22r%22%3A%22http%3A%2F%2Fm.facebook.com
%22%2C%22us%22%3A%22Facebook%22%2C%22um%22%3A%22Acquisition%22%2C%22ut%22%3A%22ACQ_WMN%22%2C%22ua%22%3A%22BlackSheepWrap_Booyah%22%2C%22ts%22%3A1511907219368%2C%22d%22%3A%
22www.betabrand.com%22%2C%22h%22%3A%22%2Fwomens%2Fpants%2Fdress-pant-yoga-pants-collection%22%7D;
__zlcmid=jigvnRPnf0yTRh; _uetsid=_uetf208ea35; _bbli=0; fs_uid=
www.fullstory.com`1bTW`5677711459876864:5629499534213120;
frontend=deadbeefdeadbeefdeadbeefaa;
uuid=a1d09da5-7506-41c2-94b6-6b5acdba0f9c;
betabrand-campaign-last=BlackSheepWrap_Booyah,Facebook,Acquisition,ACQ_WMN,undefined,Tue
Nov 28 2017 17:14:57 GMT-0500 (EST);
betabrand-campaign-session=BlackSheepWrap_Booyah,Facebook,Acquisition,ACQ_WMN,undefined,Tue
Nov 28 2017 17:14:57 GMT-0500 (EST);
optimizelyEndUserId=oeu1444061797597r0.2150009439792484;
optimizelySegments=%7B%22172841198%22%3A%22campaign%22%2C%22172864896%22%3A%22unknown%22%2C%22173083061%22%3A%22true%22%2C%223972941021%22%3A%22blacksheepwrap_booya%22%7D;
sailthru_visitor=94d5d458-c946-4228-9886-1b57e519beeb;
3060738.3440491=67c18f82-2818-44ef-ab9b-f2667055fada;
__qca=P0-1450722180-1511907220479;
tracker_device=ce48cfcc-5b82-483b-8a1b-562959838be8;
__utmt_UA-17580748-1=1; betabrand-timing=1; EcommTest1=2;
betabrand-campaign-first=BlackSheepWrap_Booyah,Facebook,Acquisition,ACQ_WMN,undefined,Tue
Nov 28 2017 17:13:23 GMT-0500 (EST); betabrand-introduction=true;
__cfduid=dcc19056430e6c8dfd790803695cfb1bf1511907198; _gr_ep_sent=1;
_gr_er_sent=1; granify.lasts at 809=1444061806057;
granify.uuid=53534c95-2046-47f9-b9f8-556dbd6e4b4d;
a1ashgd=c9de5eb25c19369cd2a04d2babab5888\",
\"CF-Connecting-IP: 208.64.112.35\",
\"X-Forwarded-Proto: https\",
\"Connection: close\",
\"X-Request-Start: t=1511907453131\",
\"X-Queue-Start: t=1511907453131\",
\"X-Unique-ID: 0A800027:8911_0A2C2B04:01BB_5A1DE07D_30A0ADB:0009\",
\"X-Forwarded-For: 208.64.112.35, 10.128.0.39, 10.44.43.4,
10.44.43.4\",
\"X-PSA-Blocking-Rewrite: betabrand-pagespeed\",
\"Accept: application/json\",
},
},
vcl = {
name = \"boot\",
busy = 135,
discard = 0,
state = auto,
temp = warm,
conf = {
srcname = {
\"/etc/varnish/default.vcl\",
\"Builtin\",
},
},
},
vmods = {
std = {Varnish 5.2.0 4c4875cbf, 0.0},
directors = {Varnish 5.2.0 4c4875cbf, 0.0},
},
flags = {
},
},
thr.busyobj = (nil) {
},
On Tue, Nov 28, 2017 at 11:48 AM, Hugues Alary <hugues at betabrand.com> wrote:
> Hi there!
>
> In the last 18 days, varnish reports its child died 8 times.
>
> We had more traffic than usual those past few days, but I don't think it
> was anything Varnish can't handle.
>
> We run Varnish on Kubernetes on GCP.
>
> I was able to retrieve the last panic via varnishadm (see below). The
> other panics are the same as the one below (same assertion failing, same
> line number). (I would send them too, but it's amazingly difficult to get
> logs out of google cloud).
>
> If the panic is not enough, I can send our configuration file on request.
>
> Panic at: Tue, 28 Nov 2017 13:44:30 GMT
> Assert error in HSH_Lookup(), cache/cache_hash.c line 432:
> Condition((vary) != 0) not true.
> version = varnish-5.2.0 revision 4c4875cbf, vrt api = 6.1
> ident = Linux,4.4.64+,x86_64,-junix,-smalloc,-smalloc,-hcritbit,epoll
> now = 1755374.451854 (mono), 1511876668.180978 (real)
> Backtrace:
> 0x556f4d169e36: varnishd(+0x4ae36) [0x556f4d169e36]
> 0x556f4d1b4b80: varnishd(VAS_Fail+0x40) [0x556f4d1b4b80]
> 0x556f4d15f1b2: varnishd(HSH_Lookup+0xcb2) [0x556f4d15f1b2]
> 0x556f4d16e14f: varnishd(CNT_Request+0xedf) [0x556f4d16e14f]
> 0x556f4d18dda2: varnishd(+0x6eda2) [0x556f4d18dda2]
> 0x556f4d18525c: varnishd(+0x6625c) [0x556f4d18525c]
> 0x556f4d185780: varnishd(+0x66780) [0x556f4d185780]
> 0x7f13aa27c494: /lib/x86_64-linux-gnu/libpthread.so.0(+0x7494)
> [0x7f13aa27c494]
> 0x7f13a9fbeaff: /lib/x86_64-linux-gnu/libc.so.6(clone+0x3f)
> [0x7f13a9fbeaff]
> thread = (cache-worker)
> thr.req = 0x7f0c615bf020 {
> vxid = 59420066, transport = HTTP/1 {
> state = HTTP1::Proc
> }
> step = R_STP_LOOKUP,
> req_body = R_BODY_NONE,
> restarts = 0, esi_level = 0,
> sp = 0x7f12be06de20 {
> fd = 82, vxid = 59420065,
> t_open = 1511876667.587429,
> t_idle = 1511876667.587429,
> transport = HTTP/1 {
> state = HTTP1::Proc
> }
> client = 10.44.11.6 50566,
> privs = 0x7f12be06de88 {
> },
> },
> worker = 0x7f1374734dd0 {
> stack = {0x7f1374735000 -> 0x7f13746a2000},
> ws = 0x7f1374734e78 {
> id = \"wrk\",
> {s, f, r, e} = {0x7f1374734190, +0, (nil), +2040},
> },
> VCL::method = DELIVER,
> VCL::return = deliver,
> VCL::methods = {},
> },
> ws = 0x7f0c615bf208 {
> id = \"req\",
> {s, f, r, e} = {0x7f0c615c1008, +3920, +516080, +516080},
> },
> http_conn = 0x7f0c615bf130 {
> fd = 82 (@0x7f12be06de38),
> doclose = NULL,
> ws = 0x7f0c615bf208 {
> [Already dumped, see above]
> },
> {rxbuf_b, rxbuf_e} = {0x7f0c615c1008, 0x7f0c615c1e27},
> {pipeline_b, pipeline_e} = {(nil), (nil)},
> content_length = -1,
> body_status = none,
> first_byte_timeout = 0.000000,
> between_bytes_timeout = 0.000000,
> },
> http[req] = 0x7f0c615bf2a0 {
> ws = 0x7f0c615bf208 {
> [Already dumped, see above]
> },
> hdrs {
> \"GET\",
> \"/api/rest/reviews/product/6430\",
> \"HTTP/1.1\",
> \"Host: www.betabrand.com\",
> \"Accept-Encoding: gzip\",
> \"CF-IPCountry: US\",
> \"CF-RAY: 3c4dc30d9aae55b2-ORD\",
> \"CF-Visitor: {\"scheme\":\"https\"}\",
> \"user-agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0_3 like Mac OS
> X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A432
> Safari/604.1\",
> \"accept-language: en-us\",
> \"referer: https://www.betabrand.com/womens/pants/womens-black-
> boot-flare-dress-pant-yoga-pants\",
> \"cookie: optimizelyPendingLogEvents=%5B%22n%3Dimage_impression%26u%
> 3Doeu1511755011199r0.44405056492852124%26wxhr%3Dtrue%26time%3D1511876665.
> 858%26f%3D8443224466%2C8247617066%2C8411440184%
> 2C7780900332%2C8106662898%26g%3D%22%5D; sailthru_content=
> d9d08cd819b052bb46ae11d6f4f23820f9f47e5805766810fea9aeef1937
> 27f0f94155821eef8e7bdb5607beb554d9f12fd3caaaf239d3b03d10e1dc4b5fe514;
> NaN_hash=d1015888FXVD1484095712135476; __utma=207700493.554833224.
> 1511755020.1511755021.1511876545.2; __utmb=207700493.5.10.1511876545;
> __utmc=207700493; __utmz=207700493.1511876545.2.2.utmcsr=Google|utmgclid=
> EAIaIQobChMIwbet_7Lh1wIVk7fACh0pGAaaEAAYASAAEgL
> FrPD_BwE|utmccn=Brand_General|utmcmd=Retention|utmctr=(not%20provided);
> _ga=GA1.2.554833224.1511755020; _gac_UA-17580748-1=1.
> 1511876545.EAIaIQobChMIwbet_7Lh1wIVk7fACh0pGAaaEAAYASAAEgLFrPD_BwE;
> _gac_UA-17580748-8=1.1511876545.EAIaIQobChMIwbet_
> 7Lh1wIVk7fACh0pGAaaEAAYASAAEgLFrPD_BwE; _gid=GA1.2.1787521728.1511876545;
> _hp2_id.777713614=%7B%22userId%22%3A%221402506040644741%22%2C%
> 22pageviewId%22%3A%226182215729724641%22%2C%22sessionId%22%3A%
> 224386307704701158%22%2C%22identity%22%3Anull%2C%
> 22trackerVersion%22%3A%223.0%22%7D; _hp2_ses_props.777713614=%7B%
> 22r%22%3A%22https%3A%2F%2Fwww.google.com%2F%22%2C%22us%22%
> 3A%22Google%22%2C%22um%22%3A%22Retention%22%2C%22ua%22%3A%
> 22Brand_General%22%2C%22ts%22%3A1511876545351%2C%22d%22%3A%2
> 2www.betabrand.com%22%2C%22h%22%3A%22%2Fsale%22%7D;
> _sp_id.ac93=0115679b-fdb3-417c-9994-b1387273d811.1511755021.2.1511876663.
> 1511755037.2396b5db-5ad9-4e6e-9061-f19c13780fc6; _sp_ses.ac93=*;
> optimizelyBuckets=%7B%228443224466%22%3A%228449793114%22%2C%
> 228247617066%22%3A%228249041800%22%2C%228411440184%22%3A%
> 228407050657%22%2C%228106662898%22%3A%228109860945%22%7D;
> _gat_UA-17580748-8=1; frontend=deadbeefdeadbeefdeadbeefaa;
> __zlcmid=jhgvWpO31r9clR; __utmt_UA-17580748-1=1; _uetsid=_ueta633ebda;
> _bbli=0; fs_uid=www.fullstory.com`1bTW`4950547299041280:5668600916475904;
> uuid=740c5206-78d1-4902-8896-a5e8e90eb7c8; bb-seed=0.01224735359031548;
> betabrand-campaign-first=Brand_General,Google,
> Retention,undefined,undefined,Tue Nov 28 2017 07:42:23 GMT-0600 (CST);
> betabrand-campaign-last=Brand_General,Google,Retention,undefined,undefined,Tue
> Nov 28 2017 07:42:23 GMT-0600 (CST); betabrand-campaign-session=
> Brand_General,Google,Retention,undefined,undefined,Tue Nov 28 2017
> 07:42:23 GMT-0600 (CST); optimizelyEndUserId=oeu1511755011199r0.44405056492852124;
> optimizelySegments=%7B%22172841198%22%3A%22campaign%
> 22%2C%22172864896%22%3A%22safari%22%2C%22173083061%22%
> 3A%22true%22%2C%223972941021%22%3A%22brand_general%22%7D;
> __qca=P0-1874833837-1511755021423; sailthru_visitor=b78950af-8d6d-4cb6-8d54-a497f97dd6ba;
> betabrand-timing=1; 3060738.3440491=e5d3965e-bb79-4cde-b307-d532ffdec44c;
> tracker_device=34b4b287-87e9-4418-bef5-28ab27d9038c; EcommTest1=2;
> __cfduid=dd480648d530b3001597f1e574200f3611511755011\",
> \"CF-Connecting-IP: 73.44.212.19\",
> \"X-Forwarded-Proto: https\",
> \"Connection: close\",
> \"X-Request-Start: t=1511876667586\",
> \"X-Queue-Start: t=1511876667586\",
> \"X-Unique-ID: 0A80002E:8771_0A2C0B06:01BB_5A1D683B_F1ABD22:0009\",
> \"X-Forwarded-For: 73.44.212.19, 10.128.0.46, 10.44.11.6,
> 10.44.11.6\",
> \"X-PSA-Blocking-Rewrite: betabrand-pagespeed\",
> \"Accept: application/json\",
> },
> },
> vcl = {
> name = \"boot\",
> busy = 92,
> discard = 0,
> state = auto,
> temp = warm,
> conf = {
> srcname = {
> \"/etc/varnish/default.vcl\",
> \"Builtin\",
> },
> },
> },
> vmods = {
> std = {Varnish 5.2.0 4c4875cbf, 0.0},
> directors = {Varnish 5.2.0 4c4875cbf, 0.0},
> },
> flags = {
> },
> },
> thr.busyobj = (nil) {
> },
>
> Thanks for your help!
> Cheers,
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20171128/210c3d50/attachment-0001.html>
More information about the varnish-misc
mailing list