Varnish Close the connection on long files

Goncalo Garcia ghfg at co.sapo.pt
Wed Jun 27 14:29:36 CEST 2012


On Tue, 26 Jun 2012 22:55:49 -0700
Abraham Cruz Sustaita <abraham.sustaita at buscacorp.com> wrote:

> We have a lot of videos. Some of them of 500Mb, and when the users
> want to dowload them, varnish close the connection before the file is
> fully downloaded. We know this is the problem cause everytime we
> receive the same file but with differents sizes. If the connection is
> slow, the file is smaller. I really can't figure where the problem
> is...

Hi,

have you looked at the send_timeout option in varnishd?

% man varnishd

"Send  timeout  for  client  connections. If the HTTP response hasn't
been transmitted in this many seconds the session is closed.  See
setsockopt(2) under SO_SNDTIMEO for more information."

We had a similar situation with some videos, after 60 seconds(default)
the download was interrupted.

ghfg
 
> This is the configuration I have:
> 
> backend app01 {
>   .host = "app01.xxxxxx.com";
>   .port = "80";
>   .connect_timeout = 1.5s;
>   .first_byte_timeout = 45s;
>   .between_bytes_timeout = 30s;
>   .probe = {
>     .url = "/";
>     .interval = 5s;
>     .timeout = 1s;
>     .window = 5;
>     .threshold = 3;
>   }
> }
> director app_director round-robin {
>   { .backend = app01; }
> }
> 
> #acl internal {
> #  "localhost";
> #  "127.0.0.1";
> #}
> 
> acl purge {
>         "localhost";
>         "192.168.11.0"/24;
> }
> 
> sub vcl_recv {
> 
>   if (req.request == "PURGE") {
>     if (!client.ip ~ purge) {
>       error 405 "Not allowed.";
>     }
>     ban("req.url = " + req.url);
>     return (lookup);
>   }
> 
>   # Normalize the host header
>   # --
>   if (req.http.Host ~ "(?i)^(www.)?xxxxx.com") {
>     set req.http.host = "www.xxxxxxx.com";
>   }
> 
>   # Use anonymous, cached pages if all backends are down.
>   if (! req.backend.healthy) {
>     unset req.http.Cookie;
>     set req.grace = 1h;
>   } else {
>     set req.grace = 30s;
>   }
> 
>   # Add the X-Forwarded-For header
>   # --
>   if (req.restarts == 0) {
>     if (req.http.x-forwarded-for) {
>       set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " +
> client.ip;
>     } else {
>       set req.http.X-Forwarded-For = client.ip;
>     }
>   }
> 
>   # Sanely handle the request based on the type
>   # --
>   if (req.request != "GET" &&
>    req.request != "HEAD" &&
>    req.request != "PUT" &&
>    req.request != "POST" &&
>    req.request != "TRACE" &&
>    req.request != "OPTIONS" &&
>    req.request != "DELETE") {
>     /* Non-RFC2616 or CONNECT which is weird. */
>     return (pipe);
>   }
> 
>   # Request by post needs no cache
>   # --
>   if (req.request == "POST") {
>     return (pass);
>   }
> 
>   # Admin needs no cache
>   # --
>   if (req.http.Host ~ "(?i)^(admin.)?xxxxxx.com") {
>     return (pass);
>   }
> 
>   # Always cache the following file types for all users.
>   if (req.url ~
> "(?i)\.(png|gif|jpeg|jpg|ico|swf|flv|avi|mov|wmv|css|js|html|htm)(\?[a-z0-9]+)?$")
> {
>     unset req.http.Cookie;
>     return (lookup);
>   }
> 
>   # Login and logout needs no cache
>   # --
>   if (req.url ~ "^/usuario/salir" || req.url ~
> "^/usuario/redirectWindow" || req.url ~ "^/usuario/al") {
>     return (pass);
>   }
> 
>   # No save at cache the user profile
>   if (req.url ~ "^/usuario/([0-9]+)/(.*?)/") {
>     return (pass);
>   }
> 
>   # Requests to /no-cache/ needs no cache
>   # --
>   if (req.request == "GET" && req.url ~ "^/no-cache") {
>     return (pass);
>   }
>   if (req.request == "GET" && req.url ~ "^/rss/xxxxx") {
>     return (pass);
>   }
> 
>   # Not sure what could match here, but it's in the default.vcl
>   # --
>   if (req.request != "GET" && req.request != "HEAD") {
>     /* We only deal with GET and HEAD by default */
>     return (pass);
>   }
> 
>   # Handle compression correctly. Different browsers send different
>   # "Accept-Encoding" headers, even though they mostly all support
> the same # compression mechanisms. By consolidating these compression
> headers into # a consistent format, we can reduce the size of the
> cache and get more hits.=
>   # @see: http:// varnish.projects.linpro.no/wiki/FAQ/Compression
>   if (req.http.Accept-Encoding) {
>     if (req.http.Accept-Encoding ~ "gzip") {
>       # If the browser supports it, we'll use gzip.
>       set req.http.Accept-Encoding = "gzip";
>     }
>     else if (req.http.Accept-Encoding ~ "deflate") {
>       # Next, try deflate if it is supported.
>       set req.http.Accept-Encoding = "deflate";
>     }
>     else {
>       # Unknown algorithm. Remove it and send unencoded.
>       unset req.http.Accept-Encoding;
>     }
>   }
> 
>   # Cache all requests by default, overriding the standard Varnish
> behavior if (req.request == "GET" || req.request == "HEAD") {
>     /* We only deal with GET and HEAD by default */
>     #unset req.http.Cookie;
>     return (lookup);
>   }
> 
>   # If there's a cookie left aft this point do not cache
>   # or is this easier to keep in the cookie deletion above?
>   # --
>   if (req.http.Authorization || req.http.Cookie) {
>     /* Not cacheable by default */
>     return (pass);
>   }
> 
>   # If we get here, try the cache
>   # --
>   return (lookup);
> 
> }
> 
> sub vcl_pipe {
>   return (pipe);
> }
> 
> sub vcl_pass {
>   return (pass);
> }
> 
> sub vcl_hash {
>   # Include cookie in cache hash.
>   # This check is unnecessary because we already pass on all cookies.
>   #if (req.http.Cookie) {
>   #  #set req.hash += req.http.Cookie;
>   #  hash_data(req.http.Cookie);
>   #}
> }
> 
> sub vcl_hit {
>   if (req.request == "PURGE") {
>     purge;
>     error 200 "Purged";
>   }
> }
> 
> sub vcl_miss {
>   if (req.request == "PURGE") {
>     purge;
>     error 200 "Purged";
>   }
> }
> 
> sub vcl_fetch {
> 
> 
>   # remove all cookies
>   # unset beresp.http.set-cookie;
> 
>   # cache for 12 hours
>   # set beresp.ttl = 2h;
> 
>   # Don't allow static files to set cookies.
>   if (req.url ~
> "(?i)\.(png|gif|jpeg|jpg|ico|swf|css|js|html|htm|mp4|flv)(\?[a-z0-9]+)?$")
> { unset beresp.http.set-cookie;
>     set beresp.ttl = 12h;
>   } else {
>     set beresp.ttl = 30m;
>   }
> 
>   # If the backend server doesn't return properly, don't send another
> connection to it
>   # for 60s and try another backend via restart.
>   #
>   #
> https://www.varnish-cache.org/docs/trunk/tutorial/handling_misbehaving_servers.html
>   # --
>   if(beresp.status == 500 || beresp.status == 503) {
>   if (req.restarts > 3) {
>       set beresp.saintmode = 5m;
>     }
>     if (req.request != "POST") {
>       return(restart);
>     } else {
>       error 500 "Failed";
>     }
>   }
> 
>   # Allow items to be stale if needed.
>   set beresp.grace = 1h;
> 
> }
> 
> sub vcl_deliver {
>  # if (resp.http.magicmarker) {
>  #   unset resp.http.magicmarker;
>  #   set resp.http.age = "0";
>  # }
> 
>   if (obj.hits > 0) {
>     set resp.http.X-Cache = "HIT";
>   } else {
>     set resp.http.X-Cache = "MISS";
>   }
>   return (deliver);
> }
> 
> # In the event of an error, show friendlier messages.
> sub vcl_error {
>   # Otherwise redirect to the homepage, which will likely be in the
> cache. set obj.http.Content-Type = "text/html; charset=utf-8";
>   synthetic {"
> <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "
> http://www.w3.org/TR/html4/loose.dtd">
> <html>
> <head>
> <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
> <title>..:: XXXXX :: Por el momento estamos en mantenimiento
> ::..</title>
> 
> </head>
> <body>
> 
>  <div class="logo">En mantenimiento</div>
> <div class="text">Estamos experimentando problemas. Vuelve
>  más tarde.</div>
>     <div style="display:none;">(Error "} + obj.status + " " +
> obj.response
> + {")</div>
> </body>
> </html>
> "};
>   return (deliver);
> }
> 
> sub vcl_init {
>   return (ok);
> }
> 
> sub vcl_fini {
>   return (ok);
> }



-- 
Chaos umpire sits, 
And by decision more embroils the fray
By which he reigns: next him high arbiter
Chance governs all.
 -- Milton, Paradise Lost
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 198 bytes
Desc: not available
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20120627/bc08bb91/attachment.pgp>


More information about the varnish-misc mailing list