Varnish and mod_pagespeed

[Miguel Gonzalez]

Dear all,

   I have Varnish as a front-end and an Apache/PHP running a Symfony application as a backend.

   I had mod_pagespeed enabled in Apache long time before I decided to put Varnish in front of it. It worked well.

   Since we decided to put Varnish in front, we are getting many errors logged in error_log file of this kind:

   Fri May 10 08:29:15 2013] [warn] [mod_pagespeed 1.3.25.4-2941 @17719] Fetch 
failed for http://www.midomain.com/images//image2.jpg.pagespeed.ic.t1diDHbOw9.jpg, 
status=404

  and others like:
   

   [warn] [mod_pagespeed 1.1.23.2-2258 @28956] Fetch timed out: http://127.0.0.1:8000/media/cache/thumb_image_user6.jpg (35) waiting for 50 ms

   the result is that the webpage is not loading as fast as it did without varnish.

   I have googled around and I found webpages like this one:

   http://serverfault.com/questions/290776/mod-pagespeed-varnish-and-apache-cache-issues-after-new-code-pushes

   but since I'm quite new to Varnish I don't really know what I should do.

   Regards,

   Miguel

here is my default.vcl

backend default {
  .host = "127.0.0.1";
  .port = "8000";
  .connect_timeout = 600s;
  .first_byte_timeout = 600s;
  .between_bytes_timeout = 600s;
}

acl purge {
  "localhost";
}

acl ban {
  "localhost";
}

sub vcl_recv {
  // Strip cookies for static files:
  if (req.url ~ "\.(jpg|jpeg|gif|png|ico|css|zip|tgz|gz|rar|bz2|pdf|txt|tar|wav|bmp|rtf|js|flv|swf|html|htm|xml)$") {
    unset req.http.Cookie;
  }

  // Remove specific cookies:
  set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
  set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
  set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
  if (req.http.cookie ~ "^ *$") { unset req.http.cookie; }
  if (req.http.Cookie ~ "^\s*$") { unset req.http.Cookie; }
  if (req.http.Cookie == "") { remove req.http.Cookie; }

  // Normalize Accept-Encoding header (straight from the manual: https://www.varnish-cache.org/docs/3.0/tutorial/vary.html)
  if (req.http.Accept-Encoding) {
    if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
      // No point in compressing these
      remove req.http.Accept-Encoding;
    } elsif (req.http.Accept-Encoding ~ "gzip") {
      set req.http.Accept-Encoding = "gzip";
    } elsif (req.http.Accept-Encoding ~ "deflate") {
      set req.http.Accept-Encoding = "deflate";
    } else {
      // unkown algorithm
      remove req.http.Accept-Encoding;
    }
  }

  if (req.restarts == 0) {
    if (req.http.x-forwarded-for) {
      set req.http.X-Forwarded-For = req.http.X-Forwarded-For;
    } else {
      set req.http.X-Forwarded-For = client.ip;
    }
  }

  if (req.request != "GET" &&
    req.request != "HEAD" &&
    req.request != "PUT" &&
    req.request != "POST" &&
    req.request != "TRACE" &&
    req.request != "OPTIONS" &&
    req.request != "DELETE") {
      /* Non-RFC2616 or CONNECT which is weird. */
      return (pipe);
  }

  if (req.request != "GET" && req.request != "HEAD") {
    /* We only deal with GET and HEAD by default */
    return (pass);
  }

  if (req.http.Authorization || req.http.Cookie) {
    /* Not cacheable by default */
    return (pass);
  }

  if (req.request == "PURGE") {
    if (!client.ip ~ purge) {
      error 405 "Not allowed.";
    }
    return (lookup);
  }

  if (req.request == "BAN") {
    if (!client.ip ~ ban) {
      error 405 "Not allowed.";
    }
    ban("req.http.host == " + req.http.host + "&& req.url == " + req.url);
    error 200 "Ban added";
  }

  return (lookup);
}

sub vcl_pipe {
  set bereq.http.connection = "close";
  return (pipe);
}

sub vcl_pass {
  return (pass);
}

sub vcl_hash {
  hash_data(req.url);
  if (req.http.host) {
    hash_data(req.http.host);
  } else {
    hash_data(server.ip);
  }

  // If the client supports compression, keep that in a different cache
  if (req.http.Accept-Encoding) {
    hash_data(req.http.Accept-Encoding);
  }

  return (hash);
}

sub vcl_hit {
  if (req.request == "PURGE") {
    purge;
    error 200 "Purged.";
  }

  //Listen to browser force refresh
  if (req.http.Cache-Control ~ "no-cache") {
    if (! (req.http.Via || req.http.User-Agent ~ "bot|MSIE")) {
      set obj.ttl = 0s;
      return (restart);
    }
  }

  return (deliver);
}

sub vcl_miss {
  if (req.request == "PURGE") {
    purge;
    error 200 "Purged.";
  }

  return (fetch);
}

sub vcl_fetch {
  if (beresp.ttl <= 0s ||
    beresp.http.Set-Cookie ||
    beresp.http.Vary == "*") {
      set beresp.ttl = 120 s;
      return (hit_for_pass);
  }
  return (deliver);
}

sub vcl_deliver {
  // Debugging
  if (obj.hits > 0) {
    set resp.http.X-Cache = "HIT";
  } else {
    set resp.http.X-Cache = "MISS";
  }

  // Remove some headers: PHP version
  unset resp.http.X-Powered-By;

  // Remove some headers: Apache version & OS
  unset resp.http.Server;

  return (deliver);
}

sub vcl_error {
  set obj.http.Content-Type = "text/html; charset=utf-8";
  set obj.http.Retry-After = "5";
  synthetic {"
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html>
  <head>
    <title>"} + obj.status + " " + obj.response + {"</title>
  </head>
  <body>
    <h1>Error "} + obj.status + " " + obj.response + {"</h1>
    <p>"} + obj.response + {"</p>
    <h3>Guru Meditation:</h3>
    <p>XID: "} + req.xid + {"</p>
    <hr>
    <p>Varnish cache server</p>
  </body>
</html>
"};
  return (deliver);
}

sub vcl_init {
  return (ok);
}

sub vcl_fini {
  return (ok);
}
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20130510/7042b655/attachment.html>