best node placement for varnish accelaration

Tim Dunphy bluethundr at gmail.com
Mon Apr 14 03:21:03 CEST 2014


>
> I cannot understand what you want to accomplish with your vcl, but in your
> conditions a classic configuration is:
> - F5 balance the vip on the two cache
> - the cache balance on the 3 web servers as backend


Hello,

 Thanks for your input. That was exactly what needed to confirm what I was
thinking we'd ought to do. I'm going to go ahead and recommend that we take
the web servers out of the vip pool and instead point the vip at the two
varnish cache nodes. I'm thinking we'll need a heartbeat established
between the two (something like keepalived) to enable the failover so that
each node can assume the identity of the VIP ip.

But I'm sorry if what I posted from my config was unclear.

All I am really still curious about at this point is whether I should post
this section on my first node:

if (req.restarts == 0) {
        if (client.ip == "10.10.40.8" || client.ip == "10.10.40.9") {
            set req.backend = www;
        } elsif (server.ip == "10.10.40.8") {
            set req.backend = varnish2;
        } else {
            set req.backend = varnish1;
        }
    } elsif (req.restarts >= 2) {
        return (pass);

And this configuration on the second node:

if (client.ip == "10.10.40.8" || client.ip == "10.10.40.9") {
            set req.backend = www;
        } elsif (server.ip == "10.10.40.9") {
            set req.backend = varnish1;
        } else {
            set req.backend = varnish2;
        }


And I am wondering what the purpose of that stanza is. In the config I am
inheriting the first version of the stanza that I show (the one on top) is
present in the same exact way on both varnish nodes. However what I am
thinking is that it needs to be varied the way that I am demonstrating here
from one machine to the other.

In the demo I'm showing the first varnish node is 10.10.40.8 and the second
varnish node is 10.10.40.9.

I'd really love to be clear on what that stanza is trying to accomplish.

Here's the full config to provide some context.


backend web1 {
    .host = "10.10.40.42";
    .port = "80";
    .connect_timeout = 45s;
    .first_byte_timeout = 45s;
    .between_bytes_timeout = 45s;
    .max_connections = 70;
    .probe = {
        .url = "/healthcheck.php";
        .timeout = 5s;
        .interval = 30s;
        .window = 10;
        .threshold = 1;
    }
}

backend web2 {
    .host = "10.10.40.10";
    .port = "80";
    .connect_timeout = 45s;
    .first_byte_timeout = 45s;
    .between_bytes_timeout = 45s;
    .max_connections = 70;
    .probe = {
        .url = "/healthcheck.php";
        .timeout = 5s;
        .interval = 30s;
        .window = 10;
        .threshold = 1;
    }
}

backend web3 {
    .host = "10.10.40.11";
    .port = "80";
    .connect_timeout = 45s;
    .first_byte_timeout = 45s;
    .between_bytes_timeout = 45s;
    .max_connections = 70;
    .probe = {
        .url = "/healthcheck.php";
        .timeout = 5s;
        .interval = 30s;
        .window = 10;
        .threshold = 1;
    }
}

backend web4 {
    .host = "10.10.40.12";
    .port = "80";
    .connect_timeout = 45s;
    .first_byte_timeout = 45s;
    .between_bytes_timeout = 45s;
    .max_connections = 70;
    .probe = {
        .url = "/healthcheck.php";
        .timeout = 5s;
        .interval = 30s;
        .window = 10;
        .threshold = 1;
    }
}

acl purge {
    "localhost";
    "127.0.0.1";
    "10.10.40.8";
    "10.10.40.9";
}

director www round-robin {
    { .backend = web1; }
    { .backend = web2; }
    { .backend = web3; }
    { .backend = web4; }
}

sub vcl_recv {
    set req.backend = www;
    set req.grace = 6h;
    if (!req.backend.healthy) {
        set req.grace = 24h;
    }
    set req.http.X-Forwarded-For = req.http.X-Forwarded-For ", " client.ip;
    if (req.http.host ~ "^origin\.(.+\.|)my_site_tv\.com$") {
      return (pass);
    }
    if (req.http.host ~ ".*\.my_site_tv.com|my_site_tv.com") {
        /* allow (origin.)stage.m.my_site_tv.com to be a separate host */
        if (req.http.host != "stage.m.my_site_tv.com") {
            set req.http.host = "stage.my_site_tv.com";
        }
    } else {
        return (pass);
    }
    if (req.request == "PURGE") {
        if (!client.ip ~ purge) {
            error 405 "Not allowed.";
        }
        return (lookup);
    }
    if (req.request != "GET" &&
        req.request != "HEAD" &&
        req.request != "PUT" &&
        req.request != "POST" &&
        req.request != "TRACE" &&
        req.request != "OPTIONS" &&
        req.request != "DELETE") {
            return (pipe);
    }
    if (req.request != "GET" && req.request != "HEAD") {
        return (pass);
    }
    if (req.url ~ "sites/all/modules/custom/my_site__ad/ads.html\?.*") {
      set req.url = "/sites/all/modules/custom/my_site__ad/ads.html";
    }
    if (req.url ~ "eyeblaster/addineyeV2.html\?.*") {
        set req.url = "/eyeblaster/addineyeV2.html";
    }
    if (req.url ~
"ahah_helper\.php|my_site__points\.php|install\.php|update\.php|cron\.php|/json(:?\?.*)?$")
{
        return (pass);
    }
    if (req.http.Authorization) {
        return (pass);
    }
    if (req.url ~ "login" || req.url ~ "logout") {
        return (pass);
    }
    if (req.url ~ "^/admin/" || req.url ~ "^/node/add/") {
        return (pass);
    }
    if (req.http.Cache-Control ~ "no-cache") {
        // return (pass);
    }
    if (req.http.Cookie ~
"(VARNISH|DRUPAL_UID|LOGGED_IN|SESS|_twitter_sess)") {
        set req.http.Cookie = regsuball(req.http.Cookie,
"(^|;\s*)(__[a-z]+|has_js)=[^;]*", "");
        set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", "");
    } else {
        unset req.http.Cookie;
    }
    /* removed varnish cache backend logic */
    if (req.restarts == 0) {
        set req.backend = www;
    } elsif (req.restarts >= 2) {
        return (pass);
    }
    if (req.restarts >= 2) {
        return (pass);
    }
    if (req.url ~
"\.(ico|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|ICO|JPG|JPEG|PNG|GIF|GZ|TGZ|BZ2|TBZ|MP3|OOG|SWF)")
{
        unset req.http.Accept-Encoding;
    }
    if (req.url ~
"^/(sites/all/modules/my_site_tv_admanager/includes/ads.php|doubleclick/DARTIframe.html)(\?.*|)$")
{
        set req.url = regsub(req.url, "\?.*$", "");
    }
    if (req.http.Accept-Encoding ~ "gzip") {
        set req.http.Accept-Encoding = "gzip";
    } elsif (req.http.Accept-Encoding ~ "deflate") {
        set req.http.Accept-Encoding = "deflate";
    } else {
        unset req.http.Accept-Encoding;
    }
    return (lookup);
}

sub vcl_pipe {
    set bereq.http.connection = "close";
    return (pipe);
}

sub vcl_pass {
    return (pass);
}

sub vcl_hash {
    set req.hash += req.url;
    set req.hash += req.http.host;
    if (req.http.Cookie ~ "VARNISH|DRUPAL_UID|LOGGED_IN") {
        set req.hash += req.http.Cookie;
    }
    return (hash);
}

sub vcl_hit {
    if (req.request == "PURGE") {
        set obj.ttl = 0s;
        error 200 "Purged.";
    }
}

sub vcl_fetch {
    if (beresp.status == 500) {
        set req.http.X-Varnish-Error = "1";
        restart;
    }
    set beresp.grace = 6h;
    # Set a short circuit cache lifetime for resp codes above 302
    if (beresp.status > 302) {
    set beresp.ttl = 60s;
    set beresp.http.Cache-Control = "max-age = 60";
    }
    if (beresp.http.Edge-control ~ "no-store") {
        set beresp.http.storage = "1";
        set beresp.cacheable = false;
        return (pass);
    }
    if (beresp.status >= 300 || !beresp.cacheable) {
        set beresp.http.Varnish-X-Cacheable = "Not Cacheable";
        set beresp.http.storage = "1";
        return (pass);
    }
    if (beresp.http.Set-Cookie) {
        return (pass);
    }
    if (beresp.cacheable) {
        unset beresp.http.expires;
        set beresp.ttl = 600s;
        set beresp.http.Cache-Control = "max-age = 600";
        if (req.url ~
"\.(ico|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|ICO|JPG|JPEG|PNG|GIF|GZ|TGZ|BZ2|TBZ|MP3|OOG|SWF)")
{
            set beresp.ttl = 43829m;
            set beresp.http.Cache-Control = "max-age = 1000000";
        }
    }
    return (deliver);
}

sub vcl_deliver {
    if (obj.hits > 0) {
        set resp.http.Varnish-X-Cache = "HIT";
        set resp.http.Varnish-X-Cache-Hits = obj.hits;
    } else {
        set resp.http.Varnish-X-Cache = "MISS";
    }
    return (deliver);
}

sub vcl_error {
    if (req.restarts == 0) {
        return (restart);
    }
    if (req.http.X-Varnish-Error != "1") {
        set req.http.X-Varnish-Error = "1";
        return (restart);
    }
    set obj.http.Content-Type = "text/html; charset=utf-8";
    synthetic {"

    <?xml version="1.0" encoding="utf-8"?>
    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
    <html>
        <head>
            <title>my_site_tv.com "} obj.status " " obj.response {"</title>
            <style type="text/css">
                body {background-color: #fff; margin: auto;}
                #page {width: 1000px; margin: 0 auto; background-color:
#fff;}
            </style>
        </head>
        <body>
            <div id="page">
                <img src="
http://www.my_site_tv.com/media/site-unavailable/site-unavailable.jpg"
width="1000" height="700" />
                <!--
                <h4>Debug</h4>
                <pre>
                    Status: "} obj.status {"
                    Response: "} obj.response {"
                    XID: "} req.xid {"
                </pre>
                -->
            </div>
        </body>
    </html>

    "};
    return (deliver);
}


Thanks
Tim


On Sun, Apr 13, 2014 at 4:15 AM, emilio brambilla <emilio at antispam.it>wrote:

> hello,
>
>
> On 2014/04/12 22:00, Tim Dunphy wrote:
>
>>
>>  So if the load balancer is handling all the traffic into the site and
>> the caching hosts are not referenced in the load balancer, don't things
>> need to be structured differently in order for the site to benefit from the
>> acceleration they are trying to use?
>>
> I cannot understand what you want to accomplish with your vcl, but in your
> conditions a classic configuration is:
>
> - F5 balance the vip on the two cache
> - the cache balance on the 3 web servers as backend
>
> --
> bye,
> emilio
>
>
> _______________________________________________
> varnish-misc mailing list
> varnish-misc at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-misc
>



-- 
GPG me!!

gpg --keyserver pool.sks-keyservers.net --recv-keys F186197B
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20140413/8d834789/attachment-0001.html>


More information about the varnish-misc mailing list