HELP !

Thu Apr 28 06:40:49 CEST 2011

i can't figure out why but all the sudden whenever my varnish server
talks to one of it's configured backends it's adding a trailing '/'
after every request . none the less this is breaking many of my sites
and i'm at a total loss as to why . i've tried to comment out as much
as i can and now am at nearly the bare minimum and it's still broken .

i've tested my backend nginx server and it's serving things correctly
but in my logs whenever varnish connects to a backend i get log
entries that look like this :
"184.73.176.218 - - [28/Apr/2011:04:33:32 +0000]  "GET /css/main.css/
HTTP/1.1" 404 18 "http://congrelate.org/" "Mozilla/5.0 (Macintosh; U;
Intel Mac OS X 10_6_7; en-US) AppleWebKit/534.3 (KHTML, like Gecko)
Chrome/6.0.472.33 Safari/534.3"

not it's asking for /css/main.css/ which is totally wrong ! i don't
want to have to put some crazy logic into my nginx configs to fix this
, but i have no idea why .

find below all of my complete vcl

any help would be much appreciated

--timball

#
# default.vcl
#            --timball at sunlighfoundation.com
#
# main varnish default.vcl for
#
# Wed Apr 27 14:45:21 EDT 2011
# $Id$
#

# Default backend definition.  Set this to point to your content
# server.
backend default {
    .host = "10.122.223.11";
    .port = "80";
}

backend live {
    .host = "10.203.43.106";
    .port = "80";
}

backend bdad {
    .host = "10.242.213.188";
    .port = "80";
}

backend cupcake {
    .host = "10.196.241.236";
    .port = "80";
}

backend capwords {
    .host = "10.126.35.19";
    .port = "80";
}

backend committeewatch {
    .host = "67.207.135.137";
    .port = "80";
}

backend congrelate {
    .host = "67.207.135.137";
    .port = "80";
}

backend natdatcat {
    .host = "66.135.42.55";
    .port = "80";
}

backend earmarkwatch {
    .host = "127.0.0.1";
    .port = "80";
}

backend fara {
    .host = "67.207.135.137";
    .port = "80";
}

#backend default {
#    .host = "127.0.0.1";
#    .port = "8080";
#}
#
# Below is a commented-out copy of the default VCL logic.  If you
# redefine any of these subroutines, the built-in logic will be
# appended to your code.
#
sub vcl_recv {

    if (req.restarts == 0) {
	# remove req.http.X-Forwarded-For;
	# set    req.http.X-Forwarded-For = req.http.rlnclientipaddr;

        if (req.http.x-forwarded-for) {
            set req.http.X-Forwarded-For =
            req.http.X-Forwarded-For ", " client.ip;
        } else {
            set req.http.X-Forwarded-For = client.ip;
        }
    }
    if (req.request != "GET"     &&
        req.request != "HEAD"    &&
        req.request != "PUT"     &&
        req.request != "POST"    &&
        req.request != "TRACE"   &&
        req.request != "OPTIONS" &&
        req.request != "DELETE") {
        /* Non-RFC2616 or CONNECT which is weird. */
            return (pipe);
    }
    if (req.request != "GET" && req.request != "HEAD") {
        /* We only deal with GET and HEAD by default */
        return (pass);
    }
     # LOGIC TO ROUTE HOSTS
     # remove req.http.X-Forwarded-For;
     # set    req.http.X-Forwarded-For = req.http.rlnclientipaddr;
     set req.grace = 30s;

     if (req.request == "GET" && req.url ~ "\.(js)") {
        remove req.http.Cookie;
        remove req.http.Authorization;
        return(lookup);
     }

     ## images
     if (req.request == "GET" && req.url ~
"\.(gif|jpg|jpeg|bmp|png|tiff|tif|ico|img|tga|wmf)$") {
        remove req.http.Cookie;
        remove req.http.Authorization;
        return(lookup);
     }

     ## various other content pages
     if (req.request == "GET" && req.url ~ "\.(css|html)$") {
        remove req.http.Cookie;
        remove req.http.Authorization;
        return(lookup);
     }

     if ( req.http.host ~ "^sunlightfoundation\.com") {
        set req.backend = default;
        set req.http.host = "sunlightfoundation.com";
     } elsif ( req.http.host ~ "(www)\.sunlightfoundation\.com") {
        error 750 "http://sunlightfoundation.com";

     # sunlight live
     } elsif ( req.http.host ~ "live\.sunlightlabs\.com") {
        set req.backend = live;
        set req.http.host = "live.sunlightlabs.com";

     # this host's external ip
     } elsif ( req.http.host ~ "184\.73\.176\.218") {
        error 750 "http://sunlightfoundation.com";

     # old blog links
     } elsif ( req.http.host ~ "blog\.sunlightfoundation\.com") {
        error 750 "http://sunlightfoundation.com/blog";

     # SUNLGIHT CAMPAIGN AD MONITORING
     } elsif ( req.http.host ~ "(www\.)?sunlightcam\.(net|org)") {
        error 750 "http://sunlightcam.com";
     } elsif ( req.http.host ~ "www\.sunlightcam\.com") {
        error 750 "http://sunlightcam.com";
     } elsif ( req.http.host ~ "^sunlightcam\.com") {
        set req.backend = cupcake;
        set req.http.host = "sunlightcam.com";

     } elsif ( req.http.host ~ "(www\.)?campaignadmonitor\.(net|com|org)") {
        error 750 "http://sunlightcam.com";

     # clearspending
     } elsif ( req.http.host ~ "(www\.)?clearspending\.(org|net|com)") {
        error 750 "http://sunlightfoundation.com/clearspending/";

     # poligraft
     #} elsif ( req.http.host ~ "(www\.)?poligraft\.net") {
     #   error 750 "http://poligraft.com";
     } elsif ( req.http.host ~ "(www\.)?poligraft\.(net|org|com)") {
        error 750 "http://poligraft.com";

     # transparencyjobs
     } elsif ( req.http.host ~ "(www\.)?transparencyjobs\.(net|org|com)") {
        error 750 "http://transparencyjobs.com";

     # congrelate
     } elsif ( req.http.host ~ "(www\.)?congrelate\.(net|com)") {
        error 750 "http://congrelate.org";
     } elsif ( req.http.host ~ "www\.congrelate\.org") {
        error 750 "http://congrelate.org";
     } elsif ( req.http.host ~ "^congrelate\.org") {
        set req.backend = congrelate;
        set req.http.host = "congrelate.org";

     # capitolwords
     } elsif ( req.http.host ~ "(www\.)?capitalwords\.(net|com)") {
        error 750 "http://capitolwords.org";
     } elsif ( req.http.host ~ "www\.capitolwords\.org") {
        error 750 "http://capitolwords.org";
     } elsif ( req.http.host ~ "capitolwords\.org") {
        set req.backend = capwords;
        set req.http.host = "capitolwords.org";

     # sunlightmediaservices
     } elsif ( req.http.host ~ "(www\.)?sunlightmediaservices\.(net|org|com)") {
        error 750 "http://sunlightmediaservices.com";

     # transparencycaucus
     } elsif ( req.http.host ~ "(www\.)?transparencycaucus.(net|org|com|us)") {
        error 750 "http://transparencycaucus.org";

     # transparencycaucus
     } elsif ( req.http.host ~ "(www\.)?subsidyscope\.(net|org|com)") {
        error 750 "http://subsidyscope.org";

     # oxtail
     } elsif ( req.http.host ~ "inbox\.influenceexplorer\.com") {
        error 750 "https://inbox.influenceexplorer.com";
     # this gets delt w/ via pound

     # publicmarkup
     } elsif ( req.http.host ~ "(www\.)?publicmarkup\.(net|com)") {
        error 750 "http://publicmarkup.org";

     # politicalpartytime
     } elsif ( req.http.host ~ "(www\.)?politicalpartytime\.(net|org|com)") {
        error 750 "http://politicalpartytime.org";
     } elsif ( req.http.host ~ "partytime\.sunlightfoundation\.com") {
        error 750 "http://politicalpartytime.org";

     # sunlightlabs
     } elsif ( req.http.host ~ "(www\.)?sunlightlabs\.(net|org|com)") {
        error 750 "http://sunlightlabs.com";
     # wiki.sunlightlabs.org
     # XXX think we can axe this XXX
     } elsif ( req.http.host ~ "wiki\.sunlightlabs\.(net|org|com)") {
        error 750 "http://wiki.sunlightlabs.com";
     # XXX think we can axe this XXX
     } elsif ( req.http.host ~ "blog\.sunlightlabs\.(net|org|com)") {
        error 750 "http://sunlightlabs.com/blog";

     # publicequalsonline
     } elsif ( req.http.host ~ "(www\.)?publicequalsonline\.(net|org|com)") {
        error 750 "http://publicequalsonline.com";
     } elsif ( req.http.host ~ "(www\.)?publicmeansonline\.(net|org|com)") {
        error 750 "http://publicequalsonline.com";

     # thepoia
     } elsif ( req.http.host ~ "(www\.)?thepoia\.(net|org|com)") {
        error 750 "http://sunlightfoundation.com/policy/poia/";

     # realtime.sunlightprojects
     } elsif ( req.http.host ~
"(www\.)?realtime.sunlightprojects\.(net|org|com)") {
        error 750 "http://reporting.sunlightfoundation.com/";

     # benefitwiki
     } elsif ( req.http.host ~ "(www\.)?benefitwiki\.(net|org|com)") {
        error 750 "http://www.opencongress.org/wiki/Project:Benefit_Wiki";

     # fara
     } elsif ( req.http.host ~ "(www\.)?foreignlobbying.org\.(net|org|com)") {
        error 750 "http://foreignlobbying.org/";
     } elsif ( req.http.host ~ "fara(db)?\.sunlightfoundation\.com") {
        error 750 "http://foreignlobbying.org/";

     # readthebill
     } elsif ( req.http.host ~ "(www\.)?readthebill\.(net|org|com|info)") {
        error 750 "http://readthebill.org";

     # congresspedia XXX NEED TO FIXME w/ correct redirects
     # rewrite ^/[wW]iki(.*) http://www.opencongress.org/wiki$1 permanent;
     } elsif ( req.http.host ~ "(www\.)?congresspedia\.(net|org|com|info)") {
        error 750 "http://www.opencongress.org/wiki";

     # transparencycorps
     } elsif ( req.http.host ~ "(www\.)?transparencycorps\.(net|org|com)") {
        error 750 "http://transparencycorps.org";

     # pass482
     } elsif ( req.http.host ~ "(www\.)?pass482\.(net|org|com)") {
        error 750 "http://sunlightfoundation.com/pass482/";
     } elsif ( req.http.host ~ "(www\.)?pass223\.(net|org|com)") {
        error 750 "http://sunlightfoundation.com/pass482/";
     } elsif ( req.http.host ~ "(www\.)?72hourrule\.(net|org|com)") {
        error 750 "http://sunlightfoundation.com/pass482/";

     # fortune535
     } elsif ( req.http.host ~ "(www\.)?fortune535\.(net|org|com)") {
        error 750 "http://sunlightfoundation.com/projects/2007/fortune535/";
     } elsif ( req.http.host ~ "fortune535\.sunlightprojects\.org") {
        error 750 "http://sunlightfoundation.com/projects/2007/fortune535/";

     # letourcongresstweet
     } elsif ( req.http.host ~ "(www\.)?letourcongresstweet\.(net|org|com)") {
        error 750 "http://www.sunlightfoundation.com/capitoltweets/";
     } elsif ( req.http.host ~ "(www\.)?capitoltweets\.(net|org|com)") {
        error 750 "http://www.sunlightfoundation.com/capitoltweets/";

     # sunlightalinazinosec
     } elsif ( req.http.host ~ "(www\.)?sunlightalinazinosec\.(net|org|com)") {
        error 750 "http://www.youtube.com/watch?v=dtiMa_xcPLY";

     # TransparencyHub
     } elsif ( req.http.host ~ "(www\.)?TransparencyHub\.(net|org|com)") {
        error 750 "http://www.opencongress.org/wiki/Project:Transparency_Hub";

     # m.transparencycamp.org
     } elsif ( req.http.host ~ "m.transparencycamp\.(net|org|com)") {
        error 750 "http://transparencycamp.org/mobile/";
     } elsif ( req.http.host ~ "m.tcamp\.(net|org|com|us)") {
        error 750 "http://transparencycamp.org/mobile/";

     # old api site
     } elsif ( req.http.host ~ "api\.sunlightlabs\.(net|org|com)") {
        error 750 "http://services.sunlightlabs.com/api/";

     # punchclockmap
     } elsif ( req.http.host ~
"^punchclock(map)?\.sunlight(s|projects)\.(net|org|com)") {
        error 750 "http://sunlightfoundation.com/projects/2007/punchclockmap/";

     # superdelegateinfo
     } elsif ( req.http.host ~ "(www\.)?superdelegateinfo\.(net|org|com)") {
        error 750
"http://www.sourcewatch.org/index.php?title=Portal:Superdelegate_Transparency_Project";

     # transparencycamp.com
     } elsif ( req.http.host ~ "(www\.)?transparencycamp\.(net|org|com)") {
        error 750 "http://transparencycamp.org";
     } elsif ( req.http.host ~ "(www\.)?tcamp\.(net|org|com|us)") {
        error 750 "http://transparencycamp.org";

     # littlesis
     } elsif ( req.http.host ~ "(www\.)?littlesis\.(net|org|com)") {
        error 750 "http://littlesis.org";

     # appsforamerica
     } elsif ( req.http.host ~ "(www\.)?appsforamerica\.(net|org|com|us)") {
        error 750 "http://sunlightlabs.com/contests/appsforamerica/";

     # bdad -- labs olympics 2010
     } elsif ( req.http.host ~ "(www\.)?betterdrawadistrict\.(net|org|com)") {
        error 750 "http://betterdrawadistrict.com";
     } elsif ( req.http.host ~ "www\.betterdrawadistrict\.com") {
        error 750 "http://betterdrawadistrict.com";
     } elsif ( req.http.host ~ "betterdrawadistrict\.com") {
        set req.backend = bdad;
        set req.http.host = "http://betterdrawadistrict.com";

     # committeewatch
     } elsif ( req.http.host ~ "(www\.)?committeewatch\.(net|com)") {
        error 750 "http://committeewatch.org";
     } elsif ( req.http.host ~ "www\.committeewatch\.org") {
        error 750 "http://committeewatch.org";
     } elsif ( req.http.host ~ "(www\.)?committeewatch\.org") {
        set req.backend = committeewatch;
        set req.http.host = "committeewatch.org";

     # natdatcat
     } elsif ( req.http.host ~ "(www\.)?datacatalog\.(net|org|com|us)") {
        error 750 "http://nationaldatacatalog.com/";
     } elsif ( req.http.host ~ "(www\.)?nationaldatacatalog\.(net|org)") {
        error 750 "http://nationaldatacatalog.com/";
     } elsif ( req.http.host ~ "www\.nationaldatacatalog\.com") {
        error 750 "http://nationaldatacatalog.com/";
     } elsif ( req.http.host ~ "nationaldatacatalog\.com") {
        set req.backend = natdatcat;
        set req.http.host = "nationaldatacatalog.com";

     # earmarkwatch
     } elsif ( req.http.host ~ "(db|www)?(\.)?earmarkwatch\.(net|com)") {
        error 750 "http://earmarkwatch.org";
     } elsif ( req.http.host ~ "^www\.earmarkwatch\.org") {
        error 750 "http://earmarkwatch.org";
     } elsif ( req.http.host ~ "^earmarkwatch\.org") {
        set req.backend = earmarkwatch;
        set req.http.host = "earmarkwatch.org";

     # elenasinbox
     } elsif ( req.http.host ~ "(www\.)?elenasinbox\.(net|org|com)") {
        error 750
"http://sunlightfoundation.com/blog/2010/06/25/top-25-viewed-pages-in-elenas-inbox/";

     # foreignlobbying
     } elsif ( req.http.host ~ "(www\.)?foreignlobbying\.(net|com)") {
        error 750 "http://foreignlobbying.org";
     } elsif ( req.http.host ~ "www\.foreignlobbying\.org") {
        error 750 "http://foreignlobbying.org";
     } elsif ( req.http.host ~ "foreignlobbying\.org") {
        set req.backend = fara;
        set req.http.host = "foreignlobbying.org";

     # sunlightlive
     } elsif ( req.http.host ~ "(www\.)?sunlightlive\.(net|org|com)") {
        error 750 "http://sunlightfoundation.com/live";

     # XXXX
     } elsif ( req.http.host ~ "(www\.)?XXXX\.(net|org|com)") {
        error 750 "http://XXXX.com";

     # everything else goes to foundation site
     } else {
        # greatjobbobbauer.org
        # greatamericanhackathon.(net|com|org)
        # fedsubsidywatch.(net|com|org)
        # fedsubsidy.(net|com|org)
        # fedsubsidieswatch.(net|com|org)
        # fedsubsidies.(net|com|org)
        # datatransparency.org
        # datajam.org
        # crooknotes.(net|com|org)
        # congresscommons.(com|org)
        # bearsareawesome.com
        # sunlightprojects.org
        # research.sunlightprojects.org
        # thesunlightfoundation.com;
        # news.sunlightfoundtion.com
        # press.sunlightfoundation.com

        # set req.backend = default;
        # set req.http.host = "sunlightfoundation.com";
        # error 404 "NONE SHALL PASS! Unknown virtual host ... srykbyenow";
        error 750 "http://sunlightfoundation.com";
     }

    if (req.http.Authorization || req.http.Cookie) {
        /* Not cacheable by default */
        return (pass);
    }
    return (lookup);
}
#
sub vcl_pipe {
    # Note that only the first request to the backend will have
    # X-Forwarded-For set.  If you use X-Forwarded-For and want to
    # have it set for all requests, make sure to have:
    # set bereq.http.connection = "close";
    # here.  It is not set by default as it might break some broken web
    # applications, like IIS with NTLM authentication.
    return (pipe);
}
#
sub vcl_pass {
    return (pass);
}

#sub vcl_hash {
#    # Normally it hashes on URL and Host but we rewrite the host
#    # into a VirtualHostBase URL. Therefore we can hash on URL alone.
#    set req.hash += req.url;
#
#    # One needs to include compression state normalised above
#    if (req.http.Accept-Encoding) {
#        set req.hash += req.http.Accept-Encoding;
#    }
#
#    # Differentiate based on login cookie too
#    #set req.hash += req.http.cookie;
#
#    return (hash);
#}
#sub vcl_hash {
#    set req.hash += req.url;
#    if (req.http.host) {
#        set req.hash += req.http.host;
#    } else {
#        set req.hash += server.ip;
#    }
#    return (hash);
#}

sub vcl_hit {
    if (!obj.cacheable) {
        return (pass);
    }
    return (deliver);
}
#
sub vcl_miss {
    return (fetch);
}
#
sub vcl_fetch {
        set req.grace = 30s;
        if (req.url ~ "\.(png|gif|jpg|swf|css|js)$") {
            unset beresp.http.set-cookie;
        }

    #if ( req.url ~ "^/admin") {
    #   return(pass);
    #}
    return(pass);
}
# sub vcl_fetch {
#     if (!beresp.cacheable) {
#         return (pass);
#     }
#     if (beresp.http.Set-Cookie) {
#         return (pass);
#     }
#     return (deliver);
# }
#
sub vcl_deliver {
    return (deliver);
}
#

## deals w/ errors .. 750 from above means trigger this redirect
sub vcl_error {
      if (obj.status == 750) {
          set obj.http.Location = obj.response;
          set obj.status = 301;
          return(deliver);
      }
}

# sub vcl_error {
#     set obj.http.Content-Type = "text/html; charset=utf-8";
#     synthetic {"
# <?xml version="1.0" encoding="utf-8"?>
# <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
#  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
# <html>
#   <head>
#     <title>"} obj.status " " obj.response {"</title>
#   </head>
#   <body>
#     <h1>Error "} obj.status " " obj.response {"</h1>
#     <p>"} obj.response {"</p>
#     <h3>Guru Meditation:</h3>
#     <p>XID: "} req.xid {"</p>
#     <hr>
#     <p>Varnish cache server</p>
#   </body>
# </html>
# "};
#     return (deliver);
# }

# vim:set background=dark:expandtab:shiftwidth=5:tabstop=5:

-- 
        GPG key available on pgpkeys.mit.edu
pub  1024D/511FBD54 2001-07-23 Timothy Lu Hu Ball <timball at tux.org>
Key fingerprint = B579 29B0 F6C8 C7AA 3840  E053 FE02 BB97 511F BD54