Varnish not respecting pass to backend for specified http hosts
Stephen Reese
rsreese at gmail.com
Tue Mar 31 21:12:14 UTC 2020
Running Varnish 6.0.6 in a Docker container for several Wordpress sites. I
have several domains that I would like to pass to the backend verse having
them cached, but the configuration is not behaving as intended. Looking to
understand why I am unable to specify which sites I would like to pass. If
I do something like:
if ((req.http.host ~ "(domain.com) || (dev.domain.com)")) {
set req.backend_hint = default;
} else {
return (pass);
}
then every hostname's content is cached where I would expect only the two
specified domains to cache, and everything not defined, to pass. Also, if I
do not specify the configuration, all sites are cached (as expected). If I
use something like:
if ((req.http.host ~ "(domain.com) || (dev.domain.com)")) {
return (pass);
}
then no sites are cached where I would expect everything to cache except
for the two domains. What might be causing this behavior? I looked at the
requests with varnishlog, the undefined domains are indeed being fetched
from the backend verse being cached:
- VCL_call RECV
- VCL_acl NO_MATCH forbidden
- VCL_return pass
- VCL_call HASH
- VCL_return lookup
- VCL_call PASS
- VCL_return fetch
Varnish configuration is attached.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20200331/ee1af07d/attachment-0001.html>
-------------- next part --------------
vcl 4.0;
# Set the default backend web server
backend default {
.host = "app-proxy";
.port = "8080";
# Increase guru timeout
# http://vincentfretin.ecreall.com/articles/varnish-guru-meditation-on-timeout
.first_byte_timeout = 300s;
}
# Forbidden IP ACL
acl forbidden {
}
# Purge ACL
acl purge {
"app-proxy";
"192.168.0.0"/16;
"127.0.0.1";
"localhost";
"172.16.0.0"/16;
"10.0.0.0"/8;
}
# This function is used when a request is send by a HTTP client (Browser)
sub vcl_recv {
# Block the forbidden IP addresse
if (client.ip ~ forbidden) {
return (synth(403, "Forbidden"));
}
if ((req.http.host ~ "(domain.com) || (dev.domain.com)")) {
return (pass);
}
# Compatibility with Apache format log
if (req.restarts == 0) {
if (req.http.X-Pss-Loop == "pagespeed_proxy") {
set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
# Normalize the header, remove the port (in case you're testing this on various TCP ports)
set req.http.Host = regsub(req.http.Host, ":[0-9]+", "");
# Allow purging from ACL
if (req.method == "PURGE") {
# If not allowed then a error 405 is returned
if (!client.ip ~ purge) {
return (synth(405, "This IP is not allowed to send PURGE requests."));
}
#return (purge);
ban("req.http.host == " + req.http.host +
" && req.url == " + req.url);
# Throw a synthetic page so the
# request won't go to the backend.
return(synth(200, "Purge added"));
}
if (req.method == "BAN") {
# Same ACL check as above:
if (!client.ip ~ purge) {
return(synth(403, "Not allowed."));
}
ban("req.http.host == " + req.http.host +
" && req.url == " + req.url);
# Throw a synthetic page so the
# request won't go to the backend.
return(synth(200, "Ban added"));
}
# Only deal with "normal" types
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "PATCH" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# Only cache GET or HEAD requests. This makes sure the POST requests are always passed.
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# Configure grace period, in case the backend goes down
#set req.grace = 15s;
#if (std.healthy(req.backend)) {
# set req.grace = 30s;
#} else {
# unset req.http.Cookie;
# set req.grace = 6h;
#}
# --- Wordpress specific configuration
# Do not cache the RSS feed
if (req.url ~ "/feed") {
return (pass);
}
# Dont Cache WordPress post pages and edit pages
if (req.url ~ "(wp-admin|post\.php|edit\.php|wp-login)") {
return(pass);
}
if (req.url ~ "/wp-cron.php" || req.url ~ "preview=true") {
return (pass);
}
# Pass through the WooCommerce dynamic pages
if (req.url ~ "^/(cart|my-account/*|checkout|wc-api/*|addons|logout|lost-password)") {
return (pass);
}
# Pass through the WooCommerce add to cart
if ( req.url ~ "\?add-to-cart=" ) {
return (pass);
}
# Pass through the WooCommerce API
if (req.url ~ "\?wc-api=" ) {
return (pass);
}
# Remove the "has_js" cookie
set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", "");
# Remove any Google Analytics based cookies
set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
# Remove any Google Analytics based cookies
set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "_gid=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmctr=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmcmd.=[^;]+(; )?", "");
set req.http.Cookie = regsuball(req.http.Cookie, "utmccn.=[^;]+(; )?", "");
# Remove the Disqus cookie
set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(disqus_unique)=[^;]*", "");
# Remove the Quant Capital cookies (added by some plugin, all __qca)
set req.http.Cookie = regsuball(req.http.Cookie, "__qc.=[^;]+(; )?", "");
# Remove the wp-settings-1 cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-1=[^;]+(; )?", "");
# Remove the wp-settings-time-1 cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wp-settings-time-1=[^;]+(; )?", "");
# Remove the wp test cookie
set req.http.Cookie = regsuball(req.http.Cookie, "wordpress_test_cookie=[^;]+(; )?", "");
# Are there cookies left with only spaces or that are empty?
if (req.http.cookie ~ "^ *$") {
unset req.http.cookie;
}
if (!(req.url ~ "(wp-login|wp-admin|cart|my-account|checkout|addons|wordpress-social-login|wp-login\.php|forumPM|members)")) {
unset req.http.cookie;
}
# Cache all static files by Removing all cookies for static files
if (req.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|ico|jpeg|jpg|js|less|pdf|png|rtf|swf|txt|woff|xml)(\?.*)?$") {
unset req.http.Cookie;
return (hash);
}
# Check the cookies for wordpress-specific items
if (req.http.Cookie ~ "wordpress_" || req.http.Cookie ~ "comment_") {
return (pass);
}
if (!req.http.cookie) {
unset req.http.cookie;
}
# Ban outside access to wp-admin
#if (req.url ~ "wp-(login|admin)" && !client.ip ~ purge) {
# error 403 "Forbidden";
#}
# Cache all others requests
# --- End of Wordpress specific configuration
# Normalize Accept-Encoding header and compression
# https://www.varnish-cache.org/docs/3.0/tutorial/vary.html
if (req.http.Accept-Encoding) {
# Do no compress compressed files...
if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";
} else {
unset req.http.Accept-Encoding;
}
}
# Large static files should be piped, so they are delivered directly to the end-user without
# waiting for Varnish to fully read the file first.
# TODO: once the Varnish Streaming branch merges with the master branch, use streaming here to avoid locking.
if (req.url ~ "^[^?]*\.(mp[34]|rar|tar|tgz|gz|wav|zip)(\?.*)?$") {
unset req.http.Cookie;
return (pipe);
}
# Do not cache HTTP authentication and HTTP Cookie
if (req.http.Authorization || req.http.Cookie) {
return (pass);
}
# Exclude caching Ajax requests
if (req.http.X-Requested-With == "XMLHttpRequest") {
return(pass);
}
# Cache all others requests
return (hash);
}
sub vcl_pipe {
# Note that only the first request to the backend will have
# X-Forwarded-For set. If you use X-Forwarded-For and want to
# have it set for all requests, make sure to have:
# set bereq.http.connection = "close";
# here. It is not set by default as it might break some broken web
# applications, like IIS with NTLM authentication.
set bereq.http.Connection = "Close";
return (pipe);
}
# The data on which the hashing will take place
sub vcl_hash {
hash_data(req.url);
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
# If the client supports compression, keep that in a different cache
if (req.http.Accept-Encoding) {
hash_data(req.http.Accept-Encoding);
}
return (lookup);
}
sub vcl_hit {
# Allow purges
if (req.method == "PURGE") {
#purge;
return (synth(200, "Purged Hit"));
}
return (deliver);
}
sub vcl_miss {
# Allow purges
if (req.method == "PURGE") {
#purge;
return (synth(200, "Purged Miss"));
}
return (fetch);
}
# This function is used when a request is sent by our backend (Nginx server)
sub vcl_backend_response {
set beresp.ttl = 1800s;
# Cache static files
if (bereq.url ~ "^[^?]*\.(bmp|bz2|css|doc|eot|flv|gif|ico|jpeg|jpg|js|less|mp[34]|pdf|png|rar|rtf|swf|tar|tgz|txt|wav|woff|xml|zip)(\?.*)?$") {
unset beresp.http.set-cookie;
}
return (deliver);
}
# The routine when we deliver the HTTP request to the user
# Last chance to modify headers that are sent to the client
sub vcl_deliver {
if (obj.hits > 0) {
set resp.http.X-Cache = "cached";
} else {
set resp.http.X-Cache = "uncached";
}
# Remove some headers: PHP version
unset resp.http.X-Powered-By;
# Remove some headers: Apache version & OS
unset resp.http.Server;
# Remove Varnish version
unset resp.http.X-Varnish;
unset resp.http.Via;
# Remove Google ModPageSpeed
unset resp.http.X-Mod-Pagespeed;
return (deliver);
}
More information about the varnish-misc
mailing list