Problem with Amazon S3 and varnish
Abraham Cruz Sustaita
abraham.sustaita at buscacorp.com
Mon Jul 2 20:24:42 CEST 2012
I was able to fix this problem.
Amazon have some issues when using -website-us-east-1 so the only thing I
had to do was remove that from the name of all the buckets and now it is
working as it have to work. My vcl now is this:
backend app01 {
.host = "app01.site.com";
.port = "80";
.connect_timeout = 1.5s;
.first_byte_timeout = 45s;
.between_bytes_timeout = 30s;
.probe = {
.url = "/";
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
backend app02 {
.host = "app02.site.com";
.port = "80";
.connect_timeout = 1.5s;
.first_byte_timeout = 45s;
.between_bytes_timeout = 30s;
.probe = {
.url = "/";
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
backend app03 {
.host = "app03.site.com";
.port = "80";
.connect_timeout = 1.5s;
.first_byte_timeout = 45s;
.between_bytes_timeout = 30s;
.probe = {
.url = "/";
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
backend app04 {
.host = "app04.site.com";
.port = "80";
.connect_timeout = 1.5s;
.first_byte_timeout = 45s;
.between_bytes_timeout = 30s;
.probe = {
.url = "/";
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
backend app05 {
.host = "app05.site.com";
.port = "80";
.connect_timeout = 1.5s;
.first_byte_timeout = 45s;
.between_bytes_timeout = 30s;
.probe = {
.url = "/";
.interval = 5s;
.timeout = 1s;
.window = 5;
.threshold = 3;
}
}
director app_director round-robin {
{ .backend = app01; }
{ .backend = app02; }
{ .backend = app03; }
{ .backend = app04; }
{ .backend = app05; }
}
# Amazon S3 Backend site4
# --
backend s3 {
.host = "site4.s3.amazonaws.com";
.port = "80";
}
# Amazon S3 Backend site3
# --
backend s3sdr {
.host = "site3.s3.amazonaws.com";
.port = "80";
}
# Amazon S3 Backend site2
# --
backend s3lvl {
.host = "site2.s3.amazonaws.com";
.port = "80";
}
# Amazon S3 Backend site1
# --
backend s3gam {
.host = "site1.s3.amazonaws.com";
.port = "80";
}
#acl internal {
# "localhost";
# "127.0.0.1";
#}
acl purge {
"localhost";
"192.168.11.0"/24;
}
sub vcl_recv {
if (req.request == "PURGE") {
if (!client.ip ~ purge) {
error 405 "Not allowed.";
}
ban("req.url = " + req.url);
return (lookup);
}
set req.backend = app_director;
# Peticion a site4
if (req.http.Host == "s3.site4.site.com" && req.url ~
"\.(flv|mp4|jpg|xml|png|gif)(\?[a-z0-9]+)$") {
unset req.http.cookie;
unset req.http.cache-control;
unset req.http.pragma;
unset req.http.expires;
unset req.http.etag;
unset req.http.X-Forwarded-For;
set req.backend = s3;
set req.http.host = "site4.s3.amazonaws.com";
return (lookup);
}
#Peticion a site3
if (req.http.Host == "s3.site3.site.com" && req.url ~
"\.(flv|mp4|jpg|xml|png|gif)(\?[a-z0-9]+)$") {
unset req.http.cookie;
unset req.http.cache-control;
unset req.http.pragma;
unset req.http.expires;
unset req.http.etag;
unset req.http.X-Forwarded-For;
set req.backend = s3sdr;
set req.http.host = "site3.s3.amazonaws.com";
return (lookup);
}
#Peticion a site2
if (req.http.Host == "s3.site2.site.com" && req.url ~
"\.(flv|mp4|jpg|xml|png|gif)(\?[a-z0-9]+)$") {
unset req.http.cookie;
unset req.http.cache-control;
unset req.http.pragma;
unset req.http.expires;
unset req.http.etag;
unset req.http.X-Forwarded-For;
set req.backend = s3lvl;
set req.http.host = "site2.s3.amazonaws.com";
return (lookup);
}
#Peticion a site1
if (req.http.Host == "s3.site1.site.com" && req.url ~
"\.(flv|mp4|jpg|xml|png|gif)(\?[a-z0-9]+)$") {
unset req.http.cookie;
unset req.http.cache-control;
unset req.http.pragma;
unset req.http.expires;
unset req.http.etag;
unset req.http.X-Forwarded-For;
set req.backend = s3gam;
set req.http.host = "site1.s3.amazonaws.com";
return (lookup);
}
# Use anonymous, cached pages if all backends are down.
if (! req.backend.healthy) {
unset req.http.Cookie;
set req.grace = 1h;
} else {
set req.grace = 30s;
}
# Add the X-Forwarded-For header
# --
if (req.restarts == 0) {
if (req.http.x-forwarded-for) {
set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " +
client.ip;
} else {
set req.http.X-Forwarded-For = client.ip;
}
}
# Sanely handle the request based on the type
# --
if (req.request != "GET" &&
req.request != "HEAD" &&
req.request != "PUT" &&
req.request != "POST" &&
req.request != "TRACE" &&
req.request != "OPTIONS" &&
req.request != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# Request by post needs no cache
# --
if (req.request == "POST") {
return (pass);
}
# Admin needs no cache
# --
if (req.http.Host ~ "(?i)^(admin.)?site.com") {
return (pass);
}
# Always cache the following file types for all users.
if (req.url ~
"(?i)\.(png|gif|jpeg|jpg|ico|swf|flv|avi|mov|wmv|css|js|html|htm)(\?[a-z0-9]+)?$")
{
unset req.http.Cookie;
return (lookup);
}
# Login and logout needs no cache
# --
if (req.url ~ "^/usuario/salir" || req.url ~ "^/usuario/redirectWindow"
|| req.url ~ "^/usuario/al") {
return (pass);
}
# No save at cache the user profile
if (req.url ~ "^/usuario/([0-9]+)/(.*?)/") {
return (pass);
}
# Requests to /no-cache/ needs no cache
# --
if (req.request == "GET" && req.url ~ "^/no-cache") {
return (pass);
}
if (req.request == "GET" && req.url ~ "^/rss/") {
return (pass);
}
# Not sure what could match here, but it's in the default.vcl
# --
if (req.request != "GET" && req.request != "HEAD") {
/* We only deal with GET and HEAD by default */
return (pass);
}
# Handle compression correctly. Different browsers send different
# "Accept-Encoding" headers, even though they mostly all support the same
# compression mechanisms. By consolidating these compression headers into
# a consistent format, we can reduce the size of the cache and get more
hits.=
# @see: http:// varnish.projects.linpro.no/wiki/FAQ/Compression
if (req.http.Accept-Encoding) {
if (req.http.Accept-Encoding ~ "gzip") {
# If the browser supports it, we'll use gzip.
set req.http.Accept-Encoding = "gzip";
}
else if (req.http.Accept-Encoding ~ "deflate") {
# Next, try deflate if it is supported.
set req.http.Accept-Encoding = "deflate";
}
else {
# Unknown algorithm. Remove it and send unencoded.
unset req.http.Accept-Encoding;
}
}
# Cache all requests by default, overriding the standard Varnish behavior
if (req.request == "GET" || req.request == "HEAD") {
/* We only deal with GET and HEAD by default */
#unset req.http.Cookie;
return (lookup);
}
# If there's a cookie left aft this point do not cache
# or is this easier to keep in the cookie deletion above?
# --
if (req.http.Authorization || req.http.Cookie) {
/* Not cacheable by default */
return (pass);
}
# If we get here, try the cache
# --
return (lookup);
}
sub vcl_pipe {
return (pipe);
}
sub vcl_pass {
return (pass);
}
sub vcl_hash {
# Include cookie in cache hash.
# This check is unnecessary because we already pass on all cookies.
#if (req.http.Cookie) {
# #set req.hash += req.http.Cookie;
# hash_data(req.http.Cookie);
#}
}
sub vcl_hit {
if (req.request == "PURGE") {
purge;
error 200 "Purged";
}
}
sub vcl_miss {
if (req.request == "PURGE") {
purge;
error 200 "Purged";
}
}
sub vcl_fetch {
# S3 Site4
# --
if (req.http.Host == "s3.site4.site.com" && req.url ~
"\.(flv|mp4|jpg)(\?[a-z0-9]+)$") {
remove beresp.http.x-amz-id-2;
remove beresp.http.x-amz-request-id;
set beresp.ttl = 1w;
set beresp.grace = 30s;
if (beresp.status != 200) {
return (error);
}
}
# S3 Site3
# --
if (req.http.Host == "s3.site3.site.com" && req.url ~
"\.(flv|mp4|jpg|xml|png|gif)(\?[a-z0-9]+)$") {
remove beresp.http.x-amz-id-2;
remove beresp.http.x-amz-request-id;
set beresp.ttl = 1w;
set beresp.grace = 30s;
if (beresp.status != 200) {
return (error);
}
}
# S3 site2
# --
if (req.http.Host == "s3.site2.site.com" && req.url ~
"\.(flv|mp4|jpg|xml|png|gif)(\?[a-z0-9]+)$") {
remove beresp.http.x-amz-id-2;
remove beresp.http.x-amz-request-id;
set beresp.ttl = 1w;
set beresp.grace = 30s;
if (beresp.status != 200) {
return (error);
}
}
# S3 site1
# --
if (req.http.Host == "s3.site1.site.com" && req.url ~
"\.(flv|mp4|jpg|xml|png|gif)(\?[a-z0-9]+)$") {
remove beresp.http.x-amz-id-2;
remove beresp.http.x-amz-request-id;
set beresp.ttl = 1w;
set beresp.grace = 30s;
if (beresp.status != 200) {
return (error);
}
}
# remove all cookies
# unset beresp.http.set-cookie;
# cache for 12 hours
# set beresp.ttl = 2h;
# Don't allow static files to set cookies.
if (req.url ~
"(?i)\.(png|gif|jpeg|jpg|ico|swf|css|js|html|htm|mp4|flv)(\?[a-z0-9]+)?$") {
unset beresp.http.set-cookie;
set beresp.ttl = 12h;
} else {
set beresp.ttl = 30m;
}
# If the backend server doesn't return properly, don't send another
connection to it
# for 60s and try another backend via restart.
#
#
https://www.varnish-cache.org/docs/trunk/tutorial/handling_misbehaving_servers.html
# --
if(beresp.status == 500 || beresp.status == 503) {
if (req.restarts > 3) {
set beresp.saintmode = 5m;
}
if (req.request != "POST") {
return(restart);
} else {
error 500 "Failed";
}
}
# Allow items to be stale if needed.
set beresp.grace = 1h;
}
sub vcl_deliver {
# if (resp.http.magicmarker) {
# unset resp.http.magicmarker;
# set resp.http.age = "0";
# }
if (obj.hits > 0) {
set resp.http.X-Cache = "HIT";
} else {
set resp.http.X-Cache = "MISS";
}
return (deliver);
}
# In the event of an error, show friendlier messages.
sub vcl_error {
# Otherwise redirect to the homepage, which will likely be in the cache.
set obj.http.Content-Type = "text/html; charset=utf-8";
synthetic {"
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "
http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>..:: Site :: Por el momento estamos en mantenimiento
::..</title>
</head>
<body>
<div class="logo">En mantenimiento</div>
<div class="text">Estamos experimentando problemas. Vuelve
más tarde.</div>
<div style="display:none;">(Error "} + obj.status + " " + obj.response
+ {")</div>
</body>
</html>
"};
return (deliver);
}
sub vcl_init {
return (ok);
}
sub vcl_fini {
return (ok);
}
2012/6/28 Hugo Cisneiros (Eitch) <hugo.cisneiros at gmail.com>
> On Wed, Jun 27, 2012 at 8:42 PM, Abraham Cruz Sustaita
> <abraham.sustaita at buscacorp.com> wrote:
> > I have this configuration:
> >
> > # Amazon S3 Backend Metatube
> > # --
> > backend s3 {
> > .host = "metatube.s3-website-us-east-1.amazonaws.com";
> > .port = "80";
> > }
> >
> > But the amazon url sometimes resolves to more of one IP (I guess is a
> normal
> > behaviour from Amazon), so I have to try several times to load the
> > configuration file (via varnishadm) until it compiles correctly. The
> problem
> > is I have a 5 different domains with amazon, so when finally one of them
> > resolves to just one IP, the others resolve to more than one, so I can
> never
> > compile the configuration file. Is there anyway to fix this via
> > configuration?
>
> Currently varnish does not support multiple IPs on one backend.
> There's no easy and native way to do this. Please see the mailing list
> archives for many discussions and some approaches about this issue :)
>
> --
> []'s
> Hugo
> www.devin.com.br
>
> _______________________________________________
> varnish-misc mailing list
> varnish-misc at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-misc
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-misc/attachments/20120702/09f133c2/attachment-0001.html>
More information about the varnish-misc
mailing list