[master] 4094cf5 retire shard .key() and .reconfigure() algorithm choice
Nils Goroll
nils.goroll at uplex.de
Mon Mar 5 19:26:08 UTC 2018
commit 4094cf5b6cf2718d137dd63ac57e2085e52de377
Author: Nils Goroll <nils.goroll at uplex.de>
Date: Sun Dec 10 17:07:00 2017 +0100
retire shard .key() and .reconfigure() algorithm choice
Ref: #2500
diff --git a/bin/varnishtest/tests/d00017.vtc b/bin/varnishtest/tests/d00017.vtc
index 4d49563..a8f33b1 100644
--- a/bin/varnishtest/tests/d00017.vtc
+++ b/bin/varnishtest/tests/d00017.vtc
@@ -25,6 +25,7 @@ server s3 {
varnish v1 -vcl+backend {
import std;
import directors;
+ import blob;
sub vcl_init {
new vd = directors.shard();
@@ -44,8 +45,9 @@ varnish v1 -vcl+backend {
}
sub vcl_recv {
- set req.backend_hint = vd.backend(by=KEY,
- key=vd.key(req.url, CRC32));
+ set req.backend_hint = vd.backend(by=BLOB,
+ key_blob=blob.decode(HEX, encoded=
+ regsub(req.url, "^/", "")));
return(pass);
}
@@ -142,15 +144,15 @@ logexpect l1 -v v1 -g raw -d 1 {
} -start
client c1 {
- txreq -url /eishoSu2
+ txreq -url /68b902f7
rxresp
expect resp.body == "ech3Ooj"
- txreq -url /Zainao9d
+ txreq -url /39dc4613
rxresp
expect resp.body == "ieQu2qua"
- txreq -url /Aunah3uo
+ txreq -url /c7793505
rxresp
expect resp.body == "xiuFi3Pe"
} -run
diff --git a/bin/varnishtest/tests/d00019.vtc b/bin/varnishtest/tests/d00019.vtc
index 5635a1c..9695457 100644
--- a/bin/varnishtest/tests/d00019.vtc
+++ b/bin/varnishtest/tests/d00019.vtc
@@ -1,4 +1,4 @@
-varnishtest "shard director SHA256 (default)"
+varnishtest "shard director by req.url (default)"
server s1 {
rxreq
diff --git a/bin/varnishtest/tests/d00020.vtc b/bin/varnishtest/tests/d00020.vtc
deleted file mode 100644
index 1fe91c7..0000000
--- a/bin/varnishtest/tests/d00020.vtc
+++ /dev/null
@@ -1,50 +0,0 @@
-varnishtest "shard director RS"
-
-server s1 {
- rxreq
- txresp -body "ech3Ooj"
-} -start
-
-server s2 {
- rxreq
- txresp -body "ieQu2qua"
-} -start
-
-server s3 {
- rxreq
- txresp -body "xiuFi3Pe"
-} -start
-
-varnish v1 -vcl+backend {
- import directors;
-
- sub vcl_init {
- new vd = directors.shard();
- vd.add_backend(s1);
- vd.add_backend(s2);
- vd.add_backend(s3);
- vd.reconfigure(replicas=25);
- }
-
- sub vcl_recv {
- set req.backend_hint = vd.backend(by=KEY,
- key=vd.key(req.url, alg=RS));
- return(pass);
- }
-
-} -start
-
-
-client c1 {
- txreq -url /we0eeTho
- rxresp
- expect resp.body == "ech3Ooj"
-
- txreq -url /mae8ooNu
- rxresp
- expect resp.body == "ieQu2qua"
-
- txreq -url /oob3dahS
- rxresp
- expect resp.body == "xiuFi3Pe"
-} -run
diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc
deleted file mode 100644
index 627f717..0000000
--- a/bin/varnishtest/tests/d00021.vtc
+++ /dev/null
@@ -1,81 +0,0 @@
-varnishtest "shard director key function"
-
-server s1 {
- rxreq
- txresp -body "ech3Ooj"
- rxreq
- txresp -body "ech3Ooj"
- rxreq
- txresp -body "ech3Ooj"
-} -start
-
-server s2 {
- rxreq
- txresp -body "ieQu2qua"
-} -start
-
-server s3 {
- rxreq
- txresp -body "xiuFi3Pe"
-} -start
-
-varnish v1 -vcl+backend {
- import directors;
-
- sub vcl_init {
- new vd = directors.shard();
- vd.add_backend(s1);
- vd.add_backend(s2);
- vd.add_backend(s3);
- vd.reconfigure(25);
- }
-
- sub recv_sub {
- set req.backend_hint = vd.backend(by=KEY,
- key=vd.key(req.http.X-Hash, RS));
- }
-
- sub vcl_recv {
- if (req.url == "/1") {
- set req.backend_hint = vd.backend(by=KEY,
- key=vd.key(alg=CRC32, string="/eishoSu2"));
- } else if (req.url == "/2") {
- set req.backend_hint = vd.backend(by=KEY,
- key=vd.key("/eishoSu2"));
- } else if (req.url == "/3") {
- set req.http.X-Hash = "/oob3dahS";
- call recv_sub;
- } else if (req.url == "/null_by_string") {
- set req.backend_hint = vd.backend(by=KEY,
- key=vd.key(req.http.NonExistent));
- } else if (req.url == "/null_by_string_hash") {
- set req.backend_hint = vd.backend(by=KEY,
- key=vd.key(req.http.NonExistent, SHA256));
- }
- return(pass);
- }
-
-} -start
-
-
-client c1 {
- txreq -url /1
- rxresp
- expect resp.body == "ech3Ooj"
-
- txreq -url /2
- rxresp
- expect resp.body == "ieQu2qua"
-
- txreq -url /3
- rxresp
- expect resp.body == "xiuFi3Pe"
-
- txreq -url /null_by_string
- rxresp
- expect resp.body == "ech3Ooj"
-
- txreq -url /null_by_string_hash
- rxresp
- expect resp.body == "ech3Ooj"
-} -run
diff --git a/bin/varnishtest/tests/d00022.vtc b/bin/varnishtest/tests/d00022.vtc
index 3d0e9e1..08130a1 100644
--- a/bin/varnishtest/tests/d00022.vtc
+++ b/bin/varnishtest/tests/d00022.vtc
@@ -40,7 +40,7 @@ varnish v1 -vcl+backend {
sub vcl_recv {
set req.backend_hint = vd.backend(by=KEY,
- key=vd.key("/eishoSu2", CRC32),
+ key=1756955383,
alt=req.restarts,
healthy=ALL);
diff --git a/bin/varnishtest/tests/d00023.vtc b/bin/varnishtest/tests/d00023.vtc
index e427264..36ebe8e 100644
--- a/bin/varnishtest/tests/d00023.vtc
+++ b/bin/varnishtest/tests/d00023.vtc
@@ -1,8 +1,6 @@
varnishtest "shard director Unhealthy"
server s1 {
- rxreq
- txresp -body "ech3Ooj"
} -start
server s2 {
@@ -32,7 +30,7 @@ varnish v1 -vcl+backend {
sub vcl_recv {
set req.backend_hint = vd.backend(by=KEY,
- key=vd.key("/eishoSu2", CRC32));
+ key=1756955383);
set req.http.healthy = std.healthy(req.backend_hint);
return(pass);
}
diff --git a/bin/varnishtest/tests/d00024.vtc b/bin/varnishtest/tests/d00024.vtc
index 57f76da..3d5606a 100644
--- a/bin/varnishtest/tests/d00024.vtc
+++ b/bin/varnishtest/tests/d00024.vtc
@@ -35,7 +35,7 @@ varnish v1 -vcl+backend {
sub vcl_recv {
set req.backend_hint = vd.backend(by=KEY,
- key=vd.key(alg=CRC32, string="/eishoSu2"));
+ key=1756955383);
return(pass);
}
} -start
diff --git a/bin/varnishtest/tests/d00026.vtc b/bin/varnishtest/tests/d00026.vtc
index 64d63bc..0cfd816 100644
--- a/bin/varnishtest/tests/d00026.vtc
+++ b/bin/varnishtest/tests/d00026.vtc
@@ -1,4 +1,4 @@
-varnishtest "shard director - same as v01000.vtc but setting backend in fetch"
+varnishtest "shard director - same as d00017.vtc but setting backend in fetch"
server s1 {
rxreq
@@ -17,6 +17,7 @@ server s3 {
varnish v1 -vcl+backend {
import directors;
+ import blob;
sub vcl_init {
new vd = directors.shard();
@@ -27,8 +28,9 @@ varnish v1 -vcl+backend {
}
sub vcl_backend_fetch {
- set bereq.backend = vd.backend(by=KEY,
- key=vd.key(bereq.url, CRC32));
+ set bereq.backend = vd.backend(by=BLOB,
+ key_blob=blob.decode(HEX, encoded=
+ regsub(bereq.url, "^/", "")));
return(fetch);
}
@@ -36,15 +38,15 @@ varnish v1 -vcl+backend {
client c1 {
- txreq -url /eishoSu2
+ txreq -url /68b902f7
rxresp
expect resp.body == "ech3Ooj"
- txreq -url /Zainao9d
+ txreq -url /39dc4613
rxresp
expect resp.body == "ieQu2qua"
- txreq -url /Aunah3uo
+ txreq -url /c7793505
rxresp
expect resp.body == "xiuFi3Pe"
} -run
diff --git a/doc/changes.rst b/doc/changes.rst
index 6648e52..e366f63 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -59,6 +59,51 @@ VCL and bundled VMODs
* added ``return(restart)`` from ``vcl_recv{}``
+* The ``alg`` argument of the ``shard`` director ``.reconfigure()``
+ method has been removed - the consistent hashing ring is now always
+ generated using the last 32 bits of a SHA256 hash of ``"ident%d"``
+ as with ``alg=SHA256`` or the default.
+
+ We believe that the other algorithms did not yield sufficiently
+ dispersed placement of backends on the consistent hashing ring and
+ thus retire this option without replacement.
+
+ Users of ``.reconfigure(alg=CRC32)`` or ``.reconfigure(alg=RS)`` be
+ advised that when upgrading and removing the ``alg`` argument,
+ consistent hashing values for all backends will change once and only
+ once.
+
+* The ``alg`` argument of the ``shard`` director ``.key()`` method has
+ been removed - it now always hashes its arguments using SHA256 and
+ returns the last 32 bits for use as a shard key.
+
+ Backwards compatibility is provided through `vmod blobdigest`_ with
+ the ``key_blob`` argument of the ``shard`` director ``.backend()``
+ method:
+
+ * for ``alg=CRC32``, replace::
+
+ <dir>.backend(by=KEY, key=<dir>.key(<string>, CRC32))
+
+ with::
+
+ <dir>.backend(by=BLOB, key_blob=blobdigest.hash(ICRC32,
+ blob.decode(encoded=<string>)))
+
+ `Note:` The `vmod blobdigest`_ hash method corresponding to the
+ shard director CRC32 method is called **I**\ CRC32
+
+.. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst
+
+ * for ``alg=RS``, replace::
+
+ <dir>.backend(by=KEY, key=<dir>.key(<string>, RS))
+
+ with::
+
+ <dir>.backend(by=BLOB, key_blob=blobdigest.hash(RS,
+ blob.decode(encoded=<string>)))
+
Logging / statistics
--------------------
diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index ff0c244..d9fce2f 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -12,8 +12,6 @@ libvmod_directors_la_SOURCES = \
shard_cfg.h \
shard_dir.c \
shard_dir.h \
- shard_hash.c \
- shard_hash.h \
shard_parse_vcc_enums.h \
shard_parse_vcc_enums.c
diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 620126f..de841ef 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -39,7 +39,6 @@
#include "shard_dir.h"
#include "shard_cfg.h"
-#include "shard_hash.h"
/*lint -esym(749, shard_change_task_e::*) */
enum shard_change_task_e {
@@ -232,11 +231,12 @@ circlepoint_compare(const struct shard_circlepoint *a,
}
static void
-shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg)
+shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
{
int i, j;
const char *ident;
- int len;
+ const int len = 12; // log10(UINT32_MAX) + 2;
+ char s[len];
CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
AZ(shardd->hashcircle);
@@ -259,14 +259,10 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas, enum alg_e alg)
assert(ident[0] != '\0');
- len = strlen(ident) + 12; // log10(UINT32_MAX) + 2;
-
- char s[len];
-
for (j = 0; j < replicas; j++) {
- assert(snprintf(s, len, "%s%d", ident, j) < len);
+ assert(snprintf(s, len, "%d", j) < len);
shardd->hashcircle[i * replicas + j].point =
- shard_hash_f[alg](s);
+ sharddir_sha256(ident, s, vrt_magic_string_end);
shardd->hashcircle[i * replicas + j].host = i;
}
/* not used in current interface */
@@ -574,7 +570,7 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
VCL_BOOL
shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
- struct sharddir *shardd, VCL_INT replicas, enum alg_e alg)
+ struct sharddir *shardd, VCL_INT replicas)
{
struct shard_change *change;
@@ -607,7 +603,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
return 0;
}
- shardcfg_hashcircle(shardd, replicas, alg);
+ shardcfg_hashcircle(shardd, replicas);
sharddir_unlock(shardd);
return (1);
}
diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h
index 8d1d88c..2ce1817 100644
--- a/lib/libvmod_directors/shard_cfg.h
+++ b/lib/libvmod_directors/shard_cfg.h
@@ -34,7 +34,7 @@ VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
VCL_BOOL shardcfg_clear(VRT_CTX, struct vmod_priv *priv,
const struct sharddir *shardd);
VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
- struct sharddir *shardd, VCL_INT replicas, enum alg_e alg_e);
+ struct sharddir *shardd, VCL_INT replicas);
VCL_VOID shardcfg_set_warmup(struct sharddir *shardd, VCL_REAL ratio);
VCL_VOID shardcfg_set_rampup(struct sharddir *shardd,
VCL_DURATION duration);
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 36c901b..ff5f9af 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -41,6 +41,8 @@
#include "vbm.h"
#include "vrnd.h"
+#include "vsha256.h"
+#include "vend.h"
#include "shard_dir.h"
@@ -87,6 +89,47 @@ sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...)
va_end(ap);
}
+uint32_t
+sharddir_sha256v(const char *s, va_list ap)
+{
+ struct VSHA256Context sha256;
+ union {
+ unsigned char digest[32];
+ uint32_t uint32_digest[8];
+ } sha256_digest;
+ uint32_t r;
+ const char *p;
+
+ VSHA256_Init(&sha256);
+ p = s;
+ while (p != vrt_magic_string_end) {
+ if (p != NULL && *p != '\0')
+ VSHA256_Update(&sha256, p, strlen(p));
+ p = va_arg(ap, const char *);
+ }
+ VSHA256_Final(sha256_digest.digest, &sha256);
+
+ /*
+ * use low 32 bits only
+ * XXX: Are these the best bits to pick?
+ */
+ vle32enc(&r, sha256_digest.uint32_digest[7]);
+ return (r);
+}
+
+uint32_t
+sharddir_sha256(const char *s, ...)
+{
+ va_list ap;
+ uint32_t r;
+
+ va_start(ap, s);
+ r = sharddir_sha256v(s, ap);
+ va_end(ap);
+
+ return (r);
+}
+
static int
shard_lookup(const struct sharddir *shardd, const uint32_t key)
{
diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 35a947c..c893c3d 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -103,6 +103,8 @@ sharddir_backend_ident(const struct sharddir *shardd, int host)
void sharddir_debug(struct sharddir *shardd, const uint32_t flags);
void sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...);
+uint32_t sharddir_sha256v(const char *s, va_list ap);
+uint32_t sharddir_sha256(const char *s, ...);
void sharddir_new(struct sharddir **sharddp, const char *vcl_name);
void sharddir_delete(struct sharddir **sharddp);
void sharddir_wrlock(struct sharddir *shardd);
diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c
deleted file mode 100644
index 9d0dc94..0000000
--- a/lib/libvmod_directors/shard_hash.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/*-
- * Copyright 2009-2013 UPLEX - Nils Goroll Systemoptimierung
- * All rights reserved.
- *
- * Authors: Nils Goroll <nils.goroll at uplex.de>
- * Geoffrey Simmons <geoff.simmons at uplex.de>
- * Julian Wiesener <jw at uplex.de>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <string.h>
-
-#include "cache/cache.h"
-
-#include "vsha256.h"
-#include "vend.h"
-
-#include "shard_parse_vcc_enums.h"
-#include "shard_hash.h"
-
-/*
- * XXX use the crc32 from libvgz, but declare it here to avoid an include
- * dependency nightmare (at least for now)
- */
-
-unsigned long crc32(unsigned long, const unsigned char *buf, unsigned len);
-
-static uint32_t v_matchproto_(hash_func)
-shard_hash_crc32(VCL_STRING s)
-{
- uint32_t crc;
- crc = crc32(~0U, (const unsigned char *)s, strlen(s));
- crc ^= ~0U;
- return (crc);
-}
-
-static uint32_t v_matchproto_(hash_func)
-shard_hash_sha256(VCL_STRING s)
-{
- struct VSHA256Context sha256;
- union {
- unsigned char digest[32];
- uint32_t uint32_digest[8];
- } sha256_digest;
- uint32_t r;
-
- VSHA256_Init(&sha256);
- VSHA256_Update(&sha256, s, strlen(s));
- VSHA256_Final(sha256_digest.digest, &sha256);
-
- /*
- * use low 32 bits only
- * XXX: Are these the best bits to pick?
- */
- vle32enc(&r, sha256_digest.uint32_digest[7]);
- return (r);
-}
-
-static uint32_t v_matchproto_(hash_func)
-shard_hash_rs(VCL_STRING s)
-{
- uint32_t res = 0;
- /* hash function from Robert Sedgwicks 'Algorithms in C' book */
- const uint32_t b = 378551;
- uint32_t a = 63689;
-
- while (*s) {
- res = res * a + (*s++);
- a *= b;
- }
-
- return (res);
-}
-
-static uint32_t v_matchproto_(hash_func)
-_shard_hash_invalid(VCL_STRING s)
-{
- (void)s;
- WRONG("invalid hash fp _ALG_E_ENVALID");
- NEEDLESS(return(0));
-}
-
-const hash_func shard_hash_f[_ALG_E_MAX] = {
- [_ALG_E_INVALID] = _shard_hash_invalid,
- [CRC32] = shard_hash_crc32,
- [SHA256] = shard_hash_sha256,
- [RS] = shard_hash_rs
-};
diff --git a/lib/libvmod_directors/shard_hash.h b/lib/libvmod_directors/shard_hash.h
deleted file mode 100644
index 838032a..0000000
--- a/lib/libvmod_directors/shard_hash.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*-
- * Copyright 2009-2013 UPLEX - Nils Goroll Systemoptimierung
- * All rights reserved.
- *
- * Author: Julian Wiesener <jw at uplex.de>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-typedef uint32_t (*hash_func)(VCL_STRING);
-extern const hash_func shard_hash_f[_ALG_E_MAX];
diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 869f62e..4737284 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -307,10 +307,11 @@ Method
``````
When ``.reconfigure()`` is called, a consistent hashing circular data
-structure gets built from hash values of "ident%d" (default ident
-being the backend name) for each backend and for a running number from
-1 to n (n is the number of `replicas`). Hashing creates the seemingly
-random order for placement of backends on the consistent hashing ring.
+structure gets built from the last 32 bits of SHA256 hash values of
+`<ident>`\ `<n>` (default `ident` being the backend name) for each
+backend and for a running number `n` from 1 to `replicas`. Hashing
+creates the seemingly random order for placement of backends on the
+consistent hashing ring.
When ``.backend()`` is called, a load balancing key gets generated
unless provided. The smallest hash value in the circle is looked up
@@ -388,19 +389,24 @@ Remove all backends from the director.
NOTE: Backend changes need to be finalized with `shard.reconfigure()`
and are only supported on one shard director at a time.
-$Method BOOL .reconfigure(PRIV_TASK, INT replicas=67,
- ENUM { CRC32, SHA256, RS } alg="SHA256")
+$Method BOOL .reconfigure(PRIV_TASK, INT replicas=67)
Reconfigure the consistent hashing ring to reflect backend changes.
This method must be called at least once before the director can be
used.
-$Method INT .key(STRING string, ENUM { CRC32, SHA256, RS } alg="SHA256")
+$Method INT .key(STRING_LIST)
-Utility method to generate a sharding key for use with the
-``shard.backend()`` method by hashing `string` with hash algorithm
-`alg`.
+Convenience method to generate a sharding key for use with the `key`
+argument to the ``shard.backend()`` method by hashing the given string
+with SHA256.
+
+To generate sharding keys using other hashes, use a custom vmod like
+`vmod blobdigest`_ with the `key_blob` argument of the
+``shard.backend()`` method.
+
+.. _vmod blobdigest: https://code.uplex.de/uplex-varnish/libvmod-blobdigest/blob/master/README.rst
$Method BACKEND .backend(
ENUM {HASH, URL, KEY, BLOB} by="HASH",
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index e5ddb58..6a8660a 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -40,7 +40,6 @@
#include "vcc_if.h"
#include "shard_dir.h"
#include "shard_cfg.h"
-#include "shard_hash.h"
struct vmod_directors_shard {
unsigned magic;
@@ -56,7 +55,7 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp,
VCL_INT t1;
uint32_t t2a, t2b;
- /* see vmod_key comment */
+ /* we put our uint32 key in a VCL_INT container */
assert(sizeof(VCL_INT) >= sizeof(uint32_t));
t2a = UINT32_MAX;
t1 = (VCL_INT)t2a;
@@ -84,22 +83,20 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp)
FREE_OBJ(vshard);
}
-/*
- * our key is a uint32_t, but VCL_INT is a (signed) long. We cast back and
- * forth, asserting in vmod_shard__init() that VCL_INT is a large enough
- * container
- */
VCL_INT v_matchproto_(td_directors_shard_key)
- vmod_shard_key(VRT_CTX, struct vmod_directors_shard *vshard,
- VCL_STRING s, VCL_ENUM alg_s)
+vmod_shard_key(VRT_CTX, struct vmod_directors_shard *vshard, const char *s, ...)
{
- enum alg_e alg = parse_alg_e(alg_s);
- hash_func hash_fp = shard_hash_f[alg];
+ va_list ap;
+ uint32_t r;
(void)ctx;
- (void)vshard;;
+ (void)vshard;
+
+ va_start(ap, s);
+ r = sharddir_sha256v(s, ap);
+ va_end(ap);
- return (VCL_INT)hash_fp(s ? s : "");
+ return ((VCL_INT)r);
}
VCL_VOID v_matchproto_(td_directors_set_warmup)
@@ -169,11 +166,9 @@ vmod_shard_clear(VRT_CTX, struct vmod_directors_shard *vshard,
VCL_BOOL v_matchproto_(td_directors_shard_reconfigure)
vmod_shard_reconfigure(VRT_CTX, struct vmod_directors_shard *vshard,
- struct vmod_priv *priv, VCL_INT replicas, VCL_ENUM alg_s)
+ struct vmod_priv *priv, VCL_INT replicas)
{
- enum alg_e alg = parse_alg_e(alg_s);
-
- return shardcfg_reconfigure(ctx, priv, vshard->shardd, replicas, alg);
+ return shardcfg_reconfigure(ctx, priv, vshard->shardd, replicas);
}
static inline uint32_t
@@ -198,7 +193,8 @@ get_key(VRT_CTX, enum by_e by, VCL_INT key_int, VCL_BLOB key_blob)
AN(ctx->http_bereq);
AN(http = ctx->http_bereq);
}
- return (shard_hash_f[SHA256](http->hd[HTTP_HDR_URL].b));
+ return (sharddir_sha256(http->hd[HTTP_HDR_URL].b,
+ vrt_magic_string_end));
case BY_KEY:
return ((uint32_t)key_int);
case BY_BLOB:
More information about the varnish-commit
mailing list