[master] d6ad52f5f Change the way we calculate the hash key for the cache.
Poul-Henning Kamp
phk at FreeBSD.org
Thu Feb 11 13:34:07 UTC 2021
commit d6ad52f5ff95daa3668fdad28b4c97e59b4c49d3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date: Thu Feb 11 13:31:10 2021 +0000
Change the way we calculate the hash key for the cache.
This lifts internal restrictions on the hashkey production,
but does not change how things work from a VCL perspective.
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index daf8a9cf7..a5dae200d 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -183,19 +183,6 @@ HSH_DeleteObjHead(const struct worker *wrk, struct objhead *oh)
FREE_OBJ(oh);
}
-void
-HSH_AddString(struct req *req, void *ctx, const char *str)
-{
-
- CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
- AN(ctx);
- if (str != NULL) {
- VSHA256_Update(ctx, str, strlen(str));
- VSLb(req->vsl, SLT_Hash, "%s", str);
- } else
- VSHA256_Update(ctx, &str, 1);
-}
-
/*---------------------------------------------------------------------
* This is a debugging hack to enable testing of boundary conditions
* in the hash algorithm.
diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h
index bc1782379..af1797462 100644
--- a/bin/varnishd/cache/cache_objhead.h
+++ b/bin/varnishd/cache/cache_objhead.h
@@ -71,7 +71,6 @@ int HSH_DerefObjCore(struct worker *, struct objcore **, int rushmax);
enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **);
void HSH_Ref(struct objcore *o);
-void HSH_AddString(struct req *, void *ctx, const char *str);
unsigned HSH_Purge(struct worker *, struct objhead *, vtim_real ttl_now,
vtim_dur ttl, vtim_dur grace, vtim_dur keep);
struct objcore *HSH_Private(const struct worker *wrk);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9b5a4b8a5..d82955a96 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -50,7 +50,6 @@
#include "storage/storage.h"
#include "common/heritage.h"
#include "vcl.h"
-#include "vsha256.h"
#include "vtim.h"
#define REQ_STEPS \
@@ -886,7 +885,6 @@ static enum req_fsm_nxt v_matchproto_(req_state_f)
cnt_recv(struct worker *wrk, struct req *req)
{
unsigned recv_handling;
- struct VSHA256Context sha256ctx;
const char *ci, *cp, *endpname;
CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -957,13 +955,12 @@ cnt_recv(struct worker *wrk, struct req *req)
}
}
- VSHA256_Init(&sha256ctx);
- VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx);
+ memset(req->digest, 0, sizeof req->digest);
+ VCL_hash_method(req->vcl, wrk, req, NULL, NULL);
if (wrk->handling == VCL_RET_FAIL)
recv_handling = wrk->handling;
else
assert(wrk->handling == VCL_RET_LOOKUP);
- VSHA256_Final(req->digest, &sha256ctx);
switch (recv_handling) {
case VCL_RET_VCL:
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 0a729c0e4..6d323a22c 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -689,19 +689,29 @@ VRT_fail(VRT_CTX, const char *fmt, ...)
VCL_VOID
VRT_hashdata(VRT_CTX, VCL_STRANDS s)
{
+ struct VSHA256Context sha256ctx;
int i;
CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
- AN(ctx->specific);
+ AZ(ctx->specific);
+ VSHA256_Init(&sha256ctx);
+ VSHA256_Update(&sha256ctx, ctx->req->digest, sizeof ctx->req->digest);
AN(s);
- for (i = 0; i < s->n; i++)
- HSH_AddString(ctx->req, ctx->specific, s->p[i]);
+ for (i = 0; i < s->n; i++) {
+ if (s->p[i] != NULL) {
+ VSHA256_Update(&sha256ctx, s->p[i], strlen(s->p[i]));
+ VSLb(ctx->req->vsl, SLT_Hash, "%s", s->p[i]);
+ } else {
+ VSHA256_Update(&sha256ctx, "", 1);
+ }
+ }
/*
* Add a 'field-separator' to make it more difficult to
* manipulate the hash.
*/
- HSH_AddString(ctx->req, ctx->specific, NULL);
+ VSHA256_Update(&sha256ctx, "", 1);
+ VSHA256_Final(ctx->req->digest, &sha256ctx);
}
/*--------------------------------------------------------------------*/
diff --git a/bin/varnishtest/tests/b00051.vtc b/bin/varnishtest/tests/b00051.vtc
index 3681ad2b3..8119dc6b5 100644
--- a/bin/varnishtest/tests/b00051.vtc
+++ b/bin/varnishtest/tests/b00051.vtc
@@ -22,5 +22,5 @@ client c1 {
rxresp
expect resp.http.req_hash ~ "[[:xdigit:]]{64}"
expect resp.http.req_hash == resp.http.bereq_hash
- expect resp.http.req_hash-sf == ":3k0f0yRKtKt7akzkyNsTGSDOJAZOQowTwKWhu5+kIu0=:"
+ expect resp.http.req_hash-sf == ":0jkH41nfmD0PRFsKpM1m7ucOApnxFc62B//mQWxOnmQ=:"
} -run
diff --git a/bin/varnishtest/tests/d00020.vtc b/bin/varnishtest/tests/d00020.vtc
index 0c42285d7..9f70ff2d3 100644
--- a/bin/varnishtest/tests/d00020.vtc
+++ b/bin/varnishtest/tests/d00020.vtc
@@ -218,9 +218,9 @@ client c1 {
client c2 {
txreq -url /b/def
rxresp
- expect resp.http.hash == "93d1c4ad76396c91dd97fa310f7f26445332662c89393dbeeb77fe49f9111ee4"
+ expect resp.http.hash == "2e03d4fee2722154a84036faa3e4b6851e003830eadbe0d2874e3df09c8efe55"
expect resp.http.by == "HASH"
- expect resp.http.key -eq 0x93d1c4ad
+ expect resp.http.key -eq 0x2e03d4fe
expect resp.http.alt == 0
expect resp.http.warmup == "-1.000"
expect resp.http.rampup == "true"
@@ -228,9 +228,9 @@ client c2 {
txreq -url /b/hash
rxresp
- expect resp.http.hash == "e47da20ea4db49d4f22acdadc69f02f445002be520a2865cd3351272add62540"
+ expect resp.http.hash == "49f33e9019891091d3dcf6edab6d9433b756678bdf5202dd41b8a667c89887b1"
expect resp.http.by == "HASH"
- expect resp.http.key -eq 0xe47da20e
+ expect resp.http.key -eq 0x49f33e90
expect resp.http.alt == 1
expect resp.http.warmup == "-1.000"
expect resp.http.rampup == "true"
@@ -268,9 +268,9 @@ client c2 {
client c3 {
txreq -url /b/c/hash/def
rxresp
- expect resp.http.hash == "df9a465f8a0455c334b24c1638d3adda0f6e64fbe759029ab83602e3b9138884"
+ expect resp.http.hash == "dd70dcbbf385c398ee3b53a849f12d0d846fc21292349fb45d37b2d9d8eca25e"
expect resp.http.by == "HASH"
- expect resp.http.key -eq 0xdf9a465f
+ expect resp.http.key -eq 0xdd70dcbb
expect resp.http.alt == 7
expect resp.http.warmup == "-1.000"
expect resp.http.rampup == "true"
@@ -278,9 +278,9 @@ client c3 {
txreq -url /b/c/hash/hash
rxresp
- expect resp.http.hash == "0eb35bc1fab5aad5902fd1bac86540bd13d43aa31c6c46f54e776b43392e66e6"
+ expect resp.http.hash == "41d09b9877cd0ac0eab888359b0ad54f0bf41da0ac03dc1b8ae12aff18465a8d"
expect resp.http.by == "HASH"
- expect resp.http.key -eq 0x0eb35bc1
+ expect resp.http.key -eq 0x41d09b98
expect resp.http.alt == 8
expect resp.http.warmup == "-1.000"
expect resp.http.rampup == "true"
@@ -288,9 +288,9 @@ client c3 {
txreq -url /b/c/hash/url
rxresp
- expect resp.http.hash == "1eb67b701ea07151cac5bea1f11b6267b9de15a3ff83cec995590480cbc2c750"
+ expect resp.http.hash == "dcac849e02b3322f5fd3dddf9b9f5fc26d295733e6f1c51b190dfb7239a56e28"
expect resp.http.by == "HASH"
- expect resp.http.key -eq 0x1eb67b70
+ expect resp.http.key -eq 0xdcac849e
expect resp.http.alt == 9
expect resp.http.warmup == "0.500"
expect resp.http.rampup == "true"
@@ -298,9 +298,9 @@ client c3 {
txreq -url /b/c/hash/key
rxresp
- expect resp.http.hash == "a11b617e21aa7db22b6205d7612002e595b1b00d8c11602017f65456a1be3a35"
+ expect resp.http.hash == "112393761506e85f0c700a5d669a48b54001c870eb2e8d95f4d2f6fdccbe80a3"
expect resp.http.by == "HASH"
- expect resp.http.key -eq 0xa11b617e
+ expect resp.http.key -eq 0x11239376
expect resp.http.alt == 10
expect resp.http.warmup == "-1.000"
expect resp.http.rampup == "false"
@@ -308,9 +308,9 @@ client c3 {
txreq -url /b/c/hash/blob
rxresp
- expect resp.http.hash == "d7eecc0ac83e1727332dcd8c7c8ae9f3114123abb2bf7e3fb15ecea8c84bb239"
+ expect resp.http.hash == "5ef050c1185ac02a66d9f79703b8cd5f0636abf3b1f15b9f22e0fe64df985d28"
expect resp.http.by == "HASH"
- expect resp.http.key -eq 0xd7eecc0a
+ expect resp.http.key -eq 0x5ef050c1
expect resp.http.alt == 11
expect resp.http.warmup == "-1.000"
expect resp.http.rampup == "true"
diff --git a/include/vrt.h b/include/vrt.h
index 08d217dd8..64b400068 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -53,6 +53,7 @@
* binary/load-time compatible, increment MAJOR version
*
* 13.0 (2021-03-15)
+ * VRT_hashdata() produces different hash-keys.
* Move VRT_synth_page() to deprecated status
* Add VRT_synth_strands() and VRT_synth_blob()
* struct vrt_type now produced by generate.py
More information about the varnish-commit
mailing list