From hermunn at varnish-software.com Fri Jun 1 09:40:25 2018 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Fri, 1 Jun 2018 09:40:25 +0000 (UTC) Subject: [4.1] 7906a41 ban lurker should back off on seeing a busy object Message-ID: <20180601094025.15DDBBAA5E@lists.varnish-cache.org> commit 7906a41782a39a407f41d831da05326c4b2d225c Author: Nils Goroll Date: Thu May 17 17:40:53 2018 +0200 ban lurker should back off on seeing a busy object HSH_Unbusy() calls BAN_NewObjCore() not holding the objhead lock, so the ban lurker may race and grab the ban mtx just after the new oc has been inserted, but the busy flag not yet cleared. While it would be correct to call BAN_NewObjCore() with the objhead mtx held, doing so would increase the pressure on the combined ban & objhead mtx. If the ban lurker encounters a busy object, we know that there must be an unbusy in progress and it would be wiser to rather back off in favor of the it. Fixes #2681 diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index ada0c05..4465ed2 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -108,7 +108,7 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt) oh = oc->objhead; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (!Lck_Trylock(&oh->mtx)) { - if (oc->refcnt == 0) { + if (oc->refcnt == 0 || oc->flags & OC_F_BUSY) { Lck_Unlock(&oh->mtx); } else { /* From phk at FreeBSD.org Mon Jun 4 08:44:27 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 4 Jun 2018 08:44:27 +0000 (UTC) Subject: [master] ccdfb59 Drag the "Makefile.phk" experiment behind the barn and kill it. Message-ID: <20180604084427.65BC6A4FD4@lists.varnish-cache.org> commit ccdfb59588f03158d8daa814e0e2af79acf73d75 Author: Poul-Henning Kamp Date: Mon Jun 4 08:42:59 2018 +0000 Drag the "Makefile.phk" experiment behind the barn and kill it. diff --git a/Makefile.inc.phk b/Makefile.inc.phk deleted file mode 100644 index ba9202f..0000000 --- a/Makefile.inc.phk +++ /dev/null @@ -1,245 +0,0 @@ - -.PHONY: default -default: all - -WARNS ?= 1 - -.PHONY: depend all clean install test -depend all clean install test: - @$(MAKE) --no-print-directory -f Makefile.phk TGT=$@ real-$@ \ - $(shell env CC=$(CC) WARNS=$(WARNS) \ - sh ${TOPDIR}/config.phk $(TOPDIR) $(CURDIR) ) - -CFLAGS += $(CF_CFLAGS) $(CF_CWFLAGS) -CFLAGS += -I$(CURDIR) -CFLAGS += -I$(TOPDIR) -CFLAGS += -I$(TOPDIR)/include -CFLAGS += -I$(TOPDIR)/lib/libvgz -CFLAGS += -I/usr/local/include - -SHLIB_LDFLAGS += -shared -Wl,-x -Wl,--fatal-warnings -Wl,--warn-shared-textrel -VMOD_LDFLAGS += ${SHLIB_LDFLAGS} - -####################################################################### - -INSTALL_BASE ?= $(TOPDIR)/_install - -####################################################################### - -LIB_VARNISH = -L $(TOPDIR)/lib/libvarnish -lvarnish -LIB_VARNISHAPI = $(TOPDIR)/lib/libvarnishapi/libvarnishapi.a -LIB_VCC = -L $(TOPDIR)/lib/libvcc -lvcc -LIB_VGZ = -L $(TOPDIR)/lib/libvgz -lvgz -LIB_PCRE = -L /usr/local/lib -lpcre -LIB_EXECINFO = ${CFG_EXECINFO} -LIB_CURSES = -lcurses -LIB_READLINE = ${CFG_LINEDISC} - -####################################################################### -ifdef USE_THREADS -CFLAGS += -pthread -D_THREAD_SAFE -g -O2 -PROG_LDFLAGS += -pthread -D_THREAD_SAFE -g -O2 -endif - -ifdef USE_DLOPEN -PROG_LDFLAGS += -Wl,-E -PROG_LIBS += ${CFG_LIBDL} -endif - -####################################################################### -ifdef SUBDIRS -ifeq "$(findstring k,$(MAKEFLAGS))" "" - EOPT=set -e -else - EOPT=set +e -endif -define run-subdirs - $(EOPT) ; for dir in $(SUBDIRS); do \ - echo "->> $$dir $(@:real-%=%)" ; \ - $(MAKE) -C $$dir -f Makefile.phk $(@:real-%=%) ; \ - done -endef -else -define run-subdirs -endef -endif - -####################################################################### -ifdef NOWORK -ifdef SUBDIRS - -.PHONY: subdirs $(SUBDIRS) - -subdirs: $(SUBDIRS) - -$(SUBDIRS): - $(MAKE) -C $@ -f Makefile.phk $(@:real-%=%) - -TODO += subdirs -endif -endif - -####################################################################### -.SUFFIXES: .So -.c.So: - ${CC} $(CFLAGS) -fPIC -c $< -o $@ - - -####################################################################### - -TODO_DEPEND += $(MADE_FILES) -TODO_ALL += $(MADE_FILES) -CLEAN_FILES += $(MADE_FILES) - -MADE_SRC = $(filter %.c, $(MADE_FILES)) - -####################################################################### -ifdef PROG_SRC -PROGNAME = $(notdir $(CURDIR)) -MANNAME=$(PROGNAME).1 -TODO_DEPEND += _.depprog -TODO_ALL += $(PROGNAME) -TODO_INSTALL += prog_install -PROG_SRC += $(MADE_SRC) -PROG_OBJ = $(notdir $(PROG_SRC:.c=.o)) -CLEAN_FILES += $(PROGNAME) $(PROG_OBJ) _.depprog $(MANNAME) - -_.depprog: ${PROG_SRC} ${MADE_FILES} - $(CC) $(CFLAGS) -MM $^ > _.depprog - -ifeq ($(TGT), all) -include _.depprog -endif - -$(PROGNAME): $(PROG_OBJ) - $(CC) $(PROG_LDFLAGS) -o $(PROGNAME) $^ $(PROG_LIBS) $(LD_ADD) - -.PHONY: prog_install -prog_install: $(PROGNAME) - @cp $(PROGNAME) $(INSTALL_BASE)/bin/ - @[ ! -f $(MANNAME) ] || cp $(MANNAME) $(INSTALL_BASE)/man/ - -endif - -####################################################################### -ifdef SHLIB_SRC -SHLIBNAME = $(notdir $(CURDIR)) -TODO_ALL += $(SHLIBNAME).so -TODO_DEPEND += _.depshlib -TODO_INSTALL += shlib_install -SHLIB_SRC += $(MADE_SRC) -SHLIB_OBJ = $(notdir $(SHLIB_SRC:.c=.So)) -CLEAN_FILES += $(SHLIB_OBJ) $(SHLIBNAME).so _.depshlib - -_.depshlib: ${SHLIB_SRC} ${MADE_FILES} - $(CC) $(CFLAGS) -MM $^ |sed 's/o:/So:/' > _.depshlib - -ifeq ($(TGT), all) -include _.depshlib -endif - -$(SHLIBNAME).so: $(SHLIB_OBJ) - $(CC) $(SHLIB_LDFLAGS) -o $(SHLIBNAME).so $^ - -.PHONY: shlib_install -shlib_install: $(SHLIBNAME).so - @cp $(SHLIBNAME).so $(INSTALL_BASE)/lib/ - -endif - -####################################################################### -ifdef LIB_SRC -LIBNAME = $(notdir $(CURDIR)) -TODO_ALL += $(LIBNAME).a -TODO_DEPEND += _.deplib -LIB_SRC += $(MADE_SRC) -LIB_OBJ = $(notdir $(LIB_SRC:.c=.o)) -CLEAN_FILES += $(LIB_OBJ) $(LIBNAME).a _.deplib - -_.deplib: ${LIB_SRC} ${MADE_FILES} - $(CC) $(CFLAGS) -MM $^ > _.deplib - -ifeq ($(TGT), all) -include _.deplib -endif - -$(LIBNAME).a: $(LIB_OBJ) - $(AR) -rc $(LIBNAME).a $^ - ranlib $(LIBNAME).a - -endif - -####################################################################### -ifdef VMOD_SRC -#VMODNAME = $(subst libvmod_,,$(notdir $(CURDIR))) -VMODNAME = $(notdir $(CURDIR)) -TODO_ALL += vcc_if.h vcc_if.c $(VMODNAME).so -TODO_DEPEND += _.depvmod -TODO_INSTALL += vmod_install -VMOD_SRC += $(MADE_SRC) -VMOD_SRC += vcc_if.c -VMOD_OBJ = $(notdir $(VMOD_SRC:.c=.So)) -CLEAN_FILES += $(VMOD_OBJ) vcc_if.c vcc_if.h $(VMODNAME).so _.depvmod - -CFLAGS += -I$(TOPDIR)/bin/varnishd - -_.depvmod: ${VMOD_SRC} ${MADE_FILES} - $(CC) $(CFLAGS) -MM $^ |sed 's/o:/So:/' > _.depvmod - -ifeq ($(TGT), all) -include _.depvmod -endif - -vcc_if.c vcc_if.h: vmod.vcc $(TOPDIR)/lib/libvcc/vmodtool.py - ${PYTHON} $(TOPDIR)/lib/libvcc/vmodtool.py - -$(VMODNAME).so: $(VMOD_OBJ) - $(CC) $(VMOD_LDFLAGS) -o $(VMODNAME).so $^ - mkdir -p .libs - (cd .libs && ln -sf ../$(VMODNAME).so) - -.PHONY: vmod_install -vmod_install: $(VMODNAME).so - @cp $(VMODNAME).so $(INSTALL_BASE)/libexec/ - -endif - -####################################################################### -ifdef RST_MAN -ifdef HAVE_RST2HTML -TODO_ALL += ${MANNAME} - -${MANNAME}: ${RST_MAN} - rst2man ${RST_MAN} $(MANNAME) - -endif -endif - -####################################################################### - -.PHONY: real-depend -real-depend: $(TODO) $(TODO_DEPEND) - @$(run-subdirs) - @true - -.PHONY: real-all -real-all: $(TODO) $(TODO_ALL) - @$(run-subdirs) - @true - -.PHONY: real-clean -real-clean: $(TODO) $(TODO_CLEAN) - @$(run-subdirs) - @[ "x$(CLEAN_FILES)" = "x" ] || rm -f $(CLEAN_FILES) - -.PHONY: real-install -real-install: $(TODO_INSTALL) - @$(run-subdirs) - @true - -.PHONY: real-test -real-test: $(TODO_TEST) - @$(run-subdirs) - @true - -####################################################################### diff --git a/Makefile.phk b/Makefile.phk deleted file mode 100644 index afeeeb6..0000000 --- a/Makefile.phk +++ /dev/null @@ -1,14 +0,0 @@ -SUBDIRS = lib bin doc - -CLEAN_FILES += config.h include/vcs_version. include/vmod_abi.h _.cache - -TODO_INSTALL += top_install - -TOPDIR = $(CURDIR) -include $(TOPDIR)/Makefile.inc.phk - -.PHONY: top_install -top_install: - @for d in man doc etc bin lib libexec ; do \ - mkdir -p $(INSTALL_BASE)/$$d ; \ - done diff --git a/bin/Makefile.phk b/bin/Makefile.phk deleted file mode 100644 index 66816ff..0000000 --- a/bin/Makefile.phk +++ /dev/null @@ -1,11 +0,0 @@ -SUBDIRS += varnishadm -SUBDIRS += varnishd -SUBDIRS += varnishlog -SUBDIRS += varnishncsa -SUBDIRS += varnishtest -SUBDIRS += varnishstat -SUBDIRS += varnishhist -SUBDIRS += varnishtop - -TOPDIR = $(CURDIR)/.. -include $(TOPDIR)/Makefile.inc.phk diff --git a/bin/varnishadm/Makefile.phk b/bin/varnishadm/Makefile.phk deleted file mode 100644 index 368b80a..0000000 --- a/bin/varnishadm/Makefile.phk +++ /dev/null @@ -1,12 +0,0 @@ -PROG_SRC = varnishadm.c - -LD_ADD += ${LIB_VARNISHAPI} -LD_ADD += ${LIB_VARNISH} -LD_ADD += ${LIB_PCRE} -LD_ADD += ${LIB_READLINE} -LD_ADD += -lm - -RST_MAN += $(TOPDIR)/doc/sphinx/reference/varnishadm.rst - -TOPDIR = $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk diff --git a/bin/varnishd/Makefile.phk b/bin/varnishd/Makefile.phk deleted file mode 100644 index 6882faa..0000000 --- a/bin/varnishd/Makefile.phk +++ /dev/null @@ -1,149 +0,0 @@ -VPATH += cache common mgt waiter storage hash http1 proxy - -PROG_SRC += cache/cache_acceptor.c -PROG_SRC += cache/cache_backend.c -PROG_SRC += cache/cache_backend_cfg.c -PROG_SRC += cache/cache_backend_tcp.c -PROG_SRC += cache/cache_backend_probe.c -PROG_SRC += cache/cache_ban.c -PROG_SRC += cache/cache_ban_build.c -PROG_SRC += cache/cache_ban_lurker.c -PROG_SRC += cache/cache_busyobj.c -PROG_SRC += cache/cache_cli.c -PROG_SRC += cache/cache_director.c -PROG_SRC += cache/cache_esi_deliver.c -PROG_SRC += cache/cache_esi_fetch.c -PROG_SRC += cache/cache_esi_parse.c -PROG_SRC += cache/cache_expire.c -PROG_SRC += cache/cache_fetch.c -PROG_SRC += cache/cache_fetch_proc.c -PROG_SRC += cache/cache_gzip.c -PROG_SRC += cache/cache_hash.c -PROG_SRC += cache/cache_http.c -PROG_SRC += http1/cache_http1_deliver.c -PROG_SRC += http1/cache_http1_fetch.c -PROG_SRC += http1/cache_http1_fsm.c -PROG_SRC += http1/cache_http1_proto.c -PROG_SRC += http1/cache_http1_pipe.c -PROG_SRC += http1/cache_http1_line.c -PROG_SRC += http1/cache_http1_vfp.c -PROG_SRC += cache/cache_lck.c -PROG_SRC += cache/cache_main.c -PROG_SRC += cache/cache_mempool.c -PROG_SRC += cache/cache_obj.c -PROG_SRC += cache/cache_panic.c -PROG_SRC += cache/cache_pool.c -PROG_SRC += cache/cache_range.c -PROG_SRC += cache/cache_req.c -PROG_SRC += cache/cache_req_fsm.c -PROG_SRC += cache/cache_req_body.c -PROG_SRC += cache/cache_rfc2616.c -PROG_SRC += cache/cache_session.c -PROG_SRC += cache/cache_shmlog.c -PROG_SRC += cache/cache_vary.c -PROG_SRC += cache/cache_vcl.c -PROG_SRC += cache/cache_vrt.c -PROG_SRC += cache/cache_vrt_re.c -PROG_SRC += cache/cache_vrt_var.c -PROG_SRC += cache/cache_vrt_vmod.c -PROG_SRC += cache/cache_vrt_priv.c -PROG_SRC += cache/cache_wrk.c -PROG_SRC += cache/cache_ws.c -PROG_SRC += cache/cache_deliver_proc.c - -PROG_SRC += proxy/cache_proxy_proto.c - -PROG_SRC += common/common_vsc.c -PROG_SRC += common/common_vsm.c - -PROG_SRC += hash/hash_classic.c -PROG_SRC += hash/hash_critbit.c -PROG_SRC += hash/mgt_hash.c -PROG_SRC += hash/hash_simple_list.c - -PROG_SRC += mgt/mgt_child.c -PROG_SRC += mgt/mgt_cli.c -PROG_SRC += mgt/mgt_main.c -PROG_SRC += mgt/mgt_param.c -PROG_SRC += mgt/mgt_param_bits.c -PROG_SRC += mgt/mgt_param_tbl.c -PROG_SRC += mgt/mgt_param_tcp.c -PROG_SRC += mgt/mgt_param_tweak.c -PROG_SRC += mgt/mgt_pool.c -PROG_SRC += mgt/mgt_jail.c -PROG_SRC += mgt/mgt_jail_unix.c -PROG_SRC += mgt/mgt_jail_solaris.c -PROG_SRC += mgt/mgt_shmem.c -PROG_SRC += mgt/mgt_vcc.c -PROG_SRC += mgt/mgt_vcl.c -PROG_SRC += mgt/mgt_acceptor.c - -PROG_SRC += storage/mgt_stevedore.c -PROG_SRC += storage/mgt_storage_persistent.c -PROG_SRC += storage/stevedore.c -PROG_SRC += storage/stevedore_utils.c -PROG_SRC += storage/storage_file.c -PROG_SRC += storage/storage_lru.c -PROG_SRC += storage/storage_malloc.c -PROG_SRC += storage/storage_persistent.c -PROG_SRC += storage/storage_persistent_silo.c -PROG_SRC += storage/storage_persistent_subr.c -PROG_SRC += storage/storage_simple.c -PROG_SRC += storage/storage_umem.c - -PROG_SRC += waiter/cache_waiter.c -PROG_SRC += waiter/cache_waiter_epoll.c -PROG_SRC += waiter/cache_waiter_kqueue.c -PROG_SRC += waiter/cache_waiter_poll.c -PROG_SRC += waiter/cache_waiter_ports.c -PROG_SRC += waiter/mgt_waiter.c - -USE_THREADS = YES -USE_DLOPEN = YES - -LD_ADD += ${LIB_VARNISH} -LD_ADD += ${LIB_VCC} -LD_ADD += ${LIB_PCRE} -LD_ADD += ${LIB_EXECINFO} -LD_ADD += ${LIB_VGZ} -LD_ADD += -lm - -RST_MAN += $(TOPDIR)/doc/sphinx/reference/varnishd.rst - -MADE_FILES += builtin_vcl.h - -TODO_INSTALL += install_etc - -PARST=$(TOPDIR)/doc/sphinx/include/params.rst - -TODO_ALL += $(PARST) -CLEAN_FILES += $(PARST) - -CFLAGS += -DVARNISHD_IS_NOT_A_VMOD - -TOPDIR = $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk - -$(PARST): $(PROGNAME) - -ls -l $(PARST) $(PROGNAME) - mkdir -p $(TOPDIR)/doc/sphinx/include - ./varnishd -x parameter > $(PARST) - -# -# Turn the builtin.vcl file into a C-string we can include in the program. -# -builtin_vcl.h: builtin.vcl - echo '/*' > $@ - echo ' * NB: This file is machine generated, DO NOT EDIT!' >> $@ - echo ' *' >> $@ - echo ' * Edit builtin.vcl instead and run make' >> $@ - echo ' *' >> $@ - echo ' */' >> $@ - echo '' >> $@ - sed -e 's/"/\\"/g' \ - -e 's/$$/\\n"/' \ - -e 's/^/ "/' builtin.vcl >> $@ - -.PHONY: install_etc -install_etc: builtin_vcl.h - @cp builtin_vcl.h $(INSTALL_BASE)/etc/ diff --git a/bin/varnishhist/Makefile.phk b/bin/varnishhist/Makefile.phk deleted file mode 100644 index c2df17f..0000000 --- a/bin/varnishhist/Makefile.phk +++ /dev/null @@ -1,9 +0,0 @@ -PROG_SRC = varnishhist.c -PROG_SRC += varnishhist_options.c - -LD_ADD += ${LIB_VARNISHAPI} -LD_ADD += ${LIB_VARNISH} -LD_ADD += -lpthread -lncurses -lm ${LIB_PCRE} - -TOPDIR = $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk diff --git a/bin/varnishlog/Makefile.phk b/bin/varnishlog/Makefile.phk deleted file mode 100644 index c024046..0000000 --- a/bin/varnishlog/Makefile.phk +++ /dev/null @@ -1,14 +0,0 @@ -PROG_SRC = varnishlog.c -PROG_SRC = varnishlog_options.c - -LD_ADD += ${LIB_VARNISHAPI} -LD_ADD += ${LIB_VARNISH} -LD_ADD += ${LIB_PCRE} -LD_ADD += -lm - -TODO_ALL += build_man - -TOPDIR = $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk - -.PHONY: build_man diff --git a/bin/varnishncsa/Makefile.phk b/bin/varnishncsa/Makefile.phk deleted file mode 100644 index fd73e25..0000000 --- a/bin/varnishncsa/Makefile.phk +++ /dev/null @@ -1,15 +0,0 @@ -PROG_SRC += base64.c -PROG_SRC += varnishncsa.c -PROG_SRC += varnishncsa_options.c - -LD_ADD += ${LIB_VARNISHAPI} -LD_ADD += ${LIB_VARNISH} -LD_ADD += ${LIB_PCRE} -LD_ADD += -lm - -TODO_ALL += build_man - -TOPDIR = $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk - -.PHONY: build_man diff --git a/bin/varnishstat/Makefile.phk b/bin/varnishstat/Makefile.phk deleted file mode 100644 index 3a89de7..0000000 --- a/bin/varnishstat/Makefile.phk +++ /dev/null @@ -1,13 +0,0 @@ -PROG_SRC += varnishstat.c -PROG_SRC += varnishstat_curses.c - -LD_ADD += ${LIB_VARNISHAPI} -LD_ADD += ${LIB_VARNISH} -LD_ADD += ${LIB_PCRE} -LD_ADD += ${LIB_CURSES} -LD_ADD += -lm - -RST_MAN += $(TOPDIR)/doc/sphinx/reference/varnishstat.rst - -TOPDIR = $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk diff --git a/bin/varnishtest/Makefile.phk b/bin/varnishtest/Makefile.phk deleted file mode 100644 index b406cce..0000000 --- a/bin/varnishtest/Makefile.phk +++ /dev/null @@ -1,31 +0,0 @@ -PROG_SRC += vtc.c -PROG_SRC += vtc_client.c -PROG_SRC += vtc_http.c -PROG_SRC += vtc_log.c -PROG_SRC += vtc_logexp.c -PROG_SRC += vtc_main.c -PROG_SRC += vtc_barrier.c -PROG_SRC += vtc_server.c -PROG_SRC += vtc_syslog.c -PROG_SRC += vtc_varnish.c -PROG_SRC += vtc_process.c - -LD_ADD += ${LIB_VARNISHAPI} -LD_ADD += ${LIB_VARNISH} -LD_ADD += ${LIB_PCRE} -LD_ADD += ${LIB_VGZ} -LD_ADD += -lm - -USE_THREADS = yes - -RST_MAN += $(TOPDIR)/doc/sphinx/reference/varnishtest.rst - -TODO_TEST += all_tests - -TOPDIR = $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk - -.PHONY: all_tests - -all_tests: - ./varnishtest -i -j 24 tests/*.vtc diff --git a/bin/varnishtop/Makefile.phk b/bin/varnishtop/Makefile.phk deleted file mode 100644 index 12d4de3..0000000 --- a/bin/varnishtop/Makefile.phk +++ /dev/null @@ -1,14 +0,0 @@ -PROG_SRC += varnishtop.c -PROG_SRC += varnishtop_options.c - -LD_ADD += ${LIB_VARNISHAPI} -LD_ADD += ${LIB_VARNISH} -LD_ADD += ${LIB_PCRE} -LD_ADD += -lm -lpthread -lncurses - -TODO_ALL += build_man - -TOPDIR = $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk - -.PHONY: build_man diff --git a/config.phk b/config.phk deleted file mode 100644 index 4111c35..0000000 --- a/config.phk +++ /dev/null @@ -1,303 +0,0 @@ -#!/bin/sh - -set -e - -####################################################################### -# Administrative settings - -ADM_PROJECT=varnish -ADM_VERSION=trunk - -CONFIG_H=phk_hack/config.h - -####################################################################### -# We always rebuild the config when make is run in the toplevel directory -# Otherwise, we try to use a cached config, if we have one. -# -# We know we're in the toplevel directory, because we get passed -# TOPDIR and CURDIR as arguments - -if [ "x$1" = "x" -o "x$2" = "x" ] ; then - echo "Missing arguments, assuming topdir" 1>&2 - set `pwd` `pwd` -fi - -if [ "x$1" = "x$2" ] ; then - rm -f _.cache -elif [ "x$1" != "x" -a -d $1 ] ; then - cd $1 -fi - -####################################################################### -# autocrap co-existence -# We put our config.h somewhere else and delete autocraps. -# Autocrap regenerates its own config.h and doesn't know about ours - -mkdir -p phk_hack -rm -f config.h - -####################################################################### - - -if [ -f _.cache -a -f ${CONFIG_H} ] ; then - cat _.cache - exit 0 -fi - -echo "Building Config" 1>&2 - -####################################################################### -# Look for #include files for HAVE_ etc. - -if true ; then - rm -f ${CONFIG_H}_ - echo '' > ${CONFIG_H}_ - - for i in \ - curses.h \ - endian.h \ - execinfo.h \ - ncurses.h \ - ncurses/curses.h \ - ncursesw.h \ - ncursesw/curses.h \ - priv.h \ - pthread_np.h \ - edit/readline/readline.h \ - readline/history.h \ - editline/readline.h \ - readline/readline.h \ - sys/endian.h \ - sys/filio.h \ - sys/mount.h \ - sys/statvfs.h \ - sys/vfs.h \ - umem.h - do - if [ -f /usr/include/$i ] ; then - n=`echo $i | tr '[a-z/.]' '[A-Z__]'` - echo "#define HAVE_$n 1" >> ${CONFIG_H}_ - else - echo "#include <$i>" > _.c - if cc -E _.c > /dev/null 2>& 1 ; then - n=`echo $i | tr '[a-z/.]' '[A-Z__]'` - echo "#define HAVE_$n 1" >> ${CONFIG_H}_ - fi - rm -f _.c - fi - done - echo "#define PACKAGE_TARNAME \"${ADM_PROJECT}\"" >> ${CONFIG_H}_ - echo "#define PACKAGE_VERSION \"${ADM_VERSION}\"" >> ${CONFIG_H}_ - - echo ' - #define VARNISH_VCL_DIR "/etc/varnish" - #define VARNISH_STATE_DIR "/tmp/phk/" - #define VARNISH_VMOD_DIR "/usr/lib/varnish/vmods" - ' >> ${CONFIG_H}_ - if [ ! -f ${CONFIG_H} ] ; then - mv ${CONFIG_H}_ ${CONFIG_H} - elif ! cmp -s ${CONFIG_H} ${CONFIG_H}_ ; then - mv ${CONFIG_H}_ ${CONFIG_H} - else - rm -f ${CONFIG_H}_ - fi -fi - -####################################################################### -# Create files depending on VCS (git) output - -VCSF=include/vcs_version.h -VMAV=include/vmod_abi.h -V=NOGIT - -if [ -d ./.git ] ; then - V=`git show -s --pretty=format:%h` -fi - -cat > ${VCSF}_ < ${VMAV} -fi - -####################################################################### -# Ask the compiler about stuff - -ask_compiler() { - a=`echo "$2" | - ${CC-cc} -E - -DWARNS=${WARNS-1} | - sed -e '/^#/d' -e '/^$/d' -e 's/"//g' -e 's/~/"/g' ` - echo "$1='" $a "'" >> _.cache -} - -# Warning flags -ask_compiler CF_CWFLAGS ' - #if WARNS >= 1 - "-Wall" - "-Werror" - #endif - #if WARNS >= 2 - "-W" - "-fstack-protector" - "-Wno-format-y2k" - "-Wstrict-prototypes" - "-Wmissing-prototypes" - "-Wpointer-arith" - "-Wreturn-type" - "-Wcast-qual" - "-Wwrite-strings" - "-Wswitch" - "-Wshadow" - "-Wunused-parameter" - "-Wcast-align" - "-Wchar-subscripts" - "-Winline" - "-Wnested-externs" - "-Wno-pointer-sign" - "-Wno-empty-body" - "-Wextra" - "-Wno-missing-field-initializers" - "-Wno-sign-compare" - #if defined(__clang__) - "-Wmissing-variable-declarations" - "-Wno-string-plus-int" - #endif - #endif - /* - * Write is marked with this on some Linux, and while that is - * well intentioned, they have implemented it so that the usual - * (void)bla; - * marker of intent does not work. - * I dont want to write bogo-code just to slip the best effort - * 4xx responses in cache_http1_fetch.c past the compiler. - */ - "-Wno-unused-result" - ' - -# Configuration options -ask_compiler CF_CFLAGS ' - #if defined(__SVR4) && defined(sun) - // Solaris and OmniOS - "-DHAVE_GETHRTIME" - "-DHAVE_PORT_CREATE" - "-DHAVE_SETPPRIV" - #endif - - #if !defined(__APPLE__) - "-DHAVE_DAEMON" - #endif - - // Where does this not work ? - "-DSO_SNDTIMEO_WORKS" - "-DSO_RCVTIMEO_WORKS" - "-DHAVE_TCP_KEEP" - - #if defined(__linux__) - "-D_GNU_SOURCE=1" - #endif - - "-DVCC_CC=~\~exec $(CC) -D_THREAD_SAFE -std=gnu99 -g " \ - "-O2 -Wall -Werror -pthread -fpic -shared -Wl,-x -o %o %s\~~" - - "-DVCC_WARNS=~\~$(CF_CWFLAGS)\~~" - - "-I$(TOPDIR)/phk_hack" - - ' - -####################################################################### -# Find a Python interpreter -# - -for i in 3.2 2.7 "" 2.5 2.6 3.0 3.1 -do - if python$i < /dev/null > /dev/null 2>&1 ; then - echo PYTHON=python$i >> _.cache - break - fi -done - -####################################################################### -# Find a rst2* tools -# - -if echo | rst2html > /dev/null 2>&1 ; then - echo "HAVE_RST2HTML=1" >> _.cache -fi - -####################################################################### -# Check for libedit - -if [ -f /usr/include/edit/readline/readline.h ] ; then - echo "CFG_LINEDISC=-ledit" >> _.cache -elif [ -f /usr/include/editline/readline.h ] ; then - echo "CFG_LINEDISC=-ledit" >> _.cache -elif [ -f /usr/include/readline/readline.h ] ; then - echo "CFG_LINEDISC=-lreadline" >> _.cache -else - echo "CFG_LINEDISC=" >> _.cache -fi - -####################################################################### -# Check for libexecinfo for FreeBSD::backtrace() - -if [ -f /usr/lib/libexecinfo.so ] ; then - echo "CFG_EXECINFO=-lexecinfo" >> _.cache -fi - -####################################################################### -# Check for libdl for Linux::dlopen() - -if [ `uname -s` = "Linux" ] ; then - echo "CFG_LIBDL=-ldl" >> _.cache -fi - -####################################################################### -# Done... -cat _.cache - -exit 0 - - -# HAVE_ACCEPT_FILTERS + -# HAVE_BACKTRACE + -# HAVE_CLOCK_GETTIME + -# HAVE_DAEMON + -# HAVE_KQUEUE + -# HAVE_NANOSLEEP + -# HAVE_PTHREAD_SET_NAME_NP + -# HAVE_PTHREAD_SETNAME_NP + -# HAVE_SETPROCTITLE + -# HAVE_SRANDOMDEV + -# HAVE_TCP_KEEP + -# HAVE_TIMEGM + -# PACKAGE_TARNAME + -# PACKAGE_VERSION + -# SO_RCVTIMEO_WORKS + -# SO_SNDTIMEO_WORKS + -# VCC_CC + -# HAVE_EPOLL_CTL - -# HAVE_LIBUMEM - -# USE_PCRE_JIT - -# _FILE_OFFSET_BITS - -# -# #echo 'CF_CFLAGS="-Wall"' diff --git a/doc/Makefile.phk b/doc/Makefile.phk deleted file mode 100644 index fe6059e..0000000 --- a/doc/Makefile.phk +++ /dev/null @@ -1,7 +0,0 @@ - -ifdef HAVE_RST2HTML -SUBDIRS += sphinx -endif - -TOPDIR = .. -include $(TOPDIR)/Makefile.inc.phk diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk deleted file mode 100644 index ba77075..0000000 --- a/doc/sphinx/Makefile.phk +++ /dev/null @@ -1,239 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = build - -# Internal variables. -PAPEROPT_a4 = -D latex_elements.papersize=a4 -PAPEROPT_letter = -D latex_elements.papersize=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(CURDIR) - -.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest - -all: conf.py html - -conf.py: conf.py.in - cp conf.py.in conf.py - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -.PHONY: depend install test -depend install test: - @true - -clean: - -rm -rf $(BUILDDIR)/* conf.py - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Varnish.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Varnish.qhc" - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -EXTRA_DIST = \ - conf.py \ - index.rst \ - glossary/index.rst \ - installation/bugs.rst \ - installation/help.rst \ - installation/index.rst \ - installation/install.rst \ - installation/platformnotes.rst \ - installation/prerequisites.rst \ - phk/autocrap.rst \ - phk/backends.rst \ - phk/barriers.rst \ - phk/gzip.rst \ - phk/http20.rst \ - phk/index.rst \ - phk/ipv6suckage.rst \ - phk/platforms.rst \ - phk/spdy.rst \ - phk/sphinx.rst \ - phk/ssl.rst \ - phk/thetoolsweworkwith.rst \ - phk/thoughts.rst \ - phk/three-zero.rst \ - phk/varnish_does_not_hash.rst \ - phk/vcl_expr.rst \ - phk/wanton_destruction.rst \ - reference/index.rst \ - reference/varnish-cli.rst \ - reference/varnishadm.rst \ - reference/varnishd.rst \ - reference/varnishhist.rst \ - reference/varnishlog.rst \ - reference/varnishncsa.rst \ - reference/varnishstat.rst \ - reference/varnishtest.rst \ - reference/varnishtop.rst \ - reference/vcl.rst \ - reference/vmod.rst \ - reference/vmod_std.rst \ - reference/vsl-query.rst \ - reference/vsl.rst \ - reference/vsm.rst \ - tutorial/backend_servers.rst \ - tutorial/index.rst \ - tutorial/introduction.rst \ - tutorial/now_what.rst \ - tutorial/peculiarities.rst \ - tutorial/putting_varnish_on_port_80.rst \ - tutorial/starting_varnish.rst \ - users-guide/command-line.rst \ - users-guide/compression.rst \ - users-guide/devicedetection.rst \ - users-guide/esi.rst \ - users-guide/increasing-your-hitrate.rst \ - users-guide/index.rst \ - users-guide/intro.rst \ - users-guide/operation-logging.rst \ - users-guide/operation-statistics.rst \ - users-guide/performance.rst \ - users-guide/purging.rst \ - users-guide/report.rst \ - users-guide/run_cli.rst \ - users-guide/run_security.rst \ - users-guide/running.rst \ - users-guide/sizing-your-cache.rst \ - users-guide/storage-backends.rst \ - users-guide/troubleshooting.rst \ - users-guide/vcl-actions.rst \ - users-guide/vcl-backends.rst \ - users-guide/vcl-built-in-subs.rst \ - users-guide/vcl-example-acls.rst \ - users-guide/vcl-example-manipulating-headers.rst \ - users-guide/vcl-example-manipulating-responses.rst \ - users-guide/vcl-example-websockets.rst \ - users-guide/vcl-examples.rst \ - users-guide/vcl-hashing.rst \ - users-guide/vcl-inline-c.rst \ - users-guide/vcl-saint-and-grace.rst \ - users-guide/vcl-syntax.rst \ - users-guide/vcl-variables.rst \ - users-guide/vcl.rst \ - include/params.rst \ - include/varnishncsa_options.rst \ - include/varnishncsa_synopsis.rst \ - include/varnishlog_options.rst \ - include/varnishlog_synopsis.rst \ - include/varnishtop_options.rst \ - include/varnishtop_synopsis.rst \ - include/varnishhist_options.rst \ - include/varnishhist_synopsis.rst \ - reference/vmod_std.generated.rst \ - reference/vmod_directors.generated.rst - - -dist-hook: - $(MAKE) html - cp -r $(BUILDDIR) $(distdir) - -distclean-local: - rm -rf $(BUILDDIR) - -# XXX: here be dragons -include/params.rst: ../../bin/varnishd/varnishd - ../../bin/varnishd/varnishd -x parameter > $@ - - -# only sphinx needs the opt2rst stuff anyway -%_opt2rst: ../../bin/%/%_options.h %_options.c - -include/varnishncsa_options.rst: varnishncsa - ../../bin/varnishncsa/varnishncsa --options > $@ - -include/varnishncsa_synopsis.rst: varnishncsa - ../../bin/varnishncsa/varnishncsa --synopsis > $@ - -include/varnishlog_options.rst: varnishlog - ../../bin/varnishlog/varnishlog --options > $@ - -include/varnishlog_synopsis.rst: varnishlog - ../../bin/varnishlog/varnishlog --synopsis > $@ - -include/varnishtop_options.rst: varnishtop - ../../bin/varnishtop/varnishtop --options > $@ - -include/varnishtop_synopsis.rst: varnishtop - ../../bin/varnishtop/varnishtop --synopsis > $@ - -include/varnishhist_options.rst: varnishhist - ./varnishhist --options > $@ - -include/varnishhist_synopsis.rst: varnishhist - ./varnishhist --synopsis > $@ - -reference/vmod_std.generated.rst: reference ../../lib/libvmod_std/vmod_std.rst - cp ../../lib/libvmod_std/vmod_std.rst $@ - -reference/vmod_directors.generated.rst: reference ../../lib/libvmod_directors/vmod_directors.rst - cp ../../lib/libvmod_directors/vmod_directors.rst $@ - diff --git a/lib/Makefile.phk b/lib/Makefile.phk deleted file mode 100644 index 86ea172..0000000 --- a/lib/Makefile.phk +++ /dev/null @@ -1,6 +0,0 @@ -SUBDIRS = libvcc libvarnish libvarnishapi libvcc -SUBDIRS += libvgz -SUBDIRS += libvmod_debug libvmod_directors libvmod_std - -TOPDIR = $(CURDIR)/.. -include $(TOPDIR)/Makefile.inc.phk diff --git a/lib/libvarnish/Makefile.phk b/lib/libvarnish/Makefile.phk deleted file mode 100644 index 501a350..0000000 --- a/lib/libvarnish/Makefile.phk +++ /dev/null @@ -1,29 +0,0 @@ - -LIB_SRC += binary_heap.c -LIB_SRC += cli_auth.c -LIB_SRC += cli_common.c -LIB_SRC += cli_serve.c -LIB_SRC += vas.c -LIB_SRC += vav.c -LIB_SRC += vct.c -LIB_SRC += version.c -LIB_SRC += vev.c -LIB_SRC += vfil.c -LIB_SRC += vfl.c -LIB_SRC += vin.c -LIB_SRC += vlu.c -LIB_SRC += vmb.c -LIB_SRC += vnum.c -LIB_SRC += vpf.c -LIB_SRC += vre.c -LIB_SRC += vrnd.c -LIB_SRC += vsa.c -LIB_SRC += vsb.c -LIB_SRC += vsha256.c -LIB_SRC += vss.c -LIB_SRC += vsub.c -LIB_SRC += vtcp.c -LIB_SRC += vtim.c - -TOPDIR= $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk diff --git a/lib/libvarnishapi/Makefile.phk b/lib/libvarnishapi/Makefile.phk deleted file mode 100644 index 3c4eca9..0000000 --- a/lib/libvarnishapi/Makefile.phk +++ /dev/null @@ -1,38 +0,0 @@ - -LIB_SRC += vsc.c -LIB_SRC += vsl.c -LIB_SRC += vsl2rst.c -LIB_SRC += vsl_arg.c -LIB_SRC += vsl_cursor.c -LIB_SRC += vsl_dispatch.c -LIB_SRC += vsl_query.c -LIB_SRC += vsm.c -LIB_SRC += vut.c -LIB_SRC += vxp.c -LIB_SRC += vxp_fixed_token.c -LIB_SRC += vxp_lexer.c -LIB_SRC += vxp_parse.c - - -# We add more stuff to the SHLIB version to make it self-contained -SHLIB_SRC = $(LIB_SRC) -VPATH += ../libvarnish -SHLIB_SRC += vas.c -SHLIB_SRC += vav.c -SHLIB_SRC += vfl.c -SHLIB_SRC += vin.c -SHLIB_SRC += vpf.c -SHLIB_SRC += vre.c -SHLIB_SRC += vsb.c -SHLIB_SRC += vtim.c - -#LIB_SRC += vsl_glob_test.c -#LIB_SRC += vxp_test.c - -MADE_FILES += vxp_fixed_token.c vxp_tokens.h - -TOPDIR= $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk - -$(MADE_FILES): generate.py - python generate.py diff --git a/lib/libvcc/Makefile.phk b/lib/libvcc/Makefile.phk deleted file mode 100644 index 333bad4..0000000 --- a/lib/libvcc/Makefile.phk +++ /dev/null @@ -1,25 +0,0 @@ - -LIB_SRC += vcc_acl.c -LIB_SRC += vcc_action.c -LIB_SRC += vcc_backend.c -LIB_SRC += vcc_backend_util.c -LIB_SRC += vcc_compile.c -LIB_SRC += vcc_expr.c -LIB_SRC += vcc_parse.c -LIB_SRC += vcc_storage.c -LIB_SRC += vcc_symb.c -LIB_SRC += vcc_token.c -LIB_SRC += vcc_utils.c -LIB_SRC += vcc_var.c -LIB_SRC += vcc_vmod.c -LIB_SRC += vcc_xref.c - -MADE_FILES += vcc_fixed_token.c vcc_obj.c vcc_token_defs.h - -CLEAN_FILES += $(MADE_FILES) - -TOPDIR= $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk - -$(MADE_FILES): generate.py - $(PYTHON) generate.py diff --git a/lib/libvgz/Makefile.phk b/lib/libvgz/Makefile.phk deleted file mode 100644 index b938312..0000000 --- a/lib/libvgz/Makefile.phk +++ /dev/null @@ -1,17 +0,0 @@ -LIB_SRC += adler32.c -LIB_SRC += compress.c -LIB_SRC += crc32.c -LIB_SRC += deflate.c -LIB_SRC += infback.c -LIB_SRC += inffast.c -LIB_SRC += inflate.c -LIB_SRC += inftrees.c -LIB_SRC += trees.c -LIB_SRC += uncompr.c -LIB_SRC += zutil.c - -CFLAGS += -DZLIB_CONST -D_LARGEFILE64_SOURCE=1 - -TOPDIR= $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk - diff --git a/lib/libvmod_debug/Makefile.phk b/lib/libvmod_debug/Makefile.phk deleted file mode 100644 index 35135cb..0000000 --- a/lib/libvmod_debug/Makefile.phk +++ /dev/null @@ -1,7 +0,0 @@ - -VMOD_SRC += vmod_debug.c -VMOD_SRC += vmod_debug_obj.c -VMOD_SRC += vmod_debug_dyn.c - -TOPDIR= $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk diff --git a/lib/libvmod_directors/Makefile.phk b/lib/libvmod_directors/Makefile.phk deleted file mode 100644 index bf0a921..0000000 --- a/lib/libvmod_directors/Makefile.phk +++ /dev/null @@ -1,9 +0,0 @@ - -VMOD_SRC += fall_back.c -VMOD_SRC += hash.c -VMOD_SRC += random.c -VMOD_SRC += round_robin.c -VMOD_SRC += vdir.c - -TOPDIR= $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk diff --git a/lib/libvmod_std/Makefile.phk b/lib/libvmod_std/Makefile.phk deleted file mode 100644 index 376a6f0..0000000 --- a/lib/libvmod_std/Makefile.phk +++ /dev/null @@ -1,8 +0,0 @@ - -VMOD_SRC += vmod_std.c -VMOD_SRC += vmod_std_conversions.c -VMOD_SRC += vmod_std_fileread.c -VMOD_SRC += vmod_std_querysort.c - -TOPDIR= $(CURDIR)/../.. -include $(TOPDIR)/Makefile.inc.phk From nils.goroll at uplex.de Mon Jun 4 12:02:25 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 4 Jun 2018 12:02:25 +0000 (UTC) Subject: [master] c66a5a2 Handle sigbus like sigsegv Message-ID: <20180604120225.348DDA95E2@lists.varnish-cache.org> commit c66a5a2620a07aefb1a65394ad25470153dc37f1 Author: Nils Goroll Date: Mon Jun 4 14:01:15 2018 +0200 Handle sigbus like sigsegv For the purpose of our signal handler (and, in particular, the stack overflow detection heuristic) they are identical. Fixes #2695 (as confirmed by @lkarsten) diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index bfc76b4..8b7a63e 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -243,9 +243,10 @@ child_signal_handler(int s, siginfo_t *si, void *c) memset(&sa, 0, sizeof sa); sa.sa_handler = SIG_DFL; (void)sigaction(SIGSEGV, &sa, NULL); + (void)sigaction(SIGBUS, &sa, NULL); (void)sigaction(SIGABRT, &sa, NULL); - while (s == SIGSEGV) { + while (s == SIGSEGV || s == SIGBUS) { req = THR_GetRequest(); if (req == NULL || req->wrk == NULL) break; diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc index f33a8bd..09a9c1c 100644 --- a/bin/varnishtest/tests/c00057.vtc +++ b/bin/varnishtest/tests/c00057.vtc @@ -76,7 +76,7 @@ client c2 -connect ${v2_sock} { expect_close } -run -varnish v2 -cliexpect "Segmentation [fF]ault" "panic.show" +varnish v2 -cliexpect "[bB]us error|Segmentation [fF]ault" "panic.show" varnish v2 -cliok "panic.clear" From nils.goroll at uplex.de Tue Jun 5 06:06:28 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Jun 2018 06:06:28 +0000 (UTC) Subject: [master] f4a6413 vcp: move open calls to VCP function, use function pointers Message-ID: <20180605060628.5CFF792B12@lists.varnish-cache.org> commit f4a64133a29223517135c32eef11749f23053e37 Author: Nils Goroll Date: Tue Jun 5 08:02:38 2018 +0200 vcp: move open calls to VCP function, use function pointers diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index fd416e8..c92ba8c 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -368,6 +368,21 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) } } +/*-------------------------------------------------------------------- + * Open a new connection from pool. + */ + +static int +VCP_Open(const struct conn_pool *cp, double tmo, const void **privp) +{ + int r; + + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); + + r = cp->methods->open(cp, tmo, privp); + + return (r); +} /*-------------------------------------------------------------------- * Close a connection. @@ -406,7 +421,7 @@ VCP_Close(struct pfd **pfdp) } /*-------------------------------------------------------------------- - * Get a connection + * Get a connection, possibly recycled */ static struct pfd * @@ -444,7 +459,7 @@ VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk, INIT_OBJ(pfd->waited, WAITED_MAGIC); pfd->state = PFD_STATE_USED; pfd->conn_pool = cp; - pfd->fd = cp->methods->open(cp, tmo, &pfd->priv); + pfd->fd = VCP_Open(cp, tmo, &pfd->priv); if (pfd->fd < 0) { FREE_OBJ(pfd); Lck_Lock(&cp->mtx); @@ -718,17 +733,13 @@ VTP_Rel(struct tcp_pool **tpp) } /*-------------------------------------------------------------------- - * Open a new connection from pool. This is a distinct function since - * probing cannot use a recycled connection. + * Open a new connection from pool. */ int VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp) { - - if (tp->uds != NULL) - return (vus_open(tp->cp, tmo, privp)); - return (vtp_open(tp->cp, tmo, privp)); + return (VCP_Open(tp->cp, tmo, privp)); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Tue Jun 5 07:45:17 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Jun 2018 07:45:17 +0000 (UTC) Subject: [master] 84ad0a7 Be a bit less Postel-y about http header charactersets. Message-ID: <20180605074517.821A696508@lists.varnish-cache.org> commit 84ad0a7236a43c5b6fef2a7be37901b230fc8999 Author: Poul-Henning Kamp Date: Tue Jun 5 07:43:55 2018 +0000 Be a bit less Postel-y about http header charactersets. diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c index ad44218..9d1c072 100644 --- a/bin/varnishd/http1/cache_http1_proto.c +++ b/bin/varnishd/http1/cache_http1_proto.c @@ -116,7 +116,7 @@ static uint16_t http1_dissect_hdrs(struct http *hp, char *p, struct http_conn *htc, unsigned maxhdr) { - char *q, *r; + char *q, *r, *s; assert(p > htc->rxbuf_b); assert(p <= htc->rxbuf_e); @@ -188,7 +188,14 @@ http1_dissect_hdrs(struct http *hp, char *p, struct http_conn *htc, q--; *q = '\0'; - if (strchr(p, ':') == NULL) { + for (s = p; *s != ':' && s < q; s++) { + if (!vct_istchar(*s)) { + VSLb(hp->vsl, SLT_BogoHeader, + "Illegal char 0x%02x in header name", *s); + return (400); + } + } + if (*s != ':') { VSLb(hp->vsl, SLT_BogoHeader, "Header without ':' %.*s", (int)(q - p > 20 ? 20 : q - p), p); return (400); @@ -204,18 +211,6 @@ http1_dissect_hdrs(struct http *hp, char *p, struct http_conn *htc, (int)(q - p > 20 ? 20 : q - p), p); return (400); } - - for (; p < q; p++) { - if (vct_islws(*p)) { - VSLb(hp->vsl, SLT_BogoHeader, - "Space in header '%.*s'", - (int)Tlen(hp->hd[hp->nhd - 1]), - hp->hd[hp->nhd - 1].b); - return (400); - } - if (*p == ':') - break; - } } if (p < htc->rxbuf_e) p += vct_skipcrlf(p); diff --git a/bin/varnishtest/tests/b00049.vtc b/bin/varnishtest/tests/b00049.vtc index 7d2e289..af3cac7 100644 --- a/bin/varnishtest/tests/b00049.vtc +++ b/bin/varnishtest/tests/b00049.vtc @@ -7,6 +7,11 @@ server s1 { varnish v1 -vcl+backend { } -start +logexpect l1 -v v1 -g raw { + expect * 1004 BogoHeader "Illegal char 0x20 in header name" + expect * 1006 BogoHeader "Illegal char 0x2f in header name" +} -start + client c1 { send "GET / HTTP/1.1\r\n" send "Host: foo\r\n" @@ -30,3 +35,14 @@ client c1 { rxresp expect resp.status == 400 } -run + +client c1 { + send "GET / HTTP/1.1\r\n" + send "Host: foo\r\n" + send "Accept/Encoding: gzip\r\n" + send "\r\n" + rxresp + expect resp.status == 400 +} -run + +logexpect l1 -wait From phk at FreeBSD.org Tue Jun 5 08:12:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Jun 2018 08:12:10 +0000 (UTC) Subject: [master] 66a6183 Cover H2's illegal header detection code. Message-ID: <20180605081210.D8CCF970D9@lists.varnish-cache.org> commit 66a618352483e2d04c9a65fefe2e817c98512de1 Author: Poul-Henning Kamp Date: Tue Jun 5 08:09:01 2018 +0000 Cover H2's illegal header detection code. diff --git a/bin/varnishtest/tests/t02002.vtc b/bin/varnishtest/tests/t02002.vtc index f2cfb50..c1652b1 100644 --- a/bin/varnishtest/tests/t02002.vtc +++ b/bin/varnishtest/tests/t02002.vtc @@ -40,3 +40,18 @@ varnish v1 -expect MEMPOOL.req0.live == 0 varnish v1 -expect MEMPOOL.req1.live == 0 varnish v1 -expect MEMPOOL.sess0.live == 0 varnish v1 -expect MEMPOOL.sess1.live == 0 + +client c1 { + stream 11 { + txreq -hdr sna[]fu foo.bar -pad cotton + rxrst + } -run +} -run + +client c1 { + stream 13 { + txreq -hdr snaFu foo.bar -pad cotton + rxrst + } -run +} -run + From daghf at varnish-software.com Tue Jun 5 12:06:10 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 5 Jun 2018 12:06:10 +0000 (UTC) Subject: [master] 2c5dfd8 Clear bo->was_304 on return(retry) Message-ID: <20180605120610.DF51AA53DB@lists.varnish-cache.org> commit 2c5dfd8dae37331252c90124f45a3b75d7ab5ea0 Author: Dag Haavi Finstad Date: Tue Jun 5 14:04:57 2018 +0200 Clear bo->was_304 on return(retry) Fixes: #2700 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 06d9ad9..0941826 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -240,6 +240,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo) bo->do_esi = 0; bo->do_stream = 1; bo->filter_list = NULL; + bo->was_304 = 0; // XXX: BereqEnd + BereqAcct ? VSL_ChgId(bo->vsl, "bereq", "retry", VXID_Get(wrk, VSL_BACKENDMARKER)); diff --git a/bin/varnishtest/tests/r02700.vtc b/bin/varnishtest/tests/r02700.vtc new file mode 100644 index 0000000..ca4598d --- /dev/null +++ b/bin/varnishtest/tests/r02700.vtc @@ -0,0 +1,47 @@ +varnishtest "#2700: IMS and return (retry)" + +server s1 { + rxreq + txresp -hdr {Etag: "foo"} -body "1" + + rxreq + expect req.http.If-None-Match == {"foo"} + expect req.http.retries == "0" + txresp -status 304 + + rxreq + expect req.http.retries == "1" + expect req.http.If-None-Match == {"foo"} + txresp -status 304 +} -start + + +varnish v1 -vcl+backend { + sub vcl_backend_fetch { + set bereq.http.retries = bereq.retries; + } + sub vcl_backend_response { + set beresp.ttl = 1ms; + set beresp.grace = 0s; + set beresp.keep = 1h; + + if (beresp.was_304 && bereq.retries == 0) { + return (retry); + } + + set beresp.http.was-304 = beresp.was_304; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.http.was-304 == "false" + + delay 0.1 + + txreq + rxresp + expect resp.http.was-304 == "true" +} -run From daghf at varnish-software.com Tue Jun 5 12:12:08 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 5 Jun 2018 12:12:08 +0000 (UTC) Subject: [4.1] 49e6b53 Clear bo->was_304 on return(retry) Message-ID: <20180605121208.67DBEA5662@lists.varnish-cache.org> commit 49e6b53c55d48c8fb8eacf5dbe52d1aee7c2e821 Author: Dag Haavi Finstad Date: Tue Jun 5 14:04:57 2018 +0200 Clear bo->was_304 on return(retry) Fixes: #2700 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index eb3d58e..146426e 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -236,6 +236,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo) bo->storage_hint = NULL; bo->do_esi = 0; bo->do_stream = 1; + bo->was_304 = 0; /* reset fetch processors */ VFP_Setup(vfc); diff --git a/bin/varnishtest/tests/r02700.vtc b/bin/varnishtest/tests/r02700.vtc new file mode 100644 index 0000000..ca4598d --- /dev/null +++ b/bin/varnishtest/tests/r02700.vtc @@ -0,0 +1,47 @@ +varnishtest "#2700: IMS and return (retry)" + +server s1 { + rxreq + txresp -hdr {Etag: "foo"} -body "1" + + rxreq + expect req.http.If-None-Match == {"foo"} + expect req.http.retries == "0" + txresp -status 304 + + rxreq + expect req.http.retries == "1" + expect req.http.If-None-Match == {"foo"} + txresp -status 304 +} -start + + +varnish v1 -vcl+backend { + sub vcl_backend_fetch { + set bereq.http.retries = bereq.retries; + } + sub vcl_backend_response { + set beresp.ttl = 1ms; + set beresp.grace = 0s; + set beresp.keep = 1h; + + if (beresp.was_304 && bereq.retries == 0) { + return (retry); + } + + set beresp.http.was-304 = beresp.was_304; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.http.was-304 == "false" + + delay 0.1 + + txreq + rxresp + expect resp.http.was-304 == "true" +} -run From phk at FreeBSD.org Wed Jun 6 06:32:29 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 06:32:29 +0000 (UTC) Subject: [master] c8b2279 A spot of FlexeLinting Message-ID: <20180606063229.70B3F61957@lists.varnish-cache.org> commit c8b2279b62a779928d6f487dbfa08a9eef13e90e Author: Poul-Henning Kamp Date: Wed Jun 6 06:31:14 2018 +0000 A spot of FlexeLinting diff --git a/bin/varnishtest/teken.c b/bin/varnishtest/teken.c index ad06c57..bf5f782 100644 --- a/bin/varnishtest/teken.c +++ b/bin/varnishtest/teken.c @@ -457,6 +457,8 @@ teken_state_numbers(teken_t *t, teken_char_t c) return (0); } +//lint -save -emacro((835),k,K) '|' has zero RHS + #define k TC_BLACK #define b TC_BLUE #define y TC_BROWN @@ -642,6 +644,8 @@ static const teken_color_t teken_256to16tab[] = { #undef R #undef W +//lint -restore + teken_color_t teken_256to8(teken_color_t c) { diff --git a/bin/varnishtest/teken_wcwidth.h b/bin/varnishtest/teken_wcwidth.h index 6482305..7340169 100644 --- a/bin/varnishtest/teken_wcwidth.h +++ b/bin/varnishtest/teken_wcwidth.h @@ -104,7 +104,7 @@ static int teken_wcwidth(teken_char_t ucs) /* if we arrive here, ucs is not a combining or C0/C1 control character */ return 1 + - (ucs >= 0x1100 && + (int)(ucs >= 0x1100 && (ucs <= 0x115f || /* Hangul Jamo init. consonants */ ucs == 0x2329 || ucs == 0x232a || (ucs >= 0x2e80 && ucs <= 0xa4cf && diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index fccc8e3..764710f 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -77,6 +77,7 @@ enum h2_type { TYPE_MAX }; +//lint -save -e849 Same enum value enum { ACK = 0x1, END_STREAM = 0x1, @@ -84,6 +85,7 @@ enum { END_HEADERS = 0x4, PRIORITY = 0x20, }; +//lint -restore struct stream { unsigned magic; @@ -103,7 +105,7 @@ struct stream { VTAILQ_HEAD(, frame) fq; char *body; - int bodylen; + long bodylen; struct hpk_hdr req[MAX_HDR]; struct hpk_hdr resp[MAX_HDR]; @@ -868,7 +870,7 @@ do { \ #define RETURN_BUFFED(val) \ do { \ - snprintf(buf, 20, "%d", val); \ + snprintf(buf, 20, "%ld", (long)val); \ return (buf); \ } while (0) From phk at FreeBSD.org Wed Jun 6 07:32:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 07:32:14 +0000 (UTC) Subject: [master] 80b041c Flexelinting Message-ID: <20180606073214.AEC8062B8E@lists.varnish-cache.org> commit 80b041ca3cc5f28a2955a25f49f8ec87b2c1dbcd Author: Poul-Henning Kamp Date: Wed Jun 6 07:01:31 2018 +0000 Flexelinting diff --git a/lib/libvcc/flint.lnt b/lib/libvcc/flint.lnt index 7d7d60d..e6c7cbd 100644 --- a/lib/libvcc/flint.lnt +++ b/lib/libvcc/flint.lnt @@ -3,3 +3,5 @@ -emacro(835, EXPR_VAR) // Info 835: A zero has been given as right argument to operator '<<' -esym(755, VCL_40) // Not used (right now) + +-esym(768, token) // FLINTBUG: global struct member 'token' not ref From phk at FreeBSD.org Wed Jun 6 07:32:14 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 07:32:14 +0000 (UTC) Subject: [master] e86fe83 More flexelinting Message-ID: <20180606073214.CCE3462B91@lists.varnish-cache.org> commit e86fe835bb684753829e4f83c8b93d0e065a5edd Author: Poul-Henning Kamp Date: Wed Jun 6 07:25:32 2018 +0000 More flexelinting diff --git a/flint.lnt b/flint.lnt index a585940..1d2482b 100644 --- a/flint.lnt +++ b/flint.lnt @@ -136,6 +136,11 @@ -esym(534, VSB_putc) /////////////////////////////////////////////////////////////////////// +// + +-emacro(801, VRB_*) // goto considered bad + +/////////////////////////////////////////////////////////////////////// // -esym(755, VLIST_*) // Global macro not ref. diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 5a04770..1b1eb01 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -254,9 +254,9 @@ VUT_Signaled(struct VUT *vut, int sig) { CHECK_OBJ_NOTNULL(vut, VUT_MAGIC); - vut->sighup |= (sig == SIGHUP); - vut->sigint |= (sig == SIGINT || sig == SIGTERM); - vut->sigusr1 |= (sig == SIGUSR1); + vut->sighup |= (int)(sig == SIGHUP); + vut->sigint |= (int)(sig == SIGINT || sig == SIGTERM); + vut->sigusr1 |= (int)(sig == SIGUSR1); } void From phk at FreeBSD.org Wed Jun 6 07:49:15 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 07:49:15 +0000 (UTC) Subject: [master] 83c1257 More FlexeLintery Message-ID: <20180606074915.DB0EC63128@lists.varnish-cache.org> commit 83c125720d79ed84ea693e306a5d415d7c394e8c Author: Poul-Henning Kamp Date: Wed Jun 6 07:48:28 2018 +0000 More FlexeLintery diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index ca89438..9372c5e 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -75,12 +75,12 @@ struct top { double count; }; -static float period = 60; /* seconds */ +static int period = 60; /* seconds */ static unsigned ntop; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static int f_flag = 0; static unsigned maxfieldlen = 0; -static char *ident; +static const char *ident; static volatile sig_atomic_t end_of_file = 0; static volatile sig_atomic_t quit = 0; @@ -206,7 +206,7 @@ update(int p) int l, len, eof; double t = 0; static time_t last = 0; - static unsigned n; + static unsigned n = 0; const char *q; time_t now; diff --git a/flint.lnt b/flint.lnt index 1d2482b..99dfe62 100644 --- a/flint.lnt +++ b/flint.lnt @@ -139,6 +139,8 @@ // -emacro(801, VRB_*) // goto considered bad +-esym(534, *_VRB_REMOVE) // ignore retval +-esym(534, *_VRB_INSERT) // ignore retval /////////////////////////////////////////////////////////////////////// // diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 1b1eb01..03cd9f0 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -275,7 +275,8 @@ VUT_Setup(struct VUT *vut) VUT_Error(vut, 1, "Only one of -n and -r options may be used"); /* Create and validate the query expression */ - vut->vslq = VSLQ_New(vut->vsl, NULL, vut->g_arg, vut->q_arg); + vut->vslq = VSLQ_New(vut->vsl, NULL, + (enum VSL_grouping_e)vut->g_arg, vut->q_arg); if (vut->vslq == NULL) VUT_Error(vut, 1, "Query expression error:\n%s", VSL_Error(vut->vsl)); From phk at FreeBSD.org Wed Jun 6 08:33:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 08:33:11 +0000 (UTC) Subject: [master] 9015fda Whitespace OCD Message-ID: <20180606083311.A1E52640F3@lists.varnish-cache.org> commit 9015fda0aa4a44cc15ef8d04bec4cba031004128 Author: Poul-Henning Kamp Date: Wed Jun 6 08:27:24 2018 +0000 Whitespace OCD diff --git a/bin/varnishtest/tests/r02700.vtc b/bin/varnishtest/tests/r02700.vtc index ca4598d..68dd28f 100644 --- a/bin/varnishtest/tests/r02700.vtc +++ b/bin/varnishtest/tests/r02700.vtc @@ -30,7 +30,7 @@ varnish v1 -vcl+backend { } set beresp.http.was-304 = beresp.was_304; - } + } } -start client c1 { From phk at FreeBSD.org Wed Jun 6 08:33:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 08:33:11 +0000 (UTC) Subject: [master] b02d927 Include a timestamp in all CLI JSON responses. Message-ID: <20180606083311.E3D4B640FB@lists.varnish-cache.org> commit b02d92735d82e2e017cc6f00db3d888ce62274d7 Author: Poul-Henning Kamp Date: Wed Jun 6 08:28:53 2018 +0000 Include a timestamp in all CLI JSON responses. diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index a80f751..ed1309d 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -370,7 +370,7 @@ cli_backend_list(struct cli *cli, const char * const *av, void *priv) return; } if (la->j) { - VCLI_JSON_begin(cli, 1, av); + VCLI_JSON_begin(cli, 2, av); VCLI_Out(cli, ",\n"); VCLI_Out(cli, "{\n"); VSB_indent(cli->sb, 2); diff --git a/lib/libvarnish/vcli_serve.c b/lib/libvarnish/vcli_serve.c index b1084a1..de7ebdb 100644 --- a/lib/libvarnish/vcli_serve.c +++ b/lib/libvarnish/vcli_serve.c @@ -50,6 +50,7 @@ #include "vav.h" #include "vcli_serve.h" #include "vsb.h" +#include "vtim.h" struct VCLS_fd { unsigned magic; @@ -159,7 +160,7 @@ VCLS_func_help_json(struct cli *cli, const char * const *av, void *priv) cs = cli->cls; CHECK_OBJ_NOTNULL(cs, VCLS_MAGIC); - VCLI_JSON_begin(cli, 1, av); + VCLI_JSON_begin(cli, 2, av); VTAILQ_FOREACH(clp, &cs->funcs, list) { if (clp->auth > cli->auth) continue; @@ -674,7 +675,7 @@ VCLI_JSON_begin(struct cli *cli, unsigned ver, const char * const * av) if (av[i + 1] != NULL) VCLI_Out(cli, ", "); } - VCLI_Out(cli, "]"); + VCLI_Out(cli, "], %.3f", VTIM_real()); VSB_indent(cli->sb, 2); } From phk at FreeBSD.org Wed Jun 6 08:36:08 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 08:36:08 +0000 (UTC) Subject: [master] 5c636fc Fix copy&paste sloppyness in http_resp_size documentation Message-ID: <20180606083608.191046424F@lists.varnish-cache.org> commit 5c636fcb6c7c9a468850e498c86dace58d5d8bc7 Author: Poul-Henning Kamp Date: Wed Jun 6 08:34:37 2018 +0000 Fix copy&paste sloppyness in http_resp_size documentation Fixes #2684 Reported by: ernestojpg at github diff --git a/include/tbl/params.h b/include/tbl/params.h index 8cfb7f0..e2ecff4 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -699,10 +699,10 @@ PARAM( /* s-text */ "Maximum number of bytes of HTTP backend response we will deal " "with. This is a limit on all bytes up to the double blank line " - "which ends the HTTP request.\n" - "The memory for the request is allocated from the backend workspace " + "which ends the HTTP response.\n" + "The memory for the response is allocated from the backend workspace " "(param: workspace_backend) and this parameter limits how much " - "of that the request is allowed to take up.", + "of that the response is allowed to take up.", /* l-text */ "", /* func */ NULL ) From phk at FreeBSD.org Wed Jun 6 13:44:15 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 13:44:15 +0000 (UTC) Subject: [master] 27d538b Remove spurios old prototype Message-ID: <20180606134415.89AD2978CF@lists.varnish-cache.org> commit 27d538b45785a7a29239c3e68e9332cedd1232b4 Author: Poul-Henning Kamp Date: Wed Jun 6 12:55:14 2018 +0000 Remove spurios old prototype diff --git a/include/vfil.h b/include/vfil.h index 48d6f62..25f3b3a 100644 --- a/include/vfil.h +++ b/include/vfil.h @@ -34,7 +34,6 @@ struct vfil_path; void VFIL_null_fd(int); -int seed_random(void); char *VFIL_readfile(const char *pfx, const char *fn, ssize_t *sz); int VFIL_writefile(const char *pfx, const char *fn, const char *buf, size_t sz); int VFIL_nonblocking(int fd); From phk at FreeBSD.org Wed Jun 6 13:44:15 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 13:44:15 +0000 (UTC) Subject: [master] 81696ed Use the string VRND for VSM segment names. Message-ID: <20180606134415.9EED7978D1@lists.varnish-cache.org> commit 81696eda467c0bfb37c758df15c5b09e74ea8c6f Author: Poul-Henning Kamp Date: Wed Jun 6 12:55:38 2018 +0000 Use the string VRND for VSM segment names. diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 0aa5477..2c2337d 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -151,13 +151,14 @@ static void vsmw_mkent(const struct vsmw *vsmw, const char *pfx) { int fd; + uint64_t rn; AN(pfx); while (1) { VSB_clear(vsmw->vsb); VSB_printf(vsmw->vsb, "_.%s", pfx); - VSB_printf(vsmw->vsb, ".%08lx", VRND_RandomTestable()); - VSB_printf(vsmw->vsb, "%08lx", VRND_RandomTestable()); + AZ(VRND_RandomCrypto(&rn, sizeof rn)); + VSB_printf(vsmw->vsb, ".%016jx", (uintmax_t)rn); AZ(VSB_finish(vsmw->vsb)); fd = openat(vsmw->vdirfd, VSB_data(vsmw->vsb), O_RDONLY); if (fd < 0 && errno == ENOENT) @@ -192,7 +193,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx) CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC); - VTAILQ_REMOVE(&vsmw->segs, seg, list); + TAILQ_REMOVE(&vsmw->segs, seg, list); REPLACE(seg->class, NULL); REPLACE(seg->id, NULL); FREE_OBJ(seg); From phk at FreeBSD.org Wed Jun 6 13:44:15 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 13:44:15 +0000 (UTC) Subject: [master] ca1eca7 Use strong VRND for silo unique number Message-ID: <20180606134415.E7963978D7@lists.varnish-cache.org> commit ca1eca7838d5d17bb9e301944e40b4bf685e5738 Author: Poul-Henning Kamp Date: Wed Jun 6 13:11:08 2018 +0000 Use strong VRND for silo unique number diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c index 05ce659..9f8d202 100644 --- a/bin/varnishd/storage/storage_persistent_subr.c +++ b/bin/varnishd/storage/storage_persistent_subr.c @@ -45,6 +45,7 @@ #include "storage/storage.h" +#include "vrnd.h" #include "vsha256.h" #include "storage/storage_persistent.h" @@ -280,7 +281,7 @@ smp_newsilo(struct smp_sc *sc) assert(strlen(SMP_IDENT_STRING) < sizeof si->ident); /* Choose a new random number */ - sc->unique = random(); + AZ(VRND_RandomCrypto(&sc->unique, sizeof sc->unique)); smp_reset_sign(&sc->idn); si = sc->ident; From phk at FreeBSD.org Wed Jun 6 13:44:15 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 13:44:15 +0000 (UTC) Subject: [master] 7618f59 Typo Message-ID: <20180606134415.C1C7D978D4@lists.varnish-cache.org> commit 7618f59ece8c5c7bedbbfcd3f7d31c5d4f10c2cc Author: Poul-Henning Kamp Date: Wed Jun 6 12:59:05 2018 +0000 Typo diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 2c2337d..6d4545f 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -193,7 +193,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx) CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC); - TAILQ_REMOVE(&vsmw->segs, seg, list); + VTAILQ_REMOVE(&vsmw->segs, seg, list); REPLACE(seg->class, NULL); REPLACE(seg->id, NULL); FREE_OBJ(seg); From phk at FreeBSD.org Wed Jun 6 13:44:16 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 13:44:16 +0000 (UTC) Subject: [master] 84b14ba Wrap the testable VRND in program supplied locking, because it is not thread-safe. Message-ID: <20180606134416.2E034978DA@lists.varnish-cache.org> commit 84b14baa085d6d20051bb8af04af09e3c466aa76 Author: Poul-Henning Kamp Date: Wed Jun 6 13:42:51 2018 +0000 Wrap the testable VRND in program supplied locking, because it is not thread-safe. Switch from random(3) to testable VRND for same reason. diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index acc1104..b1f993d 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -39,6 +39,8 @@ #include "cache_esi.h" +#include "vrnd.h" + /*--------------------------------------------------------------------- */ @@ -198,7 +200,7 @@ vfp_esi_gzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, *lp = 0; l = vef->ibuf_sz - (vef->ibuf_i - vef->ibuf); if (DO_DEBUG(DBG_ESI_CHOP)) { - d = (random() & 3) + 1; + d = (VRND_RandomTestable() & 3) + 1; if (d < l) l = d; } @@ -259,7 +261,7 @@ vfp_esi_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, ssize_t *lp) AN(p); AN(lp); if (DO_DEBUG(DBG_ESI_CHOP)) { - d = (random() & 3) + 1; + d = (VRND_RandomTestable() & 3) + 1; if (d < *lp) *lp = d; } diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 8b7a63e..4683a3f 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -52,6 +52,19 @@ volatile struct params *cache_param; +static pthread_mutex_t cache_vrnd_mtx; + +static void +cache_vrnd_lock(void) +{ + AZ(pthread_mutex_lock(&cache_vrnd_mtx)); +} + +static void +cache_vrnd_unlock(void) +{ + AZ(pthread_mutex_unlock(&cache_vrnd_mtx)); +} /*-------------------------------------------------------------------- * Per thread storage for the session currently being processed by @@ -339,6 +352,10 @@ child_main(int sigmagic, size_t altstksz) THR_SetName("cache-main"); + AZ(pthread_mutex_init(&cache_vrnd_mtx, NULL)); + VRND_Lock = cache_vrnd_lock; + VRND_Unlock = cache_vrnd_unlock; + VSM_Init(); /* First, LCK needs it. */ LCK_Init(); /* Second, locking */ diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 69b2d3d..2765665 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -40,6 +40,7 @@ #include "vtc.h" #include "vav.h" +#include "vrnd.h" #include "vtim.h" #define MAX_TOKENS 200 @@ -50,6 +51,20 @@ pthread_t vtc_thread; int ign_unknown_macro = 0; static struct vtclog *vltop; +static pthread_mutex_t vtc_vrnd_mtx; + +static void +vtc_vrnd_lock(void) +{ + AZ(pthread_mutex_lock(&vtc_vrnd_mtx)); +} + +static void +vtc_vrnd_unlock(void) +{ + AZ(pthread_mutex_unlock(&vtc_vrnd_mtx)); +} + /********************************************************************** * Macro facility */ @@ -465,6 +480,11 @@ exec_file(const char *fn, const char *script, const char *tmpdir, (void)signal(SIGPIPE, SIG_IGN); + AZ(pthread_mutex_init(&vtc_vrnd_mtx, NULL)); + VRND_Lock = vtc_vrnd_lock; + VRND_Unlock = vtc_vrnd_unlock; + VRND_SeedAll(); + tfn = fn; vtc_loginit(logbuf, loglen); vltop = vtc_logopen("top"); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 54b2a20..741c5f1 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -45,6 +45,7 @@ #include "vfil.h" #include "vgz.h" #include "vnum.h" +#include "vrnd.h" #include "vtcp.h" #include "hpack.h" @@ -189,7 +190,7 @@ synth_body(const char *len, int rnd) k = '!'; l = k; } else if (rnd) { - b[j] = (random() % 95) + ' '; + b[j] = (VRND_RandomTestable() % 95) + ' '; } else { b[j] = (char)l; if (++l == '~') @@ -1970,7 +1971,7 @@ xxx(void) *ibuf = 0; for (j = 0; j < 7; j++) { snprintf(strchr(ibuf, 0), 5, "%x", - (unsigned)random() & 0xffff); + (unsigned)VRND_RandomTestable() & 0xffff); vz.next_in = TRUST_ME(ibuf); vz.avail_in = strlen(ibuf); vz.next_out = TRUST_ME(obuf); diff --git a/include/vrnd.h b/include/vrnd.h index 6a89762..a828af8 100644 --- a/include/vrnd.h +++ b/include/vrnd.h @@ -28,7 +28,13 @@ * Random functions */ +typedef void vrnd_lock_f(void); + +extern vrnd_lock_f *VRND_Lock; +extern vrnd_lock_f *VRND_Unlock; + int VRND_RandomCrypto(void *, size_t); + long VRND_RandomTestable(void); double VRND_RandomTestableDouble(void); void VRND_SeedTestable(unsigned int); diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c index b49f9d6..fee40c9 100644 --- a/lib/libvarnish/binary_heap.c +++ b/lib/libvarnish/binary_heap.c @@ -519,6 +519,11 @@ chk2(struct binheap *bh) } #endif +static void +vrnd_lock(void) +{ +} + int main(void) { @@ -528,6 +533,9 @@ main(void) VRND_SeedAll(); VRND_SeedTestable(1); + VRND_Lock = vrnd_lock; + VRND_Unlock = vrnd_lock; + bh = binheap_new(NULL, cmp, update); for (n = 2; n; n += n) { child(bh, n - 1, &u, &v); diff --git a/lib/libvarnish/vrnd.c b/lib/libvarnish/vrnd.c index 3dc8b18..13557b0 100644 --- a/lib/libvarnish/vrnd.c +++ b/lib/libvarnish/vrnd.c @@ -45,6 +45,10 @@ #include "vas.h" #include "vrnd.h" + +vrnd_lock_f *VRND_Lock; +vrnd_lock_f *VRND_Unlock; + /********************************************************************** * Stripped down random(3) implementation from FreeBSD, to provide * predicatable "random" numbers of testing purposes. @@ -96,23 +100,8 @@ good_rand(uint32_t ctx) return (x - 1); } -void -VRND_SeedTestable(unsigned int x) -{ - int i, lim; - - state[0] = (uint32_t)x; - for (i = 1; i < rand_deg; i++) - state[i] = good_rand(state[i - 1]); - fptr = &state[rand_sep]; - rptr = &state[0]; - lim = 10 * rand_deg; - for (i = 0; i < lim; i++) - (void)VRND_RandomTestable(); -} - -long -VRND_RandomTestable(void) +static long +vrnd_RandomTestable(void) { uint32_t i; uint32_t *f, *r; @@ -135,6 +124,35 @@ VRND_RandomTestable(void) return ((long)i); } + +void +VRND_SeedTestable(unsigned int x) +{ + int i, lim; + + state[0] = (uint32_t)x; + for (i = 1; i < rand_deg; i++) + state[i] = good_rand(state[i - 1]); + fptr = &state[rand_sep]; + rptr = &state[0]; + lim = 10 * rand_deg; + for (i = 0; i < lim; i++) + (void)vrnd_RandomTestable(); +} + +long +VRND_RandomTestable(void) +{ + long l; + + AN(VRND_Lock); + VRND_Lock(); + l = vrnd_RandomTestable(); + AN(VRND_Unlock); + VRND_Unlock(); + return (l); +} + double VRND_RandomTestableDouble(void) { From phk at FreeBSD.org Wed Jun 6 16:52:24 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 16:52:24 +0000 (UTC) Subject: [master] d8274da Fix a end-of-string mistake (spotted by FlexeLint) Message-ID: <20180606165224.A47A0A4917@lists.varnish-cache.org> commit d8274da4d3a5ab3d42536516f0bc76f14f5d3f10 Author: Poul-Henning Kamp Date: Wed Jun 6 16:46:42 2018 +0000 Fix a end-of-string mistake (spotted by FlexeLint) diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 488027d..c92b0a3 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -692,13 +692,12 @@ parse_format(const char *format) for (p = format; *p != '\0'; p++) { /* Allow the most essential escape sequences in format */ - if (*p == '\\') { - p++; - if (*p == 't') + if (*p == '\\' && p[1] != '\0') { + if (*++p == 't') AZ(VSB_putc(vsb, '\t')); else if (*p == 'n') AZ(VSB_putc(vsb, '\n')); - else if (*p != '\0') + else AZ(VSB_putc(vsb, *p)); continue; } From phk at FreeBSD.org Wed Jun 6 16:52:24 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 16:52:24 +0000 (UTC) Subject: [master] 7fc50c6 Disable FlexeLint message 850, we think we know what we're doing in our loops. Message-ID: <20180606165224.AAC7CA491B@lists.varnish-cache.org> commit 7fc50c64fc974d030c972ed5fd8315c4b1ab0b83 Author: Poul-Henning Kamp Date: Wed Jun 6 16:50:21 2018 +0000 Disable FlexeLint message 850, we think we know what we're doing in our loops. diff --git a/flint.lnt b/flint.lnt index 99dfe62..a6c0737 100644 --- a/flint.lnt +++ b/flint.lnt @@ -49,6 +49,8 @@ -esym(818, argv) // Pointer parameter '...' could be declared as pointing to const +-e850 // loop variable modified in loop + /* * va_list's are opaque for a reason, but we pretend to FlexeLint that it * is just a void*, so it proposes constification, which is not generally OK, From phk at FreeBSD.org Wed Jun 6 19:14:15 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 6 Jun 2018 19:14:15 +0000 (UTC) Subject: [master] 8af995f Supress Flexelints 835 from all macro expansions Message-ID: <20180606191415.DD672A7434@lists.varnish-cache.org> commit 8af995ff6d8dc36d9631cfb82a21322fe261b7c2 Author: Poul-Henning Kamp Date: Wed Jun 6 19:13:22 2018 +0000 Supress Flexelints 835 from all macro expansions diff --git a/bin/varnishtest/teken.c b/bin/varnishtest/teken.c index bf5f782..ad06c57 100644 --- a/bin/varnishtest/teken.c +++ b/bin/varnishtest/teken.c @@ -457,8 +457,6 @@ teken_state_numbers(teken_t *t, teken_char_t c) return (0); } -//lint -save -emacro((835),k,K) '|' has zero RHS - #define k TC_BLACK #define b TC_BLUE #define y TC_BROWN @@ -644,8 +642,6 @@ static const teken_color_t teken_256to16tab[] = { #undef R #undef W -//lint -restore - teken_color_t teken_256to8(teken_color_t c) { diff --git a/flint.lnt b/flint.lnt index a6c0737..0a9d6e9 100644 --- a/flint.lnt +++ b/flint.lnt @@ -5,6 +5,8 @@ //d__flexelint_v9__=1 +fan +-emacro((835),*) // A zero has been given as ___ argument to operator '___ + /////////////////////////////////////////////////////////////////////// // build/config related From phk at FreeBSD.org Thu Jun 7 06:53:21 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jun 2018 06:53:21 +0000 (UTC) Subject: [master] 6529b1c FlexeLinting Message-ID: <20180607065321.DE9ECB5F96@lists.varnish-cache.org> commit 6529b1cf7534534039ecd8296d5a4c918bb53b4e Author: Poul-Henning Kamp Date: Thu Jun 7 06:32:52 2018 +0000 FlexeLinting diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 6c72a65..50e4356 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -68,7 +68,7 @@ static unsigned hist_buckets; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static int end_of_file = 0; -static double delay = 1; +static int ms_delay = 1000; static unsigned rr_hist[HIST_N]; static unsigned nhist; static unsigned next_hist; @@ -82,7 +82,7 @@ static pthread_cond_t timebend_cv; static double log_ten; static char *ident; -static const int scales[] = { +static const unsigned scales[] = { 1, 2, 3, @@ -103,7 +103,7 @@ static const int scales[] = { 25000, 50000, 100000, - INT_MAX + UINT_MAX }; struct profile { @@ -182,10 +182,10 @@ update(void) VTIM_format(vsl_ts, t); mvprintw(0, 0, "1:%u, n = %u, d = %g @ %s x %g", - scale, nhist, delay, t, timebend); + scale, nhist, 1e-3 * ms_delay, t, timebend); } else mvprintw(0, 0, "1:%u, n = %u, d = %g", - scale, nhist, delay); + scale, nhist, 1e-3 * ms_delay); for (j = 2; j < LINES - 3; j += 5) mvprintw(j, 0, "%u_", ((LINES - 3) - j) * scale); @@ -299,7 +299,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[], continue; /* select bucket */ - i = HIST_RES * (log(value) / log_ten); + i = HIST_RES * lround(log(value) / log_ten); if (i < hist_low * HIST_RES) i = hist_low * HIST_RES; if (i >= hist_high * HIST_RES) @@ -398,7 +398,8 @@ do_curses(void *arg) update(); AZ(pthread_mutex_unlock(&mtx)); - timeout(delay * 1000); + assert(ms_delay > 0); + timeout(ms_delay); switch ((ch = getch())) { case ERR: break; @@ -433,15 +434,15 @@ do_curses(void *arg) case '7': case '8': case '9': - delay = 1U << (ch - '0'); + ms_delay = 1U << (ch - '0'); break; case '+': - delay /= 2; - if (delay < 1e-3) - delay = 1e-3; + ms_delay /= 2; + if (ms_delay < 1) + ms_delay = 1; break; case '-': - delay *= 2; + ms_delay *= 2; break; case '>': case '<': @@ -520,8 +521,8 @@ main(int argc, char **argv) /* Usage help */ usage(0); case 'p': - delay = strtod(optarg, NULL); - if (delay <= 0) + ms_delay = lround(1e3 * strtod(optarg, NULL)); + if (ms_delay <= 0) VUT_Error(vut, 1, "-p: invalid '%s'", optarg); break; case 'P': @@ -571,7 +572,7 @@ main(int argc, char **argv) profile_error(optarg); cli_p.name = "custom"; - cli_p.tag = match_tag; + cli_p.tag = (enum VSL_tag_e)match_tag; cli_p.hist_low = -6; cli_p.hist_high = 3; profile = NULL; From phk at FreeBSD.org Thu Jun 7 06:53:22 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jun 2018 06:53:22 +0000 (UTC) Subject: [master] 1a21c1c Fix the Y-axis ticks. Message-ID: <20180607065322.0C4F9B5F99@lists.varnish-cache.org> commit 1a21c1c2902d8bad2095ed452a6f2b0a53a6465a Author: Poul-Henning Kamp Date: Thu Jun 7 06:52:07 2018 +0000 Fix the Y-axis ticks. diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 50e4356..88ab59e 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -187,8 +187,8 @@ update(void) mvprintw(0, 0, "1:%u, n = %u, d = %g", scale, nhist, 1e-3 * ms_delay); - for (j = 2; j < LINES - 3; j += 5) - mvprintw(j, 0, "%u_", ((LINES - 3) - j) * scale); + for (j = 5; j < LINES - 2; j += 5) + mvprintw((LINES - 2) - j, 0, "%u_", j * scale); /* show them */ for (i = 0; i < n; ++i) { diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc index 90b7a3d..77270fe 100644 --- a/bin/varnishtest/tests/u00009.vtc +++ b/bin/varnishtest/tests/u00009.vtc @@ -20,4 +20,14 @@ client c1 { process p1 -expect-text 22 0 {#} +process p1 -expect-text 3 1 {20_} + +process p1 -screen_dump + +process p1 -winsz 23 80 + +delay 1.5 + +process p1 -expect-text 2 1 {20_} + process p1 -screen_dump -write {q} -wait From phk at FreeBSD.org Thu Jun 7 12:02:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jun 2018 12:02:09 +0000 (UTC) Subject: [master] fd05bb9 More coverage. Message-ID: <20180607120209.D1654BBA8C@lists.varnish-cache.org> commit fd05bb9957dbf4755d9ed1206fbce46f3a90f6ef Author: Poul-Henning Kamp Date: Thu Jun 7 12:00:51 2018 +0000 More coverage. diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 88ab59e..65f9246 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -434,7 +434,7 @@ do_curses(void *arg) case '7': case '8': case '9': - ms_delay = 1U << (ch - '0'); + ms_delay = 1000U << (ch - '0'); break; case '+': ms_delay /= 2; diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc index 77270fe..060503c 100644 --- a/bin/varnishtest/tests/u00009.vtc +++ b/bin/varnishtest/tests/u00009.vtc @@ -26,6 +26,10 @@ process p1 -screen_dump process p1 -winsz 23 80 +delay 5 + +process p1 -write {0>+-<} + delay 1.5 process p1 -expect-text 2 1 {20_} From fgsch at lodoss.net Fri Jun 8 08:50:15 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 8 Jun 2018 08:50:15 +0000 (UTC) Subject: [master] 13b8160 Skip this test if asan is used Message-ID: <20180608085015.28EA5AFA56@lists.varnish-cache.org> commit 13b816074befeab5ca62f6e620674985ef73b43e Author: Federico G. Schwindt Date: Fri Jun 8 09:40:55 2018 +0100 Skip this test if asan is used diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc index 09a9c1c..ec90cc8 100644 --- a/bin/varnishtest/tests/c00057.vtc +++ b/bin/varnishtest/tests/c00057.vtc @@ -1,5 +1,7 @@ varnishtest "test sigsegv handler" +feature cmd "test -z $(env | grep ASAN_OPTIONS)" + server s1 { rxreq txresp From fgsch at lodoss.net Fri Jun 8 09:07:10 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 8 Jun 2018 09:07:10 +0000 (UTC) Subject: [master] e2f00ac Add a comment for future reference Message-ID: <20180608090710.1D4D0B023D@lists.varnish-cache.org> commit e2f00acf036c9eec49c7e1b4291ab6377ffe4640 Author: Federico G. Schwindt Date: Fri Jun 8 10:05:03 2018 +0100 Add a comment for future reference diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc index ec90cc8..2b2f388 100644 --- a/bin/varnishtest/tests/c00057.vtc +++ b/bin/varnishtest/tests/c00057.vtc @@ -1,5 +1,7 @@ varnishtest "test sigsegv handler" +# Under ASAN, the stack layout is different and STACK OVERFLOW is +# never printed. feature cmd "test -z $(env | grep ASAN_OPTIONS)" server s1 { From phk at FreeBSD.org Fri Jun 8 16:42:21 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 8 Jun 2018 16:42:21 +0000 (UTC) Subject: [master] 4842825 Fix PROXY and HTTP/1 proto dissectors to not rely on space for an extra NUL. Message-ID: <20180608164221.86E80BA3CF@lists.varnish-cache.org> commit 4842825668bd50cd833ae4d649ca2f964ec83e5e Author: Poul-Henning Kamp Date: Fri Jun 8 16:41:15 2018 +0000 Fix PROXY and HTTP/1 proto dissectors to not rely on space for an extra NUL. diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c index 9d1c072..d1ad4d2 100644 --- a/bin/varnishd/http1/cache_http1_proto.c +++ b/bin/varnishd/http1/cache_http1_proto.c @@ -37,9 +37,6 @@ * and stops when we see the magic marker (double [CR]NL), and if we overshoot, * it keeps track of the "pipelined" data. * - * Until we see the magic marker, we have to keep the rxbuf NUL terminated - * because we use strchr(3) on it. - * * We use this both for client and backend connections. */ @@ -75,10 +72,6 @@ HTTP1_Complete(struct http_conn *htc) assert(htc->rxbuf_e >= htc->rxbuf_b); assert(htc->rxbuf_e <= htc->ws->r); - if (htc->rxbuf_e == htc->ws->r) - return (HTC_S_OVERFLOW); // No space for NUL - *htc->rxbuf_e = '\0'; - /* Skip any leading white space */ for (p = htc->rxbuf_b ; vct_islws(*p); p++) continue; @@ -95,12 +88,13 @@ HTTP1_Complete(struct http_conn *htc) * is completed. More stringent validation happens later. */ while (1) { - p = strchr(p, '\n'); + p = memchr(p, '\n', htc->rxbuf_e - p); if (p == NULL) return (HTC_S_MORE); - p++; - if (*p == '\r') - p++; + if (++p == htc->rxbuf_e) + return (HTC_S_MORE); + if (*p == '\r' && ++p == htc->rxbuf_e) + return (HTC_S_MORE); if (*p == '\n') break; } diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 32ddd6f..a10756d 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -69,13 +69,14 @@ vpx_proto1(const struct worker *wrk, const struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); - q = strchr(req->htc->rxbuf_b, '\r'); + q = memchr(req->htc->rxbuf_b, '\r', + req->htc->rxbuf_e - req->htc->rxbuf_b); if (q == NULL) return (-1); *q++ = '\0'; /* Nuke the CRLF */ - if (*q != '\n') + if (*q != '\n' || q == req->htc->rxbuf_e) return (-1); *q++ = '\0'; @@ -515,8 +516,7 @@ vpx_complete(struct http_conn *htc) return (HTC_S_JUNK); if (j == 1 && i == sizeof vpx1_sig) { assert (htc->rxbuf_e < htc->ws->r); - *htc->rxbuf_e = '\0'; - q = strchr(p + i, '\n'); + q = memchr(p + i, '\n', htc->rxbuf_e - (p + i)); if (q != NULL && (q - htc->rxbuf_b) > 107) return (HTC_S_OVERFLOW); if (q == NULL) From fgsch at lodoss.net Sat Jun 9 18:12:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 9 Jun 2018 18:12:11 +0000 (UTC) Subject: [master] 333b140 Whitespace OCD Message-ID: <20180609181211.0F386BB907@lists.varnish-cache.org> commit 333b1409af8b6a175e96311d20db8c42d2ca7078 Author: Federico G. Schwindt Date: Fri Jun 8 14:49:56 2018 +0100 Whitespace OCD diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc index 060503c..5a3bf92 100644 --- a/bin/varnishtest/tests/u00009.vtc +++ b/bin/varnishtest/tests/u00009.vtc @@ -22,7 +22,7 @@ process p1 -expect-text 22 0 {#} process p1 -expect-text 3 1 {20_} -process p1 -screen_dump +process p1 -screen_dump process p1 -winsz 23 80 diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 1d45837..6eba999 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -328,7 +328,7 @@ class vcltype(object): self.c = ctype self.internal = internal vcltypes[name] = self - + vcltype("STRINGS", "void", True) vcltype("STRING_LIST", "void*", True) From fgsch at lodoss.net Sat Jun 9 18:12:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 9 Jun 2018 18:12:11 +0000 (UTC) Subject: [master] 4e5c59c Handle the failure case correctly Message-ID: <20180609181211.2459BBB90A@lists.varnish-cache.org> commit 4e5c59c158f8b4e83c3ac1e72b18445d5beec15b Author: Federico G. Schwindt Date: Fri Jun 8 15:58:20 2018 +0100 Handle the failure case correctly diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 5b553d2..0f089f0 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -493,8 +493,14 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, be->director = VRT_AddDirector(ctx, vbe_methods, be, "%s", vrt->vcl_name); if (be->director == NULL) { - // XXX - VRT_delete_backend(ctx, &be->director); + VSC_vbe_Destroy(&be->vsc_seg); +#define DA(x) do { if (be->x != NULL) free(be->x); } while (0) +#define DN(x) /**/ + VRT_BACKEND_HANDLE(); +#undef DA +#undef DN + Lck_Delete(&be->mtx); + FREE_OBJ(be); return (NULL); } From nils.goroll at uplex.de Mon Jun 11 08:13:26 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jun 2018 08:13:26 +0000 (UTC) Subject: [master] c98edbb finalize task privs when rolling back workspaces Message-ID: <20180611081326.2C856A9161@lists.varnish-cache.org> commit c98edbb0214f6d64b7822def58036cd67e3bdb1d Author: Nils Goroll Date: Sun Jun 10 10:07:29 2018 +0200 finalize task privs when rolling back workspaces ... and introduce request functions for this purpose (for busy objects, there is only one use case yet, so we don't). Before we reset the workspace, we must ensure that there are no active references to objects on it. As PRIV_TASK and PRIV_TOP have the same lifetime as the respective workspace, they need to be destroyed. vmods must not use workspaces for storing information referenced via any of the other PRIVs unless the rollback case is considered. Note that while this bug was exposed by beeaa19cced3fe1ab79381b2b1b7b0b5594cbb18, it existed all along for any vmod priv state stored on the workspace, so if a vmod happened to access a TASK_PRIV stored on the workspace, it would likely have triggered a magic check assertion as well. I got plans for making std.rollback() more useful. While this change is required to do so, it only partly covers the planned changes. Fixes #2706 diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index a6b9ee0..d741441 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -181,6 +181,28 @@ Req_Release(struct req *req) MPL_Free(pp->mpl_req, req); } +static void +req_finalize(struct req *req) +{ + VRTPRIV_dynamic_kill(req->privs, (uintptr_t)req); + VRTPRIV_dynamic_kill(req->privs, (uintptr_t)&req->top); + assert(VTAILQ_EMPTY(&req->privs->privs)); +} + +/*---------------------------------------------------------------------- + * TODO: + * - check for code duplication with cnt_recv_prep + * - re-check if complete + */ + +void +Req_Rollback(struct req *req) +{ + req_finalize(req); + HTTP_Copy(req->http, req->http0); + WS_Reset(req->ws, req->ws_req); +} + /*---------------------------------------------------------------------- * TODO: remove code duplication with cnt_recv_prep */ @@ -207,9 +229,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) req->vcl = NULL; } - VRTPRIV_dynamic_kill(req->privs, (uintptr_t)req); - VRTPRIV_dynamic_kill(req->privs, (uintptr_t)&req->top); - assert(VTAILQ_EMPTY(&req->privs->privs)); + req_finalize(req); /* Charge and log byte counters */ if (req->vsl->wid) { diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index b36e85d..5a5e5eb 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -208,8 +208,8 @@ cnt_vclfail(const struct worker *wrk, struct req *req) AZ(req->objcore); AZ(req->stale_oc); - HTTP_Copy(req->http, req->http0); - WS_Reset(req->ws, req->ws_req); + Req_Rollback(req); + req->err_code = 503; req->err_reason = "VCL failed"; req->req_step = R_STP_SYNTH; @@ -856,8 +856,7 @@ cnt_recv(struct worker *wrk, struct req *req) VCL_recv_method(req->vcl, wrk, req, NULL, NULL); if (wrk->handling == VCL_RET_VCL && req->restarts == 0) { - HTTP_Copy(req->http, req->http0); - WS_Reset(req->ws, req->ws_req); + Req_Rollback(req); cnt_recv_prep(req, ci); VCL_recv_method(req->vcl, wrk, req, NULL, NULL); } diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index d2f35e3..292e336 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -314,6 +314,7 @@ void VRG_dorange(struct req *req, const char *r); /* cache_req.c */ struct req *Req_New(const struct worker *, struct sess *); void Req_Release(struct req *); +void Req_Rollback(struct req *req); void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req); void Req_Fail(struct req *req, enum sess_close reason); void Req_AcctLogCharge(struct VSC_main *, struct req *); diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index adeb349..bc92daa 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -518,10 +518,11 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp) CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); if (hp == ctx->http_req) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - HTTP_Copy(ctx->req->http, ctx->req->http0); - WS_Reset(ctx->req->ws, ctx->req->ws_req); + Req_Rollback(ctx->req); } else if (hp == ctx->http_bereq) { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + // -> VBO_Rollback ? + VRTPRIV_dynamic_kill(ctx->bo->privs, (uintptr_t)ctx->bo); HTTP_Copy(ctx->bo->bereq, ctx->bo->bereq0); WS_Reset(ctx->bo->bereq->ws, ctx->bo->ws_bo); WS_Reset(ctx->bo->ws, ctx->bo->ws_bo); diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 7bdaa02..5c0fa75 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -16,7 +16,15 @@ varnish v1 -vcl+backend { debug.vsc_new(); } + sub vcl_synth { + set req.http.overwrite = "the workspace " + + "to ensure we notice any unfinished privs"; + } sub vcl_recv { + if (req.url == "/fail") { + debug.test_priv_task("foo"); + return (fail); + } debug.rot52(req); debug.vsc_count(); } @@ -52,6 +60,9 @@ client c1 { expect resp.http.encrypted == "ROT52" expect resp.http.what >= 16 expect resp.http.not == -1 + txreq -url "/fail" + rxresp + expect resp.status == 503 } -run varnish v1 -expect DEBUG.count == 1 From nils.goroll at uplex.de Mon Jun 11 08:29:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jun 2018 08:29:10 +0000 (UTC) Subject: [master] 314b17c Statistics for backend open errors Message-ID: <20180611082910.B2D9CA97B7@lists.varnish-cache.org> commit 314b17cccd92b4328da06fa9bcea102cd6a1037e Author: Nils Goroll Date: Tue Jun 5 08:04:14 2018 +0200 Statistics for backend open errors Previously, we had zero stats on the cause of backend connection errors, which made it close to impossible to diagnose such issues in retrospect (only via log mining). We now pass an optional backend vsc to vcp and record errors per backend. Open errors are really per vcp entry (ip + port or udc path), which can be shared amongst backends (and even vcls), but we maintain the counters per backend (and, consequently, per vcl) for simplicity. It should be noted though that errors for shared endpoints affect all backends using them. Ref #2622 diff --git a/bin/varnishd/VSC_vbe.vsc b/bin/varnishd/VSC_vbe.vsc index c5ca37f..b215909 100644 --- a/bin/varnishd/VSC_vbe.vsc +++ b/bin/varnishd/VSC_vbe.vsc @@ -78,5 +78,50 @@ :level: info :oneliner: Backend requests sent -.. varnish_vsc_end:: vbe +.. + === Anything below is actually per VCP entry, but collected per + === backend for simplicity + +.. varnish_vsc:: fail + :type: counter + :level: info + :oneliner: Connections failed + + Counter of failed opens. Detailed reasons are given in the + fail_* counters (DIAG level) and in Debug VSL. + + This counter is the sum of all detailed fail_* counters. + + All fail_* counters may be slightly inaccurate for efficiency. + +.. varnish_vsc:: fail_eacces + :type: counter + :level: diag + :oneliner: Connections failed with EACCES or EPERM + +.. varnish_vsc:: fail_eaddrnotavail + :type: counter + :level: diag + :oneliner: Connections failed with EADDRNOTAVAIL + +.. varnish_vsc:: fail_econnrefused + :type: counter + :level: diag + :oneliner: Connections failed with ECONNREFUSED + +.. varnish_vsc:: fail_enetunreach + :type: counter + :level: diag + :oneliner: Connections failed with ENETUNREACH +.. varnish_vsc:: fail_etimedout + :type: counter + :level: diag + :oneliner: Connections failed ETIMEDOUT + +.. varnish_vsc:: fail_other + :type: counter + :level: diag + :oneliner: Connections failed for other reason + +.. varnish_vsc_end:: vbe diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 0f089f0..4513474 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -113,12 +113,11 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, bo->htc->doclose = SC_NULL; FIND_TMO(connect_timeout, tmod, bo, bp); - pfd = VTP_Get(bp->tcp_pool, tmod, wrk, force_fresh); + pfd = VTP_Get(bp->tcp_pool, tmod, wrk, force_fresh, bp->vsc); if (pfd == NULL) { VSLb(bo->vsl, SLT_FetchError, "backend %s: fail errno %d (%s)", VRT_BACKEND_string(bp->director), errno, strerror(errno)); - // XXX: Per backend stats ? VSC_C_main->backend_fail++; bo->htc = NULL; return (NULL); diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index e9f886e..2c87327 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -272,7 +272,8 @@ vbp_poke(struct vbp_target *vt) t_start = t_now = VTIM_real(); t_end = t_start + vt->timeout; - s = VTP_Open(vt->tcp_pool, t_end - t_now, (const void **)&sa); + s = VTP_Open(vt->tcp_pool, t_end - t_now, (const void **)&sa, + vt->backend->vsc); if (s < 0) { /* Got no connection: failed */ return; diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index c92ba8c..f5fa2a3 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -45,6 +45,8 @@ #include "cache_tcp_pool.h" #include "cache_pool.h" +#include "VSC_vbe.h" + struct conn_pool; /*-------------------------------------------------------------------- @@ -373,7 +375,8 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) */ static int -VCP_Open(const struct conn_pool *cp, double tmo, const void **privp) +VCP_Open(const struct conn_pool *cp, double tmo, const void **privp, + struct VSC_vbe *vsc) { int r; @@ -381,6 +384,32 @@ VCP_Open(const struct conn_pool *cp, double tmo, const void **privp) r = cp->methods->open(cp, tmo, privp); + if (r >= 0 || vsc == NULL) + return (r); + + /* stats access unprotected */ + switch (errno) { + case EACCES: + case EPERM: + vsc->fail_eacces++; + break; + case EADDRNOTAVAIL: + vsc->fail_eaddrnotavail++; + break; + case ECONNREFUSED: + vsc->fail_econnrefused++; + break; + case ENETUNREACH: + vsc->fail_enetunreach++; + break; + case ETIMEDOUT: + vsc->fail_etimedout++; + break; + default: + vsc->fail_other++; + } + vsc->fail++; + return (r); } @@ -426,7 +455,7 @@ VCP_Close(struct pfd **pfdp) static struct pfd * VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk, - unsigned force_fresh) + unsigned force_fresh, struct VSC_vbe *vsc) { struct pfd *pfd; @@ -459,7 +488,7 @@ VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk, INIT_OBJ(pfd->waited, WAITED_MAGIC); pfd->state = PFD_STATE_USED; pfd->conn_pool = cp; - pfd->fd = VCP_Open(cp, tmo, &pfd->priv); + pfd->fd = VCP_Open(cp, tmo, &pfd->priv, vsc); if (pfd->fd < 0) { FREE_OBJ(pfd); Lck_Lock(&cp->mtx); @@ -737,9 +766,10 @@ VTP_Rel(struct tcp_pool **tpp) */ int -VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp) +VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp, + struct VSC_vbe *vsc) { - return (VCP_Open(tp->cp, tmo, privp)); + return (VCP_Open(tp->cp, tmo, privp, vsc)); } /*-------------------------------------------------------------------- @@ -770,10 +800,10 @@ VTP_Close(struct pfd **pfdp) struct pfd * VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk, - unsigned force_fresh) + unsigned force_fresh, struct VSC_vbe *vsc) { - return VCP_Get(tp->cp, tmo, wrk, force_fresh); + return VCP_Get(tp->cp, tmo, wrk, force_fresh, vsc); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index d554a36..88b131e 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -50,6 +50,8 @@ void PFD_RemoteName(const struct pfd *, char *, unsigned, char *, unsigned); * Prototypes */ +struct VSC_vbe; + struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds, const void *id); /* @@ -70,9 +72,11 @@ void VTP_Rel(struct tcp_pool **); * the pool is destroyed and all cached connections closed. */ -int VTP_Open(const struct tcp_pool *, double tmo, const void **); +int VTP_Open(const struct tcp_pool *, double tmo, const void **, + struct VSC_vbe *); /* * Open a new connection and return the adress used. + * Errors will be accounted in the optional vsc */ void VTP_Close(struct pfd **); @@ -86,9 +90,10 @@ void VTP_Recycle(const struct worker *, struct pfd **); */ struct pfd *VTP_Get(struct tcp_pool *, double tmo, struct worker *, - unsigned force_fresh); + unsigned force_fresh, struct VSC_vbe *); /* * Get a (possibly) recycled connection. + * Open errors will be accounted in the optional vsc */ int VTP_Wait(struct worker *, struct pfd *, double tmo); From nils.goroll at uplex.de Mon Jun 11 08:29:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jun 2018 08:29:10 +0000 (UTC) Subject: [master] e02e31e holddown times for certain backend connection errors Message-ID: <20180611082911.05333A97BB@lists.varnish-cache.org> commit e02e31ed2ad9a58fdee60628a9c4450bd1266179 Author: Nils Goroll Date: Tue Jun 5 10:13:44 2018 +0200 holddown times for certain backend connection errors This is similar to the vca pace: Depending on the backend connection error, it does not make sense to re-try in rapid succession, instead not attempting the failed connection again for some time will save resources both locally and remotely, where applicable, and should thus help improve the overall situation. Fixes #2622 diff --git a/bin/varnishd/VSC_vbe.vsc b/bin/varnishd/VSC_vbe.vsc index b215909..5212119 100644 --- a/bin/varnishd/VSC_vbe.vsc +++ b/bin/varnishd/VSC_vbe.vsc @@ -124,4 +124,13 @@ :level: diag :oneliner: Connections failed for other reason +.. varnish_vsc:: helddown + :type: counter + :level: diag + :oneliner: Connection opens not attempted + + Connections not attempted during the backend_local_error_holddown + or backend_remote_error_holddown interval after a fundamental + connection issue. + .. varnish_vsc_end:: vbe diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index f5fa2a3..d792735 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -101,6 +101,9 @@ struct conn_pool { int n_kill; int n_used; + + double holddown; + int holddown_errno; }; struct tcp_pool { @@ -238,6 +241,7 @@ VCP_New(struct conn_pool *cp, const void *id, void *priv, cp->priv = priv; cp->methods = cm; cp->refcnt = 1; + cp->holddown = 0; Lck_New(&cp->mtx, lck_tcp_pool); VTAILQ_INIT(&cp->connlist); VTAILQ_INIT(&cp->killlist); @@ -375,32 +379,59 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) */ static int -VCP_Open(const struct conn_pool *cp, double tmo, const void **privp, +VCP_Open(struct conn_pool *cp, double tmo, const void **privp, struct VSC_vbe *vsc) { int r; + double h; CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); + while (cp->holddown > 0) { + Lck_Lock(&cp->mtx); + if (cp->holddown == 0) { + Lck_Unlock(&cp->mtx); + break; + } + + if (VTIM_mono() >= cp->holddown) { + cp->holddown = 0; + Lck_Unlock(&cp->mtx); + break; + } + + if (vsc) + vsc->helddown++; + errno = cp->holddown_errno; + Lck_Unlock(&cp->mtx); + return (-1); + } + r = cp->methods->open(cp, tmo, privp); if (r >= 0 || vsc == NULL) return (r); + h = 0; + /* stats access unprotected */ switch (errno) { case EACCES: case EPERM: vsc->fail_eacces++; + h = cache_param->backend_local_error_holddown; break; case EADDRNOTAVAIL: vsc->fail_eaddrnotavail++; + h = cache_param->backend_local_error_holddown; break; case ECONNREFUSED: vsc->fail_econnrefused++; + h = cache_param->backend_remote_error_holddown; break; case ENETUNREACH: vsc->fail_enetunreach++; + h = cache_param->backend_remote_error_holddown; break; case ETIMEDOUT: vsc->fail_etimedout++; @@ -410,6 +441,18 @@ VCP_Open(const struct conn_pool *cp, double tmo, const void **privp, } vsc->fail++; + if (h == 0) + return (r); + + Lck_Lock(&cp->mtx); + h += VTIM_mono(); + if (cp->holddown == 0 || h < cp->holddown) { + cp->holddown = h; + cp->holddown_errno = errno; + } + + Lck_Unlock(&cp->mtx); + return (r); } @@ -766,7 +809,7 @@ VTP_Rel(struct tcp_pool **tpp) */ int -VTP_Open(const struct tcp_pool *tp, double tmo, const void **privp, +VTP_Open(struct tcp_pool *tp, double tmo, const void **privp, struct VSC_vbe *vsc) { return (VCP_Open(tp->cp, tmo, privp, vsc)); diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 88b131e..9f640da 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -72,7 +72,7 @@ void VTP_Rel(struct tcp_pool **); * the pool is destroyed and all cached connections closed. */ -int VTP_Open(const struct tcp_pool *, double tmo, const void **, +int VTP_Open(struct tcp_pool *, double tmo, const void **, struct VSC_vbe *); /* * Open a new connection and return the adress used. diff --git a/include/tbl/params.h b/include/tbl/params.h index e2ecff4..054b2f4 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -286,6 +286,45 @@ PARAM( ) PARAM( + /* name */ backend_local_error_holddown, + /* typ */ timeout, + /* min */ "0.000", + /* max */ NULL, + /* default */ "10.000", + /* units */ "seconds", + /* flags */ EXPERIMENTAL, + /* s-text */ + "When connecting to backends, certain error codes " + "(EADDRNOTAVAIL, EACCESS, EPERM) signal a local resource shortage " + "or configuration issue for which retrying connection attempts " + "may worsen the situation due to the complexity of the operations " + "involved in the kernel.\n" + "This parameter prevents repeated connection attempts for the " + "configured duration.", + /* l-text */ "", + /* func */ NULL +) + +PARAM( + /* name */ backend_remote_error_holddown, + /* typ */ timeout, + /* min */ "0.000", + /* max */ NULL, + /* default */ "0.250", + /* units */ "seconds", + /* flags */ EXPERIMENTAL, + /* s-text */ + "When connecting to backends, certain error codes (ECONNREFUSED, " + "ENETUNREACH) signal fundamental connection issues such as the backend " + "not accepting connections or routing problems for which repeated " + "connection attempts are considered useless\n" + "This parameter prevents repeated connection attempts for the " + "configured duration.", + /* l-text */ "", + /* func */ NULL +) + +PARAM( /* name */ cli_limit, /* typ */ bytes_u, /* min */ "128b", From nils.goroll at uplex.de Mon Jun 11 08:29:11 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jun 2018 08:29:11 +0000 (UTC) Subject: [master] 561579a add the long missing per-backend busy and unhealthy counters Message-ID: <20180611082911.3C61FA97BF@lists.varnish-cache.org> commit 561579a3e12b2307e19e5da7b4cb37fd4ee1b74e Author: Nils Goroll Date: Tue Jun 5 10:51:45 2018 +0200 add the long missing per-backend busy and unhealthy counters diff --git a/bin/varnishd/VSC_vbe.vsc b/bin/varnishd/VSC_vbe.vsc index 5212119..cc05598 100644 --- a/bin/varnishd/VSC_vbe.vsc +++ b/bin/varnishd/VSC_vbe.vsc @@ -78,6 +78,18 @@ :level: info :oneliner: Backend requests sent +.. varnish_vsc:: unhealthy + :type: counter + :level: info + :oneliner: Fetches not attempted due to backend being unhealthy + +.. varnish_vsc:: busy + :type: counter + :level: info + :oneliner: Fetches not attempted due to backend being busy + + Number of times the max_connections limit was reached + .. === Anything below is actually per VCP entry, but collected per === backend for simplicity diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 4513474..a6e1760 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -90,7 +90,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (bp->director->sick) { VSLb(bo->vsl, SLT_FetchError, "backend %s: unhealthy", VRT_BACKEND_string(bp->director)); - // XXX: per backend stats ? + bp->vsc->unhealthy++; VSC_C_main->backend_unhealthy++; return (NULL); } @@ -98,7 +98,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, if (bp->max_connections > 0 && bp->n_conn >= bp->max_connections) { VSLb(bo->vsl, SLT_FetchError, "backend %s: busy", VRT_BACKEND_string(bp->director)); - // XXX: per backend stats ? + bp->vsc->busy++; VSC_C_main->backend_busy++; return (NULL); } From nils.goroll at uplex.de Mon Jun 11 12:04:14 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jun 2018 12:04:14 +0000 (UTC) Subject: [master] 09a75da Backend_health: report error details Message-ID: <20180611120414.57C06B0C08@lists.varnish-cache.org> commit 09a75daaf302eba6c4f81ff30f394f013d93f8c5 Author: Nils Goroll Date: Mon Jun 11 13:55:01 2018 +0200 Backend_health: report error details Previously, tracing the root cause of probe failures was unnecessarily complicated by the fact that the probe window bits and timing information were the only source of information when no HTTP status line was logged and for the case of all the bits being zero, almost impossible (e.g. differentiating between a local and remote connection open failure). We now re-use the response field for failing probes also. diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 2c87327..fa823d4 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -39,6 +39,8 @@ #include #include +#include +#include #include #include "cache_varnishd.h" @@ -223,8 +225,15 @@ vbp_write(struct vbp_target *vt, int *sock, const void *buf, size_t len) i = write(*sock, buf, len); if (i != len) { - if (i < 0) + if (i < 0) { vt->err_xmit |= 1; + bprintf(vt->resp_buf, "Write error %d (%s)", + errno, strerror(errno)); + } else { + bprintf(vt->resp_buf, + "Short write (%d/%zu) error %d (%s)", + i, len, errno, strerror(errno)); + } VTCP_close(sock); return (-1); } @@ -275,7 +284,8 @@ vbp_poke(struct vbp_target *vt) s = VTP_Open(vt->tcp_pool, t_end - t_now, (const void **)&sa, vt->backend->vsc); if (s < 0) { - /* Got no connection: failed */ + bprintf(vt->resp_buf, "Open error %d (%s)", + errno, strerror(errno)); return; } @@ -292,7 +302,9 @@ vbp_poke(struct vbp_target *vt) t_now = VTIM_real(); tmo = (int)round((t_end - t_now) * 1e3); if (tmo <= 0) { - /* Spent too long time getting it */ + bprintf(vt->resp_buf, + "Open timeout %.3fs exceeded by %.3fs", + vt->timeout, t_now - t_end); VTCP_close(&s); return; } @@ -304,8 +316,10 @@ vbp_poke(struct vbp_target *vt) proxy_header = -1; Lck_Unlock(&vbp_mtx); - if (proxy_header < 0) + if (proxy_header < 0) { + bprintf(vt->resp_buf, "%s", "No backend"); return; + } /* Send the PROXY header */ assert(proxy_header <= 2); @@ -329,9 +343,17 @@ vbp_poke(struct vbp_target *vt) tmo = (int)round((t_end - t_now) * 1e3); if (tmo > 0) i = poll(pfd, 1, tmo); - if (i == 0 || tmo <= 0) { - if (i == 0) - vt->err_recv |= 1; + if (i == 0) { + vt->err_recv |= 1; + bprintf(vt->resp_buf, "Poll error %d (%s)", + errno, strerror(errno)); + VTCP_close(&s); + return; + } + if (tmo <= 0) { + bprintf(vt->resp_buf, + "Poll (read) timeout %.3fs exceeded by %.3fs", + vt->timeout, t_now - t_end); VTCP_close(&s); return; } @@ -340,20 +362,27 @@ vbp_poke(struct vbp_target *vt) sizeof vt->resp_buf - rlen); else i = read(s, buf, sizeof buf); - if (i <= 0) + if (i <= 0) { + if (i < 0) + bprintf(vt->resp_buf, "Read error %d (%s)", + errno, strerror(errno)); break; + } rlen += i; } VTCP_close(&s); if (i < 0) { + /* errno reported above */ vt->err_recv |= 1; return; } - if (rlen == 0) + if (rlen == 0) { + bprintf(vt->resp_buf, "%s", "Empty response"); return; + } /* So we have a good receive ... */ t_now = VTIM_real(); diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 1bafbf1..e56b7c8 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -389,7 +389,7 @@ SLTM(Backend_health, 0, "Backend health check", "The format is::\n\n" "\t%s %s %s %u %u %u %f %f %s\n" "\t| | | | | | | | |\n" - "\t| | | | | | | | +- Probe HTTP response\n" + "\t| | | | | | | | +- Probe HTTP response / error information\n" "\t| | | | | | | +---- Average response time\n" "\t| | | | | | +------- Response time\n" "\t| | | | | +---------- Probe window size\n" From martin at varnish-software.com Mon Jun 11 13:25:13 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 11 Jun 2018 13:25:13 +0000 (UTC) Subject: [master] 9d1b135 Fixups from 'Fix PROXY and HTTP/1 proto dissectors' Message-ID: <20180611132513.D9BB9B2D40@lists.varnish-cache.org> commit 9d1b135dd5c33796fb675f1e046f6b87c284dfc8 Author: Martin Blix Grydeland Date: Mon Jun 11 14:33:26 2018 +0200 Fixups from 'Fix PROXY and HTTP/1 proto dissectors' Remove old and now invalid assert. Change order of evaluation in if-statement to make sure we don't step outside rxbuf_e. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index a10756d..fc4c451 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -76,7 +76,7 @@ vpx_proto1(const struct worker *wrk, const struct req *req) *q++ = '\0'; /* Nuke the CRLF */ - if (*q != '\n' || q == req->htc->rxbuf_e) + if (q == req->htc->rxbuf_e || *q != '\n') return (-1); *q++ = '\0'; @@ -515,7 +515,6 @@ vpx_complete(struct http_conn *htc) if (j == 0) return (HTC_S_JUNK); if (j == 1 && i == sizeof vpx1_sig) { - assert (htc->rxbuf_e < htc->ws->r); q = memchr(p + i, '\n', htc->rxbuf_e - (p + i)); if (q != NULL && (q - htc->rxbuf_b) > 107) return (HTC_S_OVERFLOW); From martin at varnish-software.com Mon Jun 11 13:25:14 2018 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Mon, 11 Jun 2018 13:25:14 +0000 (UTC) Subject: [master] ed5c43b New fix for #2285 and #2624 Message-ID: <20180611132514.048A4B2D43@lists.varnish-cache.org> commit ed5c43be0fdb120d19c1b703233d26b7593eb3b8 Author: Martin Blix Grydeland Date: Tue Jun 5 17:02:55 2018 +0200 New fix for #2285 and #2624 Previous fix for #2285 (and the duplicate #2624) was missdiagnosed. The problem stems from a wrong assumption that the number of bytes already pipelined will be less than maxbytes, with maxbytes beeing the maximum number of bytes the HTC_RxStuff may need to get a full work unit. That assumption may fail during the H/1 to H/2 upgrade path where maxbytes change with the context, or during runtime changing of parameters. This patch makes HTC_RxStuff not assert if the pipelined data turned out to exceed maxbytes, but return overflow if we run out of workspace. (#2624 has received a workaround in the H/2 code that perhaps should be reverted). diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index a6c8f83..0500bce 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -239,6 +239,11 @@ HTC_RxPipeline(struct http_conn *htc, void *p) /*---------------------------------------------------------------------- * Receive a request/packet/whatever, with timeouts * + * maxbytes is the maximum number of bytes the caller expects to need to + * reach a complete work unit. Note that due to pipelining the actual + * number of bytes passed to func in htc->rxbuf_b through htc->rxbuf_e may + * be larger. + * * t0 is when we start * *t1 becomes time of first non-idle rx * *t2 becomes time of complete rx @@ -261,6 +266,7 @@ HTC_RxStuff(struct http_conn *htc, htc_complete_f *func, AN(htc->ws->r); AN(htc->rxbuf_b); assert(htc->rxbuf_b <= htc->rxbuf_e); + assert(htc->rxbuf_e <= htc->ws->r); AZ(isnan(tn)); if (t1 != NULL) @@ -273,7 +279,7 @@ HTC_RxStuff(struct http_conn *htc, htc_complete_f *func, } z = (htc->ws->r - htc->rxbuf_b); if (z < maxbytes) - maxbytes = z; + maxbytes = z; /* Cap maxbytes at available WS */ while (1) { now = VTIM_real(); @@ -308,8 +314,9 @@ HTC_RxStuff(struct http_conn *htc, htc_complete_f *func, if (!isnan(ti) && ti < tn && hs == HTC_S_EMPTY) tmo = ti - now; z = maxbytes - (htc->rxbuf_e - htc->rxbuf_b); - assert(z >= 0); - if (z == 0) { + if (z <= 0) { + /* maxbytes reached but not HTC_S_COMPLETE. Return + * overflow. */ WS_ReleaseP(htc->ws, htc->rxbuf_b); return (HTC_S_OVERFLOW); } diff --git a/bin/varnishtest/tests/r02219.vtc b/bin/varnishtest/tests/r02219.vtc index 7187abc..ff1a924 100644 --- a/bin/varnishtest/tests/r02219.vtc +++ b/bin/varnishtest/tests/r02219.vtc @@ -5,6 +5,8 @@ server s1 { txresp rxreq txresp + rxreq + txresp } -start varnish v1 -arg "-p workspace_client=9k" -proto PROXY -vcl+backend { @@ -59,3 +61,8 @@ client c2 { } rxresp } -run + +client c3 { + send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC HTTP/1.1\r\n\r\n" + rxresp +} -run From daghf at varnish-software.com Tue Jun 12 09:39:20 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 12 Jun 2018 09:39:20 +0000 (UTC) Subject: [master] ee3abf8 New take on #2624 after Martin's RxStuff overflow fixes Message-ID: <20180612093920.2CDC2A897C@lists.varnish-cache.org> commit ee3abf8eb30ae03e6e518945fa7927362cd81d1d Author: Dag Haavi Finstad Date: Tue Jun 12 11:28:59 2018 +0200 New take on #2624 after Martin's RxStuff overflow fixes diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 64d55a6..c43a65f 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -882,8 +882,6 @@ h2_frame_complete(struct http_conn *htc) u = vbe32dec(htc->rxbuf_b) >> 8; if (l >= u + 9) return (HTC_S_COMPLETE); - else if (l > h2->local_settings.max_frame_size + 9) - return (HTC_S_OVERFLOW); return (HTC_S_MORE); } @@ -1068,7 +1066,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) hs = HTC_RxStuff(h2->htc, h2_frame_complete, NULL, NULL, NAN, h2->sess->t_idle + cache_param->timeout_idle, - 16384 + 9); // rfc7540,l,4228,4228 + h2->local_settings.max_frame_size + 9); switch (hs) { case HTC_S_COMPLETE: break; From phk at FreeBSD.org Tue Jun 12 12:28:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 12 Jun 2018 12:28:10 +0000 (UTC) Subject: [master] e57f3e1 Minor polish Message-ID: <20180612122810.96A39AF368@lists.varnish-cache.org> commit e57f3e187c1ad8de81f6c717cfdc8b345a8289fb Author: Poul-Henning Kamp Date: Tue Jun 12 12:27:16 2018 +0000 Minor polish diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h index 124ff2e..bf7c37f 100644 --- a/bin/varnishd/cache/cache_director.h +++ b/bin/varnishd/cache/cache_director.h @@ -31,8 +31,6 @@ * */ -struct vdi_methods; - struct vcldir { unsigned magic; #define VCLDIR_MAGIC 0xbf726c7d @@ -40,14 +38,11 @@ struct vcldir { struct vcl *vcl; const struct vdi_methods *methods; VTAILQ_ENTRY(vcldir) list; - unsigned health; const struct vdi_ahealth *admin_health; double health_changed; char *cli_name; }; -/* cache_director.c */ - #define VBE_AHEALTH_LIST \ VBE_AHEALTH(healthy, HEALTHY, 1) \ VBE_AHEALTH(sick, SICK, 0) \ From dridi at varni.sh Wed Jun 13 07:10:01 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 13 Jun 2018 09:10:01 +0200 Subject: [master] ce0ccf5 Introduce optional resolve arg to std.ip In-Reply-To: <20180212125306.AA6F2B6AEF@lists.varnish-cache.org> References: <20180212125306.AA6F2B6AEF@lists.varnish-cache.org> Message-ID: > -$Function IP ip(STRING s, IP fallback) > +$Function IP ip(STRING s, IP fallback, BOOL resolve = 1) > > Description > Converts the string *s* to the first IP number returned by > the system library function getaddrinfo(3). If conversion > fails, *fallback* will be returned. > + > + If *resolve* is true, getaddrinfo() is called using *AI_NUMERICHOST* > + to avoid network lookups. This makes "pure" IP strings cheaper to > + convert. > + Isn't it the other way around? If *resolve* is false (default is true), getaddrinfo() is called using *AI_NUMERICHOST* to avoid network lookups. This makes "pure" IP strings cheaper to convert. Also, can we make fallback default to the bogo IP in the current state of things? Dridi From phk at FreeBSD.org Wed Jun 13 07:36:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 13 Jun 2018 07:36:10 +0000 (UTC) Subject: [master] fda1241b4 Add asserts to ensure the (hypothetical?) race in #2528 doesn't happen. Message-ID: <20180613073610.8CC9F107A21@lists.varnish-cache.org> commit fda1241b4f4d358174788f90d4b0803e0bc055cf Author: Poul-Henning Kamp Date: Wed Jun 13 07:35:00 2018 +0000 Add asserts to ensure the (hypothetical?) race in #2528 doesn't happen. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 09418262a..0f81e544e 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -927,6 +927,10 @@ vbf_fetch_thread(struct worker *wrk, void *priv) while (stp != F_STP_DONE) { CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); assert(bo->fetch_objcore->boc->refcount >= 1); + if (bo->fetch_objcore->boc->state < BOS_REQ_DONE) + AN(bo->req); + else + AZ(bo->req); switch (stp) { #define FETCH_STEP(l, U, arg) \ case F_STP_##U: \ From phk at FreeBSD.org Wed Jun 13 10:03:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 13 Jun 2018 10:03:13 +0000 (UTC) Subject: [master] 7cfe66d4c Add missing VTCP_Close() Message-ID: <20180613100313.8C9C910ABB1@lists.varnish-cache.org> commit 7cfe66d4c84d0e475f3f0515a4207b4f070be35b Author: Poul-Henning Kamp Date: Wed Jun 13 10:02:28 2018 +0000 Add missing VTCP_Close() diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index fa823d47f..dfa594ce8 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -318,6 +318,7 @@ vbp_poke(struct vbp_target *vt) if (proxy_header < 0) { bprintf(vt->resp_buf, "%s", "No backend"); + VTCP_close(&s); return; } From phk at FreeBSD.org Wed Jun 13 16:52:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 13 Jun 2018 16:52:13 +0000 (UTC) Subject: [master] e4fa9f4e8 Remember to set the REQ_DONE state when we unset the bereq.body; Message-ID: <20180613165213.AFA502BD2@lists.varnish-cache.org> commit e4fa9f4e82af70ca9539d86616141d05517d0968 Author: Poul-Henning Kamp Date: Wed Jun 13 16:50:55 2018 +0000 Remember to set the REQ_DONE state when we unset the bereq.body; Found by: fgs & ASAN diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 32055736c..ea8ea5e6e 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -499,6 +499,8 @@ VRT_u_bereq_body(VRT_CTX) if (ctx->bo->req != NULL) { CHECK_OBJ_NOTNULL(ctx->bo->req, REQ_MAGIC); ctx->bo->req = NULL; + ObjSetState(ctx->bo->wrk, + ctx->bo->fetch_objcore, BOS_REQ_DONE); http_Unset(ctx->bo->bereq, H_Content_Length); } } From fgsch at users.noreply.github.com Wed Jun 13 23:58:08 2018 From: fgsch at users.noreply.github.com (Federico G. Schwindt) Date: Wed, 13 Jun 2018 23:58:08 +0000 (UTC) Subject: [master] fc0ee48e9 Add JSON ping output Message-ID: <20180613235808.D866464220@lists.varnish-cache.org> commit fc0ee48e9ee2f6bc7993f7f3958789acfa83d4ce Author: Lucas Guardalben Date: Wed May 16 15:52:17 2018 +0100 Add JSON ping output This adds -j support to the command varnishadm ping -j Tested on RFC4627/RFC7159/ECMA404 standards (all valid JSON) diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c index bdaa4ae33..35dac06a6 100644 --- a/bin/varnishd/cache/cache_cli.c +++ b/bin/varnishd/cache/cache_cli.c @@ -111,7 +111,7 @@ CLI_Run(void) /*--------------------------------------------------------------------*/ static struct cli_proto cli_cmds[] = { - { CLICMD_PING, "i", VCLS_func_ping }, + { CLICMD_PING, "i", VCLS_func_ping, VCLS_func_ping_json }, { CLICMD_HELP, "i", VCLS_func_help, VCLS_func_help_json }, { NULL } }; diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index 3304ef03c..3677eb620 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -327,7 +327,7 @@ mcf_help_json(struct cli *cli, const char * const *av, void *priv) static struct cli_proto cli_auth[] = { { CLICMD_HELP, "", mcf_help, mcf_help_json }, - { CLICMD_PING, "", VCLS_func_ping }, + { CLICMD_PING, "", VCLS_func_ping, VCLS_func_ping_json }, { CLICMD_AUTH, "", mcf_auth }, { CLICMD_QUIT, "", VCLS_func_close }, { NULL } diff --git a/bin/varnishtest/tests/b00008.vtc b/bin/varnishtest/tests/b00008.vtc index 1ec361155..6134ff0e6 100644 --- a/bin/varnishtest/tests/b00008.vtc +++ b/bin/varnishtest/tests/b00008.vtc @@ -1,4 +1,4 @@ -varnishtest "Test CLI help and parameter functions" +varnishtest "Test CLI commands and parameter functions" varnish v1 -arg "-b ${bad_ip}:9080" @@ -28,6 +28,14 @@ varnish v1 -cliok "help" varnish v1 -clijson "help -j" +varnish v1 -cliok "backend.list" + +varnish v1 -clijson "backend.list -j" + +varnish v1 -cliok "ping" + +varnish v1 -clijson "ping -j" + varnish v1 -clierr 106 "param.set waiter HASH(0x8839c4c)" varnish v1 -cliok "param.set cli_limit 128" diff --git a/include/vcli_serve.h b/include/vcli_serve.h index 8378993c4..00b364ff4 100644 --- a/include/vcli_serve.h +++ b/include/vcli_serve.h @@ -105,3 +105,4 @@ cli_func_t VCLS_func_close; cli_func_t VCLS_func_help; cli_func_t VCLS_func_help_json; cli_func_t VCLS_func_ping; +cli_func_t VCLS_func_ping_json; diff --git a/lib/libvarnish/vcli_serve.c b/lib/libvarnish/vcli_serve.c index de7ebdbd8..fc8578c50 100644 --- a/lib/libvarnish/vcli_serve.c +++ b/lib/libvarnish/vcli_serve.c @@ -103,6 +103,16 @@ VCLS_func_ping(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "PONG %jd 1.0", (intmax_t)t); } +void v_matchproto_(cli_func_t) +VCLS_func_ping_json(struct cli *cli, const char * const *av, void *priv) +{ + (void)av; + (void)priv; + VCLI_JSON_begin(cli, 2, av); + VCLI_Out(cli, ", \"PONG\"\n"); + VCLI_JSON_end(cli); +} + /*--------------------------------------------------------------------*/ void v_matchproto_(cli_func_t) From phk at FreeBSD.org Thu Jun 14 07:49:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 14 Jun 2018 07:49:12 +0000 (UTC) Subject: [master] f5ada80b6 Fix a case I overlooked in the recent change to remove the NUL termination of the rxbuf. Message-ID: <20180614074912.DBC6E9A258@lists.varnish-cache.org> commit f5ada80b6c603adc253c5fe2605e72803ada7d54 Author: Poul-Henning Kamp Date: Thu Jun 14 07:47:58 2018 +0000 Fix a case I overlooked in the recent change to remove the NUL termination of the rxbuf. Found by: fgs diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c index d1ad4d28e..ac1a2a86e 100644 --- a/bin/varnishd/http1/cache_http1_proto.c +++ b/bin/varnishd/http1/cache_http1_proto.c @@ -73,7 +73,7 @@ HTTP1_Complete(struct http_conn *htc) assert(htc->rxbuf_e <= htc->ws->r); /* Skip any leading white space */ - for (p = htc->rxbuf_b ; vct_islws(*p); p++) + for (p = htc->rxbuf_b ; p < htc->rxbuf_e && vct_islws(*p); p++) continue; if (p == htc->rxbuf_e) return (HTC_S_EMPTY); From phk at FreeBSD.org Thu Jun 14 08:39:07 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 14 Jun 2018 08:39:07 +0000 (UTC) Subject: [master] 3c0b8768a The backend probes can only access the backend under lock, because the backend might be going away, and since we cannot afford to hold the lock over VTP_Open(), we have to pull the VBE_vsc knowledge one level back up. Message-ID: <20180614083908.122949BAF9@lists.varnish-cache.org> commit 3c0b8768afda0cc657fb902d1d390bdd0e8b7ce5 Author: Poul-Henning Kamp Date: Thu Jun 14 08:37:05 2018 +0000 The backend probes can only access the backend under lock, because the backend might be going away, and since we cannot afford to hold the lock over VTP_Open(), we have to pull the VBE_vsc knowledge one level back up. Overlooked by: slink, phk Spotted by: Coverity diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index a6e176049..4f542f548 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -58,6 +58,41 @@ static struct lock backends_mtx; /*--------------------------------------------------------------------*/ +void +VBE_Connect_Error(struct VSC_vbe *vsc, int err) +{ + + switch(err) { + case 0: + /* + * This is kind of brittle, but zero is the only + * value of errno we can trust to have no meaning. + */ + vsc->helddown++; + break; + case EACCES: + case EPERM: + vsc->fail_eacces++; + break; + case EADDRNOTAVAIL: + vsc->fail_eaddrnotavail++; + break; + case ECONNREFUSED: + vsc->fail_econnrefused++; + break; + case ENETUNREACH: + vsc->fail_enetunreach++; + break; + case ETIMEDOUT: + vsc->fail_etimedout++; + break; + default: + vsc->fail_other++; + } +} + +/*--------------------------------------------------------------------*/ + #define FIND_TMO(tmx, dst, bo, be) \ do { \ CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); \ @@ -77,7 +112,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, unsigned force_fresh) { struct pfd *pfd; - int *fdp; + int *fdp, err; double tmod; char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE]; char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE]; @@ -113,11 +148,12 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo, bo->htc->doclose = SC_NULL; FIND_TMO(connect_timeout, tmod, bo, bp); - pfd = VTP_Get(bp->tcp_pool, tmod, wrk, force_fresh, bp->vsc); + pfd = VTP_Get(bp->tcp_pool, tmod, wrk, force_fresh, &err); if (pfd == NULL) { + VBE_Connect_Error(bp->vsc, err); VSLb(bo->vsl, SLT_FetchError, "backend %s: fail errno %d (%s)", - VRT_BACKEND_string(bp->director), errno, strerror(errno)); + VRT_BACKEND_string(bp->director), err, strerror(err)); VSC_C_main->backend_fail++; bo->htc = NULL; return (NULL); diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 940e2426d..706fea428 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -82,3 +82,5 @@ void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, void VBP_Remove(struct backend *b); void VBP_Control(const struct backend *b, int stop); void VBP_Status(struct vsb *, const struct backend *, int details, int json); +void VBE_Connect_Error(struct VSC_vbe *, int err); + diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index dfa594ce8..a5e67c021 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -271,7 +271,7 @@ vbp_write_proxy_v1(struct vbp_target *vt, int *sock) static void vbp_poke(struct vbp_target *vt) { - int s, tmo, i, proxy_header; + int s, tmo, i, proxy_header, err; double t_start, t_now, t_end; unsigned rlen, resp; char buf[8192], *p; @@ -281,11 +281,13 @@ vbp_poke(struct vbp_target *vt) t_start = t_now = VTIM_real(); t_end = t_start + vt->timeout; - s = VTP_Open(vt->tcp_pool, t_end - t_now, (const void **)&sa, - vt->backend->vsc); + s = VTP_Open(vt->tcp_pool, t_end - t_now, (const void **)&sa, &err); if (s < 0) { - bprintf(vt->resp_buf, "Open error %d (%s)", - errno, strerror(errno)); + bprintf(vt->resp_buf, "Open error %d (%s)", err, strerror(err)); + Lck_Lock(&vbp_mtx); + if (vt->backend) + VBE_Connect_Error(vt->backend->vsc, err); + Lck_Unlock(&vbp_mtx); return; } diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index d79273586..f33668b2f 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -45,8 +45,6 @@ #include "cache_tcp_pool.h" #include "cache_pool.h" -#include "VSC_vbe.h" - struct conn_pool; /*-------------------------------------------------------------------- @@ -379,13 +377,13 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) */ static int -VCP_Open(struct conn_pool *cp, double tmo, const void **privp, - struct VSC_vbe *vsc) +VCP_Open(struct conn_pool *cp, double tmo, const void **privp, int *err) { int r; double h; CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); + AN(err); while (cp->holddown > 0) { Lck_Lock(&cp->mtx); @@ -400,8 +398,7 @@ VCP_Open(struct conn_pool *cp, double tmo, const void **privp, break; } - if (vsc) - vsc->helddown++; + *err = 0; errno = cp->holddown_errno; Lck_Unlock(&cp->mtx); return (-1); @@ -409,7 +406,9 @@ VCP_Open(struct conn_pool *cp, double tmo, const void **privp, r = cp->methods->open(cp, tmo, privp); - if (r >= 0 || vsc == NULL) + *err = errno; + + if (r >= 0) return (r); h = 0; @@ -418,28 +417,20 @@ VCP_Open(struct conn_pool *cp, double tmo, const void **privp, switch (errno) { case EACCES: case EPERM: - vsc->fail_eacces++; h = cache_param->backend_local_error_holddown; break; case EADDRNOTAVAIL: - vsc->fail_eaddrnotavail++; h = cache_param->backend_local_error_holddown; break; case ECONNREFUSED: - vsc->fail_econnrefused++; h = cache_param->backend_remote_error_holddown; break; case ENETUNREACH: - vsc->fail_enetunreach++; h = cache_param->backend_remote_error_holddown; break; - case ETIMEDOUT: - vsc->fail_etimedout++; - break; default: - vsc->fail_other++; + break; } - vsc->fail++; if (h == 0) return (r); @@ -498,13 +489,15 @@ VCP_Close(struct pfd **pfdp) static struct pfd * VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk, - unsigned force_fresh, struct VSC_vbe *vsc) + unsigned force_fresh, int *err) { struct pfd *pfd; CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + AN(err); + *err = 0; Lck_Lock(&cp->mtx); pfd = VTAILQ_FIRST(&cp->connlist); CHECK_OBJ_ORNULL(pfd, PFD_MAGIC); @@ -531,7 +524,7 @@ VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk, INIT_OBJ(pfd->waited, WAITED_MAGIC); pfd->state = PFD_STATE_USED; pfd->conn_pool = cp; - pfd->fd = VCP_Open(cp, tmo, &pfd->priv, vsc); + pfd->fd = VCP_Open(cp, tmo, &pfd->priv, err); if (pfd->fd < 0) { FREE_OBJ(pfd); Lck_Lock(&cp->mtx); @@ -809,10 +802,9 @@ VTP_Rel(struct tcp_pool **tpp) */ int -VTP_Open(struct tcp_pool *tp, double tmo, const void **privp, - struct VSC_vbe *vsc) +VTP_Open(struct tcp_pool *tp, double tmo, const void **privp, int *err) { - return (VCP_Open(tp->cp, tmo, privp, vsc)); + return (VCP_Open(tp->cp, tmo, privp, err)); } /*-------------------------------------------------------------------- @@ -843,10 +835,10 @@ VTP_Close(struct pfd **pfdp) struct pfd * VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk, - unsigned force_fresh, struct VSC_vbe *vsc) + unsigned force_fresh, int *err) { - return VCP_Get(tp->cp, tmo, wrk, force_fresh, vsc); + return VCP_Get(tp->cp, tmo, wrk, force_fresh, err); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 9f640da73..7c25afda2 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -72,11 +72,10 @@ void VTP_Rel(struct tcp_pool **); * the pool is destroyed and all cached connections closed. */ -int VTP_Open(struct tcp_pool *, double tmo, const void **, - struct VSC_vbe *); +int VTP_Open(struct tcp_pool *, double tmo, const void **, int*); /* * Open a new connection and return the adress used. - * Errors will be accounted in the optional vsc + * errno will be returned in the last argument. */ void VTP_Close(struct pfd **); @@ -90,10 +89,10 @@ void VTP_Recycle(const struct worker *, struct pfd **); */ struct pfd *VTP_Get(struct tcp_pool *, double tmo, struct worker *, - unsigned force_fresh, struct VSC_vbe *); + unsigned force_fresh, int *err); /* * Get a (possibly) recycled connection. - * Open errors will be accounted in the optional vsc + * errno will be stored in err */ int VTP_Wait(struct worker *, struct pfd *, double tmo); From phk at FreeBSD.org Thu Jun 14 13:55:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 14 Jun 2018 13:55:11 +0000 (UTC) Subject: [master] d558ab03e Check the backend connection failures get counted Message-ID: <20180614135511.5AFA4A403C@lists.varnish-cache.org> commit d558ab03e20f486ef19a5c8882df627e33c1845c Author: Poul-Henning Kamp Date: Thu Jun 14 13:53:15 2018 +0000 Check the backend connection failures get counted diff --git a/bin/varnishtest/tests/b00015.vtc b/bin/varnishtest/tests/b00015.vtc index cc6df6d22..5f3b69c03 100644 --- a/bin/varnishtest/tests/b00015.vtc +++ b/bin/varnishtest/tests/b00015.vtc @@ -92,3 +92,6 @@ client c1 { } -run delay .1 + +varnish v1 -expect VBE.vcl1.foo.fail_econnrefused > 0 +varnish v1 -expect VBE.vcl1.foo.helddown > 0 From daghf at varnish-software.com Fri Jun 15 14:18:14 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 15 Jun 2018 14:18:14 +0000 (UTC) Subject: [master] f69063109 Fix an h/2 thread leak Message-ID: <20180615141814.C6944A17A6@lists.varnish-cache.org> commit f69063109328a8d26e5c0fd402c42d13d72af6ee Author: Dag Haavi Finstad Date: Fri Jun 15 16:03:22 2018 +0200 Fix an h/2 thread leak If we failed to schedule a thread for a stream and it's not cleaned up prior to handling a request body, the rxthread would sit around waiting in h2_rx_data indefinitely. diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index c43a65f96..729f0a4ee 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -723,7 +723,7 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) (void)wrk; ASSERT_RXTHR(h2); - if (r2 == NULL) + if (r2 == NULL || !r2->scheduled) return (0); Lck_Lock(&h2->sess->mtx); AZ(h2->mailcall); From daghf at varnish-software.com Fri Jun 15 14:18:14 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 15 Jun 2018 14:18:14 +0000 (UTC) Subject: [master] 2ea634310 h2: Avoid sending duplicate rst_stream frames on Pool_Task failure Message-ID: <20180615141814.E7867A17A9@lists.varnish-cache.org> commit 2ea6343104dce6250ab5f1853092abe17963fa85 Author: Dag Haavi Finstad Date: Fri Jun 15 16:07:37 2018 +0200 h2: Avoid sending duplicate rst_stream frames on Pool_Task failure diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 729f0a4ee..7001b60fc 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -603,6 +603,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, r2->scheduled = 1; if (Pool_Task(wrk->pool, &req->task, TASK_QUEUE_STR) != 0) { r2->scheduled = 0; + r2->state = H2_S_CLOSED; return (H2SE_REFUSED_STREAM); //rfc7540,l,3326,3329 } return (0); From daghf at varnish-software.com Fri Jun 15 14:18:15 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 15 Jun 2018 14:18:15 +0000 (UTC) Subject: [master] c7c7f52d4 Stabilize r02679 Message-ID: <20180615141815.10DE8A17AC@lists.varnish-cache.org> commit c7c7f52d435f3fa6517b3b15508cf2a1591ed4f8 Author: Dag Haavi Finstad Date: Fri Jun 15 16:12:54 2018 +0200 Stabilize r02679 Tune down h2_rx_window_low_water to make sure we don't get a window_update racing against the response frames. Fixes: #2709 diff --git a/bin/varnishtest/tests/r02679.vtc b/bin/varnishtest/tests/r02679.vtc index ca99d75e8..c0029782a 100644 --- a/bin/varnishtest/tests/r02679.vtc +++ b/bin/varnishtest/tests/r02679.vtc @@ -15,6 +15,7 @@ varnish v1 -vcl+backend { } -start varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set h2_rx_window_low_water 65535" client c1 { stream 1 { @@ -22,9 +23,7 @@ client c1 { txdata -datalen 1550 -nostrend rxwinup txdata -datalen 16000 -nostrend - rxwinup txdata -datalen 13919 - rxwinup rxresp expect resp.status == 200 } -run From phk at FreeBSD.org Mon Jun 18 07:33:13 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 18 Jun 2018 07:33:13 +0000 (UTC) Subject: [master] b591b022a Remove unnecessary #includes Message-ID: <20180618073313.E3D299B53B@lists.varnish-cache.org> commit b591b022a209cdf84223d1c930fc23c5f86bfc0e Author: Poul-Henning Kamp Date: Mon Jun 18 07:32:43 2018 +0000 Remove unnecessary #includes diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index cf3824618..ee6b27896 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -31,7 +31,6 @@ #include #include "cache/cache.h" -#include "cache/cache_director.h" #include "vbm.h" diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 77b522703..db277268a 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -33,7 +33,6 @@ #include #include "cache/cache.h" -#include "cache/cache_director.h" #include "vcl.h" #include "vend.h" From fgsch at lodoss.net Mon Jun 18 10:08:17 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 18 Jun 2018 10:08:17 +0000 (UTC) Subject: [master] 44d289316 Revert 88b2eb8e for now Message-ID: <20180618100817.4EC659E586@lists.varnish-cache.org> commit 44d289316908ead3defe7cf576ef7587a2d2d871 Author: Federico G. Schwindt Date: Mon Jun 18 11:04:20 2018 +0100 Revert 88b2eb8e for now Should address #2666 and #2711. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index d814944a2..e64bcfa41 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -244,10 +244,6 @@ tst_cb(const struct vev *ve, int what) VEV_Stop(vb, jp->evt); free(jp->evt); } - if (!jp->tst->ntodo) { - free(jp->tst->script); - FREE_OBJ(jp->tst); - } FREE_OBJ(jp); return (1); } From daghf at varnish-software.com Tue Jun 19 13:50:14 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 19 Jun 2018 13:50:14 +0000 (UTC) Subject: [master] c8bfae704 Decline DATA frames after seeing END_STREAM Message-ID: <20180619135015.0C18D97701@lists.varnish-cache.org> commit c8bfae704dcb25002daaf5206d5b4c1a9b7cf753 Author: Dag Haavi Finstad Date: Tue Jun 19 15:40:03 2018 +0200 Decline DATA frames after seeing END_STREAM If a client mistakenly sent a DATA frame on a stream where it already transmitted an END_STREAM, it would lead to the rxthread sitting around indefinitely. Big thanks to @xcir for his help in diagnosing this. Fixes: #2623 diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 7001b60fc..d10f00df2 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -726,6 +726,12 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) ASSERT_RXTHR(h2); if (r2 == NULL || !r2->scheduled) return (0); + if (r2->req->req_body_status == REQ_BODY_NONE) { + /* state REQ_BODY_NONE implies we already received an + * END_STREAM. */ + r2->error = H2SE_STREAM_CLOSED; + return (H2SE_STREAM_CLOSED); // rfc7540,l,1766,1769 + } Lck_Lock(&h2->sess->mtx); AZ(h2->mailcall); h2->mailcall = r2; diff --git a/bin/varnishtest/tests/t02014.vtc b/bin/varnishtest/tests/t02014.vtc index bc49f4e84..db72566db 100644 --- a/bin/varnishtest/tests/t02014.vtc +++ b/bin/varnishtest/tests/t02014.vtc @@ -1,6 +1,6 @@ varnishtest "Exercise h/2 sender flow control code" -barrier b1 sock 3 +barrier b1 sock 3 -cyclic server s1 { rxreq @@ -45,3 +45,19 @@ client c1 { stream 0 -wait } -run + +client c1 { + stream 0 { + barrier b1 sync + } -start + + stream 1 { + txreq + txdata -data "fail" + rxrst + expect rst.err == STREAM_CLOSED + barrier b1 sync + } -run + + stream 0 -wait +} -run From fgsch at lodoss.net Wed Jun 20 12:26:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 20 Jun 2018 12:26:11 +0000 (UTC) Subject: [master] 2f26470c6 Also revert this as part of 44d28931 Message-ID: <20180620122612.0C0AC618A9@lists.varnish-cache.org> commit 2f26470c6cfa7ec640fa2e4d977b667efb368fe2 Author: Federico G. Schwindt Date: Wed Jun 20 13:23:40 2018 +0100 Also revert this as part of 44d28931 Should help with the ASAN builds in travis. diff --git a/tools/lsan.suppr b/tools/lsan.suppr index d08ee697c..b6d916a30 100644 --- a/tools/lsan.suppr +++ b/tools/lsan.suppr @@ -1,6 +1,5 @@ # varnishtest -leak:parse_string -leak:receive_frame +leak:varnishtest # pp->{def,min,max} leak:MCF_ParamConf # pp->{def,min,max} From daghf at varnish-software.com Fri Jun 22 09:44:14 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 22 Jun 2018 09:44:14 +0000 (UTC) Subject: [master] 86d58ee31 Revert "Decline DATA frames after seeing END_STREAM" Message-ID: <20180622094414.A33FD9856C@lists.varnish-cache.org> commit 86d58ee31fe1ebaab2442f840b8b03becda0ceff Author: Dag Haavi Finstad Date: Thu Jun 21 15:09:34 2018 +0200 Revert "Decline DATA frames after seeing END_STREAM" This approach opens up to a data race. This reverts commit c8bfae704dcb25002daaf5206d5b4c1a9b7cf753. diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index d10f00df2..7001b60fc 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -726,12 +726,6 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) ASSERT_RXTHR(h2); if (r2 == NULL || !r2->scheduled) return (0); - if (r2->req->req_body_status == REQ_BODY_NONE) { - /* state REQ_BODY_NONE implies we already received an - * END_STREAM. */ - r2->error = H2SE_STREAM_CLOSED; - return (H2SE_STREAM_CLOSED); // rfc7540,l,1766,1769 - } Lck_Lock(&h2->sess->mtx); AZ(h2->mailcall); h2->mailcall = r2; diff --git a/bin/varnishtest/tests/t02014.vtc b/bin/varnishtest/tests/t02014.vtc index db72566db..bc49f4e84 100644 --- a/bin/varnishtest/tests/t02014.vtc +++ b/bin/varnishtest/tests/t02014.vtc @@ -1,6 +1,6 @@ varnishtest "Exercise h/2 sender flow control code" -barrier b1 sock 3 -cyclic +barrier b1 sock 3 server s1 { rxreq @@ -45,19 +45,3 @@ client c1 { stream 0 -wait } -run - -client c1 { - stream 0 { - barrier b1 sync - } -start - - stream 1 { - txreq - txdata -data "fail" - rxrst - expect rst.err == STREAM_CLOSED - barrier b1 sync - } -run - - stream 0 -wait -} -run From daghf at varnish-software.com Fri Jun 22 09:44:14 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 22 Jun 2018 09:44:14 +0000 (UTC) Subject: [master] ace8e8fe1 Don't transition to CLOS_REM state until we've seen END_STREAM Message-ID: <20180622094414.AD2549856F@lists.varnish-cache.org> commit ace8e8fe1a3601f0a6b760a2b50cd3cbb4fc75fe Author: Dag Haavi Finstad Date: Fri Jun 22 11:20:12 2018 +0200 Don't transition to CLOS_REM state until we've seen END_STREAM Previously we've been incorrectly transtitioning to CLOS_REM on END_HEADERS, which prevents us from seeing if a client incorrectly transmitted a DATA frame on a closed stream. This slightly complicates things in that we can now be in state OPEN with an inactive hpack decoding state, and we need to make sure on cleanup if that has already been finalized. This would be simpler if the h/2 spec had split the OPEN state in two parts, with an extra state transition on END_HEADERS. Again big thanks to @xcir for his help in diagnosing this. Fixes: #2623 diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 7001b60fc..2a0d73b81 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -225,7 +225,7 @@ h2_kill_req(struct worker *wrk, const struct h2_sess *h2, AZ(pthread_cond_signal(r2->cond)); r2 = NULL; } else { - if (r2->state == H2_S_OPEN) { + if (r2->state == H2_S_OPEN && r2->decode != NULL) { (void)h2h_decode_fini(h2, r2->decode); FREE_OBJ(r2->decode); } @@ -559,7 +559,8 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, assert(r2->state == H2_S_OPEN); h2e = h2h_decode_fini(h2, r2->decode); FREE_OBJ(r2->decode); - r2->state = H2_S_CLOS_REM; + if (h2->rxf_flags & H2FF_HEADERS_END_STREAM) + r2->state = H2_S_CLOS_REM; h2->new_req = NULL; if (h2e != NULL) { Lck_Lock(&h2->sess->mtx); @@ -693,7 +694,7 @@ h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) h2_error h2e; ASSERT_RXTHR(h2); - if (r2 == NULL || r2->state != H2_S_OPEN) + if (r2 == NULL || r2->state != H2_S_OPEN || r2->req != h2->new_req) return (H2CE_PROTOCOL_ERROR); // XXX spec ? req = r2->req; h2e = h2h_decode_bytes(h2, r2->decode, h2->rxf_data, h2->rxf_len); @@ -726,6 +727,10 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) ASSERT_RXTHR(h2); if (r2 == NULL || !r2->scheduled) return (0); + if (r2->state >= H2_S_CLOS_REM) { + r2->error = H2SE_STREAM_CLOSED; + return (H2SE_STREAM_CLOSED); // rfc7540,l,1766,1769 + } Lck_Lock(&h2->sess->mtx); AZ(h2->mailcall); h2->mailcall = r2; @@ -780,6 +785,7 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp) *lp = 0; Lck_Lock(&h2->sess->mtx); + assert (r2->state == H2_S_OPEN); r2->cond = &vc->wrk->cond; while (h2->mailcall != r2 && h2->error == 0 && r2->error == 0) AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0)); @@ -803,8 +809,10 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp) return (VFP_OK); } if (h2->rxf_len == 0) { - if (h2->rxf_flags & H2FF_DATA_END_STREAM) + if (h2->rxf_flags & H2FF_DATA_END_STREAM) { retval = VFP_END; + r2->state = H2_S_CLOS_REM; + } } h2->mailcall = NULL; AZ(pthread_cond_signal(h2->cond)); diff --git a/bin/varnishtest/tests/t02014.vtc b/bin/varnishtest/tests/t02014.vtc index bc49f4e84..17489d362 100644 --- a/bin/varnishtest/tests/t02014.vtc +++ b/bin/varnishtest/tests/t02014.vtc @@ -1,8 +1,8 @@ varnishtest "Exercise h/2 sender flow control code" -barrier b1 sock 3 +barrier b1 sock 3 -cyclic -server s1 { +server s1 -repeat 2 { rxreq txresp -bodylen 66300 } -start @@ -45,3 +45,59 @@ client c1 { stream 0 -wait } -run + +client c1 { + stream 0 { + barrier b1 sync + } -start + + stream 1 { + txreq + txdata -data "fail" + rxrst + expect rst.err == STREAM_CLOSED + barrier b1 sync + } -run + + stream 0 -wait +} -run + +client c1 { + stream 0 { + barrier b1 sync + barrier b1 sync + delay .5 + txwinup -size 256 + delay .5 + txwinup -size 256 + delay .5 + txwinup -size 256 + + } -start + + stream 1 { + txreq -req "POST" -nostrend + txdata -data "ok" + rxwinup + txdata -data "fail" + rxrst + expect rst.err == STREAM_CLOSED + barrier b1 sync + } -run + + stream 3 { + txreq + barrier b1 sync + delay .5 + txwinup -size 256 + delay .5 + txwinup -size 256 + delay .5 + txwinup -size 256 + rxresp + expect resp.status == 200 + expect resp.bodylen == 66300 + } -run + + stream 0 -wait +} -run From daghf at varnish-software.com Fri Jun 22 13:41:11 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 22 Jun 2018 13:41:11 +0000 (UTC) Subject: [master] da0e6c3d2 Proper END_STREAM handling Message-ID: <20180622134111.D4DDF9EC5E@lists.varnish-cache.org> commit da0e6c3d29dcd1c491ba99546c543917d06675b0 Author: Dag Haavi Finstad Date: Fri Jun 22 15:32:10 2018 +0200 Proper END_STREAM handling The previous commit made the assumption that END_STREAM is in the last of the frames in a header block. This is not necessarily the case. diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 2a0d73b81..9dede792e 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -559,9 +559,12 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, assert(r2->state == H2_S_OPEN); h2e = h2h_decode_fini(h2, r2->decode); FREE_OBJ(r2->decode); - if (h2->rxf_flags & H2FF_HEADERS_END_STREAM) - r2->state = H2_S_CLOS_REM; h2->new_req = NULL; + if (r2->req->req_body_status == REQ_BODY_NONE) { + /* REQ_BODY_NONE implies one of the frames in the + * header block contained END_STREAM */ + r2->state = H2_S_CLOS_REM; + } if (h2e != NULL) { Lck_Lock(&h2->sess->mtx); VSLb(h2->vsl, SLT_Debug, "HPACK/FINI %s", h2e->name); From fgsch at lodoss.net Sat Jun 23 10:29:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 23 Jun 2018 10:29:12 +0000 (UTC) Subject: [master] 50d4b6f1e Apply workaround for macos Message-ID: <20180623102912.D74CD62785@lists.varnish-cache.org> commit 50d4b6f1e72b00db0211ddc9c739fb438056673f Author: Federico G. Schwindt Date: Sat Jun 23 11:24:23 2018 +0100 Apply workaround for macos A bound socket will timeout instead of refusing the connection. Should fix b00015.vtc under macos. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index e64bcfa41..e85ae92b7 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -492,6 +492,15 @@ ip_magic(void) VTCP_myname(fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); extmacro_def("localhost", "%s", abuf); +#if defined (__APPLE__) + /* + * In MacOS a bound socket that is not listening will timeout + * instead of refusing the connection so close it and hope + * for the best. + */ + VTCP_close(&fd); +#endif + /* Expose a backend that is forever down. */ extmacro_def("bad_backend", "%s %s", abuf, pbuf); From nils.goroll at uplex.de Mon Jun 25 12:36:14 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 25 Jun 2018 12:36:14 +0000 (UTC) Subject: [master] 6cbd0d9f0 Accurate ban statistics except for a few remaining corner cases Message-ID: <20180625123614.6D1D7A02C7@lists.varnish-cache.org> commit 6cbd0d9f0f01a9c302fb22211e16dbb8b8a06d4c Author: Nils Goroll Date: Wed Jun 20 13:39:09 2018 +0200 Accurate ban statistics except for a few remaining corner cases For ban statistics, we updated VSC_C_main directly, so if we raced Pool_Sumstat(), that could undo our changes. This patch fixes statistics by using the per-worker statistics cache except for the following remaining corner cases: * bans_persisted_* counters receive absolut updates, which does not seem to fit the incremental updates via the per-worker stats. I've kept these cases untouched and marked with comments. Worst that should happen here are temporary inconsistencies until the next absolute update. * For BAN_Reload(), my understanding is that it should only happen during init, so we continue to update VSC_C_main directly. * For bans via the cli, we would need to grab the wstat lock, which, at the moment, is private to the worker implementation. Until we make a change here, we could miss a ban increment from the cli. * for VCL bans from vcl_init / fini, we do not have access to the worker struct at the moment, so for now we also accept an inconsistency here. Fixes #2716 for relevant cases diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 02d042005..551a3e1d0 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -571,7 +571,7 @@ struct sess { struct ban_proto *BAN_Build(void); const char *BAN_AddTest(struct ban_proto *, const char *, const char *, const char *); -const char *BAN_Commit(struct ban_proto *b); +const char *BAN_Commit(struct ban_proto *b, struct VSC_main *stats); void BAN_Abandon(struct ban_proto *b); /* cache_cli.c [CLI] */ diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 0393deea6..b3203a042 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -152,7 +152,7 @@ ban_equal(const uint8_t *bs1, const uint8_t *bs2) } void -ban_mark_completed(struct ban *b) +ban_mark_completed(struct ban *b, struct VSC_main *stats) { unsigned ln; @@ -166,8 +166,12 @@ ban_mark_completed(struct ban *b) b->spec[BANS_FLAGS] |= BANS_FLAG_COMPLETED; VWMB(); vbe32enc(b->spec + BANS_LENGTH, BANS_HEAD_LEN); - VSC_C_main->bans_completed++; + stats->bans_completed++; bans_persisted_fragmentation += ln - ban_len(b->spec); + /* + * XXX absolute update of gauges - may be inaccurate for + * Pool_Sumstat race + */ VSC_C_main->bans_persisted_fragmentation = bans_persisted_fragmentation; } @@ -315,6 +319,10 @@ ban_export(void) assert(VSB_len(vsb) == ln); STV_BanExport((const uint8_t *)VSB_data(vsb), VSB_len(vsb)); VSB_destroy(&vsb); + /* + * XXX absolute update of gauges - may be inaccurate for Pool_Sumstat + * race + */ VSC_C_main->bans_persisted_bytes = bans_persisted_bytes = ln; VSC_C_main->bans_persisted_fragmentation = @@ -359,7 +367,7 @@ ban_reload(const uint8_t *ban, unsigned len) struct ban *b, *b2; int duplicate = 0; double t0, t1, t2 = 9e99; - + struct VSC_main *stats = VSC_C_main; // XXX accurate? ASSERT_CLI(); Lck_AssertHeld(&ban_mtx); @@ -378,8 +386,8 @@ ban_reload(const uint8_t *ban, unsigned len) duplicate = 1; } - VSC_C_main->bans++; - VSC_C_main->bans_added++; + stats->bans++; + stats->bans_added++; b2 = ban_alloc(); AN(b2); @@ -387,18 +395,22 @@ ban_reload(const uint8_t *ban, unsigned len) AN(b2->spec); memcpy(b2->spec, ban, len); if (ban[BANS_FLAGS] & BANS_FLAG_REQ) { - VSC_C_main->bans_req++; + stats->bans_req++; b2->flags |= BANS_FLAG_REQ; } if (duplicate) - VSC_C_main->bans_dups++; + stats->bans_dups++; if (duplicate || (ban[BANS_FLAGS] & BANS_FLAG_COMPLETED)) - ban_mark_completed(b2); + ban_mark_completed(b2, stats); if (b == NULL) VTAILQ_INSERT_TAIL(&ban_head, b2, list); else VTAILQ_INSERT_BEFORE(b, b2, list); bans_persisted_bytes += len; + /* + * XXX absolute update of gauges - may be inaccurate for Pool_Sumstat + * race + */ VSC_C_main->bans_persisted_bytes = bans_persisted_bytes; /* Hunt down older duplicates */ @@ -406,8 +418,8 @@ ban_reload(const uint8_t *ban, unsigned len) if (b->flags & BANS_FLAG_COMPLETED) continue; if (ban_equal(b->spec, ban)) { - ban_mark_completed(b); - VSC_C_main->bans_dups++; + ban_mark_completed(b, stats); + stats->bans_dups++; } } } @@ -536,13 +548,16 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req) struct vsl_log *vsl; struct ban *b0, *bn; unsigned tests; + struct VSC_main *stats; + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); Lck_AssertHeld(&oc->objhead->mtx); assert(oc->refcnt > 0); vsl = req->vsl; + stats = wrk->stats; CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC); @@ -585,8 +600,8 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req) Lck_Lock(&ban_mtx); bn->refcount--; - VSC_C_main->bans_tested++; - VSC_C_main->bans_tests_tested += tests; + stats->bans_tested++; + stats->bans_tests_tested += tests; if (b == bn) { /* not banned */ @@ -609,7 +624,7 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req) return (0); } else { VSLb(vsl, SLT_ExpBan, "%u banned lookup", ObjGetXID(wrk, oc)); - VSC_C_main->bans_obj_killed++; + stats->bans_obj_killed++; return (1); } } @@ -655,8 +670,10 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv) break; } - if (err == NULL) - err = BAN_Commit(bp); + if (err == NULL) { + // XXX racy - grab wstat lock? + err = BAN_Commit(bp, VSC_C_main); + } if (err != NULL) { VCLI_Out(cli, "%s", err); @@ -804,9 +821,9 @@ BAN_Init(void) bp = BAN_Build(); AN(bp); AZ(pthread_cond_init(&ban_lurker_cond, NULL)); - AZ(BAN_Commit(bp)); + AZ(BAN_Commit(bp, VSC_C_main)); Lck_Lock(&ban_mtx); - ban_mark_completed(VTAILQ_FIRST(&ban_head)); + ban_mark_completed(VTAILQ_FIRST(&ban_head), VSC_C_main); Lck_Unlock(&ban_mtx); } diff --git a/bin/varnishd/cache/cache_ban.h b/bin/varnishd/cache/cache_ban.h index 6267c4eed..d534f6e99 100644 --- a/bin/varnishd/cache/cache_ban.h +++ b/bin/varnishd/cache/cache_ban.h @@ -109,7 +109,7 @@ extern pthread_cond_t ban_lurker_cond; extern uint64_t bans_persisted_bytes; extern uint64_t bans_persisted_fragmentation; -void ban_mark_completed(struct ban *b); +void ban_mark_completed(struct ban *b, struct VSC_main *stats); unsigned ban_len(const uint8_t *banspec); void ban_info_new(const uint8_t *ban, unsigned len); void ban_info_drop(const uint8_t *ban, unsigned len); diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c index 717f5c506..95f82bcc8 100644 --- a/bin/varnishd/cache/cache_ban_build.c +++ b/bin/varnishd/cache/cache_ban_build.c @@ -246,7 +246,7 @@ BAN_AddTest(struct ban_proto *bp, */ const char * -BAN_Commit(struct ban_proto *bp) +BAN_Commit(struct ban_proto *bp, struct VSC_main *stats) { struct ban *b, *bi; ssize_t ln; @@ -294,15 +294,19 @@ BAN_Commit(struct ban_proto *bp) VTAILQ_INSERT_HEAD(&ban_head, b, list); ban_start = b; - VSC_C_main->bans++; - VSC_C_main->bans_added++; + stats->bans++; + stats->bans_added++; bans_persisted_bytes += ln; + /* + * XXX absolute update of gauges - may be inaccurate for Pool_Sumstat + * race + */ VSC_C_main->bans_persisted_bytes = bans_persisted_bytes; if (b->flags & BANS_FLAG_OBJ) - VSC_C_main->bans_obj++; + stats->bans_obj++; if (b->flags & BANS_FLAG_REQ) - VSC_C_main->bans_req++; + stats->bans_req++; if (bi != NULL) ban_info_new(b->spec, ln); /* Notify stevedores */ @@ -313,8 +317,8 @@ BAN_Commit(struct ban_proto *bp) bi = VTAILQ_NEXT(bi, list)) { if (!(bi->flags & BANS_FLAG_COMPLETED) && ban_equal(b->spec, bi->spec)) { - ban_mark_completed(bi); - VSC_C_main->bans_dups++; + ban_mark_completed(bi, stats); + stats->bans_dups++; } } } diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index f59b888ce..343009dd5 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -62,7 +62,7 @@ ban_kick_lurker(void) */ static int -ban_cleantail(const struct ban *victim) +ban_cleantail(const struct ban *victim, struct VSC_main *stats) { struct ban *b, *bt; struct banhead_s freelist = VTAILQ_HEAD_INITIALIZER(freelist); @@ -78,17 +78,21 @@ ban_cleantail(const struct ban *victim) if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) { assert(VTAILQ_EMPTY(&b->objcore)); if (b->flags & BANS_FLAG_COMPLETED) - VSC_C_main->bans_completed--; + stats->bans_completed--; if (b->flags & BANS_FLAG_OBJ) - VSC_C_main->bans_obj--; + stats->bans_obj--; if (b->flags & BANS_FLAG_REQ) - VSC_C_main->bans_req--; - VSC_C_main->bans--; - VSC_C_main->bans_deleted++; + stats->bans_req--; + stats->bans--; + stats->bans_deleted++; VTAILQ_REMOVE(&ban_head, b, list); VTAILQ_INSERT_TAIL(&freelist, b, list); bans_persisted_fragmentation += ban_len(b->spec); + /* + * XXX absolute update of gauges - may be inaccurate for + * Pool_Sumstat race + */ VSC_C_main->bans_persisted_fragmentation = bans_persisted_fragmentation; ban_info_drop(b->spec, ban_len(b->spec)); @@ -121,7 +125,7 @@ ban_cleantail(const struct ban *victim) */ static struct objcore * -ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt) +ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt, struct VSC_main *stats) { struct objhead *oh; struct objcore *oc, *noc; @@ -151,7 +155,7 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt) /* hold off to give lookup a chance and reiterate */ Lck_Unlock(&ban_mtx); - VSC_C_main->bans_lurker_contention++; + stats->bans_lurker_contention++; VSL_Flush(vsl, 0); VTIM_sleep(cache_param->ban_lurker_holdoff); Lck_Lock(&ban_mtx); @@ -205,6 +209,10 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, struct objcore *oc; unsigned tests; int i; + struct VSC_main *stats; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + stats = wrk->stats; /* * First see if there is anything to do, and if so, insert markers @@ -224,7 +232,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, VTIM_sleep(cache_param->ban_lurker_sleep); ban_batch = 0; } - oc = ban_lurker_getfirst(vsl, bt); + oc = ban_lurker_getfirst(vsl, bt, stats); if (oc == NULL) return; i = 0; @@ -248,21 +256,21 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, tests = 0; i = ban_evaluate(wrk, bl->spec, oc, NULL, &tests); - VSC_C_main->bans_lurker_tested++; - VSC_C_main->bans_lurker_tests_tested += tests; + stats->bans_lurker_tested++; + stats->bans_lurker_tests_tested += tests; } if (i) { if (kill) { VSLb(vsl, SLT_ExpBan, "%u killed for lurker cutoff", ObjGetXID(wrk, oc)); - VSC_C_main-> + stats-> bans_lurker_obj_killed_cutoff++; } else { VSLb(vsl, SLT_ExpBan, "%u banned by lurker", ObjGetXID(wrk, oc)); - VSC_C_main->bans_lurker_obj_killed++; + stats->bans_lurker_obj_killed++; } HSH_Kill(oc); break; @@ -303,10 +311,14 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) struct banhead_s obans; double d, dt, n; unsigned count = 0, cutoff = UINT_MAX; + struct VSC_main *stats; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + stats = wrk->stats; dt = 49.62; // Random, non-magic if (cache_param->ban_lurker_sleep == 0) { - (void)ban_cleantail(NULL); + (void)ban_cleantail(NULL, stats); return (dt); } if (cache_param->ban_cutoff > 0) @@ -346,7 +358,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) * containted the first oban, all obans were on the tail and we're * done. */ - if (ban_cleantail(VTAILQ_FIRST(&obans))) + if (ban_cleantail(VTAILQ_FIRST(&obans), stats)) return (dt); if (VTAILQ_FIRST(&obans) == NULL) @@ -374,7 +386,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) Lck_Lock(&ban_mtx); VTAILQ_FOREACH(b, &obans, l_list) { - ban_mark_completed(b); + ban_mark_completed(b, stats); if (b == bd) break; } diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index bc92daa8a..8765c64fa 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -557,6 +557,26 @@ VRT_synth_page(VRT_CTX, const char *str, ...) /*--------------------------------------------------------------------*/ +static struct VSC_main * +vrt_stats(VRT_CTX) +{ + struct worker *wrk; + + if (ctx->req) { + CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + wrk = ctx->req->wrk; + } else if (ctx->bo) { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + wrk = ctx->bo->wrk; + } else { + // XXX + return (VSC_C_main); + } + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + return (wrk->stats); +} + VCL_VOID VRT_ban_string(VRT_CTX, VCL_STRING str) { @@ -612,7 +632,7 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) break; } if (av[++i] == NULL) { - err = BAN_Commit(bp); + err = BAN_Commit(bp, vrt_stats(ctx)); if (err == NULL) bp = NULL; else diff --git a/include/vrt.h b/include/vrt.h index 8e8f169b4..d4c97c346 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -125,6 +125,7 @@ struct vsc_seg; struct vsmw_cluster; struct vsl_log; struct ws; +struct VSC_main; struct strands { int n; From nils.goroll at uplex.de Tue Jun 26 08:42:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 26 Jun 2018 08:42:10 +0000 (UTC) Subject: [master] 344d09ce2 Shard director: add documentation from 9b8f6e21809bf867eb5bd792331484d0e70eb367 Message-ID: <20180626084210.316D063626@lists.varnish-cache.org> commit 344d09ce2f614416199389061b3de2a28ed4e4ab Author: Nils Goroll Date: Tue Jun 26 10:16:32 2018 +0200 Shard director: add documentation from 9b8f6e21809bf867eb5bd792331484d0e70eb367 diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index db277268a..5e03f8a27 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -41,6 +41,44 @@ #include "shard_dir.h" #include "shard_cfg.h" +/* ------------------------------------------------------------------------- + * shard director: LAZY mode (vdi resolve function), parameter objects + * + * By associating a parameter object with a shard director, we enable LAZY + * lookups as with the other directors. Parameter objects are defined with VCL + * scope (normal vmod objects), but can be overridden per backend request using + * a task priv. + * + * We use the same concept to carry shard.backend() parameters to vdi resolve + * for LAZY mode: They get saved in a per-director task scope parameter object. + * + * Each object points to another object providing defaults for values which are + * not defined. + * + * Actual resolution of the various parameter objects does not happen before + * they are used, which enabled changing them independently (ie, shard + * .backend() parameters have precedence over an associated parameter object, + * which by itself can be overridden). + * + * Overview of parameter objects (pointers are alternatives) + * + * shard() director shard_param() object default praram + * + * ---------------------------------> vmod static + * VCL obj / -> + * .param -+---------> VCL obj / _ + * .default -------- /| + * / + * ^ / + * | / + * / + * .default / + * -------------> TASK priv / + * / / + * .default ----------------------------- + * TASK priv + */ + /* ------------------------------------------------------------------------- * method arguments and set parameters bitmask in vmod_directors_shard_param */ From nils.goroll at uplex.de Tue Jun 26 08:42:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 26 Jun 2018 08:42:10 +0000 (UTC) Subject: [master] 552607abb shard director lifetime is not identical to the vcl object Message-ID: <20180626084210.40F8563628@lists.varnish-cache.org> commit 552607abb2de6a6bfe514fa3d9ee8217f67e50e1 Author: Nils Goroll Date: Tue Jun 26 10:31:13 2018 +0200 shard director lifetime is not identical to the vcl object Prepare for use of the shard director as a dynamic backend, where the VCL_BACKEND can outlast the vmod object: * the director priv now points to the struct sharddir instead of the vmod object * move the default parameters pointer to struct sharddir * index the PRIV_TASK by the struct sharddir pointer diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index e0a876c2e..61a14bf75 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -221,7 +221,8 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy) } void -sharddir_new(struct sharddir **sharddp, const char *vcl_name) +sharddir_new(struct sharddir **sharddp, const char *vcl_name, + const struct vmod_directors_shard_param *param) { struct sharddir *shardd; @@ -232,9 +233,18 @@ sharddir_new(struct sharddir **sharddp, const char *vcl_name) AN(shardd); *sharddp = shardd; shardd->name = vcl_name; + shardd->param = param; AZ(pthread_rwlock_init(&shardd->mtx, NULL)); } +void +sharddir_set_param(struct sharddir *shardd, + const struct vmod_directors_shard_param *param) +{ + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + shardd->param = param; +} + void sharddir_delete(struct sharddir **sharddp) { diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 7bf5d756c..62bc2df53 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -65,6 +65,8 @@ struct shard_backend { uint32_t canon_point; }; +struct vmod_directors_shard_param; + #define SHDBG_LOOKUP 1 #define SHDBG_CIRCLE (1<<1) #define SHDBG_RAMPWARM (1<<2) @@ -76,13 +78,13 @@ struct sharddir { pthread_rwlock_t mtx; - const char *name; - unsigned n_backend; unsigned l_backend; struct shard_backend *backend; + const char *name; struct shard_circlepoint *hashcircle; + const struct vmod_directors_shard_param *param; VCL_DURATION rampup_duration; VCL_REAL warmup; @@ -127,7 +129,10 @@ void sharddir_debug(struct sharddir *shardd, const uint32_t flags); void sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...); uint32_t sharddir_sha256v(const char *s, va_list ap); uint32_t sharddir_sha256(const char *s, ...); -void sharddir_new(struct sharddir **sharddp, const char *vcl_name); +void sharddir_new(struct sharddir **sharddp, const char *vcl_name, + const struct vmod_directors_shard_param *param); +void sharddir_set_param(struct sharddir *shardd, + const struct vmod_directors_shard_param *param); void sharddir_delete(struct sharddir **sharddp); void sharddir_wrlock(struct sharddir *shardd); void sharddir_unlock(struct sharddir *shardd); diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 5e03f8a27..6a28721b1 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -175,7 +175,6 @@ struct vmod_directors_shard { #define VMOD_SHARD_SHARD_MAGIC 0x6e63e1bf struct sharddir *shardd; VCL_BACKEND dir; - const struct vmod_directors_shard_param *param; }; static enum by_e @@ -250,11 +249,11 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, AN(vshard); *vshardp = vshard; - sharddir_new(&vshard->shardd, vcl_name); + sharddir_new(&vshard->shardd, vcl_name, &shard_param_default); - vshard->param = &shard_param_default; vshard->dir = - VRT_AddDirector(ctx, vmod_shard_methods, vshard, "%s", vcl_name); + VRT_AddDirector(ctx, vmod_shard_methods, vshard->shardd, + "%s", vcl_name); } VCL_VOID v_matchproto_(td_directors_shard__fini) @@ -315,7 +314,8 @@ vmod_shard_associate(VRT_CTX, CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); if (b == NULL) { - vshard->param = &shard_param_default; + sharddir_set_param(vshard->shardd, + &shard_param_default); return; } @@ -326,7 +326,8 @@ vmod_shard_associate(VRT_CTX, return; } - vshard->param = ppt; + sharddir_set_param(vshard->shardd, + ppt); } VCL_BOOL v_matchproto_(td_directors_shard_add_backend) @@ -663,7 +664,8 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, assert(ctx->method & VCL_MET_TASK_B); - pp = shard_param_task(ctx, vshard, vshard->param); + pp = shard_param_task(ctx, vshard->shardd, + vshard->shardd->param); if (pp == NULL) return (NULL); pp->vcl_name = vshard->shardd->name; @@ -675,7 +677,7 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, "used in vcl_init{}/vcl_fini{}"); return (NULL); } - pp = shard_param_stack(&pstk, vshard->param, + pp = shard_param_stack(&pstk, vshard->shardd->param, vshard->shardd->name); break; default: @@ -712,31 +714,31 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, static VCL_BOOL v_matchproto_(vdi_healthy) vmod_shard_healthy(VRT_CTX, VCL_BACKEND dir, VCL_TIME *changed) { - struct vmod_directors_shard *vshard; + struct sharddir *shardd; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); - CAST_OBJ_NOTNULL(vshard, dir->priv, VMOD_SHARD_SHARD_MAGIC); - return (sharddir_any_healthy(ctx, vshard->shardd, changed)); + CAST_OBJ_NOTNULL(shardd, dir->priv, SHARDDIR_MAGIC); + return (sharddir_any_healthy(ctx, shardd, changed)); } static VCL_BACKEND v_matchproto_(vdi_resolve_f) vmod_shard_resolve(VRT_CTX, VCL_BACKEND dir) { - struct vmod_directors_shard *vshard; + struct sharddir *shardd; struct vmod_directors_shard_param pstk[1]; const struct vmod_directors_shard_param *pp; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); - CAST_OBJ_NOTNULL(vshard, dir->priv, VMOD_SHARD_SHARD_MAGIC); + CAST_OBJ_NOTNULL(shardd, dir->priv, SHARDDIR_MAGIC); - pp = vmod_shard_param_read(ctx, vshard, - vshard->param, pstk, "shard_resolve"); + pp = vmod_shard_param_read(ctx, shardd, shardd->param, + pstk, "shard_resolve"); if (pp == NULL) return (NULL); - return (sharddir_pick_be(ctx, vshard->shardd, + return (sharddir_pick_be(ctx, shardd, shard_get_key(ctx, pp), pp->alt, pp->warmup, pp->rampup, pp->healthy)); } From nils.goroll at uplex.de Tue Jun 26 08:42:10 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 26 Jun 2018 08:42:10 +0000 (UTC) Subject: [master] 78c243894 polish: use TAKE_OBJ_NOTNULL Message-ID: <20180626084210.5C0726362B@lists.varnish-cache.org> commit 78c243894bed86b3c0637fda49d47f3d33fc72b0 Author: Nils Goroll Date: Tue Jun 26 10:39:21 2018 +0200 polish: use TAKE_OBJ_NOTNULL diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 61a14bf75..bfc86f45a 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -250,11 +250,7 @@ sharddir_delete(struct sharddir **sharddp) { struct sharddir *shardd; - AN(sharddp); - shardd = *sharddp; - *sharddp = NULL; - - CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + TAKE_OBJ_NOTNULL(shardd, sharddp, SHARDDIR_MAGIC); shardcfg_delete(shardd); AZ(pthread_rwlock_destroy(&shardd->mtx)); FREE_OBJ(shardd); diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 6a28721b1..90430a066 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -259,10 +259,9 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, VCL_VOID v_matchproto_(td_directors_shard__fini) vmod_shard__fini(struct vmod_directors_shard **vshardp) { - struct vmod_directors_shard *vshard = *vshardp; + struct vmod_directors_shard *vshard; - *vshardp = NULL; - CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + TAKE_OBJ_NOTNULL(vshard, vshardp, VMOD_SHARD_SHARD_MAGIC); sharddir_delete(&vshard->shardd); VRT_DelDirector(&vshard->dir); FREE_OBJ(vshard); @@ -778,12 +777,11 @@ vmod_shard_param__init(VRT_CTX, VCL_VOID v_matchproto_(td_directors_shard_param__fini) vmod_shard_param__fini(struct vmod_directors_shard_param **pp) { - struct vmod_directors_shard_param *p = *pp; + struct vmod_directors_shard_param *p; - if (p == NULL) + if (*pp == NULL) return; - *pp = NULL; - CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); + TAKE_OBJ_NOTNULL(p, pp, VMOD_SHARD_SHARD_PARAM_MAGIC); FREE_OBJ(p); } From nils.goroll at uplex.de Tue Jun 26 08:46:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 26 Jun 2018 08:46:08 +0000 (UTC) Subject: [master] e08aa09df post-push style ocd Message-ID: <20180626084608.DEAEE63B90@lists.varnish-cache.org> commit e08aa09dfd31750c7d3168458e94662def63a7c9 Author: Nils Goroll Date: Tue Jun 26 10:45:32 2018 +0200 post-push style ocd diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 90430a066..6dacf20e6 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -251,9 +251,8 @@ vmod_shard__init(VRT_CTX, struct vmod_directors_shard **vshardp, *vshardp = vshard; sharddir_new(&vshard->shardd, vcl_name, &shard_param_default); - vshard->dir = - VRT_AddDirector(ctx, vmod_shard_methods, vshard->shardd, - "%s", vcl_name); + vshard->dir = VRT_AddDirector(ctx, vmod_shard_methods, vshard->shardd, + "%s", vcl_name); } VCL_VOID v_matchproto_(td_directors_shard__fini) @@ -313,8 +312,7 @@ vmod_shard_associate(VRT_CTX, CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); if (b == NULL) { - sharddir_set_param(vshard->shardd, - &shard_param_default); + sharddir_set_param(vshard->shardd, &shard_param_default); return; } @@ -325,8 +323,7 @@ vmod_shard_associate(VRT_CTX, return; } - sharddir_set_param(vshard->shardd, - ppt); + sharddir_set_param(vshard->shardd, ppt); } VCL_BOOL v_matchproto_(td_directors_shard_add_backend) From nils.goroll at uplex.de Tue Jun 26 10:05:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 26 Jun 2018 10:05:12 +0000 (UTC) Subject: [master] 4102ed93b vmod_directors: vmod object destruction vs. director destruction Message-ID: <20180626100512.A3AF365412@lists.varnish-cache.org> commit 4102ed93bbb0192fefa7829db208a75c2344fa26 Author: Nils Goroll Date: Tue Jun 26 11:52:34 2018 +0200 vmod_directors: vmod object destruction vs. director destruction Turn the director destruction inside-out: When the vmod object goes out of scope, we inform VRT that the director is to be destroyed. Actual destruction is moved to the respective director callback. This allows directors to outlast vmod objects. For vdir based directors, the director object is the vmod object; for shard, the two are different instances (we might want to reconsider changing this later). diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index e4a5a12ff..e6d3e7226 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -84,11 +84,22 @@ vmod_fallback_resolve(VRT_CTX, VCL_BACKEND dir) return (be); } +static void v_matchproto_(vdi_destroy_f) +vmod_fallback_destroy(VCL_BACKEND dir) +{ + struct vmod_directors_fallback *fallback; + + CAST_OBJ_NOTNULL(fallback, dir->priv, VMOD_DIRECTORS_FALLBACK_MAGIC); + vdir_delete(&fallback->vd); + FREE_OBJ(fallback); +} + static const struct vdi_methods vmod_fallback_methods[1] = {{ .magic = VDI_METHODS_MAGIC, .type = "fallback", .healthy = vmod_fallback_healthy, .resolve = vmod_fallback_resolve, + .destroy = vmod_fallback_destroy }}; @@ -113,11 +124,8 @@ vmod_fallback__fini(struct vmod_directors_fallback **fbp) { struct vmod_directors_fallback *fb; - fb = *fbp; - *fbp = NULL; - CHECK_OBJ_NOTNULL(fb, VMOD_DIRECTORS_FALLBACK_MAGIC); - vdir_delete(&fb->vd); - FREE_OBJ(fb); + TAKE_OBJ_NOTNULL(fb, fbp, VMOD_DIRECTORS_FALLBACK_MAGIC); + VRT_DelDirector(&fb->vd->dir); } VCL_VOID v_matchproto_() diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index 9a362cb94..a82e19fe3 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -46,9 +46,20 @@ struct vmod_directors_hash { struct vdir *vd; }; +static void v_matchproto_(vdi_destroy_f) +vmod_hash_destroy(VCL_BACKEND dir) +{ + struct vmod_directors_hash *rr; + + CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_HASH_MAGIC); + vdir_delete(&rr->vd); + FREE_OBJ(rr); +} + static const struct vdi_methods vmod_hash_methods[1] = {{ .magic = VDI_METHODS_MAGIC, .type = "hash", + .destroy = vmod_hash_destroy }}; @@ -72,11 +83,8 @@ vmod_hash__fini(struct vmod_directors_hash **rrp) { struct vmod_directors_hash *rr; - rr = *rrp; - *rrp = NULL; - CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC); - vdir_delete(&rr->vd); - FREE_OBJ(rr); + TAKE_OBJ_NOTNULL(rr, rrp, VMOD_DIRECTORS_HASH_MAGIC); + VRT_DelDirector(&rr->vd->dir); } VCL_VOID v_matchproto_() diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 3968e5cf0..5ad0849e9 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -72,11 +72,22 @@ vmod_random_resolve(VRT_CTX, VCL_BACKEND dir) return (be); } +static void v_matchproto_(vdi_destroy_f) +vmod_random_destroy(VCL_BACKEND dir) +{ + struct vmod_directors_random *random; + + CAST_OBJ_NOTNULL(random, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC); + vdir_delete(&random->vd); + FREE_OBJ(random); +} + static const struct vdi_methods vmod_random_methods[1] = {{ .magic = VDI_METHODS_MAGIC, .type = "random", .healthy = vmod_random_healthy, .resolve = vmod_random_resolve, + .destroy = vmod_random_destroy }}; @@ -100,11 +111,8 @@ vmod_random__fini(struct vmod_directors_random **rrp) { struct vmod_directors_random *rr; - rr = *rrp; - *rrp = NULL; - CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC); - vdir_delete(&rr->vd); - FREE_OBJ(rr); + TAKE_OBJ_NOTNULL(rr, rrp, VMOD_DIRECTORS_RANDOM_MAGIC); + VRT_DelDirector(&rr->vd->dir); } VCL_VOID v_matchproto_() diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 636d88ae7..af2aec3b8 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -81,11 +81,22 @@ vmod_rr_resolve(VRT_CTX, VCL_BACKEND dir) return (be); } +static void v_matchproto_(vdi_destroy_f) +vmod_rr_destroy(VCL_BACKEND dir) +{ + struct vmod_directors_round_robin *rr; + + CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); + vdir_delete(&rr->vd); + FREE_OBJ(rr); +} + static const struct vdi_methods vmod_rr_methods[1] = {{ .magic = VDI_METHODS_MAGIC, .type = "round-robin", .healthy = vmod_rr_healthy, .resolve = vmod_rr_resolve, + .destroy = vmod_rr_destroy }}; VCL_VOID v_matchproto_() @@ -108,11 +119,8 @@ vmod_round_robin__fini(struct vmod_directors_round_robin **rrp) { struct vmod_directors_round_robin *rr; - rr = *rrp; - *rrp = NULL; - CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); - vdir_delete(&rr->vd); - FREE_OBJ(rr); + TAKE_OBJ_NOTNULL(rr, rrp, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); + VRT_DelDirector(&rr->vd->dir); } VCL_VOID v_matchproto_() diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index ee6b27896..61f78139e 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -75,10 +75,10 @@ vdir_delete(struct vdir **vdp) TAKE_OBJ_NOTNULL(vd, vdp, VDIR_MAGIC); + AZ(vd->dir); free(vd->backend); free(vd->weight); AZ(pthread_rwlock_destroy(&vd->mtx)); - VRT_DelDirector(&vd->dir); vbit_destroy(vd->vbm); FREE_OBJ(vd); } diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 6dacf20e6..558675335 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -227,11 +227,21 @@ shard__assert(void) assert(t2a == t2b); } +static void v_matchproto_(vdi_destroy_f) +vmod_shard_destroy(VCL_BACKEND dir) +{ + struct sharddir *shardd; + + CAST_OBJ_NOTNULL(shardd, dir->priv, SHARDDIR_MAGIC); + sharddir_delete(&shardd); +} + static const struct vdi_methods vmod_shard_methods[1] = {{ .magic = VDI_METHODS_MAGIC, .type = "shard", .resolve = vmod_shard_resolve, .healthy = vmod_shard_healthy, + .destroy = vmod_shard_destroy }}; @@ -261,7 +271,6 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp) struct vmod_directors_shard *vshard; TAKE_OBJ_NOTNULL(vshard, vshardp, VMOD_SHARD_SHARD_MAGIC); - sharddir_delete(&vshard->shardd); VRT_DelDirector(&vshard->dir); FREE_OBJ(vshard); } From daghf at varnish-software.com Wed Jun 27 11:41:12 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 27 Jun 2018 11:41:12 +0000 (UTC) Subject: [master] 812b13613 Ensure that only the rxthread gets to use h2->cond in h2_send_get Message-ID: <20180627114112.59A3C63C1A@lists.varnish-cache.org> commit 812b136135eac0592d8d8f43d4e5511725d9579b Author: Dag Haavi Finstad Date: Wed Jun 27 13:36:04 2018 +0200 Ensure that only the rxthread gets to use h2->cond in h2_send_get Future-proofing to avoid mistakenly introducing another race down the line. diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 7c290bf9d..054796e4b 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -45,6 +45,8 @@ h2_send_get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); Lck_AssertHeld(&h2->sess->mtx); + if (&wrk->cond == h2->cond) + ASSERT_RXTHR(h2); r2->wrk = wrk; VTAILQ_INSERT_TAIL(&h2->txqueue, r2, tx_list); while (VTAILQ_FIRST(&h2->txqueue) != r2) From daghf at varnish-software.com Wed Jun 27 11:41:12 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 27 Jun 2018 11:41:12 +0000 (UTC) Subject: [master] 51e6ded88 Use a separate condvar for connection-level flow control updates Message-ID: <20180627114112.368D863C17@lists.varnish-cache.org> commit 51e6ded883b7655091bd6a8ffb8869fa7d40b071 Author: Dag Haavi Finstad Date: Wed Jun 27 13:27:34 2018 +0200 Use a separate condvar for connection-level flow control updates The current flow control code's use of h2->cond is racy. h2->cond is already used for handing over a DATA frame to a stream thread. In the event that we have both streams waiting on this condvar for window updates and at the same time the rxthread gets signaled for a DATA frame, we could end up waking up the wrong thread and the rxthread gets stuck forever. This commit addresses this by using a separate condvar for window updates. An alternative would be to always issue a broadcast on h2->cond instead of signal, but I found this approach much cleaner. Probably fixes: #2623 diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index a1d6f849b..fbc190535 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -145,6 +145,7 @@ struct h2_sess { pthread_t rxthr; struct h2_req *mailcall; pthread_cond_t *cond; + pthread_cond_t winupd_cond[1]; struct sess *sess; int refcnt; diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 9dede792e..e5106e8a4 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -369,7 +369,7 @@ h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) Lck_Lock(&h2->sess->mtx); r2->t_window += wu; if (r2 == h2->req0) - AZ(pthread_cond_broadcast(h2->cond)); + AZ(pthread_cond_broadcast(h2->winupd_cond)); else if (r2->cond != NULL) AZ(pthread_cond_signal(r2->cond)); Lck_Unlock(&h2->sess->mtx); diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index cfbb51be7..7c290bf9d 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -213,7 +213,7 @@ h2_do_window(struct worker *wrk, struct h2_req *r2, r2->cond = NULL; } while (h2->req0->t_window <= 0 && h2_errcheck(r2, h2) == 0) { - AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0)); + AZ(Lck_CondWait(h2->winupd_cond, &h2->sess->mtx, 0)); } if (h2_errcheck(r2, h2) == 0) { diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index b350f05c3..c0545265c 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -123,6 +123,7 @@ h2_init_sess(const struct worker *wrk, struct sess *sp, h2->htc->rfd = &sp->fd; h2->sess = sp; h2->rxthr = pthread_self(); + AZ(pthread_cond_init(h2->winupd_cond, NULL)); VTAILQ_INIT(&h2->streams); VTAILQ_INIT(&h2->txqueue); h2_local_settings(&h2->local_settings); @@ -150,6 +151,7 @@ h2_del_sess(struct worker *wrk, struct h2_sess *h2, enum sess_close reason) assert(VTAILQ_EMPTY(&h2->streams)); VHT_Fini(h2->dectbl); + AZ(pthread_cond_destroy(h2->winupd_cond)); req = h2->srq; AZ(req->ws->r); sp = h2->sess; @@ -404,7 +406,7 @@ h2_new_session(struct worker *wrk, void *arg) if (r2->cond != NULL) AZ(pthread_cond_signal(r2->cond)); } - AZ(pthread_cond_broadcast(h2->cond)); + AZ(pthread_cond_broadcast(h2->winupd_cond)); Lck_Unlock(&h2->sess->mtx); while (1) { again = 0; From nils.goroll at uplex.de Thu Jun 28 08:12:05 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jun 2018 08:12:05 +0000 (UTC) Subject: [master] 73ad1590c tell the truth about vmod filenames Message-ID: <20180628081205.DD327A34AB@lists.varnish-cache.org> commit 73ad1590c44f88fee308b32ae4768cc707fec078 Author: Nils Goroll Date: Wed Jun 27 19:52:39 2018 +0200 tell the truth about vmod filenames diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index 4e11ce5fe..ffd039ebc 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -93,7 +93,8 @@ VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, void *ptr, int len, const char *nm, dlhdl = dlopen(backup, RTLD_NOW | RTLD_LOCAL); if (dlhdl == NULL) { - VSB_printf(ctx->msg, "Loading vmod %s from %s:\n", nm, backup); + VSB_printf(ctx->msg, "Loading vmod %s from %s (%s):\n", + nm, backup, path); VSB_printf(ctx->msg, "dlopen() failed: %s\n", dlerror()); return (1); } @@ -113,8 +114,8 @@ VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, void *ptr, int len, const char *nm, if (d == NULL || d->file_id == NULL || strcmp(d->file_id, file_id)) { - VSB_printf(ctx->msg, - "Loading vmod %s from %s:\n", nm, path); + VSB_printf(ctx->msg, "Loading vmod %s from %s (%s):\n", + nm, backup, path); VSB_printf(ctx->msg, "This is no longer the same file seen by" " the VCL-compiler.\n"); @@ -129,8 +130,8 @@ VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, void *ptr, int len, const char *nm, d->func_len <= 0 || d->proto == NULL || d->json == NULL) { - VSB_printf(ctx->msg, - "Loading VMOD %s from %s:\n", nm, path); + VSB_printf(ctx->msg, "Loading vmod %s from %s (%s):\n", + nm, backup, path); VSB_printf(ctx->msg, "VMOD data is mangled.\n"); (void)dlclose(v->hdl); FREE_OBJ(v); From nils.goroll at uplex.de Thu Jun 28 12:29:07 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jun 2018 12:29:07 +0000 (UTC) Subject: [master] 4c86d54f5 Get the probe's VTP reference in the probe code Message-ID: <20180628122907.8A336A7B91@lists.varnish-cache.org> commit 4c86d54f5e5be4bd19ae279057c53bf043757a98 Author: Nils Goroll Date: Thu Jun 28 14:27:28 2018 +0200 Get the probe's VTP reference in the probe code ... not in the caller diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 4f542f548..f50928cb2 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -550,10 +550,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, vrt->path, vbe_proto_ident); Lck_Unlock(&backends_mtx); - if (vbp != NULL) { - VTP_AddRef(be->tcp_pool); + if (vbp != NULL) VBP_Insert(be, vbp, be->tcp_pool); - } return (be->director); } diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index a5e67c021..d8337216b 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -663,6 +663,7 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp, XXXAN(vt); vt->tcp_pool = tp; + VTP_AddRef(vt->tcp_pool); vt->backend = b; b->probe = vt; From nils.goroll at uplex.de Thu Jun 28 14:24:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jun 2018 14:24:08 +0000 (UTC) Subject: [master] e9eb047b4 vbp_update_backend: do not assume a director Message-ID: <20180628142408.73FF24E3A@lists.varnish-cache.org> commit e9eb047b4be4dd92a6bbe425ceaef4461bb12117 Author: Nils Goroll Date: Thu Jun 28 15:31:28 2018 +0200 vbp_update_backend: do not assume a director To get out of a catch-22, see second next commit diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index d8337216b..b5a665a9d 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -153,42 +153,50 @@ vbp_has_poked(struct vbp_target *vt) static void vbp_update_backend(struct vbp_target *vt) { - unsigned i; + unsigned i = 0; char bits[10]; const char *logmsg; CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); Lck_Lock(&vbp_mtx); - if (vt->backend != NULL) { - i = 0; + if (vt->backend == NULL) { + Lck_Unlock(&vbp_mtx); + return; + } + #define BITMAP(n, c, t, b) \ - bits[i++] = (vt->n & 1) ? c : '-'; + bits[i++] = (vt->n & 1) ? c : '-'; #include "tbl/backend_poll.h" - bits[i] = '\0'; - assert(i < sizeof bits); - - if (vt->good >= vt->threshold) { - if (vt->backend->director->sick) { - logmsg = "Back healthy"; - VRT_SetHealth(vt->backend->director, 1); - } else { - logmsg = "Still healthy"; - } + bits[i] = '\0'; + assert(i < sizeof bits); + + if (vt->backend->director == NULL) { + Lck_Unlock(&vbp_mtx); + return; + } + + if (vt->good >= vt->threshold) { + if (vt->backend->director->sick) { + logmsg = "Back healthy"; + VRT_SetHealth(vt->backend->director, 1); } else { - if (vt->backend->director->sick) { - logmsg = "Still sick"; - } else { - logmsg = "Went sick"; - VRT_SetHealth(vt->backend->director, 0); - } + logmsg = "Still healthy"; + } + } else { + if (vt->backend->director->sick) { + logmsg = "Still sick"; + } else { + logmsg = "Went sick"; + VRT_SetHealth(vt->backend->director, 0); } - VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", - vt->backend->director->vcl_name, logmsg, bits, - vt->good, vt->threshold, vt->window, - vt->last, vt->avg, vt->resp_buf); - VBE_SetHappy(vt->backend, vt->happy); } + VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s", + vt->backend->director->vcl_name, logmsg, bits, + vt->good, vt->threshold, vt->window, + vt->last, vt->avg, vt->resp_buf); + VBE_SetHappy(vt->backend, vt->happy); + Lck_Unlock(&vbp_mtx); } From nils.goroll at uplex.de Thu Jun 28 14:24:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jun 2018 14:24:08 +0000 (UTC) Subject: [master] ee5b6458f VRT_new_backend_clustered: swap success and error handling Message-ID: <20180628142408.8BF004E3D@lists.varnish-cache.org> commit ee5b6458f04d977e9489f5142c0f36a78d0e2e63 Author: Nils Goroll Date: Thu Jun 28 16:16:53 2018 +0200 VRT_new_backend_clustered: swap success and error handling ... to make the next commit more readable diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index f50928cb2..c56081332 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -527,33 +527,32 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, be->director = VRT_AddDirector(ctx, vbe_methods, be, "%s", vrt->vcl_name); - if (be->director == NULL) { - VSC_vbe_Destroy(&be->vsc_seg); + if (be->director != NULL) { + vbp = vrt->probe; + if (vbp == NULL) + vbp = VCL_DefaultProbe(vcl); + + Lck_Lock(&backends_mtx); + VTAILQ_INSERT_TAIL(&backends, be, list); + VSC_C_main->n_backend++; + be->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr, + vrt->path, vbe_proto_ident); + Lck_Unlock(&backends_mtx); + + if (vbp != NULL) + VBP_Insert(be, vbp, be->tcp_pool); + + return (be->director); + } + VSC_vbe_Destroy(&be->vsc_seg); #define DA(x) do { if (be->x != NULL) free(be->x); } while (0) #define DN(x) /**/ VRT_BACKEND_HANDLE(); #undef DA #undef DN - Lck_Delete(&be->mtx); - FREE_OBJ(be); - return (NULL); - } - - vbp = vrt->probe; - if (vbp == NULL) - vbp = VCL_DefaultProbe(vcl); - - Lck_Lock(&backends_mtx); - VTAILQ_INSERT_TAIL(&backends, be, list); - VSC_C_main->n_backend++; - be->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr, - vrt->path, vbe_proto_ident); - Lck_Unlock(&backends_mtx); - - if (vbp != NULL) - VBP_Insert(be, vbp, be->tcp_pool); - - return (be->director); + Lck_Delete(&be->mtx); + FREE_OBJ(be); + return (NULL); } VCL_BACKEND v_matchproto_() From nils.goroll at uplex.de Thu Jun 28 14:24:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jun 2018 14:24:08 +0000 (UTC) Subject: [master] d912ffe4f fix probe not started for already warm vcl Message-ID: <20180628142408.AEBF64E40@lists.varnish-cache.org> commit d912ffe4f7a5e643275699c94b2722d486fd9e7f Author: Nils Goroll Date: Thu Jun 28 15:57:29 2018 +0200 fix probe not started for already warm vcl For (real) backends, we used to assign the probe after adding the director via VRT_AddDirector(). For an already warm vcl, this lead to a VCL_WARM event missed because the event was issued by VRT_AddDirector() when the probe did not exist yet. We fix this by preparing all of the backend before adding the director, undoing the work for a failure doing do. In addition, if adding a backend while the vcl is cold, we need to set the initial director state based on the probe configuration after adding the director. We avoid the overhead of the vcl temperature lock because no harm is done: If, after the director was added, the temperature goes cold, we just run an additional update, and if it goes warm, the update will already have happened. Likely related to https://github.com/nigoroll/libvmod-dynamic/issues/35 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index c56081332..9f08c2385 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -43,6 +43,7 @@ #include "cache_backend.h" #include "cache_tcp_pool.h" #include "cache_transport.h" +#include "cache_vcl.h" #include "http1/cache_http1.h" #include "VSC_vbe.h" @@ -525,25 +526,38 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); AN(be->vsc); + be->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr, + vrt->path, vbe_proto_ident); + AN(be->tcp_pool); + + vbp = vrt->probe; + if (vbp == NULL) + vbp = VCL_DefaultProbe(vcl); + + if (vbp != NULL) + VBP_Insert(be, vbp, be->tcp_pool); + be->director = VRT_AddDirector(ctx, vbe_methods, be, "%s", vrt->vcl_name); + if (be->director != NULL) { - vbp = vrt->probe; - if (vbp == NULL) - vbp = VCL_DefaultProbe(vcl); + /* for cold VCL, update initial director state */ + if (be->probe != NULL && ! VCL_WARM(vcl)) + VBP_Update_Backend(be->probe); Lck_Lock(&backends_mtx); VTAILQ_INSERT_TAIL(&backends, be, list); VSC_C_main->n_backend++; - be->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr, - vrt->path, vbe_proto_ident); Lck_Unlock(&backends_mtx); - - if (vbp != NULL) - VBP_Insert(be, vbp, be->tcp_pool); - return (be->director); } + + /* undo */ + if (vbp != NULL) + VBP_Remove(be); + + VTP_Rel(&be->tcp_pool); + VSC_vbe_Destroy(&be->vsc_seg); #define DA(x) do { if (be->x != NULL) free(be->x); } while (0) #define DN(x) /**/ diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 706fea428..f5c016860 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -77,6 +77,7 @@ struct backend { void VBE_SetHappy(const struct backend *, uint64_t); /* cache_backend_probe.c */ +void VBP_Update_Backend(struct vbp_target *vt); void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, struct tcp_pool *); void VBP_Remove(struct backend *b); diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index b5a665a9d..e63cd02c4 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -150,8 +150,8 @@ vbp_has_poked(struct vbp_target *vt) vt->good = j; } -static void -vbp_update_backend(struct vbp_target *vt) +void +VBP_Update_Backend(struct vbp_target *vt) { unsigned i = 0; char bits[10]; @@ -433,7 +433,7 @@ vbp_task(struct worker *wrk, void *priv) vbp_start_poke(vt); vbp_poke(vt); vbp_has_poked(vt); - vbp_update_backend(vt); + VBP_Update_Backend(vt); Lck_Lock(&vbp_mtx); if (vt->running < 0) { @@ -637,7 +637,7 @@ VBP_Control(const struct backend *be, int enable) CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); vbp_reset(vt); - vbp_update_backend(vt); + VBP_Update_Backend(vt); Lck_Lock(&vbp_mtx); if (enable) { @@ -679,7 +679,7 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp, vbp_build_req(vt, vp, b); vbp_reset(vt); - vbp_update_backend(vt); + VBP_Update_Backend(vt); } void From daghf at varnish-software.com Fri Jun 29 14:22:07 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Fri, 29 Jun 2018 14:22:07 +0000 (UTC) Subject: [master] 952115fdb Simplify memory handling around struct h2h_decode Message-ID: <20180629142207.D7593A06C5@lists.varnish-cache.org> commit 952115fdb02ef34f17e877686f818e24c385687f Author: Dag Haavi Finstad Date: Fri Jun 29 15:04:11 2018 +0200 Simplify memory handling around struct h2h_decode Since we verify that header blocks are not interleaved, and we zero the struct h2h_decode on every new header block, there is no need to malloc a separate struct h2h_decode per stream. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index fbc190535..792fd985a 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -127,7 +127,6 @@ struct h2_req { VTAILQ_ENTRY(h2_req) list; int64_t t_window; int64_t r_window; - struct h2h_decode *decode; /* Where to wake this stream up */ struct worker *wrk; @@ -161,6 +160,7 @@ struct h2_sess { struct ws *ws; struct http_conn *htc; struct vsl_log *vsl; + struct h2h_decode *decode; struct vht_table dectbl[1]; unsigned rxf_len; @@ -208,10 +208,10 @@ struct h2h_decode { struct vhd_decode vhd[1]; }; -void h2h_decode_init(const struct h2_sess *h2, struct h2h_decode *d); -h2_error h2h_decode_fini(const struct h2_sess *h2, struct h2h_decode *d); -h2_error h2h_decode_bytes(struct h2_sess *h2, struct h2h_decode *d, - const uint8_t *ptr, size_t len); +void h2h_decode_init(const struct h2_sess *h2); +h2_error h2h_decode_fini(const struct h2_sess *h2); +h2_error h2h_decode_bytes(struct h2_sess *h2, const uint8_t *ptr, + size_t len); /* cache_http2_send.c */ void H2_Send_Get(struct worker *, struct h2_sess *, struct h2_req *); diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 3d7c2d00b..0524f3e32 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -164,13 +164,15 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) } void -h2h_decode_init(const struct h2_sess *h2, struct h2h_decode *d) +h2h_decode_init(const struct h2_sess *h2) { + struct h2h_decode *d; CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); CHECK_OBJ_NOTNULL(h2->new_req, REQ_MAGIC); CHECK_OBJ_NOTNULL(h2->new_req->http, HTTP_MAGIC); - AN(d); + AN(h2->decode); + d = h2->decode; INIT_OBJ(d, H2H_DECODE_MAGIC); VHD_Init(d->vhd); d->out_l = WS_Reserve(h2->new_req->http->ws, 0); @@ -189,11 +191,13 @@ h2h_decode_init(const struct h2_sess *h2, struct h2h_decode *d) * is a stream level error. */ h2_error -h2h_decode_fini(const struct h2_sess *h2, struct h2h_decode *d) +h2h_decode_fini(const struct h2_sess *h2) { h2_error ret; + struct h2h_decode *d; CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + d = h2->decode; CHECK_OBJ_NOTNULL(h2->new_req, REQ_MAGIC); CHECK_OBJ_NOTNULL(d, H2H_DECODE_MAGIC); WS_ReleaseP(h2->new_req->http->ws, d->out); @@ -217,10 +221,10 @@ h2h_decode_fini(const struct h2_sess *h2, struct h2h_decode *d) * H2E_PROTOCOL_ERROR: Malformed header or duplicate pseudo-header. */ h2_error -h2h_decode_bytes(struct h2_sess *h2, struct h2h_decode *d, - const uint8_t *in, size_t in_l) +h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) { struct http *hp; + struct h2h_decode *d; size_t in_u = 0; CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); @@ -229,6 +233,7 @@ h2h_decode_bytes(struct h2_sess *h2, struct h2h_decode *d, CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); CHECK_OBJ_NOTNULL(hp->ws, WS_MAGIC); AN(hp->ws->r); + d = h2->decode; CHECK_OBJ_NOTNULL(d, H2H_DECODE_MAGIC); /* Only H2E_ENHANCE_YOUR_CALM indicates that we should continue diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index e5106e8a4..f4cc294e5 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -225,10 +225,8 @@ h2_kill_req(struct worker *wrk, const struct h2_sess *h2, AZ(pthread_cond_signal(r2->cond)); r2 = NULL; } else { - if (r2->state == H2_S_OPEN && r2->decode != NULL) { - (void)h2h_decode_fini(h2, r2->decode); - FREE_OBJ(r2->decode); - } + if (r2->state == H2_S_OPEN && h2->new_req == r2->req) + (void)h2h_decode_fini(h2); } Lck_Unlock(&h2->sess->mtx); if (r2 != NULL) @@ -557,8 +555,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); - h2e = h2h_decode_fini(h2, r2->decode); - FREE_OBJ(r2->decode); + h2e = h2h_decode_fini(h2); h2->new_req = NULL; if (r2->req->req_body_status == REQ_BODY_NONE) { /* REQ_BODY_NONE implies one of the frames in the @@ -650,9 +647,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod); http_SetH(req->http, HTTP_HDR_PROTO, "HTTP/2.0"); - ALLOC_OBJ(r2->decode, H2H_DECODE_MAGIC); - AN(r2->decode); - h2h_decode_init(h2, r2->decode); + h2h_decode_init(h2); p = h2->rxf_data; l = h2->rxf_len; @@ -668,13 +663,12 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) l -= 5; p += 5; } - h2e = h2h_decode_bytes(h2, r2->decode, p, l); + h2e = h2h_decode_bytes(h2, p, l); if (h2e != NULL) { Lck_Lock(&h2->sess->mtx); VSLb(h2->vsl, SLT_Debug, "HPACK(hdr) %s", h2e->name); Lck_Unlock(&h2->sess->mtx); - (void)h2h_decode_fini(h2, r2->decode); - FREE_OBJ(r2->decode); + (void)h2h_decode_fini(h2); AZ(r2->req->ws->r); h2_del_req(wrk, r2); return (h2e); @@ -700,14 +694,13 @@ h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) if (r2 == NULL || r2->state != H2_S_OPEN || r2->req != h2->new_req) return (H2CE_PROTOCOL_ERROR); // XXX spec ? req = r2->req; - h2e = h2h_decode_bytes(h2, r2->decode, h2->rxf_data, h2->rxf_len); + h2e = h2h_decode_bytes(h2, h2->rxf_data, h2->rxf_len); r2->req->acct.req_hdrbytes += h2->rxf_len; if (h2e != NULL) { Lck_Lock(&h2->sess->mtx); VSLb(h2->vsl, SLT_Debug, "HPACK(cont) %s", h2e->name); Lck_Unlock(&h2->sess->mtx); - (void)h2h_decode_fini(h2, r2->decode); - FREE_OBJ(r2->decode); + (void)h2h_decode_fini(h2); AZ(r2->req->ws->r); h2_del_req(wrk, r2); return (h2e); diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index c0545265c..40a111754 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -97,7 +97,7 @@ h2_local_settings(struct h2_settings *h2s) static struct h2_sess * h2_init_sess(const struct worker *wrk, struct sess *sp, - struct h2_sess *h2s, struct req *srq) + struct h2_sess *h2s, struct req *srq, struct h2h_decode *decode) { uintptr_t *up; struct h2_sess *h2; @@ -128,6 +128,7 @@ h2_init_sess(const struct worker *wrk, struct sess *sp, VTAILQ_INIT(&h2->txqueue); h2_local_settings(&h2->local_settings); h2->remote_settings = H2_proto_settings; + h2->decode = decode; AZ(VHT_Init(h2->dectbl, h2->local_settings.header_table_size)); @@ -338,6 +339,7 @@ h2_new_session(struct worker *wrk, void *arg) struct h2_req *r2, *r22; int again; uint8_t settings[48]; + struct h2h_decode decode; size_t l; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -350,7 +352,7 @@ h2_new_session(struct worker *wrk, void *arg) assert (req->err_code == H2_PU_MARKER || req->err_code == H2_OU_MARKER); h2 = h2_init_sess(wrk, sp, &h2s, - req->err_code == H2_PU_MARKER ? req : NULL); + req->err_code == H2_PU_MARKER ? req : NULL, &decode); h2->req0 = h2_new_req(wrk, h2, 0, NULL); AZ(h2->htc->priv); h2->htc->priv = h2;